Building an AI-Powered Video Segmentation Toolkit

# Core dependencies
pandas>=1.5.0
pydantic>=2.0.0
tiktoken>=0.5.0

# LLM providers (install one or both)
openai>=1.0.0
anthropic>=0.18.0

# Web application (for annotation tool)
flask>=3.0.0
flask-cors>=4.0.0

# Optional: video probing
# ffprobe (from ffmpeg) - install via system package manager

# OpenAI (for GPT-4o, o1, o3 models)
OPENAI_API_KEY=sk-your-key-here

# Anthropic (for Claude models)
ANTHROPIC_API_KEY=sk-ant-your-key-here

# Or use a local/custom endpoint
OPENAI_BASE_URL=https://your-endpoint.com/v1

from typing import Dict, List, Optional
from pydantic import BaseModel, Field, validator
from enum import Enum

class SegmentType(str, Enum):
    """Valid segment types."""
    INTRODUCTION = "introduction"
    CONCEPT_EXPLANATION = "concept_explanation"
    EXAMPLE = "example"
    DEEP_REASONING = "deep_reasoning"
    COMMON_MISTAKES = "common_mistakes"
    PRACTICE_PROBLEM = "practice_problem"
    REAL_WORLD_APPLICATION = "real_world_application"
    SUMMARY = "summary"
    Q_AND_A = "q_and_a"
    TRANSITION = "transition"

class DifficultyLevel(str, Enum):
    """Valid difficulty levels."""
    EASY = "Easy"
    MEDIUM = "Medium"
    HARD = "Hard"

class TimeRange(BaseModel):
    """Represents a time range in HH:MM:SS,mmm format."""
    start: str = Field(..., pattern=r'^\d{2}:\d{2}:\d{2}[,;]\d{2,3}$')
    end: str = Field(..., pattern=r'^\d{2}:\d{2}:\d{2}[,;]\d{2,3}$')

class SegmentSchema(BaseModel):
    """Schema for a single segment."""
    time_range: TimeRange
    segment_type: SegmentType
    title: str = Field(..., min_length=1, max_length=200)
    description: str = Field(..., min_length=1, max_length=500)
    key_concepts: List[str] = Field(default_factory=list, max_items=10)
    learning_objectives: List[str] = Field(default_factory=list, max_items=5)
    prerequisites: List[str] = Field(default_factory=list, max_items=5)
    difficulty: DifficultyLevel = Field(default=DifficultyLevel.MEDIUM)
    engagement_tips: List[str] = Field(default_factory=list, max_items=5)
    microlecture_suitable: bool = Field(default=False)
    
    class Config:
        use_enum_values = True

class SegmentationAnalysis(BaseModel):
    """Complete schema for segmentation analysis output."""
    overview: LectureOverview
    segments: List[SegmentSchema] = Field(..., min_items=1)
    interactive_opportunities: List[InteractiveOpportunity] = Field(default_factory=list)
    microlecture_recommendations: List[MicrolectureRecommendation] = Field(default_factory=list)

def tc_to_seconds(tc: str, fps: float = 30.0) -> float:
    """Convert timecode to seconds - handles all formats."""
    try:
        if ',' in tc:  # Millisecond format: HH:MM:SS,mmm
            h, m, rest = tc.split(':')
            s, ms = rest.split(',')
            return int(h) * 3600 + int(m) * 60 + int(s) + int(ms) / 1000
        else:  # Frame format: HH:MM:SS:FF or HH:MM:SS;FF
            parts = re.split(r'[:;]', tc)
            if len(parts) == 4:
                h, m, s, ff = parts
                return int(h) * 3600 + int(m) * 60 + int(s) + int(ff) / fps
            else:
                h, m, s = parts
                return int(h) * 3600 + int(m) * 60 + int(s)
    except Exception as e:
        logger.error(f"Error parsing timecode '{tc}': {e}")
        return 0.0

def seconds_to_tc(sec: float, fps: float = 30.0, mode: str = "ms") -> str:
    """Convert seconds to timecode string."""
    h, rem = divmod(sec, 3600)
    m, s_full = divmod(rem, 60)
    s = int(s_full)
    
    if mode == "frames":
        frames = int(round((s_full - s) * fps))
        return f"{int(h):02d}:{int(m):02d}:{int(s):02d}:{frames:02d}"
    else:  # milliseconds
        ms = int((sec - int(sec)) * 1000)
        return f"{int(h):02d}:{int(m):02d}:{int(s):02d},{ms:03d}"

def enhance_transcript_with_timestamps(srt_text: str) -> Tuple[str, List]:
    """
    Add timestamp markers to transcript for AI reference.
    Returns: (enhanced_text, timestamp_mapping)
    """
    enhanced_lines = []
    timestamp_mapping = []
    
    # Split into caption blocks
    blocks = re.split(r'\n\n+', srt_text.strip())
    
    for block in blocks:
        lines = block.strip().split('\n')
        if len(lines) < 3:
            continue
        
        # Extract timestamp
        timestamp_match = TIMESTAMP_RE.search(lines[1])
        if timestamp_match:
            start_tc, end_tc = timestamp_match.groups()
            
            # Get caption text (lines after timestamp)
            text = ' '.join(lines[2:])
            
            # Add to enhanced transcript with timestamp marker
            enhanced_lines.append(f"[{start_tc}] {text}")
            timestamp_mapping.append((start_tc, end_tc, text))
    
    return '\n'.join(enhanced_lines), timestamp_mapping

python estimate_limit.py --srt lecture_42.srt --model gpt-4o

# Output:
# Transcript characters : 45,230
# Exact tokens (gpt-4o) : 12,847
# ---------------------------------------------------------
# Recommended config.token_limit  : 19,764  (largest safe model window)
# Minimum-for-current heuristic   : 12,848  (fits transcript + 1-token reply)

MODEL_CONFIGS = {
    'gpt-4o': {
        'temperature': 0.1,      # Low temp for consistent output
        'max_tokens': 4000,
        'token_limit': 128000,
        'provider': 'openai'
    },
    'claude-sonnet-4': {
        'temperature': 0.1,
        'max_tokens': 4000,
        'token_limit': 200000,
        'provider': 'claude'
    },
    'o3': {
        'temperature': None,     # O-series ignores temperature
        'max_completion_tokens': 25000,
        'token_limit': 200000,
        'provider': 'openai',
        'reasoning_effort': 'high'
    }
}

You are an expert educational content analyst specializing in 
pedagogical segmentation of lecture videos.

## CRITICAL TIMESTAMP RULES
1. ONLY use timestamps that appear in [HH:MM:SS,mmm] markers in the transcript
2. Segment boundaries MUST align with actual caption timestamps
3. DO NOT interpolate or invent timestamps
4. If uncertain about a boundary, use the nearest visible timestamp

## Segment Types
- introduction: Topic setup and learning objectives
- concept_explanation: Core theory and definitions
- example: Worked problems and demonstrations
- deep_reasoning: Intuition building and "why" explanations
- common_mistakes: Pitfalls and misconceptions
- practice_problem: Student exercises
- real_world_application: Practical applications
- summary: Key takeaways and review
- q_and_a: Student questions
- transition: Topic changes and administrative content

## Output Format
Return valid JSON matching this schema:
{
  "overview": {
    "learning_objectives": ["..."],
    "prerequisites": ["..."],
    "key_takeaways": ["..."]
  },
  "segments": [
    {
      "time_range": {"start": "HH:MM:SS,mmm", "end": "HH:MM:SS,mmm"},
      "segment_type": "concept_explanation",
      "title": "Descriptive Title",
      "description": "1-2 sentence summary",
      "key_concepts": ["concept1", "concept2"],
      "difficulty": "Easy|Medium|Hard"
    }
  ]
}

Video File,YouTube Video ID,Status
"/path/to/STAT 350 - Chapter 6.3.1 Intro.mp4",dQw4w9WgXcQ,UPLOAD SUCCESS
"/path/to/STAT 350 - Chapter 6.3.2 Examples.mp4",9bZkp7q19f0,UPLOAD SUCCESS
"/path/to/STAT 350 - Chapter 7.1 Overview.mp4",kJQP7kiw5Fk,UPLOAD SUCCESS

python create_youtube_mapping_smart.py \
    --log-file youtube_upload_log.csv \
    --json-dir segmentation_reports/json \
    --output youtube_mapping.json \
    --threshold 0.7

# Output files:
# - youtube_mapping.json           (primary mapping)
# - youtube_mapping_detailed.csv   (for human review)
# - unmatched_videos.csv           (failed matches)

{
  "1": "dQw4w9WgXcQ",
  "2": "9bZkp7q19f0",
  "3": "kJQP7kiw5Fk",
  "38": "abc123xyz",
  "39": "def456uvw"
}

python generate_segmentation_report_youtube.py \
    --json-dir segmentation_reports/json \
    --youtube-mapping youtube_mapping.json \
    --out video_viewer.html

// Embedded in generated HTML
window.segmentData = {
    "1": {
        "lecture_index": 1,
        "lecture_title": "Chapter 1.1: Introduction to Statistics",
        "total_duration": 1847.5,
        "segments": [
            {
                "start_time": 0,
                "end_time": 185.2,
                "start_tc": "00:00:00,000",
                "end_tc": "00:03:05,200",
                "segment_type": "introduction",
                "title": "Course Overview",
                "description": "Introduction to the course...",
                "key_concepts": ["statistics", "data analysis"],
                "difficulty_level": "Easy"
            }
            // ... more segments
        ]
    }
};

const YOUTUBE_MAPPING = {
    "1": "dQw4w9WgXcQ",
    "2": "9bZkp7q19f0"
};

# Start the Flask server
python lecture_segment_annotator.py

# Server runs at http://localhost:5005
# Open in browser to begin annotation

python fix_json_line_breaks.py --json-dir segmentation_reports/json

# Fixes:
# - Removes \n, \r, \t sequences
# - Cleans HTML entities (<br/>)
# - Normalizes whitespace
# - Creates .bak backups

// Before (incorrectly split)
"prerequisites": [
    "High",
    "school algebra",
    "Basic probability"
]

// After (merged)
"prerequisites": [
    "High-school algebra",
    "Basic probability"
]

# Preview changes
python fix_lecture_indices.py --json-dir output/json --dry-run

# Apply changes
python fix_lecture_indices.py --json-dir output/json

# File: 038_STAT_350_Chapter_7_segments.json
# Before: lecture_index: 1
# After:  lecture_index: 38

python rebuild_course_context.py \
    --input-dir segmentation_analysis \
    --verify \
    --enhanced-summary

# Outputs:
# - course_context.json (concept tracking)
# - course_summary.md (human-readable overview)

# Create a docs folder for GitHub Pages
mkdir -p docs
cp video_viewer.html docs/index.html

# Commit and push
git add docs/
git commit -m "Deploy video viewer"
git push

# Enable GitHub Pages in repository settings
# Source: Deploy from branch → main → /docs

# Python 3
python -m http.server 8000

# Then open http://localhost:8000/video_viewer.html

def export_segmentation_edl(segmentation, output_path, video_title, fps):
    """Export segmentation as EDL file for Adobe Premiere Pro."""
    with open(output_path, 'w') as f:
        f.write(f"TITLE: {video_title}_segments\n")
        f.write("FCM: NON-DROP FRAME\n\n")
        
        for i, segment in enumerate(segmentation.segments, 1):
            start_tc = seconds_to_tc(segment.start_time, fps, 'frames')
            end_tc = seconds_to_tc(segment.end_time, fps, 'frames')
            
            f.write(f"{i:03d}  AX       V     C        ")
            f.write(f"{start_tc} {end_tc} {start_tc} {end_tc}\n")
            f.write(f"* COMMENT: TYPE={segment.segment_type} | {segment.title}\n\n")