transcript_fetcher.py

#!/usr/bin/env python3
"""
YouTube Channel Transcript Fetcher

This script fetches transcripts for all videos from a YouTube channel
within a specified date range and saves them to a JSON file with rate limiting.
"""

import json
import sys
import os
import time
import random
import argparse
import logging
from datetime import datetime
from typing import List, Dict, Optional, Tuple
from urllib.parse import urlparse, parse_qs

try:
    from youtube_transcript_api import YouTubeTranscriptApi
    import yt_dlp
    from tqdm import tqdm
except ImportError:
    print("Error: Required packages not installed.")
    print("Please install them using: pip install youtube-transcript-api yt-dlp tqdm")
    sys.exit(1)


# Configure logging
def setup_logging(verbose: bool = False, quiet: bool = False):
    """Setup logging configuration based on verbosity level."""
    if quiet:
        level = logging.ERROR
        log_format = '%(message)s'
    elif verbose:
        level = logging.DEBUG
        log_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
    else:
        level = logging.INFO
        log_format = '%(levelname)s: %(message)s'
    
    # Force output to stream immediately
    logging.basicConfig(
        level=level,
        format=log_format,
        datefmt='%Y-%m-%d %H:%M:%S',
        force=True,
        stream=sys.stdout
    )
    logger = logging.getLogger(__name__)
    # Ensure handlers flush immediately
    for handler in logger.handlers:
        handler.flush()
    return logger


def parse_date(date_str: str) -> datetime:
    """Parse date string in YYYY-MM-DD format."""
    try:
        return datetime.strptime(date_str, '%Y-%m-%d')
    except ValueError:
        raise ValueError(f"Invalid date format: {date_str}. Expected YYYY-MM-DD")


def normalize_channel_url(channel_url: str) -> str:
    """Normalize various YouTube channel URL formats to a standard format."""
    # Handle channel ID format: @channelname or channel_id
    if channel_url.startswith('@'):
        return f"https://www.youtube.com/{channel_url}"
    
    # Handle channel ID directly
    if channel_url.startswith('UC') and len(channel_url) == 24:
        return f"https://www.youtube.com/channel/{channel_url}"
    
    # Handle custom URL: /c/name or /user/name
    if channel_url.startswith('/'):
        return f"https://www.youtube.com{channel_url}"
    
    # If it's already a full URL, return as is
    if channel_url.startswith('http'):
        return channel_url
    
    # Assume it's a handle
    return f"https://www.youtube.com/@{channel_url}"


def get_video_ids_from_channel(
    channel_url: str,
    start_date: Optional[datetime] = None,
    end_date: Optional[datetime] = None,
    logger: Optional[logging.Logger] = None,
    show_progress: bool = True,
    quiet: bool = False,
    output_file: Optional[str] = None
) -> List[Dict[str, any]]:
    """
    Extract all video IDs from a YouTube channel within a date range using yt-dlp.
    
    Returns list of dictionaries with 'id' and 'upload_date' keys.
    """
    if logger is None:
        logger = logging.getLogger(__name__)
    
    normalized_url = normalize_channel_url(channel_url)
    logger.info(f"Fetching video list from channel: {normalized_url}")
    logger.debug(f"Normalized URL: {normalized_url}")
    
    if start_date:
        logger.info(f"Start date: {start_date.strftime('%Y-%m-%d')}")
    if end_date:
        logger.info(f"End date: {end_date.strftime('%Y-%m-%d')}")
    
    videos = []
    
    try:
        # Step 1: Extract flat list first (fast, avoids rate limits)
        logger.info("Step 1: Fetching video list from YouTube...")
        logger.info("Connecting and fetching video list (this may take a moment)...")
        sys.stdout.flush()
        
        ydl_opts_flat = {
            'quiet': True,
            'no_warnings': True,
            'extract_flat': True,  # Fast flat extraction - just IDs
            'skip_download': True,
            'noplaylist': False,
        }
        
        with yt_dlp.YoutubeDL(ydl_opts_flat) as ydl:
            # Use channel URL with videos
            if '/videos' not in normalized_url and '/streams' not in normalized_url:
                channel_videos_url = normalized_url.rstrip('/') + '/videos'
            else:
                channel_videos_url = normalized_url
            
            logger.debug(f"Fetching videos from: {channel_videos_url}")
            sys.stdout.flush()
            
            # Extract info - this is the blocking call
            # Using extract_flat: True should be much faster (just IDs, no metadata)
            try:
                channel_info = ydl.extract_info(channel_videos_url, download=False)
            except Exception as e:
                logger.error(f"Error during extraction: {e}")
                logger.info("Trying alternative extraction method...")
                sys.stdout.flush()
                # Try without playlistend limit
                ydl_opts_flat.pop('playlistend', None)
                channel_info = ydl.extract_info(channel_videos_url, download=False)
            
            if 'entries' not in channel_info:
                logger.warning("No entries found in channel info")
                return videos
            
            entries = channel_info['entries']
            
            if not entries:
                logger.warning("No videos found in channel")
                return videos
            
            # Process and display entries immediately as we iterate
            logger.info("Videos found (listing as processed):")
            sys.stdout.flush()
            
            entries_list = []
            entry_count = 0
            for entry in entries:
                if entry is None:
                    continue
                
                entry_count += 1
                video_id = entry.get('id', 'N/A')
                # With extract_flat: True, we may not have titles, use URL or ID
                title = entry.get('title') or entry.get('url', str(video_id))[:70]
                
                # Show each video immediately as we iterate through the list
                if video_id and video_id != 'N/A':
                    logger.info(f"  {entry_count}. [{video_id}] {title}")
                    sys.stdout.flush()
                
                entries_list.append(entry)
            
            entries = entries_list
            total_entries = len(entries)
            
            logger.info(f"\n✓ Total videos found: {total_entries}")
            sys.stdout.flush()
        
        total_entries = len(entries)
        
        # Step 2: Only fetch upload dates if date filtering is needed
        video_details = {}
        
        # Load existing checkpoint for date fetching
        date_checkpoint = {}
        if output_file:
            date_checkpoint_file = output_file.replace('.json', '_dates_checkpoint.json')
            if os.path.exists(date_checkpoint_file):
                try:
                    with open(date_checkpoint_file, 'r', encoding='utf-8') as f:
                        date_checkpoint = json.load(f)
                    logger.info(f"Loaded {len(date_checkpoint)} video dates from checkpoint")
                except Exception as e:
                    logger.debug(f"Could not load date checkpoint: {e}")
        
        if start_date or end_date:
            # Need dates for filtering
            logger.info(f"\nStep 2: Fetching upload dates and titles for {total_entries} videos...")
            logger.info("Fetching without delays (may hit rate limits but much faster)")
            sys.stdout.flush()
            
            ydl_opts_detail = {
                'quiet': True,
                'no_warnings': True,
                'skip_download': True,
            }
            
            # Collect video IDs first
            video_ids = []
            for entry in entries:
                if entry and entry.get('id'):
                    video_ids.append(entry['id'])
            
            # Fetch details with delays to avoid rate limits
            with yt_dlp.YoutubeDL(ydl_opts_detail) as ydl:
                if show_progress and not quiet:
                    id_iter = tqdm(video_ids, desc="Fetching dates", unit="video", file=sys.stdout)
                else:
                    id_iter = video_ids
                
                for i, video_id in enumerate(id_iter):
                    # Skip if already in checkpoint AND has a valid date
                    # (retry if date fetch failed previously)
                    if video_id in date_checkpoint:
                        checkpoint_entry = date_checkpoint[video_id]
                        # Only skip if we have a valid date (not empty string)
                        if checkpoint_entry.get('upload_date') and checkpoint_entry.get('upload_date').strip():
                            video_details[video_id] = checkpoint_entry
                            if not quiet:
                                logger.info(f"  [{i+1}/{len(video_ids)}] (from checkpoint) {checkpoint_entry.get('upload_date', 'No date')} - {checkpoint_entry.get('title', 'Unknown')[:60]}")
                            continue
                        # If date was empty/failed, retry fetching it
                        if not quiet:
                            logger.info(f"  [{i+1}/{len(video_ids)}] (retrying failed date fetch) {video_id}")
                            sys.stdout.flush()
                    
                    try:
                        video_info = ydl.extract_info(f"https://www.youtube.com/watch?v={video_id}", download=False)
                        if video_info:
                            upload_date = video_info.get('upload_date', '')
                            title = video_info.get('title', 'Unknown')
                            video_details[video_id] = {
                                'upload_date': upload_date,
                                'title': title
                            }
                            # Update checkpoint
                            date_checkpoint[video_id] = video_details[video_id]
                            # Save checkpoint every 10 videos
                            if output_file and (i + 1) % 10 == 0:
                                date_checkpoint_file = output_file.replace('.json', '_dates_checkpoint.json')
                                try:
                                    with open(date_checkpoint_file, 'w', encoding='utf-8') as f:
                                        json.dump(date_checkpoint, f, indent=2, ensure_ascii=False)
                                except Exception:
                                    pass  # Don't fail if checkpoint save fails
                            
                            # Show each video as it's processed
                            if not quiet:
                                logger.info(f"  [{i+1}/{len(video_ids)}] {upload_date if upload_date else 'No date'} - {title[:60]}")
                                sys.stdout.flush()
                    except Exception as e:
                        logger.debug(f"Failed to fetch details for {video_id}: {e}")
                        # Only save to checkpoint if we have a title from a previous fetch
                        # Don't save failed fetches so they can be retried
                        if video_id in date_checkpoint and date_checkpoint[video_id].get('title'):
                            # Keep existing entry but mark date as failed
                            video_details[video_id] = date_checkpoint[video_id]
                        else:
                            # New failure - don't save to checkpoint so it retries
                            video_details[video_id] = {
                                'upload_date': '',
                                'title': 'Unknown'
                            }
            
            # Save final date checkpoint
            if output_file:
                date_checkpoint_file = output_file.replace('.json', '_dates_checkpoint.json')
                try:
                    with open(date_checkpoint_file, 'w', encoding='utf-8') as f:
                        json.dump(date_checkpoint, f, indent=2, ensure_ascii=False)
                except Exception as e:
                    logger.debug(f"Could not save date checkpoint: {e}")
            
            logger.info(f"\nStep 3: Filtering {total_entries} videos by date range...")
            sys.stdout.flush()
        else:
            # No date filtering needed - skip date fetching, use basic info from flat extraction
            logger.info(f"\nStep 2: Skipping date fetch (no date filters specified)")
            logger.info(f"Processing all {total_entries} videos...")
            sys.stdout.flush()
            
            # Create basic video_details from flat extraction
            for entry in entries:
                if entry and entry.get('id'):
                    video_id = entry.get('id')
                    video_details[video_id] = {
                        'upload_date': None,
                        'title': entry.get('title', entry.get('url', 'Unknown'))
                    }
        
        # Process and filter videos
        if show_progress and not quiet:
            entry_iter = tqdm(entries, desc="Filtering", unit="video", file=sys.stdout)
        else:
            entry_iter = entries if entries else []
        
        for entry in entry_iter:
            if not entry:
                continue
            
            video_id = entry.get('id')
            if not video_id:
                continue
            
            # Get upload date and title from fetched details
            details = video_details.get(video_id, {})
            upload_date_str = details.get('upload_date', '')
            title = details.get('title', entry.get('title', 'Unknown'))
            
            # If we have date filters, we need a valid date to filter
            if start_date or end_date:
                if not upload_date_str:
                    logger.debug(f"Skipping video {video_id} (no upload date)")
                    continue
                
                try:
                    # yt-dlp returns date as YYYYMMDD
                    upload_date = datetime.strptime(upload_date_str, '%Y%m%d')
                    
                    # Filter by date range (inclusive - compare dates only, not times)
                    if start_date and upload_date.date() < start_date.date():
                        logger.debug(f"Skipping video {video_id} (before start date: {upload_date_str})")
                        continue
                    if end_date and upload_date.date() > end_date.date():
                        logger.debug(f"Skipping video {video_id} (after end date: {upload_date_str})")
                        continue
                    
                    videos.append({
                        'id': video_id,
                        'upload_date': upload_date_str,
                        'title': title
                    })
                    logger.debug(f"Added video: {video_id} - {title} ({upload_date_str})")
                    if not quiet and len(videos) % 10 == 0:
                        logger.info(f"  Found {len(videos)} videos so far...")
                        sys.stdout.flush()
                except ValueError as e:
                    logger.debug(f"Skipping video {video_id} (date parsing failed: {upload_date_str}, error: {e})")
                    continue
            else:
                # No date filters - include all videos (with or without dates)
                videos.append({
                    'id': video_id,
                    'upload_date': upload_date_str if upload_date_str else None,
                    'title': title
                })
                logger.debug(f"Added video: {video_id} - {title} ({upload_date_str if upload_date_str else 'no date'})")
                if not quiet and len(videos) % 10 == 0:
                    logger.info(f"  Found {len(videos)} videos so far...")
                    sys.stdout.flush()
        
        logger.info(f"Total videos found in date range: {len(videos)}")
        sys.stdout.flush()
        return videos
            
    except Exception as e:
        logger.error(f"Error fetching channel videos: {e}", exc_info=logger.level == logging.DEBUG)
        logger.error("Note: Make sure the channel URL is correct and accessible.")
        sys.exit(1)


def load_checkpoint(output_file: str, logger: Optional[logging.Logger] = None) -> Dict:
    """
    Load existing checkpoint file if it exists.
    
    Returns:
        Dictionary with existing results, and set of video IDs that have successful transcripts
    """
    if logger is None:
        logger = logging.getLogger(__name__)
    
    try:
        if os.path.exists(output_file):
            logger.info(f"Loading existing checkpoint from {output_file}...")
            with open(output_file, 'r', encoding='utf-8') as f:
                existing = json.load(f)
            
            # Only count videos with successful transcripts
            successful_video_ids = set()
            all_video_ids = set()
            for entry in existing.get('transcripts', []):
                video_id = entry.get('video_id')
                if video_id:
                    all_video_ids.add(video_id)
                    # Only count as processed if transcript was successfully fetched
                    if entry.get('success') and entry.get('transcript_full_text'):
                        successful_video_ids.add(video_id)
            
            logger.info(f"Found {len(all_video_ids)} videos in checkpoint ({len(successful_video_ids)} with successful transcripts)")
            return existing, successful_video_ids
        else:
            logger.debug("No existing checkpoint file found")
            return None, set()
    except Exception as e:
        logger.warning(f"Error loading checkpoint: {e}. Starting fresh.")
        return None, set()


def save_checkpoint(results: Dict, output_file: str, logger: Optional[logging.Logger] = None) -> None:
    """
    Save current results to checkpoint file.
    
    Args:
        results: Results dictionary to save
        output_file: Output file path
        logger: Logger instance
    """
    if logger is None:
        logger = logging.getLogger(__name__)
    
    try:
        # Create temporary file first, then rename (atomic write)
        temp_file = output_file + '.tmp'
        with open(temp_file, 'w', encoding='utf-8') as f:
            json.dump(results, f, indent=2, ensure_ascii=False)
        
        # Atomic rename
        os.replace(temp_file, output_file)
        logger.debug(f"Checkpoint saved to {output_file}")
    except Exception as e:
        logger.error(f"Failed to save checkpoint: {e}")


def get_video_transcript(
    video_id: str, 
    languages: List[str] = None,
    logger: Optional[logging.Logger] = None,
    max_retries: int = 3,
    retry_delay: float = 5.0
) -> Optional[List[Dict]]:
    """
    Fetch transcript for a single video with retry logic.
    
    Args:
        video_id: YouTube video ID
        languages: List of language codes to try (default: ['en', 'en-US', 'en-GB'])
        logger: Logger instance for logging
        max_retries: Maximum number of retry attempts
        retry_delay: Base delay in seconds for exponential backoff
    
    Returns:
        Transcript data or None if not available
    """
    if logger is None:
        logger = logging.getLogger(__name__)
    
    if languages is None:
        languages = ['en', 'en-US', 'en-GB']
    
    for attempt in range(max_retries + 1):
        try:
            # Try to get transcript in preferred languages
            # New API: YouTubeTranscriptApi requires instantiation
            logger.debug(f"Fetching transcript for {video_id} (languages: {languages}, attempt {attempt + 1}/{max_retries + 1})")
            api = YouTubeTranscriptApi()
            fetched_transcript = api.fetch(video_id, languages=languages)
            # Convert FetchedTranscript to list of dicts (format: [{'text': '...', 'start': ..., 'duration': ...}])
            transcript = fetched_transcript.to_raw_data()
            logger.debug(f"Successfully fetched transcript for {video_id} ({len(transcript)} entries)")
            return transcript
        except Exception as e:
            error_str = str(e).lower()
            # Check if it's an IP block
            is_ip_block = any(term in error_str for term in ['ip', 'blocked', 'requestblocked', 'ipblocked', 'too many requests'])
            
            if is_ip_block and attempt < max_retries:
                # Exponential backoff with jitter
                delay = retry_delay * (2 ** attempt) + random.uniform(0, retry_delay)
                logger.warning(f"IP blocked for {video_id}, waiting {delay:.1f}s before retry (attempt {attempt + 1}/{max_retries + 1})...")
                time.sleep(delay)
                continue
            elif attempt < max_retries:
                # Other errors - shorter retry
                delay = retry_delay * (attempt + 1)
                logger.debug(f"Retry {attempt + 1}/{max_retries} for {video_id} after {delay:.1f}s...")
                time.sleep(delay)
                continue
            else:
                # Final attempt failed
                if is_ip_block:
                    logger.warning(f"Could not fetch transcript for {video_id}: IP blocked (tried {max_retries + 1} times). You may need to wait longer or use a different network.")
                else:
                    logger.warning(f"Could not fetch transcript for {video_id}: {e}")
                return None
    
    return None


def fetch_channel_transcripts(
    channel_url: str,
    start_date: Optional[str] = None,
    end_date: Optional[str] = None,
    output_file: str = "channel_transcripts.json",
    delay_seconds: float = 1.0,
    languages: List[str] = None,
    logger: Optional[logging.Logger] = None,
    verbose: bool = False,
    quiet: bool = False
) -> None:
    """
    Fetch transcripts for all videos from a channel within a date range.
    
    Args:
        channel_url: YouTube channel URL, handle (@channelname), or channel ID
        start_date: Start date in YYYY-MM-DD format (inclusive)
        end_date: End date in YYYY-MM-DD format (inclusive)
        output_file: Output JSON file path
        delay_seconds: Delay between requests to avoid rate limiting
        languages: List of language codes to try for transcripts
        logger: Logger instance for logging
        verbose: Enable verbose logging
        quiet: Enable quiet mode (errors only)
    """
    if logger is None:
        logger = logging.getLogger(__name__)
    
    logger.info("=" * 60)
    logger.info("YouTube Channel Transcript Fetcher")
    logger.info("=" * 60)
    
    # Parse dates
    start_dt = None
    end_dt = None
    
    if start_date:
        try:
            start_dt = parse_date(start_date)
            logger.debug(f"Parsed start date: {start_dt}")
        except ValueError as e:
            logger.error(f"Invalid start date format: {e}")
            sys.exit(1)
    
    if end_date:
        try:
            end_dt = parse_date(end_date)
            logger.debug(f"Parsed end date: {end_dt}")
        except ValueError as e:
            logger.error(f"Invalid end date format: {e}")
            sys.exit(1)
    
    if start_dt and end_dt and start_dt > end_dt:
        logger.error("Start date must be before or equal to end date.")
        sys.exit(1)
    
    # Load existing checkpoint
    existing_results, existing_video_ids = load_checkpoint(output_file, logger)
    
    # Try to load video list from date checkpoint file (much faster than fetching from YouTube)
    date_checkpoint_file = output_file.replace('.json', '_dates_checkpoint.json')
    videos = []
    
    if os.path.exists(date_checkpoint_file):
        logger.info("Step 1: Loading video list from date checkpoint (skipping YouTube fetch)...")
        sys.stdout.flush()
        
        try:
            with open(date_checkpoint_file, 'r', encoding='utf-8') as f:
                date_checkpoint = json.load(f)
            
            logger.info(f"Found {len(date_checkpoint)} videos in date checkpoint")
            
            # Convert date checkpoint to video list format
            for video_id, video_data in date_checkpoint.items():
                upload_date_str = video_data.get('upload_date', '')
                title = video_data.get('title', 'Unknown')
                
                # Filter by date range if specified
                if start_dt or end_dt:
                    if upload_date_str and upload_date_str.strip():
                        try:
                            upload_date = datetime.strptime(upload_date_str, '%Y%m%d')
                            if start_dt and upload_date.date() < start_dt.date():
                                continue
                            if end_dt and upload_date.date() > end_dt.date():
                                continue
                        except ValueError:
                            # Invalid date format, skip this video
                            continue
                
                videos.append({
                    'id': video_id,
                    'upload_date': upload_date_str,
                    'title': title
                })
            
            logger.info(f"Loaded {len(videos)} videos from date checkpoint (after date filtering)")
            
        except Exception as e:
            logger.warning(f"Error loading date checkpoint: {e}. Will fetch from YouTube instead.")
            videos = []
    
    # If date checkpoint doesn't exist or failed, fetch from YouTube
    if not videos:
        logger.info("Step 1: Fetching video list from channel...")
        sys.stdout.flush()
        videos = get_video_ids_from_channel(
            channel_url, start_dt, end_dt, logger, 
            show_progress=not quiet, quiet=quiet,
            output_file=output_file
        )
        
        if not videos:
            logger.warning("No videos found in the specified date range.")
            sys.exit(1)
    
    # Initialize or load results
    if existing_results:
        results = existing_results
        logger.info(f"Resuming from checkpoint: {len(existing_video_ids)} videos already have successful transcripts")
    else:
        results = {
            'channel_url': channel_url,
            'start_date': start_date,
            'end_date': end_date,
            'total_videos': len(videos),
            'transcripts': []
        }
    
    # Filter out videos that already have successful transcripts
    videos = [v for v in videos if v['id'] not in existing_video_ids]
    logger.info(f"{len(videos)} videos remaining to process (need transcripts)")
    
    if not videos:
        logger.info("All videos already processed!")
        return
    else:
        # No existing checkpoint or no transcripts - fetch from YouTube
        logger.info("Step 1: Fetching video list from channel...")
        sys.stdout.flush()
        videos = get_video_ids_from_channel(
            channel_url, start_dt, end_dt, logger, 
            show_progress=not quiet, quiet=quiet,
            output_file=output_file
        )
        
        if not videos:
            logger.warning("No videos found in the specified date range.")
            sys.exit(1)
        
        # Initialize or load results
        if existing_results:
            results = existing_results
            logger.info(f"Resuming from checkpoint: {len(existing_video_ids)} videos already processed")
            # Filter out already processed videos
            videos = [v for v in videos if v['id'] not in existing_video_ids]
            logger.info(f"{len(videos)} videos remaining to process")
            if not videos:
                logger.info("All videos already processed!")
                return
        else:
            results = {
                'channel_url': channel_url,
                'start_date': start_date,
                'end_date': end_date,
                'total_videos': len(videos),
                'transcripts': []
            }
    
    # Create a set of existing video IDs for duplicate detection
    existing_transcript_ids = {entry.get('video_id') for entry in results.get('transcripts', []) if entry.get('video_id')}
    
    logger.info(f"Step 2: Fetching transcripts for {len(videos)} videos...")
    logger.info(f"Rate limiting: {delay_seconds} seconds between requests")
    logger.debug(f"Language preferences: {languages if languages else 'default'}")
    
    # Use progress bar for transcript fetching
    successful = 0
    failed = 0
    total_processed = len(existing_video_ids)
    
    if not quiet:
        pbar = tqdm(
            enumerate(videos, 1),
            total=len(videos),
            desc="Fetching transcripts",
            unit="video",
            bar_format='{l_bar}{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}]'
        )
    else:
        pbar = enumerate(videos, 1)
    
    for i, video_info in pbar:
        video_id = video_info['id']
        video_title = video_info.get('title', 'Unknown')
        
        if not quiet:
            pbar.set_postfix({'current': video_title[:30] + '...' if len(video_title) > 30 else video_title})
        
        logger.debug(f"Processing video [{i}/{len(videos)}]: {video_id} - {video_title}")
        
        transcript = get_video_transcript(video_id, languages, logger, max_retries=2, retry_delay=5.0)
        
        # Combine transcript text into a single string
        full_text = None
        if transcript:
            # Combine all transcript text with timestamps
            text_segments = []
            for entry in transcript:
                text_segments.append(entry.get('text', ''))
            full_text = ' '.join(text_segments)
            successful += 1
        else:
            failed += 1
        
        # Duplicate detection: skip if this video_id is already in results
        if video_id in existing_transcript_ids:
            logger.debug(f"Skipping duplicate video {video_id} - already in results")
            continue
        
        video_entry = {
            'video_id': video_id,
            'video_title': video_title,
            'upload_date': video_info.get('upload_date'),
            'video_url': f"https://www.youtube.com/watch?v={video_id}",
            'success': transcript is not None,
            'transcript_full_text': full_text
        }
        
        results['transcripts'].append(video_entry)
        existing_transcript_ids.add(video_id)  # Track this video_id to prevent duplicates
        total_processed += 1
        
        if transcript:
            successful += 1
            logger.debug(f"✓ Successfully fetched transcript for {video_id} ({len(transcript)} entries)")
        else:
            failed += 1
            logger.debug(f"✗ Failed to fetch transcript for {video_id}")
        
        # Save checkpoint after each video (incremental save)
        results['successful_transcripts'] = sum(1 for v in results['transcripts'] if v['success'])
        results['failed_transcripts'] = sum(1 for v in results['transcripts'] if not v['success'])
        results['total_videos'] = len(results['transcripts'])
        save_checkpoint(results, output_file, logger)
        
        # Rate limiting: delay between requests (except for last video)
        # Add small random jitter to avoid predictable patterns
        if i < len(videos):
            jitter = random.uniform(0, delay_seconds * 0.2)  # 0-20% jitter
            time.sleep(delay_seconds + jitter)
    
    if not quiet:
        pbar.close()
    
    # Sort transcripts by title and date before final save
    logger.info("Finalizing and sorting results...")
    
    def sort_key(entry):
        """Sort by date (descending) then by title (ascending)"""
        date_str = entry.get('upload_date', '')
        title = entry.get('video_title', '')
        try:
            if date_str:
                date_obj = datetime.strptime(date_str, '%Y%m%d')
                # Return tuple: (-date for descending, title for ascending)
                return (-date_obj.timestamp(), title.lower())
            else:
                # Videos without dates go to end
                return (0, title.lower())
        except (ValueError, TypeError):
            return (0, title.lower())
    
    results['transcripts'].sort(key=sort_key)
    
    # Final save (results already saved incrementally, but now sorted)
    save_checkpoint(results, output_file, logger)
    
    # Calculate final stats
    total_successful = sum(1 for v in results['transcripts'] if v['success'])
    total_failed = sum(1 for v in results['transcripts'] if not v['success'])
    total_videos = len(results['transcripts'])
    
    # Final summary
    logger.info("=" * 60)
    logger.info("✓ Complete!")
    logger.info(f"  Total videos processed: {total_videos}")
    logger.info(f"  Successful transcripts: {total_successful}")
    logger.info(f"  Failed transcripts: {total_failed}")
    if total_videos > 0:
        logger.info(f"  Success rate: {(total_successful/total_videos*100):.1f}%")
    logger.info(f"  Output file: {output_file}")
    logger.info(f"  Results sorted by date (newest first) and title")
    logger.info("=" * 60)


def main():
    parser = argparse.ArgumentParser(
        description='Fetch transcripts for all videos from a YouTube channel within a date range',
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Examples:
  python transcript_fetcher.py "https://www.youtube.com/@channelname" --start-date "2024-01-01" --end-date "2024-12-31"
  python transcript_fetcher.py "@channelname" --start-date "2024-01-01"
  python transcript_fetcher.py "UCxxxxx" --start-date "2024-01-01" --end-date "2024-12-31" -o my_transcripts.json
  python transcript_fetcher.py "@channelname" --start-date "2024-01-01" --delay 2.0
        """
    )
    
    parser.add_argument(
        'channel_url',
        help='YouTube channel URL, handle (@channelname), or channel ID'
    )
    
    parser.add_argument(
        '--start-date',
        type=str,
        help='Start date in YYYY-MM-DD format (inclusive). If not specified, no start limit.'
    )
    
    parser.add_argument(
        '--end-date',
        type=str,
        help='End date in YYYY-MM-DD format (inclusive). If not specified, no end limit.'
    )
    
    parser.add_argument(
        '-o', '--output',
        default='channel_transcripts.json',
        help='Output JSON file path (default: channel_transcripts.json)'
    )
    
    parser.add_argument(
        '-d', '--delay',
        type=float,
        default=2.0,
        help='Delay in seconds between requests (default: 2.0, increase if you get IP blocked)'
    )
    
    parser.add_argument(
        '-l', '--languages',
        nargs='+',
        default=['en', 'en-US', 'en-GB'],
        help='Language codes to try for transcripts (default: en en-US en-GB)'
    )
    
    parser.add_argument(
        '-v', '--verbose',
        action='store_true',
        help='Enable verbose logging (DEBUG level)'
    )
    
    parser.add_argument(
        '-q', '--quiet',
        action='store_true',
        help='Quiet mode (errors only, no progress bars)'
    )
    
    args = parser.parse_args()
    
    # Setup logging
    logger = setup_logging(args.verbose, args.quiet)
    
    if not args.start_date and not args.end_date:
        if not args.quiet:
            logger.warning("No date range specified. This will fetch ALL videos from the channel.")
            logger.warning("This may take a very long time and use a lot of resources.")
            response = input("Continue? (y/N): ")
            if response.lower() != 'y':
                logger.info("Cancelled.")
                sys.exit(0)
        else:
            logger.error("Error: Date range required. Use --start-date and/or --end-date")
            sys.exit(1)
    
    fetch_channel_transcripts(
        channel_url=args.channel_url,
        start_date=args.start_date,
        end_date=args.end_date,
        output_file=args.output,
        delay_seconds=args.delay,
        languages=args.languages,
        logger=logger,
        verbose=args.verbose,
        quiet=args.quiet
    )


if __name__ == '__main__':
    main()