bulk_indexer.py

"""
=============================================================================
  GOOGLE INDEXING API - BULK URL INDEXER SERVICE
=============================================================================
  Free tier: 200 publish requests/day/project
  
  Files:
    credentials.txt     -> Your API credentials (you fill this)
    urls_to_index.txt   -> Your URL list (one per line, or provide path)
    indexing_master.xlsx -> Master tracking file (auto-created)
    indexing_log.txt     -> Last 200 indexed URLs log (auto-created)
    indexer_state.json   -> Service state & quota tracking (auto-created)
  
  Modes:
    Interactive  -> python bulk_indexer.py
    Silent/Daily -> python bulk_indexer.py --auto
    Status check -> python bulk_indexer.py --status
=============================================================================
"""

import subprocess
import sys
import os

# ============================================================
# AUTO-INSTALL DEPENDENCIES
# ============================================================
REQUIRED_PACKAGES = {
    'pandas': 'pandas',
    'openpyxl': 'openpyxl',
    'requests': 'requests',
    'google.oauth2': 'google-auth',
    'googleapiclient': 'google-api-python-client',
    'google.auth.transport.requests': 'google-auth-httplib2',
}

def install_packages():
    """Check and install missing packages automatically."""
    missing = []
    for import_name, pip_name in REQUIRED_PACKAGES.items():
        try:
            __import__(import_name)
        except ImportError:
            missing.append(pip_name)
    
    if missing:
        print(f"\n[SETUP] Installing missing packages: {', '.join(missing)}")
        for pkg in missing:
            print(f"  -> Installing {pkg}...")
            subprocess.check_call(
                [sys.executable, '-m', 'pip', 'install', pkg, '--quiet'],
                stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
            )
        print("[SETUP] All packages installed!\n")

install_packages()

# ============================================================
# IMPORTS (safe after auto-install)
# ============================================================
import json
import time
import logging
from datetime import datetime
from pathlib import Path

import pandas as pd
import requests
from google.oauth2 import service_account
from google.auth.transport.requests import Request

# ============================================================
# PATHS & CONSTANTS
# ============================================================
SCRIPT_DIR = Path(os.path.dirname(os.path.abspath(__file__)))
CREDENTIALS_FILE = SCRIPT_DIR / "credentials.txt"
DEFAULT_URLS_FILE = SCRIPT_DIR / "urls_to_index.txt"
MASTER_FILE = SCRIPT_DIR / "indexing_master.xlsx"
LOG_FILE = SCRIPT_DIR / "indexing_log.txt"
STATE_FILE = SCRIPT_DIR / "indexer_state.json"

SCOPES = ['https://www.googleapis.com/auth/indexing']
API_ENDPOINT = 'https://indexing.googleapis.com/v3/urlNotifications:publish'
BATCH_ENDPOINT = 'https://indexing.googleapis.com/batch'
DAILY_QUOTA = 200
BATCH_SIZE = 40  # safe batch size (max 100 but 40 is more reliable)

# ============================================================
# LOGGING SETUP
# ============================================================
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s | %(levelname)-7s | %(message)s',
    datefmt='%Y-%m-%d %H:%M:%S'
)
logger = logging.getLogger('BulkIndexer')

# ============================================================
# CREDENTIALS MANAGEMENT
# ============================================================
def create_credentials_template():
    """Create credentials.txt template if it doesn't exist."""
    if not CREDENTIALS_FILE.exists():
        template = """# ============================================================
# GOOGLE INDEXING API CREDENTIALS
# ============================================================
# Fill in your service account JSON key file path below.
#
# HOW TO GET THIS:
#   1. Go to https://console.cloud.google.com/
#   2. Select your project
#   3. Go to IAM & Admin > Service Accounts
#   4. Create a service account (or use existing)
#   5. Go to Keys tab > Add Key > Create new key > JSON
#   6. Save the downloaded .json file in this folder
#   7. Put the filename below
#
# IMPORTANT: Add the service account email to your
#   Search Console property as Owner!
# ============================================================

# Path to your service account JSON key file
# Can be just the filename if it's in the same folder
service_account_json=your-service-account-key.json

# Your website URL (must match exactly as in Search Console)
site_url=https://www.example.com/
"""
        CREDENTIALS_FILE.write_text(template, encoding='utf-8')
        print(f"\n[!] Created credentials template: {CREDENTIALS_FILE}")
        print("    Please edit it with your service account JSON path and site URL.\n")
        return None
    return True


def load_credentials():
    """Load credentials from credentials.txt"""
    if not CREDENTIALS_FILE.exists():
        create_credentials_template()
        return None, None
    
    config = {}
    with open(CREDENTIALS_FILE, 'r', encoding='utf-8') as f:
        for line in f:
            line = line.strip()
            if line and not line.startswith('#') and '=' in line:
                key, value = line.split('=', 1)
                config[key.strip()] = value.strip()
    
    json_path = config.get('service_account_json', '')
    site_url = config.get('site_url', '')
    
    # Resolve JSON path
    if json_path and not os.path.isabs(json_path):
        json_path = str(SCRIPT_DIR / json_path)
    
    if not json_path or json_path.endswith('your-service-account-key.json'):
        print("\n[ERROR] Please update credentials.txt with your service account JSON file path!")
        return None, None
    
    if not os.path.exists(json_path):
        print(f"\n[ERROR] Service account JSON not found: {json_path}")
        print("        Make sure the file exists and the path is correct in credentials.txt")
        return None, None
    
    if not site_url or site_url == 'https://www.example.com/':
        print("\n[ERROR] Please update credentials.txt with your actual site URL!")
        return None, None
    
    return json_path, site_url


def get_access_token(json_path):
    """Get OAuth2 access token from service account."""
    try:
        credentials = service_account.Credentials.from_service_account_file(
            json_path, scopes=SCOPES
        )
        credentials.refresh(Request())
        return credentials.token
    except Exception as e:
        logger.error(f"Failed to get access token: {e}")
        return None

# ============================================================
# STATE MANAGEMENT (quota tracking, progress)
# ============================================================
def load_state():
    """Load service state from JSON file."""
    default_state = {
        'last_run_date': None,
        'requests_today': 0,
        'quota_reset_time': None,
        'total_indexed': 0,
        'total_failed': 0,
        'total_runs': 0,
        'urls_file_path': None,
        'last_processed_index': 0,
        'history': []
    }
    
    if STATE_FILE.exists():
        try:
            with open(STATE_FILE, 'r') as f:
                state = json.load(f)
            for key, value in default_state.items():
                if key not in state:
                    state[key] = value
            return state
        except (json.JSONDecodeError, Exception):
            logger.warning("Corrupted state file, starting fresh.")
    
    return default_state


def save_state(state):
    """Save service state to JSON file."""
    with open(STATE_FILE, 'w') as f:
        json.dump(state, f, indent=2, default=str)


def check_quota(state):
    """Check remaining daily quota. Resets on new calendar day."""
    today = datetime.now().strftime('%Y-%m-%d')
    
    if state['last_run_date'] != today:
        state['requests_today'] = 0
        state['last_run_date'] = today
        state['quota_reset_time'] = None
        save_state(state)
    
    remaining = DAILY_QUOTA - state['requests_today']
    return max(0, remaining)

# ============================================================
# URL LOADING
# ============================================================
def load_urls_from_file(file_path):
    """Load URLs from TXT, CSV, or Excel file."""
    file_path = Path(file_path)
    
    if not file_path.exists():
        logger.error(f"File not found: {file_path}")
        return []
    
    urls = []
    ext = file_path.suffix.lower()
    
    try:
        if ext == '.txt':
            with open(file_path, 'r', encoding='utf-8') as f:
                urls = [line.strip() for line in f if line.strip() and not line.startswith('#')]
        
        elif ext == '.csv':
            df = pd.read_csv(file_path, header=None)
            for col in df.columns:
                col_data = df[col].dropna().astype(str)
                url_col = col_data[col_data.str.startswith('http')]
                if len(url_col) > 0:
                    urls = url_col.tolist()
                    break
            if not urls:
                urls = df.iloc[:, 0].dropna().astype(str).tolist()
        
        elif ext in ('.xlsx', '.xls'):
            df = pd.read_excel(file_path)
            for col in df.columns:
                col_data = df[col].dropna().astype(str)
                url_col = col_data[col_data.str.startswith('http')]
                if len(url_col) > 0:
                    urls = url_col.tolist()
                    break
            if not urls:
                urls = df.iloc[:, 0].dropna().astype(str).tolist()
        
        else:
            logger.error(f"Unsupported file type: {ext}. Use .txt, .csv, or .xlsx")
            return []
        
    except Exception as e:
        logger.error(f"Error reading file: {e}")
        return []
    
    # Clean & validate URLs
    cleaned = []
    for url in urls:
        url = url.strip()
        if url.startswith('http://') or url.startswith('https://'):
            cleaned.append(url)
    
    return cleaned


def load_urls_from_sitemap(sitemap_url):
    """Fetch URLs from an XML sitemap (supports sitemap index)."""
    try:
        from xml.etree import ElementTree as ET
        
        logger.info(f"Fetching sitemap: {sitemap_url}")
        resp = requests.get(sitemap_url, timeout=30, headers={
            'User-Agent': 'Mozilla/5.0 (compatible; BulkIndexer/1.0)'
        })
        resp.raise_for_status()
        
        root = ET.fromstring(resp.content)
        ns = {'ns': 'http://www.sitemaps.org/schemas/sitemap/0.9'}
        
        urls = []
        # Check if it's a sitemap index
        sitemaps = root.findall('.//ns:sitemap/ns:loc', ns)
        if sitemaps:
            logger.info(f"Sitemap index found with {len(sitemaps)} sub-sitemaps")
            for sm in sitemaps:
                sub_urls = load_urls_from_sitemap(sm.text.strip())
                urls.extend(sub_urls)
        else:
            locs = root.findall('.//ns:url/ns:loc', ns)
            urls = [loc.text.strip() for loc in locs if loc.text]
        
        return urls
        
    except Exception as e:
        logger.error(f"Error fetching sitemap: {e}")
        return []

# ============================================================
# INDEXING API CALLS
# ============================================================
def index_single_url(url, access_token, action='URL_UPDATED'):
    """Send a single indexing request."""
    headers = {
        'Content-Type': 'application/json',
        'Authorization': f'Bearer {access_token}'
    }
    body = {
        'url': url,
        'type': action
    }
    
    try:
        resp = requests.post(API_ENDPOINT, json=body, headers=headers, timeout=30)
        
        if resp.status_code == 200:
            return {'url': url, 'status': 'SUCCESS', 'code': 200, 'message': 'OK'}
        elif resp.status_code == 429:
            return {'url': url, 'status': 'QUOTA_EXCEEDED', 'code': 429, 'message': 'Daily quota exceeded'}
        else:
            try:
                error_msg = resp.json().get('error', {}).get('message', resp.text[:200])
            except Exception:
                error_msg = resp.text[:200]
            return {'url': url, 'status': 'FAILED', 'code': resp.status_code, 'message': error_msg}
            
    except requests.exceptions.Timeout:
        return {'url': url, 'status': 'TIMEOUT', 'code': 0, 'message': 'Request timed out'}
    except Exception as e:
        return {'url': url, 'status': 'ERROR', 'code': 0, 'message': str(e)[:200]}


def index_batch_urls(urls, access_token, action='URL_UPDATED'):
    """Send batch indexing request (up to 100 URLs per HTTP call)."""
    boundary = 'batch_indexing_boundary'
    
    headers = {
        'Content-Type': f'multipart/mixed; boundary={boundary}',
        'Authorization': f'Bearer {access_token}'
    }
    
    body_parts = []
    for i, url in enumerate(urls):
        part = (
            f'--{boundary}\r\n'
            f'Content-Type: application/http\r\n'
            f'Content-ID: <item{i}>\r\n'
            f'\r\n'
            f'POST /v3/urlNotifications:publish HTTP/1.1\r\n'
            f'Content-Type: application/json\r\n'
            f'\r\n'
            f'{{"url": "{url}", "type": "{action}"}}\r\n'
        )
        body_parts.append(part)
    
    body = ''.join(body_parts) + f'--{boundary}--'
    
    results = []
    try:
        resp = requests.post(BATCH_ENDPOINT, data=body.encode('utf-8'), headers=headers, timeout=120)
        
        if resp.status_code == 200:
            # Parse multipart response to get individual statuses
            content = resp.text
            for url in urls:
                if '200' in content or '"urlNotificationMetadata"' in content:
                    results.append({'url': url, 'status': 'SUCCESS', 'code': 200, 'message': 'Batch OK'})
                else:
                    results.append({'url': url, 'status': 'FAILED', 'code': resp.status_code, 'message': 'Batch partial fail'})
        elif resp.status_code == 429:
            for url in urls:
                results.append({'url': url, 'status': 'QUOTA_EXCEEDED', 'code': 429, 'message': 'Quota exceeded'})
        else:
            logger.warning(f"Batch failed ({resp.status_code}), falling back to individual requests...")
            for url in urls:
                result = index_single_url(url, access_token, action)
                results.append(result)
                time.sleep(0.5)
    
    except Exception as e:
        logger.error(f"Batch request error: {e}")
        for url in urls:
            result = index_single_url(url, access_token, action)
            results.append(result)
            time.sleep(0.5)
    
    return results

# ============================================================
# LOGGING & REPORTING
# ============================================================
def update_log_file(results):
    """Maintain a rolling log of the last 200 indexed URLs."""
    existing = []
    if LOG_FILE.exists():
        with open(LOG_FILE, 'r', encoding='utf-8') as f:
            lines = f.readlines()
            existing = [l.strip() for l in lines if l.strip() and not l.startswith('#') and not l.startswith('=')]
    
    timestamp = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
    new_entries = []
    for r in results:
        entry = f"{timestamp} | {r['status']:>15} | {r['code']:>3} | {r['url']}"
        new_entries.append(entry)
    
    all_entries = existing + new_entries
    all_entries = all_entries[-200:]
    
    with open(LOG_FILE, 'w', encoding='utf-8') as f:
        f.write("# ============================================================\n")
        f.write(f"# INDEXING LOG - Last 200 requests (updated: {timestamp})\n")
        f.write("# Format: TIMESTAMP | STATUS | CODE | URL\n")
        f.write("# ============================================================\n")
        for entry in all_entries:
            f.write(entry + "\n")


def update_master_file(results, state):
    """Update master Excel tracking file with all indexing history."""
    timestamp = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
    
    new_rows = []
    for r in results:
        new_rows.append({
            'URL': r['url'],
            'Status': r['status'],
            'HTTP Code': r['code'],
            'Message': r['message'],
            'Submitted At': timestamp,
            'Run #': state['total_runs']
        })
    
    new_df = pd.DataFrame(new_rows)
    
    if MASTER_FILE.exists():
        try:
            existing_df = pd.read_excel(MASTER_FILE, sheet_name='All Requests')
            master_df = pd.concat([existing_df, new_df], ignore_index=True)
        except Exception:
            master_df = new_df
    else:
        master_df = new_df
    
    with pd.ExcelWriter(MASTER_FILE, engine='openpyxl') as writer:
        
        # Sheet 1: All Requests (full history)
        master_df.to_excel(writer, sheet_name='All Requests', index=False)
        
        # Sheet 2: Daily Summary
        if len(master_df) > 0:
            master_df['Date'] = pd.to_datetime(master_df['Submitted At']).dt.date
            summary = master_df.groupby(['Date', 'Status']).size().unstack(fill_value=0).reset_index()
            summary.to_excel(writer, sheet_name='Daily Summary', index=False)
        
        # Sheet 3: Latest status per URL
        if len(master_df) > 0:
            latest = master_df.sort_values('Submitted At').drop_duplicates('URL', keep='last')
            latest = latest[['URL', 'Status', 'HTTP Code', 'Submitted At']].sort_values('Status')
            latest.to_excel(writer, sheet_name='URL Status (Latest)', index=False)
        
        # Sheet 4: Failed URLs for retry
        failed = master_df[master_df['Status'].isin(['FAILED', 'ERROR', 'TIMEOUT'])]
        if len(failed) > 0:
            # Only keep URLs that haven't succeeded later
            success_urls = set(master_df[master_df['Status'] == 'SUCCESS']['URL'].unique())
            retry_df = failed[~failed['URL'].isin(success_urls)].drop_duplicates('URL', keep='last')
            if len(retry_df) > 0:
                retry_df.to_excel(writer, sheet_name='Failed (Retry)', index=False)
        
        # Sheet 5: Stats overview
        stats_data = {
            'Metric': [
                'Total Requests Sent',
                'Successful',
                'Failed / Error / Timeout',
                'Quota Exceeded',
                'Unique URLs Submitted',
                'Unique URLs Successfully Indexed',
                'Total Runs',
                'Last Run',
                'Requests Used Today',
                'Quota Remaining Today'
            ],
            'Value': [
                len(master_df),
                len(master_df[master_df['Status'] == 'SUCCESS']),
                len(master_df[master_df['Status'].isin(['FAILED', 'ERROR', 'TIMEOUT'])]),
                len(master_df[master_df['Status'] == 'QUOTA_EXCEEDED']),
                master_df['URL'].nunique(),
                master_df[master_df['Status'] == 'SUCCESS']['URL'].nunique(),
                state['total_runs'],
                timestamp,
                state['requests_today'],
                DAILY_QUOTA - state['requests_today']
            ]
        }
        pd.DataFrame(stats_data).to_excel(writer, sheet_name='Stats', index=False)
    
    logger.info(f"Master file updated: {MASTER_FILE}")

# ============================================================
# MAIN INDEXING ENGINE
# ============================================================
def run_indexing(urls, json_path, state, action='URL_UPDATED', use_batch=True):
    """Main indexing loop with quota management."""
    remaining_quota = check_quota(state)
    
    if remaining_quota <= 0:
        logger.warning("Daily quota exhausted (200/200). Run again tomorrow!")
        return []
    
    urls_to_process = urls[:remaining_quota]
    skipped = len(urls) - len(urls_to_process)
    
    if skipped > 0:
        logger.info(f"Will process {len(urls_to_process)} URLs (quota limit). {skipped} URLs queued for tomorrow.")
    
    logger.info(f"Starting: {len(urls_to_process)} URLs | Action: {action} | Batch: {use_batch}")
    logger.info(f"Quota: {state['requests_today']}/{DAILY_QUOTA} used | {remaining_quota} remaining")
    print("-" * 70)
    
    access_token = get_access_token(json_path)
    if not access_token:
        logger.error("Could not get access token. Check your service account JSON.")
        return []
    
    all_results = []
    success_count = 0
    fail_count = 0
    
    if use_batch and len(urls_to_process) > 5:
        for i in range(0, len(urls_to_process), BATCH_SIZE):
            batch = urls_to_process[i:i + BATCH_SIZE]
            batch_num = (i // BATCH_SIZE) + 1
            total_batches = (len(urls_to_process) + BATCH_SIZE - 1) // BATCH_SIZE
            
            logger.info(f"Batch {batch_num}/{total_batches} ({len(batch)} URLs)...")
            
            results = index_batch_urls(batch, access_token, action)
            all_results.extend(results)
            
            quota_hit = False
            for r in results:
                if r['status'] == 'SUCCESS':
                    success_count += 1
                elif r['status'] == 'QUOTA_EXCEEDED':
                    quota_hit = True
                else:
                    fail_count += 1
            
            state['requests_today'] += len(batch)
            save_state(state)
            
            total_done = i + len(batch)
            pct = (total_done / len(urls_to_process)) * 100
            print(f"  [{pct:5.1f}%] {total_done}/{len(urls_to_process)} | OK: {success_count} | Fail: {fail_count} | Quota: {state['requests_today']}/{DAILY_QUOTA}")
            
            if quota_hit:
                logger.warning("Quota exceeded mid-batch! Stopping.")
                state['requests_today'] = DAILY_QUOTA
                break
            
            if i + BATCH_SIZE < len(urls_to_process):
                time.sleep(2)
    
    else:
        for i, url in enumerate(urls_to_process):
            result = index_single_url(url, access_token, action)
            all_results.append(result)
            
            if result['status'] == 'SUCCESS':
                success_count += 1
            elif result['status'] == 'QUOTA_EXCEEDED':
                logger.warning("Quota exceeded! Stopping.")
                state['requests_today'] = DAILY_QUOTA
                break
            else:
                fail_count += 1
            
            state['requests_today'] += 1
            
            if (i + 1) % 10 == 0 or i == len(urls_to_process) - 1:
                pct = ((i + 1) / len(urls_to_process)) * 100
                print(f"  [{pct:5.1f}%] {i+1}/{len(urls_to_process)} | OK: {success_count} | Fail: {fail_count} | Quota: {state['requests_today']}/{DAILY_QUOTA}")
                save_state(state)
            
            time.sleep(0.3)
    
    # Update state
    state['total_indexed'] += success_count
    state['total_failed'] += fail_count
    state['total_runs'] += 1
    state['last_processed_index'] += len(all_results)
    
    state['history'].append({
        'date': datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
        'urls_processed': len(all_results),
        'success': success_count,
        'failed': fail_count,
        'quota_used': state['requests_today']
    })
    state['history'] = state['history'][-30:]
    
    save_state(state)
    update_log_file(all_results)
    update_master_file(all_results, state)
    
    print("\n" + "=" * 70)
    print(f"  INDEXING COMPLETE")
    print(f"  Processed:  {len(all_results)} URLs")
    print(f"  Success:    {success_count}")
    print(f"  Failed:     {fail_count}")
    print(f"  Quota used: {state['requests_today']}/{DAILY_QUOTA}")
    print(f"  Remaining:  {DAILY_QUOTA - state['requests_today']} requests left today")
    if skipped > 0:
        days_needed = (skipped + DAILY_QUOTA - 1) // DAILY_QUOTA
        print(f"  Queued:     {skipped} URLs (~{days_needed} more day(s) to finish)")
    print("=" * 70)
    
    return all_results

# ============================================================
# INTERACTIVE MODE
# ============================================================
def interactive_menu():
    """Full interactive CMD interface."""
    print("\n" + "=" * 70)
    print("  GOOGLE INDEXING API - BULK URL INDEXER")
    print("  Free: 200 requests/day | Batch mode | Auto-resume")
    print("=" * 70)
    
    json_path, site_url = load_credentials()
    if not json_path:
        create_credentials_template()
        input("\nPress Enter to exit...")
        return
    
    state = load_state()
    remaining = check_quota(state)
    
    print(f"\n  Site:         {site_url}")
    print(f"  Credentials:  {os.path.basename(json_path)}")
    print(f"  Quota today:  {state['requests_today']}/{DAILY_QUOTA} used | {remaining} remaining")
    print(f"  All-time:     {state['total_indexed']} indexed | {state['total_runs']} runs")
    
    if remaining <= 0:
        print("\n  [!] Daily quota exhausted (200/200). Run again tomorrow!")
        show_status(state)
        input("\n  Press Enter to exit...")
        return
    
    print("\n  MENU:")
    print("  [1] Index URLs from file  (TXT / CSV / XLSX)")
    print("  [2] Index URLs from sitemap")
    print("  [3] Retry failed URLs")
    print("  [4] View status & history")
    print("  [5] View last 200 log entries")
    print("  [6] Reset progress (start file from beginning)")
    print("  [0] Exit")
    
    choice = input("\n  > ").strip()
    
    # ----------------------------------------------------------
    if choice == '1':
        print(f"\n  Default file: {DEFAULT_URLS_FILE}")
        print("  Supports: .txt (one URL per line), .csv, .xlsx")
        file_input = input("  File path (Enter = default, or drag & drop): ").strip()
        
        if not file_input:
            file_path = DEFAULT_URLS_FILE
        else:
            file_path = Path(file_input.strip('"').strip("'"))
        
        if not file_path.exists():
            print(f"\n  [ERROR] File not found: {file_path}")
            if not DEFAULT_URLS_FILE.exists():
                DEFAULT_URLS_FILE.write_text(
                    "# URLs to index - one per line\n"
                    "# Example:\n"
                    "# https://www.yoursite.com/page1\n"
                    "# https://www.yoursite.com/page2\n",
                    encoding='utf-8'
                )
                print(f"  [+] Created template: {DEFAULT_URLS_FILE}")
                print("      Add your URLs and run again!")
            input("\n  Press Enter to exit...")
            return
        
        urls = load_urls_from_file(file_path)
        if not urls:
            print("  [ERROR] No valid URLs found in file!")
            input("\n  Press Enter to exit...")
            return
        
        print(f"\n  Found {len(urls)} URLs")
        
        # Resume check
        if state.get('urls_file_path') == str(file_path) and state.get('last_processed_index', 0) > 0:
            done = state['last_processed_index']
            if done < len(urls):
                print(f"  Previous progress: {done}/{len(urls)} done")
                resume = input(f"  Resume from #{done + 1}? (y/n, default=y): ").strip().lower()
                if resume != 'n':
                    urls = urls[done:]
                    print(f"  Resuming: {len(urls)} remaining")
                else:
                    state['last_processed_index'] = 0
        
        state['urls_file_path'] = str(file_path)
        
        print("\n  Action:")
        print("  [1] URL_UPDATED  - request indexing (default)")
        print("  [2] URL_DELETED  - request removal")
        action_choice = input("  > ").strip()
        action = 'URL_DELETED' if action_choice == '2' else 'URL_UPDATED'
        
        to_process = min(len(urls), remaining)
        print(f"\n  Will submit {to_process} URLs as {action}")
        if len(urls) > remaining:
            print(f"  ({len(urls) - remaining} more URLs will need tomorrow's quota)")
        
        confirm = input("  Proceed? (y/n): ").strip().lower()
        if confirm == 'y':
            print()
            run_indexing(urls, json_path, state, action, use_batch=True)
    
    # ----------------------------------------------------------
    elif choice == '2':
        sitemap_url = input("\n  Sitemap URL: ").strip()
        if not sitemap_url:
            print("  [ERROR] No URL entered!")
            input("\n  Press Enter to exit...")
            return
        
        urls = load_urls_from_sitemap(sitemap_url)
        if not urls:
            print("  [ERROR] No URLs found in sitemap!")
            input("\n  Press Enter to exit...")
            return
        
        print(f"  Found {len(urls)} URLs in sitemap")
        
        filter_q = input(f"  Filter to {site_url} only? (y/n, default=y): ").strip().lower()
        if filter_q != 'n':
            base = site_url.rstrip('/')
            urls = [u for u in urls if base in u]
            print(f"  After filter: {len(urls)} URLs")
        
        to_process = min(len(urls), remaining)
        confirm = input(f"\n  Submit {to_process} URLs? (y/n): ").strip().lower()
        if confirm == 'y':
            run_indexing(urls, json_path, state, 'URL_UPDATED', True)
    
    # ----------------------------------------------------------
    elif choice == '3':
        if not MASTER_FILE.exists():
            print("\n  No master file yet. Run indexing first!")
            input("\n  Press Enter to exit...")
            return
        
        try:
            failed_df = pd.read_excel(MASTER_FILE, sheet_name='Failed (Retry)')
            failed_urls = failed_df['URL'].unique().tolist()
            print(f"\n  Found {len(failed_urls)} failed URLs to retry")
            confirm = input(f"  Retry {min(len(failed_urls), remaining)}? (y/n): ").strip().lower()
            if confirm == 'y':
                run_indexing(failed_urls, json_path, state, 'URL_UPDATED', True)
        except ValueError:
            print("\n  No failed URLs found - everything succeeded!")
        except Exception as e:
            print(f"\n  Error reading master file: {e}")
    
    # ----------------------------------------------------------
    elif choice == '4':
        show_status(state)
    
    elif choice == '5':
        show_log()
    
    elif choice == '6':
        print("\n  This will reset file progress to start from the beginning.")
        confirm = input("  Are you sure? (y/n): ").strip().lower()
        if confirm == 'y':
            state['last_processed_index'] = 0
            save_state(state)
            print("  Progress reset!")
    
    elif choice == '0':
        print("\n  Bye!")
        return
    
    else:
        print("\n  Invalid choice!")
    
    input("\n  Press Enter to exit...")


def show_status(state=None):
    """Display current status and stats."""
    if state is None:
        state = load_state()
    
    remaining = check_quota(state)
    
    print("\n" + "=" * 70)
    print("  STATUS & HISTORY")
    print("=" * 70)
    print(f"  Last run:          {state.get('last_run_date', 'Never')}")
    print(f"  Quota today:       {state['requests_today']}/{DAILY_QUOTA} ({remaining} left)")
    print(f"  Total indexed:     {state['total_indexed']}")
    print(f"  Total failed:      {state['total_failed']}")
    print(f"  Total runs:        {state['total_runs']}")
    print(f"  Current file:      {state.get('urls_file_path', 'Not set')}")
    print(f"  File progress:     {state.get('last_processed_index', 0)} URLs done")
    
    if state.get('history'):
        print(f"\n  LAST 10 RUNS:")
        print(f"  {'Date':<22} {'Done':>6} {'OK':>6} {'Fail':>6} {'Quota':>6}")
        print("  " + "-" * 50)
        for h in state['history'][-10:]:
            print(f"  {h['date']:<22} {h['urls_processed']:>6} {h['success']:>6} {h['failed']:>6} {h['quota_used']:>6}")
    
    print("=" * 70)


def show_log():
    """Display the last 200 log entries."""
    if not LOG_FILE.exists():
        print("\n  No log file yet. Run indexing first!")
        return
    
    with open(LOG_FILE, 'r', encoding='utf-8') as f:
        print(f.read())

# ============================================================
# AUTO MODE (for Task Scheduler / cron - no interaction)
# ============================================================
def auto_mode():
    """Silent mode for scheduled daily runs."""
    logger.info("=" * 50)
    logger.info("BULK INDEXER - AUTO MODE")
    logger.info("=" * 50)
    
    json_path, site_url = load_credentials()
    if not json_path:
        logger.error("Credentials not configured! Edit credentials.txt first.")
        return
    
    state = load_state()
    remaining = check_quota(state)
    
    logger.info(f"Site: {site_url}")
    logger.info(f"Quota: {remaining}/{DAILY_QUOTA} remaining")
    
    if remaining <= 0:
        logger.info("Quota exhausted for today. Exiting.")
        return
    
    urls_file = state.get('urls_file_path') or str(DEFAULT_URLS_FILE)
    
    if not os.path.exists(urls_file):
        logger.error(f"URLs file not found: {urls_file}")
        logger.error("Run in interactive mode first to set the file path.")
        return
    
    urls = load_urls_from_file(urls_file)
    if not urls:
        logger.info("No URLs to process.")
        return
    
    # Resume from last position
    resume_from = state.get('last_processed_index', 0)
    if resume_from > 0 and resume_from < len(urls):
        urls = urls[resume_from:]
        logger.info(f"Resuming from #{resume_from + 1} ({len(urls)} remaining)")
    elif resume_from >= len(urls):
        logger.info(f"All {len(urls)} URLs already processed!")
        state['last_processed_index'] = 0
        save_state(state)
        return
    
    run_indexing(urls, json_path, state, 'URL_UPDATED', use_batch=True)
    logger.info("Auto mode complete.")

# ============================================================
# ENTRY POINT
# ============================================================
if __name__ == '__main__':
    os.chdir(SCRIPT_DIR)
    
    if len(sys.argv) > 1:
        arg = sys.argv[1].lower()
        
        if arg == '--auto':
            auto_mode()
        elif arg == '--status':
            show_status()
        elif arg == '--log':
            show_log()
        elif arg == '--help':
            print("""
  GOOGLE INDEXING API - BULK INDEXER
  ==================================

  Usage:
    python bulk_indexer.py            Interactive mode (full menu)
    python bulk_indexer.py --auto     Silent mode (for Task Scheduler)
    python bulk_indexer.py --status   Show quota & stats
    python bulk_indexer.py --log      Show last 200 log entries
    python bulk_indexer.py --help     This help

  Files created in script folder:
    credentials.txt     -> Your service account config (edit this!)
    urls_to_index.txt   -> Default URL list (one per line)
    indexing_master.xlsx -> Master tracking (all runs, stats, failed)
    indexing_log.txt     -> Rolling log of last 200 requests
    indexer_state.json   -> Quota counter & resume progress

  Task Scheduler setup (run daily at 9:00 AM):
    Action:    Start a program
    Program:   python
    Arguments: "C:\\path\\to\\bulk_indexer.py" --auto
    Start in:  C:\\path\\to\\
""")
        else:
            print(f"  Unknown argument: {arg}")
            print("  Use --help for options.")
    else:
        interactive_menu()