batch_fetch.py

#!/usr/bin/env python3
"""Fetch multiple URLs with Plasmate and save results as JSON."""
import subprocess
import json
import sys
import os
from datetime import datetime

URLS = [
    "https://news.ycombinator.com",
    "https://github.com/trending",
    "https://www.reddit.com/r/programming",
]

def fetch_url(url: str) -> dict:
    """Fetch a single URL and return the SOM."""
    result = subprocess.run(
        ["plasmate", "fetch", url],
        capture_output=True,
        text=True,
    )
    if result.returncode != 0:
        return {"url": url, "error": result.stderr.strip()}
    try:
        som = json.loads(result.stdout)
        return {"url": url, "som": som}
    except json.JSONDecodeError as e:
        return {"url": url, "error": f"JSON decode error: {e}"}

def main():
    urls = sys.argv[1:] if len(sys.argv) > 1 else URLS
    print(f"Fetching {len(urls)} URLs...")

    results = []
    for url in urls:
        print(f"  → {url}")
        results.append(fetch_url(url))

    output = {
        "fetched_at": datetime.utcnow().isoformat() + "Z",
        "count": len(results),
        "results": results,
    }

    output_file = "results.json"
    with open(output_file, "w") as f:
        json.dump(output, f, indent=2)

    successes = sum(1 for r in results if "som" in r)
    errors = sum(1 for r in results if "error" in r)
    print(f"\nDone: {successes} succeeded, {errors} failed")
    print(f"Results saved to {output_file}")

if __name__ == "__main__":
    main()