docker-compose.yml

services:
  scraper:
    build:
      context: .
      dockerfile: Dockerfile
      labels:
        org.opencontainers.image.title: "pdf-scraper"
        org.opencontainers.image.description: "Local PDF scraper with RAGFlow upload"
        org.opencontainers.image.revision: "local"
    container_name: pdf-scraper
    restart: unless-stopped
    security_opt:
      - no-new-privileges:true
    ports:
      - "${PORT:-5000}:5000"
    volumes:
      - ./data:/app/data
      - ./config:/app/config
      - ./logs:/app/logs
    environment:
      # Flask Configuration
      - FLASK_ENV=${FLASK_ENV:-production}
      - FLASK_DEBUG=${FLASK_DEBUG:-0}
      - SECRET_KEY=${SECRET_KEY:?SECRET_KEY must be set in .env}
      - HOST=0.0.0.0
      - PORT=5000
      # RAGFlow Configuration
      - RAGFLOW_API_URL=${RAGFLOW_API_URL:-http://localhost:9380}
      - RAGFLOW_API_KEY=${RAGFLOW_API_KEY:-}
      - RAGFLOW_DATASET_ID=${RAGFLOW_DATASET_ID:-}
      # FlareSolverr (rendered page fetching + anti-bot bypass)
      - FLARESOLVERR_URL=${FLARESOLVERR_URL:-http://flaresolverr:8191}
      # Directories (inside container)
      - DOWNLOAD_DIR=/app/data/scraped
      - METADATA_DIR=/app/data/metadata
      - STATE_DIR=/app/data/state
      - LOG_DIR=/app/data/logs
      # Scraper Configuration
      - MAX_CONCURRENT_DOWNLOADS=${MAX_CONCURRENT_DOWNLOADS:-3}
      - REQUEST_TIMEOUT=${REQUEST_TIMEOUT:-60}
      - RETRY_ATTEMPTS=${RETRY_ATTEMPTS:-3}
      - LOG_LEVEL=${LOG_LEVEL:-INFO}
      # Guardian API
      - GUARDIAN_API_KEY=${GUARDIAN_API_KEY:-}
      # Gotenberg (document → PDF conversion for archiving)
      - GOTENBERG_URL=http://gotenberg:3000
      # pgvector RAG backend
      - DATABASE_URL=${DATABASE_URL:-}
      # Redis / Valkey (job dispatch and real-time events)
      - REDIS_URL=${REDIS_URL:-}
      - EMBEDDING_BACKEND=${EMBEDDING_BACKEND:-ollama}
      - EMBEDDING_MODEL=${EMBEDDING_MODEL:-nomic-embed-text}
      - EMBEDDING_URL=${EMBEDDING_URL:-}
      - EMBEDDING_API_KEY=${EMBEDDING_API_KEY:-}
      - EMBEDDING_DIMENSIONS=${EMBEDDING_DIMENSIONS:-768}
      - EMBEDDING_TIMEOUT=${EMBEDDING_TIMEOUT:-60}
      - CHUNKING_STRATEGY=${CHUNKING_STRATEGY:-hybrid}
      - CHUNK_MAX_TOKENS=${CHUNK_MAX_TOKENS:-512}
      - CHUNK_OVERLAP_TOKENS=${CHUNK_OVERLAP_TOKENS:-64}
      # LLM enrichment
      - LLM_BACKEND=${LLM_BACKEND:-ollama}
      - LLM_MODEL=${LLM_MODEL:-llama3.1:8b}
      - LLM_URL=${LLM_URL:-}
      - LLM_API_KEY=${LLM_API_KEY:-}
      - LLM_TIMEOUT=${LLM_TIMEOUT:-120}
      - LLM_ENRICHMENT_ENABLED=${LLM_ENRICHMENT_ENABLED:-false}
      - LLM_ENRICHMENT_MAX_TOKENS=${LLM_ENRICHMENT_MAX_TOKENS:-8000}
      - CONTEXTUAL_ENRICHMENT_ENABLED=${CONTEXTUAL_ENRICHMENT_ENABLED:-false}
      - CONTEXTUAL_ENRICHMENT_WINDOW=${CONTEXTUAL_ENRICHMENT_WINDOW:-3}
      # Ntfy alerts
      - NTFY_URL=${NTFY_URL:-}
      - NTFY_TOPIC=${NTFY_TOPIC:-scraper-alerts}
      # S3 / Garage archive backend
      - S3_ENDPOINT_URL=${S3_ENDPOINT_URL:-}
      - S3_ACCESS_KEY=${S3_ACCESS_KEY:-}
      - S3_SECRET_KEY=${S3_SECRET_KEY:-}
      - S3_BUCKET=${S3_BUCKET:-}
      - S3_REGION=${S3_REGION:-us-east-1}
      # Archive backend selection
      - ARCHIVE_BACKEND=${ARCHIVE_BACKEND:-paperless}
      - PARSER_BACKEND=${PARSER_BACKEND:-docling}
      - RAG_BACKEND=${RAG_BACKEND:-pgvector}
    depends_on:
      flaresolverr:
        condition: service_healthy
      gotenberg:
        condition: service_healthy
      valkey:
        condition: service_healthy
    networks:
      - scraper-net
    deploy:
      resources:
        limits:
          memory: 1g
          cpus: "0.75"
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:5000/"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 10s
    logging:
      driver: "json-file"
      options:
        max-size: "10m"
        max-file: "3"

  # FlareSolverr for rendered page fetching and anti-bot bypass
  flaresolverr:
    image: ghcr.io/flaresolverr/flaresolverr:latest
    container_name: pdf-scraper-flaresolverr
    restart: unless-stopped
    environment:
      - LOG_LEVEL=info
      - LOG_HTML=false
      - CAPTCHA_SOLVER=none
      - TZ=Australia/Sydney
    networks:
      - scraper-net
    security_opt:
      - no-new-privileges:true
    deploy:
      resources:
        limits:
          memory: 1g
          cpus: "1.0"
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8191/health"]
      interval: 15s
      timeout: 10s
      retries: 3
      start_period: 15s
    logging:
      driver: "json-file"
      options:
        max-size: "10m"
        max-file: "3"

  gotenberg:
    image: gotenberg/gotenberg:8
    container_name: pdf-scraper-gotenberg
    restart: unless-stopped
    security_opt:
      - no-new-privileges:true
    networks:
      - scraper-net
    deploy:
      resources:
        limits:
          memory: 512m
          cpus: "0.50"
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:3000/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 10s
    logging:
      driver: "json-file"
      options:
        max-size: "10m"
        max-file: "3"

  valkey:
    image: valkey/valkey:8-alpine
    container_name: pdf-scraper-valkey
    restart: unless-stopped
    security_opt:
      - no-new-privileges:true
    networks:
      - scraper-net
    volumes:
      - valkey-data:/data
    command: valkey-server --save 60 1 --loglevel warning
    deploy:
      resources:
        limits:
          memory: 128m
          cpus: "0.25"
    healthcheck:
      test: ["CMD", "valkey-cli", "ping"]
      interval: 15s
      timeout: 5s
      retries: 3
      start_period: 5s
    logging:
      driver: "json-file"
      options:
        max-size: "10m"
        max-file: "3"

networks:
  scraper-net:
    driver: bridge

volumes:
  valkey-data: