ragflow_scraper/docker-compose.dev.yml at main · nkissick-del/ragflow_scraper · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
services:
  scraper:
    build:
      context: .
      dockerfile: Dockerfile
      target: dev-runtime
      labels:
        org.opencontainers.image.title: "pdf-scraper-dev"
        org.opencontainers.image.description: "Dev runtime for PDF scraper"
        org.opencontainers.image.revision: "local"
    container_name: pdf-scraper-dev
    restart: unless-stopped
    ports:
      - "5001:5000"
    volumes:
      # Mount source code for live reloading
      - ./app:/app/app
      - ./scripts:/app/scripts
      - ./docs:/app/docs
      - ./tests:/app/tests
      - ./data:/app/data
      - ./config:/app/config
    env_file:
      - .env
    environment:
      - FLASK_ENV=development
      - FLASK_DEBUG=1
      - SECRET_KEY=dev-secret-key
      - HOST=0.0.0.0
      - PORT=5000
      # FlareSolverr (rendered page fetching + anti-bot bypass)
      - FLARESOLVERR_URL=http://flaresolverr:8191
      # Directories (container paths)
      - DOWNLOAD_DIR=/app/data/scraped
      - METADATA_DIR=/app/data/metadata
      - STATE_DIR=/app/data/state
      - LOG_DIR=/app/data/logs
      - LOG_LEVEL=DEBUG
      # Gotenberg (document → PDF conversion for archiving)
      - GOTENBERG_URL=http://gotenberg:3000
    depends_on:
      flaresolverr:
        condition: service_healthy
      gotenberg:
        condition: service_healthy
    networks:
      - scraper-net
    deploy:
      resources:
        limits:
          memory: 1g
          cpus: "0.75"

  # FlareSolverr for rendered page fetching and anti-bot bypass
  flaresolverr:
    image: ghcr.io/flaresolverr/flaresolverr:latest
    container_name: pdf-scraper-flaresolverr-dev
    restart: unless-stopped
    ports:
      - "8191:8191"
    environment:
      - LOG_LEVEL=info
      - LOG_HTML=false
      - CAPTCHA_SOLVER=none
      - TZ=Australia/Sydney
    networks:
      - scraper-net
    security_opt:
      - no-new-privileges:true
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8191/health"]
      interval: 15s
      timeout: 10s
      retries: 3
      start_period: 15s

  gotenberg:
    image: gotenberg/gotenberg:8
    container_name: pdf-scraper-gotenberg-dev
    restart: unless-stopped
    security_opt:
      - no-new-privileges:true
    ports:
      - "3156:3000"
    networks:
      - scraper-net
    deploy:
      resources:
        limits:
          memory: 512m
          cpus: "0.50"
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:3000/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 10s

networks:
  scraper-net:
    driver: bridge