From 38757d55969af77b2a15bdc6ccfd1bfe65be2d62 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 8 Dec 2025 01:48:31 +0000 Subject: [PATCH 1/3] Initial plan From cf4e8ea235152789c839215d60e46d490773092c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 8 Dec 2025 01:59:54 +0000 Subject: [PATCH 2/3] feat: add production templates (CI/CD, Docker, K8s, n8n, LangGraph) Co-authored-by: Stacey77 <54900383+Stacey77@users.noreply.github.com> --- .env.prod.example | 81 +++ .github/workflows/cd-staging.yml | 73 +++ .github/workflows/ci.yml | 109 ++++ README.md | 4 +- docker-compose.prod.yml | 149 +++++ docs/deployment.md | 355 +++++++++++ docs/observability.md | 454 ++++++++++++++ docs/runbook.md | 557 ++++++++++++++++++ integration/api/Dockerfile | 39 ++ .../api/__pycache__/server.cpython-312.pyc | Bin 0 -> 15431 bytes integration/api/requirements.txt | 40 ++ integration/api/server.py | 352 +++++++++++ k8s/hpa.yaml | 61 ++ k8s/langgraph-deployment.yaml | 350 +++++++++++ n8n/README.md | 413 +++++++++++++ n8n/credentials/credentials_template.json | 60 ++ n8n/workflows/langgraph_trigger.json | 285 +++++++++ n8n/workflows/main_orchestrator.json | 263 +++++++++ pyproject.toml | 122 ++++ requirements.txt | 56 ++ 20 files changed, 3822 insertions(+), 1 deletion(-) create mode 100644 .env.prod.example create mode 100644 .github/workflows/cd-staging.yml create mode 100644 .github/workflows/ci.yml create mode 100644 docker-compose.prod.yml create mode 100644 docs/deployment.md create mode 100644 docs/observability.md create mode 100644 docs/runbook.md create mode 100644 integration/api/Dockerfile create mode 100644 integration/api/__pycache__/server.cpython-312.pyc create mode 100644 integration/api/requirements.txt create mode 100644 integration/api/server.py create mode 100644 k8s/hpa.yaml create mode 100644 k8s/langgraph-deployment.yaml create mode 100644 n8n/README.md create mode 100644 n8n/credentials/credentials_template.json create mode 100644 n8n/workflows/langgraph_trigger.json create mode 100644 n8n/workflows/main_orchestrator.json create mode 100644 pyproject.toml create mode 100644 requirements.txt diff --git a/.env.prod.example b/.env.prod.example new file mode 100644 index 0000000..122244e --- /dev/null +++ b/.env.prod.example @@ -0,0 +1,81 @@ +# Production Environment Configuration +# Copy this file to .env.prod and fill in the actual values + +# Application +APP_NAME=rag7-langgraph +APP_ENV=production +APP_DEBUG=false +LOG_LEVEL=INFO + +# API Configuration +API_HOST=0.0.0.0 +API_PORT=8000 +API_WORKERS=4 +API_RELOAD=false + +# LangGraph Configuration +LANGGRAPH_API_URL=http://langgraph:8123 +LANGGRAPH_CHECKPOINT_STORE=postgres +LANGGRAPH_STREAM_MODE=values + +# Database (PostgreSQL for LangGraph checkpoints) +# TODO: Replace with actual production database credentials +POSTGRES_HOST=postgres +POSTGRES_PORT=5432 +POSTGRES_DB=langgraph_checkpoints +POSTGRES_USER=langgraph +POSTGRES_PASSWORD=CHANGEME_SECURE_PASSWORD + +# Redis (for caching and rate limiting) +# TODO: Replace with actual production Redis credentials +REDIS_HOST=redis +REDIS_PORT=6379 +REDIS_PASSWORD=CHANGEME_SECURE_PASSWORD +REDIS_DB=0 + +# Authentication & Security +# TODO: Generate a secure secret key (e.g., using: openssl rand -hex 32) +SECRET_KEY=CHANGEME_GENERATE_SECURE_KEY +API_KEY_SALT=CHANGEME_GENERATE_SECURE_SALT +ALLOWED_ORIGINS=https://yourdomain.com,https://www.yourdomain.com + +# Observability +ENABLE_METRICS=true +METRICS_PORT=9090 +JAEGER_AGENT_HOST=jaeger +JAEGER_AGENT_PORT=6831 +OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 + +# LangChain/LangSmith (optional) +# TODO: Add your LangSmith API key if using +LANGCHAIN_TRACING_V2=false +LANGCHAIN_API_KEY= +LANGCHAIN_PROJECT=rag7-production + +# OpenAI API (if using OpenAI models) +# TODO: Add your OpenAI API key +OPENAI_API_KEY= + +# Other LLM Providers (as needed) +# TODO: Add your API keys for other providers +ANTHROPIC_API_KEY= +COHERE_API_KEY= +HUGGINGFACE_API_KEY= + +# Vector Store Configuration +VECTOR_STORE_TYPE=postgres # or 'pinecone', 'weaviate', 'qdrant' +VECTOR_DIMENSION=1536 + +# n8n Integration (if using) +N8N_WEBHOOK_URL=https://n8n.yourdomain.com/webhook +N8N_API_KEY=CHANGEME_N8N_API_KEY + +# Rate Limiting +RATE_LIMIT_ENABLED=true +RATE_LIMIT_PER_MINUTE=60 +RATE_LIMIT_BURST=10 + +# Feature Flags +ENABLE_ASYNC_PROCESSING=true +ENABLE_CACHING=true +CACHE_TTL_SECONDS=3600 diff --git a/.github/workflows/cd-staging.yml b/.github/workflows/cd-staging.yml new file mode 100644 index 0000000..7d3bcd9 --- /dev/null +++ b/.github/workflows/cd-staging.yml @@ -0,0 +1,73 @@ +name: CD - Deploy to Staging + +on: + push: + branches: [develop] + workflow_dispatch: + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + DEPLOYMENT_NAME: langgraph-api + NAMESPACE: staging + +jobs: + deploy: + runs-on: ubuntu-latest + environment: staging + steps: + - uses: actions/checkout@v4 + + - name: Set up kubectl + uses: azure/setup-kubectl@v3 + with: + version: 'v1.28.0' + + - name: Configure kubeconfig + run: | + mkdir -p $HOME/.kube + echo "${{ secrets.KUBECONFIG_STAGING }}" | base64 -d > $HOME/.kube/config + chmod 600 $HOME/.kube/config + + - name: Verify cluster connection + run: | + kubectl cluster-info + kubectl get nodes + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set image tag + id: image + run: | + IMAGE_TAG="${{ github.sha }}" + echo "tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT + echo "image=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${IMAGE_TAG}" >> $GITHUB_OUTPUT + + - name: Update deployment image + run: | + kubectl set image deployment/${{ env.DEPLOYMENT_NAME }} \ + langgraph=${{ steps.image.outputs.image }} \ + -n ${{ env.NAMESPACE }} + + - name: Wait for rollout + run: | + kubectl rollout status deployment/${{ env.DEPLOYMENT_NAME }} \ + -n ${{ env.NAMESPACE }} \ + --timeout=5m + + - name: Verify deployment + run: | + kubectl get pods -n ${{ env.NAMESPACE }} -l app=langgraph + kubectl get service -n ${{ env.NAMESPACE }} -l app=langgraph + + - name: Run smoke tests + run: | + # TODO: Add smoke test endpoint checks + # kubectl run smoke-test --image=curlimages/curl --rm -it --restart=Never \ + # -- curl -f http://${{ env.DEPLOYMENT_NAME }}.${{ env.NAMESPACE }}.svc.cluster.local/health + echo "Smoke tests would run here" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..6365e17 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,109 @@ +name: CI Pipeline + +on: + push: + branches: [main, develop] + pull_request: + branches: [main, develop] + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 black isort mypy + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + + - name: Lint with flake8 + run: | + # Stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # Exit-zero treats all errors as warnings + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + + - name: Check formatting with black + run: black --check . + continue-on-error: true + + - name: Check import ordering with isort + run: isort --check-only . + continue-on-error: true + + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + pip install pytest pytest-cov pytest-asyncio + + - name: Run tests + run: | + pytest --cov=. --cov-report=xml --cov-report=term + continue-on-error: true + + - name: Upload coverage reports + uses: codecov/codecov-action@v3 + continue-on-error: true + + build: + runs-on: ubuntu-latest + needs: [lint, test] + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=sha,prefix={{branch}}- + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/README.md b/README.md index f5a8ce3..6889614 100644 --- a/README.md +++ b/README.md @@ -1 +1,3 @@ -# rag7 \ No newline at end of file +# rag7 + +> **📦 Production Templates Available**: This repository now includes production-ready deployment templates, CI/CD workflows, Kubernetes manifests, n8n workflows, and comprehensive documentation. See the `docs/` directory and related files to get started with deployment. \ No newline at end of file diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml new file mode 100644 index 0000000..62973eb --- /dev/null +++ b/docker-compose.prod.yml @@ -0,0 +1,149 @@ +version: '3.8' + +services: + langgraph: + image: ghcr.io/stacey77/rag7:latest + container_name: langgraph-api + restart: unless-stopped + env_file: + - .env.prod + ports: + - "8123:8123" + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8123/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + networks: + - rag7-network + volumes: + - ./data:/app/data + deploy: + resources: + limits: + cpus: '2.0' + memory: 4G + reservations: + cpus: '1.0' + memory: 2G + + integration-api: + build: + context: ./integration/api + dockerfile: Dockerfile + container_name: integration-api + restart: unless-stopped + env_file: + - .env.prod + ports: + - "8000:8000" + depends_on: + langgraph: + condition: service_healthy + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 20s + networks: + - rag7-network + deploy: + resources: + limits: + cpus: '1.0' + memory: 2G + reservations: + cpus: '0.5' + memory: 1G + + postgres: + image: postgres:15-alpine + container_name: langgraph-postgres + restart: unless-stopped + environment: + POSTGRES_DB: ${POSTGRES_DB:-langgraph_checkpoints} + POSTGRES_USER: ${POSTGRES_USER:-langgraph} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-CHANGEME_SECURE_PASSWORD} + PGDATA: /var/lib/postgresql/data/pgdata + ports: + - "5432:5432" + volumes: + - postgres-data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-langgraph}"] + interval: 10s + timeout: 5s + retries: 5 + networks: + - rag7-network + deploy: + resources: + limits: + cpus: '1.0' + memory: 2G + reservations: + cpus: '0.5' + memory: 1G + + redis: + image: redis:7-alpine + container_name: langgraph-redis + restart: unless-stopped + command: redis-server --requirepass ${REDIS_PASSWORD:-CHANGEME_SECURE_PASSWORD} + ports: + - "6379:6379" + volumes: + - redis-data:/data + healthcheck: + test: ["CMD", "redis-cli", "--raw", "incr", "ping"] + interval: 10s + timeout: 5s + retries: 5 + networks: + - rag7-network + deploy: + resources: + limits: + cpus: '0.5' + memory: 512M + reservations: + cpus: '0.25' + memory: 256M + + nginx: + image: nginx:alpine + container_name: rag7-nginx + restart: unless-stopped + ports: + - "80:80" + - "443:443" + volumes: + - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro + - ./nginx/ssl:/etc/nginx/ssl:ro + depends_on: + - integration-api + - langgraph + networks: + - rag7-network + deploy: + resources: + limits: + cpus: '0.5' + memory: 512M + +volumes: + postgres-data: + driver: local + redis-data: + driver: local + +networks: + rag7-network: + driver: bridge diff --git a/docs/deployment.md b/docs/deployment.md new file mode 100644 index 0000000..349556b --- /dev/null +++ b/docs/deployment.md @@ -0,0 +1,355 @@ +# Deployment Guide + +This guide covers deploying the RAG7 LangGraph application to production environments. + +## Table of Contents + +- [Prerequisites](#prerequisites) +- [Environment Configuration](#environment-configuration) +- [Deployment Options](#deployment-options) + - [Docker Compose](#docker-compose) + - [Kubernetes](#kubernetes) +- [Post-Deployment](#post-deployment) +- [Troubleshooting](#troubleshooting) + +## Prerequisites + +### Required Tools + +- Docker & Docker Compose (v2.0+) +- kubectl (v1.28+) for Kubernetes deployments +- Access to container registry (GitHub Container Registry) +- Database: PostgreSQL 15+ +- Cache: Redis 7+ + +### Required Secrets + +Before deploying, you must configure the following secrets: + +1. **Database Credentials**: PostgreSQL password +2. **Secret Keys**: Application secret key and API key salt +3. **API Keys**: OpenAI, Anthropic, or other LLM provider keys +4. **Registry Access**: GitHub Container Registry token (for pulling images) +5. **Kubeconfig**: Kubernetes cluster credentials (for K8s deployments) + +## Environment Configuration + +### Step 1: Create Production Environment File + +```bash +cp .env.prod.example .env.prod +``` + +### Step 2: Update Required Values + +Edit `.env.prod` and replace all `CHANGEME_*` placeholders: + +```bash +# Generate secure secret key +openssl rand -hex 32 + +# Generate API key salt +openssl rand -hex 16 + +# Generate secure database password +openssl rand -base64 32 +``` + +### Step 3: Configure API Keys + +Add your LLM provider API keys: + +```bash +OPENAI_API_KEY=sk-your-actual-key-here +LANGCHAIN_API_KEY=ls__your-actual-key-here +``` + +## Deployment Options + +### Docker Compose + +Docker Compose is suitable for single-server deployments or staging environments. + +#### Deploy with Docker Compose + +```bash +# Pull latest images +docker-compose -f docker-compose.prod.yml pull + +# Start services +docker-compose -f docker-compose.prod.yml up -d + +# Check service status +docker-compose -f docker-compose.prod.yml ps + +# View logs +docker-compose -f docker-compose.prod.yml logs -f langgraph +``` + +#### Verify Deployment + +```bash +# Check health endpoint +curl http://localhost:8000/health + +# Check readiness endpoint +curl http://localhost:8000/ready + +# Test graph execution +curl -X POST http://localhost:8000/v1/graph/run \ + -H "Content-Type: application/json" \ + -d '{"input": {"query": "test"}}' +``` + +#### Stop Services + +```bash +docker-compose -f docker-compose.prod.yml down +``` + +### Kubernetes + +Kubernetes is recommended for production deployments requiring high availability and auto-scaling. + +#### Prerequisites + +1. **Configure kubectl** + +```bash +# Set up kubeconfig +export KUBECONFIG=/path/to/your/kubeconfig + +# Verify connection +kubectl cluster-info +kubectl get nodes +``` + +2. **Create Namespace** + +```bash +kubectl apply -f k8s/langgraph-deployment.yaml +# This creates the rag7 namespace +``` + +3. **Configure Secrets** + +Update the secrets in `k8s/langgraph-deployment.yaml` or create them via kubectl: + +```bash +# Create secrets from literal values +kubectl create secret generic langgraph-secrets \ + --namespace=rag7 \ + --from-literal=POSTGRES_PASSWORD='your-secure-password' \ + --from-literal=REDIS_PASSWORD='your-redis-password' \ + --from-literal=SECRET_KEY='your-secret-key' \ + --from-literal=API_KEY_SALT='your-api-salt' \ + --from-literal=OPENAI_API_KEY='sk-your-key' \ + --dry-run=client -o yaml | kubectl apply -f - +``` + +4. **Create Image Pull Secret** (for GHCR) + +```bash +kubectl create secret docker-registry ghcr-secret \ + --namespace=rag7 \ + --docker-server=ghcr.io \ + --docker-username=YOUR_GITHUB_USERNAME \ + --docker-password=YOUR_GITHUB_TOKEN \ + --docker-email=YOUR_EMAIL +``` + +#### Deploy to Kubernetes + +```bash +# Apply all manifests +kubectl apply -f k8s/langgraph-deployment.yaml +kubectl apply -f k8s/hpa.yaml + +# Check deployment status +kubectl get deployments -n rag7 +kubectl get pods -n rag7 +kubectl get services -n rag7 + +# Watch rollout +kubectl rollout status deployment/langgraph -n rag7 +``` + +#### Verify Kubernetes Deployment + +```bash +# Port-forward to test locally +kubectl port-forward -n rag7 svc/langgraph 8123:8123 + +# In another terminal, test endpoints +curl http://localhost:8123/health +curl http://localhost:8123/ready +``` + +#### Expose Service (Optional) + +Using an Ingress controller: + +```yaml +# ingress.yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: langgraph-ingress + namespace: rag7 + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod +spec: + ingressClassName: nginx + tls: + - hosts: + - api.yourdomain.com + secretName: langgraph-tls + rules: + - host: api.yourdomain.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: langgraph + port: + number: 8123 +``` + +```bash +kubectl apply -f ingress.yaml +``` + +## Post-Deployment + +### Database Migrations + +Run any necessary database migrations: + +```bash +# Docker Compose +docker-compose -f docker-compose.prod.yml exec langgraph python -m alembic upgrade head + +# Kubernetes +kubectl exec -n rag7 -it deployment/langgraph -- python -m alembic upgrade head +``` + +### Monitoring Setup + +1. **Check Metrics Endpoint** + +```bash +curl http://localhost:9090/metrics +``` + +2. **Configure Prometheus** (if using) + +Add scraping configuration for the metrics endpoint. + +3. **Set Up Alerts** + +Configure alerting rules for critical metrics. + +### Backup Configuration + +1. **Database Backups** + +```bash +# Set up automated PostgreSQL backups +kubectl create cronjob postgres-backup \ + --image=postgres:15-alpine \ + --schedule="0 2 * * *" \ + --restart=Never \ + -- pg_dump -h postgres.rag7.svc.cluster.local -U langgraph > /backup/db.sql +``` + +2. **Configuration Backups** + +Store ConfigMaps and Secrets in version control (encrypted). + +## Troubleshooting + +### Common Issues + +#### Pods Not Starting + +```bash +# Check pod status +kubectl get pods -n rag7 + +# View pod logs +kubectl logs -n rag7 deployment/langgraph --tail=100 + +# Describe pod for events +kubectl describe pod -n rag7 POD_NAME +``` + +#### Connection Issues + +```bash +# Test database connectivity +kubectl run -n rag7 -it --rm debug --image=postgres:15-alpine --restart=Never \ + -- psql -h postgres.rag7.svc.cluster.local -U langgraph -d langgraph_checkpoints + +# Test Redis connectivity +kubectl run -n rag7 -it --rm debug --image=redis:7-alpine --restart=Never \ + -- redis-cli -h redis.rag7.svc.cluster.local -a PASSWORD ping +``` + +#### Image Pull Errors + +```bash +# Verify image pull secret +kubectl get secret ghcr-secret -n rag7 -o yaml + +# Test image pull manually +docker pull ghcr.io/stacey77/rag7:latest +``` + +#### Resource Constraints + +```bash +# Check resource usage +kubectl top pods -n rag7 + +# Check HPA status +kubectl get hpa -n rag7 + +# Describe HPA for details +kubectl describe hpa langgraph-hpa -n rag7 +``` + +### Logs and Debugging + +```bash +# Stream all logs +kubectl logs -n rag7 -l app=langgraph --tail=100 -f + +# Get logs from specific pod +kubectl logs -n rag7 POD_NAME --tail=200 + +# Get previous crashed container logs +kubectl logs -n rag7 POD_NAME --previous +``` + +### Rollback Deployment + +```bash +# View rollout history +kubectl rollout history deployment/langgraph -n rag7 + +# Rollback to previous version +kubectl rollout undo deployment/langgraph -n rag7 + +# Rollback to specific revision +kubectl rollout undo deployment/langgraph -n rag7 --to-revision=2 +``` + +## Next Steps + +- Configure observability (see [observability.md](./observability.md)) +- Set up alerting and on-call rotation (see [runbook.md](./runbook.md)) +- Import n8n workflows (see [../n8n/README.md](../n8n/README.md)) +- Configure CI/CD pipelines +- Set up backup and disaster recovery procedures diff --git a/docs/observability.md b/docs/observability.md new file mode 100644 index 0000000..a0c3051 --- /dev/null +++ b/docs/observability.md @@ -0,0 +1,454 @@ +# Observability Guide + +This guide covers monitoring, logging, and tracing for the RAG7 LangGraph application. + +## Table of Contents + +- [Metrics](#metrics) +- [Logging](#logging) +- [Tracing](#tracing) +- [Dashboards](#dashboards) +- [Alerting](#alerting) + +## Metrics + +### Prometheus Metrics Endpoint + +The application exposes Prometheus-compatible metrics at `/metrics` on port 9090. + +#### Key Metrics to Monitor + +**Application Metrics:** +- `http_requests_total` - Total HTTP requests by endpoint and status +- `http_request_duration_seconds` - Request duration histogram +- `langgraph_execution_duration_seconds` - Graph execution time +- `langgraph_errors_total` - Total graph execution errors +- `active_graph_executions` - Currently running graph executions + +**System Metrics:** +- `process_cpu_usage` - CPU usage percentage +- `process_memory_bytes` - Memory usage in bytes +- `process_open_fds` - Open file descriptors + +**Database Metrics:** +- `db_connections_active` - Active database connections +- `db_query_duration_seconds` - Database query duration +- `db_errors_total` - Database errors + +**Cache Metrics:** +- `cache_hits_total` - Cache hit count +- `cache_misses_total` - Cache miss count +- `cache_evictions_total` - Cache eviction count + +### Scraping Configuration + +Add to your Prometheus configuration: + +```yaml +scrape_configs: + - job_name: 'langgraph' + kubernetes_sd_configs: + - role: pod + namespaces: + names: + - rag7 + relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + target_label: __address__ + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) +``` + +### Example Queries + +```promql +# Request rate per second +rate(http_requests_total[5m]) + +# 99th percentile latency +histogram_quantile(0.99, rate(http_request_duration_seconds_bucket[5m])) + +# Error rate +rate(langgraph_errors_total[5m]) / rate(http_requests_total[5m]) + +# Memory usage over time +process_memory_bytes + +# Cache hit rate +rate(cache_hits_total[5m]) / (rate(cache_hits_total[5m]) + rate(cache_misses_total[5m])) +``` + +## Logging + +### Log Levels + +The application supports the following log levels: +- `DEBUG` - Detailed diagnostic information +- `INFO` - General informational messages +- `WARNING` - Warning messages for potentially harmful situations +- `ERROR` - Error events that might still allow the app to continue +- `CRITICAL` - Critical events that may cause the app to abort + +Set log level via environment variable: +```bash +LOG_LEVEL=INFO +``` + +### Log Format + +Logs are output in JSON format for easy parsing: + +```json +{ + "timestamp": "2024-01-15T10:30:45.123Z", + "level": "INFO", + "logger": "langgraph.api", + "message": "Graph execution completed", + "request_id": "req_abc123", + "graph_id": "graph_xyz789", + "duration_ms": 1234, + "user_id": "user_456" +} +``` + +### Accessing Logs + +#### Docker Compose + +```bash +# View logs from all services +docker-compose -f docker-compose.prod.yml logs -f + +# View logs from specific service +docker-compose -f docker-compose.prod.yml logs -f langgraph + +# View last 100 lines +docker-compose -f docker-compose.prod.yml logs --tail=100 langgraph +``` + +#### Kubernetes + +```bash +# View logs from all pods +kubectl logs -n rag7 -l app=langgraph --tail=100 -f + +# View logs from specific pod +kubectl logs -n rag7 POD_NAME --tail=200 -f + +# View logs from all containers in pod +kubectl logs -n rag7 POD_NAME --all-containers=true +``` + +### Centralized Logging + +#### ELK Stack (Elasticsearch, Logstash, Kibana) + +Configure Filebeat to ship logs to Elasticsearch: + +```yaml +# filebeat.yml +filebeat.inputs: + - type: container + paths: + - '/var/lib/docker/containers/*/*.log' + processors: + - add_kubernetes_metadata: + host: ${NODE_NAME} + matchers: + - logs_path: + logs_path: "/var/lib/docker/containers/" + +output.elasticsearch: + hosts: ["elasticsearch:9200"] + index: "langgraph-logs-%{+yyyy.MM.dd}" +``` + +#### Loki (Grafana Loki) + +Deploy Promtail to collect and forward logs: + +```yaml +# promtail-config.yaml +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: + - job_name: kubernetes-pods + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: [__meta_kubernetes_namespace] + target_label: namespace + - source_labels: [__meta_kubernetes_pod_name] + target_label: pod +``` + +## Tracing + +### OpenTelemetry Integration + +The application supports OpenTelemetry for distributed tracing. + +#### Configuration + +Set environment variables: + +```bash +OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 +JAEGER_AGENT_HOST=jaeger +JAEGER_AGENT_PORT=6831 +``` + +#### Trace Context + +Each request includes trace context propagation: +- `traceparent` - W3C Trace Context +- `tracestate` - Additional trace state + +#### Trace Attributes + +Custom attributes added to spans: +- `graph.id` - LangGraph graph identifier +- `graph.execution.id` - Unique execution ID +- `user.id` - User identifier +- `llm.provider` - LLM provider (OpenAI, Anthropic, etc.) +- `llm.model` - Model name +- `llm.tokens` - Token count + +### Jaeger UI + +Access Jaeger UI to view traces: + +```bash +# Port forward Jaeger UI +kubectl port-forward -n observability svc/jaeger-query 16686:16686 + +# Open in browser +open http://localhost:16686 +``` + +### Trace Analysis + +Use traces to identify: +- Slow operations and bottlenecks +- Service dependencies +- Error propagation +- Concurrent execution patterns + +## Dashboards + +### Grafana Dashboard + +Import the pre-configured Grafana dashboard: + +```bash +# TODO: Create and include dashboard JSON +# grafana-dashboard.json +``` + +#### Key Panels + +1. **Overview** + - Request rate (requests/sec) + - Error rate (%) + - 95th/99th percentile latency + - Active executions + +2. **Performance** + - Request duration distribution + - Graph execution time + - Database query time + - Cache hit rate + +3. **Resources** + - CPU usage + - Memory usage + - Network I/O + - Disk I/O + +4. **Errors** + - Error count by type + - Error rate trend + - Failed executions + +### Custom Dashboards + +Create custom dashboards using Grafana or your preferred tool. + +#### Example: Request Rate Dashboard + +```json +{ + "title": "Request Rate", + "targets": [ + { + "expr": "rate(http_requests_total{namespace=\"rag7\"}[5m])", + "legendFormat": "{{method}} {{endpoint}}" + } + ] +} +``` + +## Alerting + +### Alert Rules + +Configure alerts for critical conditions: + +#### High Error Rate + +```yaml +groups: + - name: langgraph_alerts + interval: 30s + rules: + - alert: HighErrorRate + expr: rate(langgraph_errors_total[5m]) / rate(http_requests_total[5m]) > 0.05 + for: 5m + labels: + severity: warning + annotations: + summary: "High error rate detected" + description: "Error rate is {{ $value | humanizePercentage }} (threshold: 5%)" +``` + +#### High Latency + +```yaml + - alert: HighLatency + expr: histogram_quantile(0.99, rate(http_request_duration_seconds_bucket[5m])) > 5 + for: 5m + labels: + severity: warning + annotations: + summary: "High latency detected" + description: "99th percentile latency is {{ $value }}s (threshold: 5s)" +``` + +#### Pod Restart + +```yaml + - alert: PodRestarting + expr: rate(kube_pod_container_status_restarts_total{namespace="rag7"}[15m]) > 0 + for: 5m + labels: + severity: critical + annotations: + summary: "Pod {{ $labels.pod }} is restarting" + description: "Pod has restarted {{ $value }} times in the last 15 minutes" +``` + +#### Resource Exhaustion + +```yaml + - alert: HighMemoryUsage + expr: container_memory_usage_bytes{namespace="rag7"} / container_spec_memory_limit_bytes > 0.9 + for: 5m + labels: + severity: warning + annotations: + summary: "High memory usage detected" + description: "Memory usage is {{ $value | humanizePercentage }}" +``` + +### Alert Channels + +Configure notification channels: + +#### Slack + +```yaml +receivers: + - name: 'slack-notifications' + slack_configs: + - api_url: 'YOUR_SLACK_WEBHOOK_URL' + channel: '#alerts' + title: 'Alert: {{ .GroupLabels.alertname }}' + text: '{{ range .Alerts }}{{ .Annotations.description }}{{ end }}' +``` + +#### PagerDuty + +```yaml +receivers: + - name: 'pagerduty' + pagerduty_configs: + - service_key: 'YOUR_PAGERDUTY_KEY' + description: '{{ .GroupLabels.alertname }}: {{ .CommonAnnotations.summary }}' +``` + +#### Email + +```yaml +receivers: + - name: 'email' + email_configs: + - to: 'oncall@example.com' + from: 'alerts@example.com' + smarthost: 'smtp.example.com:587' + auth_username: 'alerts@example.com' + auth_password: 'YOUR_SMTP_PASSWORD' +``` + +## Health Checks + +### Endpoints + +- `GET /health` - Basic health check (liveness probe) +- `GET /ready` - Readiness check (includes dependencies) +- `GET /metrics` - Prometheus metrics + +### Example Health Check Response + +```json +{ + "status": "healthy", + "timestamp": "2024-01-15T10:30:45.123Z", + "checks": { + "database": "ok", + "redis": "ok", + "llm_provider": "ok" + }, + "version": "1.0.0", + "uptime_seconds": 86400 +} +``` + +### Curl Commands + +```bash +# Check health +curl -i http://localhost:8123/health + +# Check readiness +curl -i http://localhost:8123/ready + +# Fetch metrics +curl http://localhost:9090/metrics +``` + +## Best Practices + +1. **Set up alerts before issues occur** - Don't wait for production incidents +2. **Monitor the full stack** - Application, infrastructure, and dependencies +3. **Use structured logging** - Enables better searching and analysis +4. **Implement distributed tracing** - Essential for debugging microservices +5. **Create runbooks** - Document response procedures (see [runbook.md](./runbook.md)) +6. **Regular review** - Periodically review dashboards and alerts +7. **Load testing** - Test observability under realistic load conditions + +## Next Steps + +- Set up Prometheus and Grafana +- Configure alert notification channels +- Create custom dashboards for your use case +- Implement distributed tracing with Jaeger or Zipkin +- Review and test incident response procedures diff --git a/docs/runbook.md b/docs/runbook.md new file mode 100644 index 0000000..ee35bde --- /dev/null +++ b/docs/runbook.md @@ -0,0 +1,557 @@ +# Operations Runbook + +This runbook provides step-by-step procedures for common operational tasks and incident response. + +## Table of Contents + +- [Emergency Contacts](#emergency-contacts) +- [Common Incidents](#common-incidents) +- [Operational Procedures](#operational-procedures) +- [Maintenance Tasks](#maintenance-tasks) +- [Recovery Procedures](#recovery-procedures) + +## Emergency Contacts + +### On-Call Schedule + +| Role | Primary | Secondary | +|------|---------|-----------| +| Engineering | TODO: Add | TODO: Add | +| DevOps | TODO: Add | TODO: Add | +| Manager | TODO: Add | TODO: Add | + +### Escalation Path + +1. On-call engineer (15 min response time) +2. Secondary on-call (30 min response time) +3. Engineering manager (1 hour response time) + +### Communication Channels + +- **Slack**: #incidents (for incident coordination) +- **PagerDuty**: For critical alerts +- **Status Page**: TODO: Add URL + +## Common Incidents + +### High Error Rate + +**Symptoms:** +- Alert: "HighErrorRate" firing +- Increased 5xx responses +- User reports of failures + +**Diagnosis:** + +```bash +# Check error logs +kubectl logs -n rag7 -l app=langgraph --tail=100 | grep ERROR + +# Check error metrics +curl http://localhost:9090/metrics | grep error + +# Check recent deployments +kubectl rollout history deployment/langgraph -n rag7 +``` + +**Resolution Steps:** + +1. **Identify error type** + ```bash + # Group errors by type + kubectl logs -n rag7 -l app=langgraph | grep ERROR | cut -d' ' -f5- | sort | uniq -c | sort -rn + ``` + +2. **Check dependencies** + ```bash + # Test database connection + kubectl run -n rag7 -it --rm debug --image=postgres:15-alpine --restart=Never \ + -- psql -h postgres.rag7.svc.cluster.local -U langgraph -c "SELECT 1" + + # Test Redis connection + kubectl run -n rag7 -it --rm debug --image=redis:7-alpine --restart=Never \ + -- redis-cli -h redis.rag7.svc.cluster.local ping + ``` + +3. **Rollback if recent deployment** + ```bash + kubectl rollout undo deployment/langgraph -n rag7 + ``` + +4. **Scale up if capacity issue** + ```bash + kubectl scale deployment/langgraph -n rag7 --replicas=5 + ``` + +### High Latency + +**Symptoms:** +- Alert: "HighLatency" firing +- Slow response times +- Request timeouts + +**Diagnosis:** + +```bash +# Check request latency +curl http://localhost:9090/metrics | grep duration + +# Check resource usage +kubectl top pods -n rag7 + +# Check database slow queries +kubectl exec -n rag7 -it deployment/postgres -- \ + psql -U langgraph -c "SELECT query, calls, mean_exec_time FROM pg_stat_statements ORDER BY mean_exec_time DESC LIMIT 10" +``` + +**Resolution Steps:** + +1. **Check for resource constraints** + ```bash + kubectl describe pod -n rag7 POD_NAME | grep -A 5 "Limits\|Requests" + ``` + +2. **Scale horizontally** + ```bash + kubectl scale deployment/langgraph -n rag7 --replicas=8 + ``` + +3. **Check for memory leaks** + ```bash + kubectl top pods -n rag7 --sort-by=memory + ``` + +4. **Restart pods if necessary** + ```bash + kubectl rollout restart deployment/langgraph -n rag7 + ``` + +### Pod Crash Loop + +**Symptoms:** +- Alert: "PodRestarting" firing +- Pods in CrashLoopBackOff state +- Service degradation + +**Diagnosis:** + +```bash +# Check pod status +kubectl get pods -n rag7 + +# View recent logs +kubectl logs -n rag7 POD_NAME --tail=100 + +# View previous container logs +kubectl logs -n rag7 POD_NAME --previous + +# Describe pod for events +kubectl describe pod -n rag7 POD_NAME +``` + +**Resolution Steps:** + +1. **Check for configuration issues** + ```bash + kubectl get configmap langgraph-config -n rag7 -o yaml + kubectl get secret langgraph-secrets -n rag7 -o yaml + ``` + +2. **Verify image** + ```bash + kubectl get deployment langgraph -n rag7 -o jsonpath='{.spec.template.spec.containers[0].image}' + ``` + +3. **Check resource limits** + ```bash + kubectl describe pod -n rag7 POD_NAME | grep -A 5 "Limits" + ``` + +4. **Fix and redeploy** + ```bash + # Update configuration + kubectl edit configmap langgraph-config -n rag7 + + # Restart deployment + kubectl rollout restart deployment/langgraph -n rag7 + ``` + +### Database Connection Issues + +**Symptoms:** +- Database connection errors in logs +- "connection refused" or "connection timeout" +- Service unavailable + +**Diagnosis:** + +```bash +# Check PostgreSQL pod status +kubectl get pods -n rag7 -l app=postgres + +# Check PostgreSQL logs +kubectl logs -n rag7 -l app=postgres --tail=100 + +# Test connection from application pod +kubectl exec -n rag7 -it deployment/langgraph -- \ + python -c "import psycopg2; conn = psycopg2.connect('postgresql://langgraph:PASSWORD@postgres.rag7.svc.cluster.local/langgraph_checkpoints'); print('Connected')" +``` + +**Resolution Steps:** + +1. **Check database is running** + ```bash + kubectl get statefulset postgres -n rag7 + ``` + +2. **Check connection limits** + ```bash + kubectl exec -n rag7 -it statefulset/postgres -- \ + psql -U langgraph -c "SHOW max_connections" + + kubectl exec -n rag7 -it statefulset/postgres -- \ + psql -U langgraph -c "SELECT count(*) FROM pg_stat_activity" + ``` + +3. **Restart PostgreSQL if needed** + ```bash + kubectl delete pod -n rag7 -l app=postgres + ``` + +4. **Check for disk space issues** + ```bash + kubectl exec -n rag7 -it statefulset/postgres -- df -h + ``` + +### Out of Memory (OOM) + +**Symptoms:** +- Pods killed by OOMKiller +- Memory usage at 100% +- Frequent restarts + +**Diagnosis:** + +```bash +# Check memory usage +kubectl top pods -n rag7 + +# Check OOM events +kubectl get events -n rag7 --sort-by='.lastTimestamp' | grep OOM + +# Check memory limits +kubectl describe pod -n rag7 POD_NAME | grep -A 3 "Limits" +``` + +**Resolution Steps:** + +1. **Increase memory limits** + ```bash + kubectl edit deployment langgraph -n rag7 + # Update memory limits under resources + ``` + +2. **Check for memory leaks** + ```bash + # Monitor memory over time + kubectl top pods -n rag7 --watch + ``` + +3. **Scale horizontally instead** + ```bash + kubectl scale deployment/langgraph -n rag7 --replicas=6 + ``` + +### API Rate Limiting + +**Symptoms:** +- 429 Too Many Requests errors +- LLM provider rate limit errors +- Requests being throttled + +**Diagnosis:** + +```bash +# Check rate limit errors +kubectl logs -n rag7 -l app=langgraph | grep "rate limit" + +# Check metrics +curl http://localhost:9090/metrics | grep rate_limit +``` + +**Resolution Steps:** + +1. **Implement exponential backoff** (code change required) + +2. **Distribute load across multiple API keys** + ```bash + kubectl edit secret langgraph-secrets -n rag7 + # Add additional API keys + ``` + +3. **Cache responses to reduce API calls** + ```bash + # Verify Redis is working + kubectl get pods -n rag7 -l app=redis + ``` + +4. **Contact provider for rate limit increase** + +## Operational Procedures + +### Deployment Procedure + +**Standard Deployment:** + +```bash +# 1. Review changes +git diff main..feature-branch + +# 2. Merge to main +git checkout main +git merge feature-branch + +# 3. Tag release +git tag -a v1.0.1 -m "Release 1.0.1" +git push origin v1.0.1 + +# 4. Build and push image (CI does this automatically) +# CI will build and push ghcr.io/stacey77/rag7:v1.0.1 + +# 5. Update deployment +kubectl set image deployment/langgraph -n rag7 \ + langgraph=ghcr.io/stacey77/rag7:v1.0.1 + +# 6. Monitor rollout +kubectl rollout status deployment/langgraph -n rag7 + +# 7. Verify deployment +curl http://API_ENDPOINT/health +``` + +**Hotfix Deployment:** + +```bash +# 1. Create hotfix branch +git checkout -b hotfix/critical-fix main + +# 2. Make minimal fix +# ... edit files ... + +# 3. Test locally +docker build -t rag7:hotfix . +docker run -p 8123:8123 rag7:hotfix + +# 4. Deploy directly +git commit -am "Hotfix: description" +git push origin hotfix/critical-fix + +# 5. Trigger CD pipeline or deploy manually +kubectl set image deployment/langgraph -n rag7 \ + langgraph=ghcr.io/stacey77/rag7:hotfix-critical-fix + +# 6. Monitor closely +kubectl logs -n rag7 -l app=langgraph -f +``` + +### Rollback Procedure + +```bash +# 1. List rollout history +kubectl rollout history deployment/langgraph -n rag7 + +# 2. Rollback to previous version +kubectl rollout undo deployment/langgraph -n rag7 + +# 3. Or rollback to specific revision +kubectl rollout undo deployment/langgraph -n rag7 --to-revision=5 + +# 4. Monitor rollback +kubectl rollout status deployment/langgraph -n rag7 + +# 5. Verify service health +curl http://API_ENDPOINT/health +``` + +### Scaling Procedure + +**Manual Scaling:** + +```bash +# Scale up +kubectl scale deployment/langgraph -n rag7 --replicas=8 + +# Scale down +kubectl scale deployment/langgraph -n rag7 --replicas=2 + +# Verify +kubectl get pods -n rag7 -l app=langgraph +``` + +**Adjust HPA:** + +```bash +# Update HPA limits +kubectl edit hpa langgraph-hpa -n rag7 + +# Check HPA status +kubectl get hpa -n rag7 +kubectl describe hpa langgraph-hpa -n rag7 +``` + +### Backup Procedure + +**Database Backup:** + +```bash +# 1. Create backup +kubectl exec -n rag7 statefulset/postgres -- \ + pg_dump -U langgraph langgraph_checkpoints > backup_$(date +%Y%m%d_%H%M%S).sql + +# 2. Compress +gzip backup_*.sql + +# 3. Upload to storage +aws s3 cp backup_*.sql.gz s3://rag7-backups/$(date +%Y/%m/%d)/ + +# 4. Verify backup +gunzip -c backup_*.sql.gz | head -n 20 +``` + +**Configuration Backup:** + +```bash +# Export all configurations +kubectl get all,configmap,secret -n rag7 -o yaml > rag7_backup_$(date +%Y%m%d).yaml + +# Store securely +gpg -c rag7_backup_$(date +%Y%m%d).yaml +``` + +## Maintenance Tasks + +### Certificate Renewal + +```bash +# Check certificate expiration +kubectl get certificate -n rag7 + +# Renew certificate (cert-manager does this automatically) +kubectl describe certificate langgraph-tls -n rag7 + +# Manual renewal if needed +kubectl delete secret langgraph-tls -n rag7 +# cert-manager will recreate +``` + +### Database Maintenance + +```bash +# Vacuum database +kubectl exec -n rag7 statefulset/postgres -- \ + psql -U langgraph -c "VACUUM ANALYZE" + +# Check database size +kubectl exec -n rag7 statefulset/postgres -- \ + psql -U langgraph -c "SELECT pg_size_pretty(pg_database_size('langgraph_checkpoints'))" + +# Clean old checkpoints (if applicable) +kubectl exec -n rag7 statefulset/postgres -- \ + psql -U langgraph -c "DELETE FROM checkpoints WHERE created_at < NOW() - INTERVAL '30 days'" +``` + +### Log Rotation + +```bash +# Check log volume sizes +kubectl exec -n rag7 POD_NAME -- du -sh /var/log + +# Logs are automatically rotated by Kubernetes +# Configure retention in logging backend (ELK, Loki, etc.) +``` + +## Recovery Procedures + +### Disaster Recovery + +**Complete Cluster Failure:** + +1. **Provision new cluster** +2. **Restore configurations** + ```bash + kubectl apply -f rag7_backup_YYYYMMDD.yaml + ``` +3. **Restore database** + ```bash + kubectl exec -n rag7 -it statefulset/postgres -- \ + psql -U langgraph langgraph_checkpoints < backup_YYYYMMDD.sql + ``` +4. **Verify services** +5. **Update DNS/Load Balancer** + +### Data Corruption + +```bash +# 1. Stop application +kubectl scale deployment/langgraph -n rag7 --replicas=0 + +# 2. Restore database from backup +kubectl exec -n rag7 -it statefulset/postgres -- \ + dropdb -U langgraph langgraph_checkpoints +kubectl exec -n rag7 -it statefulset/postgres -- \ + createdb -U langgraph langgraph_checkpoints +kubectl exec -n rag7 -it statefulset/postgres -- \ + psql -U langgraph langgraph_checkpoints < backup_YYYYMMDD.sql + +# 3. Restart application +kubectl scale deployment/langgraph -n rag7 --replicas=2 + +# 4. Verify data integrity +# Run validation queries +``` + +## Appendix + +### Useful Commands + +```bash +# Get cluster info +kubectl cluster-info + +# Get all resources in namespace +kubectl get all -n rag7 + +# Check resource usage +kubectl top nodes +kubectl top pods -n rag7 + +# Port forward for local testing +kubectl port-forward -n rag7 svc/langgraph 8123:8123 + +# Execute command in pod +kubectl exec -n rag7 -it POD_NAME -- /bin/bash + +# Copy files from pod +kubectl cp rag7/POD_NAME:/path/to/file ./local/path + +# View events +kubectl get events -n rag7 --sort-by='.lastTimestamp' +``` + +### Monitoring Dashboards + +- **Grafana**: TODO: Add URL +- **Prometheus**: TODO: Add URL +- **Jaeger**: TODO: Add URL +- **Kibana/Loki**: TODO: Add URL + +### Documentation Links + +- [Deployment Guide](./deployment.md) +- [Observability Guide](./observability.md) +- [n8n Workflows](../n8n/README.md) +- [API Documentation](TODO) + +--- + +**Remember:** Always follow the change management process and communicate with the team during incidents. diff --git a/integration/api/Dockerfile b/integration/api/Dockerfile new file mode 100644 index 0000000..065c2ca --- /dev/null +++ b/integration/api/Dockerfile @@ -0,0 +1,39 @@ +FROM python:3.11-slim + +LABEL org.opencontainers.image.source=https://github.com/Stacey77/rag7 +LABEL org.opencontainers.image.description="RAG7 Integration API" +LABEL org.opencontainers.image.licenses=MIT + +# Set working directory +WORKDIR /app + +# Install system dependencies +RUN apt-get update && apt-get install -y \ + gcc \ + g++ \ + curl \ + && rm -rf /var/lib/apt/lists/* + +# Copy requirements first for better caching +COPY requirements.txt . + +# Install Python dependencies +RUN pip install --no-cache-dir -r requirements.txt + +# Copy application code +COPY server.py . + +# Create non-root user +RUN useradd -m -u 1000 appuser && \ + chown -R appuser:appuser /app +USER appuser + +# Expose port +EXPOSE 8000 + +# Health check +HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \ + CMD curl -f http://localhost:8000/health || exit 1 + +# Run the application +CMD ["uvicorn", "server:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/integration/api/__pycache__/server.cpython-312.pyc b/integration/api/__pycache__/server.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..038f944fc11d59d025c84e569decdfc2bfc16f02 GIT binary patch literal 15431 zcmch8Yg8Opnpjm=^}A`Hn@965Ko45tDf9w8tQG=;)7jTH6Y@I!w{4SYQ6F;dhW zil;^?p5_fED0nNbGbOqiWAjvA*-qoyhIsF_AsHf)))j9N(E7`9H?Mr~8} zQTvo*)G?Jenn%h_Vdqr-Xg-}m zjaC6v^}dh=e4!4R?zd)NXczC+p(lNrIq5mRDBVXb^kL(R`4YaA_wZ$F<#z#J_>9&G zb(6hnK06f(g<8IXcaPKfTdUOJX=)#MH(#|@tH)-mA# z%(Fdfo-O-~c$~C6q3=F_QfP&ir?OhM?bGt~?v_>x|MA!O4#6z6mccan&UEgKP(5bi z&%SL;@4zwWb^OV-2=?r~=FbWCF_u4{HJju6v^%k<-3e&-R93r_&@TGqc>INQJ73mL zpWX-hVj8+D3%Wz7;4i(>KvCj(a$j_9Qd^f!u zkOZC!WkH1lvx3NtMMbVZ5E<_k1Je^+R18iClC1XAY{5g9gG)AvlABHgXd}AS_RC&_VDOsgVmrcy1~h3CU406d7-}eTi!u@G=T}DHN0y z#uu4YEQ8b7HW2pGiiHozf*hI>ysTo@X07C19vOMM`({ueaEf_I`287}pJI~afIK5X zGsl(T!GR$`nvO;!0luAGgG0mDLOdT9ZUjV9(mNWO&MGk|){C$i*P^@-R*XF%A8P~K{1j_ z#RS_r6_Ay}e&0ZE?~w25%YHzD|N2mWtaw6}r#o9(!m#ON4?9n{9cx$YSU=Ktt$XnL zh+;U|s^t6n`v;%vzT_Vq>g(+r7*^;bkLku(Q7&x9TC>rg1F;(Vp)DuPO0qEZ*x1(G z+T0p5w(!xQ6f?Dm0%W{K#VCj5u%Orh>p?N3Z97ns1b6}*0kD5Y3@cUwkJ%BxS6(0- zj^6MCrpH4ONhwr6g8(B$DiM z!G`+7LS$T?P>cj{AeK*B0apWN#zKPl5v{_SXP`HKh>y93fu8Xg$-pQ!NiD;Aw|Z0r zqmi-D_>6i~0OjZN!dPG?Ec?d-LE!mWwO5l07IXV}KBV$G2SgA6W)CP9J?AY@EPg+- zl;5w|{QjvZKNH5h!|(t7nE+un_%i$bK!AR~m=B8+kKhU!IINU>zdsO(MBzw)X-iNK znkOY9&4^71#PV?Bh%NYy+J(qsf|}k()FVi#$7NaPVoP8;)B;-pG~YZuD;`EzOisX0 zdJB?2r0%mv?iZFUN%4mBKW*D8ytdH2*uKeLyI)neay)*bXRE4rp?C2+OTk+&ZnC}i zxq~Zw{B-{ocWvRyl4aR_I}c$GH*So?2l=hT!a~pD<)xA3`ekXk;r2@j$AL{&xbJP= zkmJvcZF$ERE-y|j$;(|U?&Zt3Ur#s>ZL#BGhd!%aT!QPu=h0t7vLBbMv1#g6dBmtw zD{tb>S+&R`HlS_$F50$aL0hw+krSTeC=+kbg2@9Ir^X{`j)GD1vS5%$^nRR#Gfdum z#ZG8$a7ISfs8!#;kv9sub-~1`qCz|hdm^F73??_8H zr2V=K46l(s$W(4B6b^?Z0cai0HoIQqpORvw-J%G*sfh?KG{%huLScbdi~?483&b{@ z7n&DhJ0@s0sPv6<6_M)1lbD>tgwS+`hRG^Sm_2N zNWVw#yGoatxcAJK>nzY;*Cu-wC~|dV133tC;C|iVRW{!F^j6(73;jzC%k8(D zHrZ$H*B)At;~g(<)xNZFb;+|_f4gdved&G$w^9^u@87CG&N#i)b?e*~drh4aPA50b z{$r^5zo1zIZ_1(`3pw@8X&N=ash^>Dpe#7=ty#6UtXi93hgv(<&N{q#it{p2Xms5o zR;+%j3JwQELsX9DSe{nCm(Cs+cFl+)9J3s%5l}ldtyt697z%t&6Rctt3>quE4&ru7 zK!Va~$vH!c{Mz?`*YU0Cr5m~={fa%9Lz8bWZFFH7CTP=1S0O>#s3EkmTC?TqP)VcX zesTG7d%W?ft>Oy{eT&j2d*Qy@v+Q1N+H#+UlWusEJ$=8tW`&8jUD_(|UKm*F+G4v^ zd}Hsq@r}Cj?*QL;M%mO83b3~y=IsEbVSd);wJUi;pn1%uQR~6aPy}S$6WA@tRAnJp zF^bqhth5VcZZv{O23;nj{wgL_4HhrKDn&%aZ$w=m04VIs(P&uZ^()wl!G!QXLFFF4 zzKpe)48l*sZARSHW%G6S;&J3_kV@_GYMV>hWZ-~Kx zrdwFwr#>{yGqNtks7bR9nR)Zf9P@pOXXmI%OQuxML5&e=aDIE9NynZ7_=NFH`q(sB zvIc67o=eZ>Es8h226O!$bBFmOqmi0tzug+=3^5pAUD>?xTdl@C$4)wP&5yUt8F}mX zS>E=cIWyBac9=Rvy?dpek_+@!2Pm;`-t_-CL$}^r=B@3wT`tzk=F)oxyPb1>Z^F*a zn_f5Fpl?t&ndhk+^uK3r&|XKZvOhY`foT9|C$Zur?nVd7RZw9TmsRb)f2S-b`>#TtY)_#?ti z8PsAqAj*CmMX_nC0yROg=+Xf!CQVIHXhA}P=F~?-;jB1dQht3m!~hh%06(b|xasey zUoKEPCdyTEr|fpwcPkdIq)b$i=L2T>%4$)fvT4)P94~E&7q%{3`2}lx=jt0*K`M7H zH?F+ATKXTVKdO#5oZg@}4*$i`KR>$Nc_q<#CEjs0UeFJMIK}20o%hS?R!(h}9o=>x zU9>zX^4#gW-S^$AD@PJVjq7Z}eSFdK3ybqT`z?FYS+W#c3a2Q#z9d;vy8I8`I{Xc;I9k(1> z|0~|Lu66ou&+6&*&W(wWrHz+XQ2AAxkWvz6_YBt z!FX}%2t(5Y+ybl|7?*-DRQj<(jvRVxklB0aWglyk=>sg?Hthz?b)@NsBX5v#4D!f! zJ(ue+(_6^tLk*7{{eW0w?rJj_rU>J)u{j$O9XLHg!Chi1?a|>H!l73MBLBdOd|B`| zlNKZy666^%B5~kAg63!#g@S^{gM@)WjaH>5a52XV)8Hl&BEgU#shuchb+>25a#)m? zB|10}i5cw#SgQ*vMfBAwrWrXHiQZ7GAt|c5QB>sx9kq&saNNu^;kiBfjN(v7@)OxB zz5?(!;U}T$qq38ea+D|YJr7*vcUo?@Y`W^VT}=sB)26E>Sy1|*wETnPKj?VB)uI zF6YYMhTU_fDH%<4@}N+a3-zx_m!8YK zb$Xsm3z;YDAiTn;r9}9>3j#0#zmxS;8%0J3n*SbI@hQqkN%Rc^O+62)2~EA*qY5Dq zmubZS#AVey%RA&eeWW>7-#=7CvRC>%?B_iDI{WUV`raefoUs{=O?GRLy0bKkle&m! z6m39x0ys$HB=nPpz^9tcIUIN{K}rIRadMQK0FW?#t5Q?#ab#SbdeMcCXvK^Y2>5V7 zd>Kl={3#@gA^M79O**K+j_`iV0CtQEMa&EKXo91c}$Ec8488DtAq<>yQqF>fo}{B-p*v-=8A~3xJ=^XB3HO23;VpOD z*ObXxox)`O*<@Ay4@Tb~-8yl69)E7E8AN0Nt* zESZ*G`>de;i-yjP)7xhT6K4j&(2YO+O#ILgRwWARlb)(2Afb5|)Z&LGd3OzeyRP{@2nU!XCV7>}sbM%e!pUzoIX;nEo{bxqn-E@dWdUfxg&g z`h;o5e48DLKRHWZY-c|?$3p&7x*6Ji+CpQ#jRu%c+e!X}smp9eX?fo9=~*-6?=fb8 zxo5L>wKDhI?yjTEy+b(Wy`x4fX=Nb)IIrc^ww7`9{o+z$nHcx4d9H)HWnaI{DiS!_^s--0l8V{Y*g}d8xFaRhMTkm zrwB1N(vmhaSG5+57~=_^h?l7Nm*VlP*n+5cYKRlT2k3kPah`ht>bx)&KOMk3g%6JV4SSX0@kv8K9N~x=8TgyPuA)^&YOP= zT@=g#UGRl-CP-Z$8ud|O?zMWaynD_(XP7gs6=k%ktX}JNxXhbZdo6~!<(N5)R01_< z-t2jHo1yYyy*hV4eCfWu7}h*GR^FsRHvSrZ_u=<5O-9;Y+h0b0o|chm`8?U6&m{8( zx%tBH3d7%OH_LM1D7{1S5q)Npje0Kg=F7A9L(Z(LIRm5TU|rI@<#$85-)dbIGP-@q zGiTB14=5q6g}^T7S||F6_dI;%K7H+?2i`VsdEIh@mebc#{=cueYM*)jwl$;od(XpH z@6%V#HScmm)C88`n6#nZq5AR#E|3)fzY!H*8G~46mfAKl5t1^Nk_<6U@FPgjipaoB z1YGlGfNBRe6o$<)+zv94jD=dBs992|{eis+F^8;CeB!v&NkDOG8DZMVslf_T0_DN* z3@AzCYS^!l(}H%5piYvY9e5uaIYAC)^{g3ina)XvY7C`Q+XM{vsZ+`tLRAwRIW`K1mZQC`=@Ley(+lKy*sQ} z1CeNCb}9;?e})7A)leUa@;TBG$(JF3ZIvfbpM?Fus65#{H3B#Vv@{^1VHk6CMd3a~ zL~a_HofcvRf$8aRC`f{cEt66-0u5Pw6AKpO)M6A{6{rYF$A!3B}08BAU zGr=GrMYniK{3ac+i4~;%RO&XPJ4<&$oQrw(j-f`4I=PPwMDJEiJRl+zjun!fLL9>RALt=soT)slDvb7+PVj~jSk zz`-V35VBxIwQe##IU5#xkkI@mT95mCeIwn^`JPW-pKxgY!)^?*5$i;$)PURl!`(yA z_H}jpuMha1_4W1pF7|g5D@Li-KwABM10&r-1HOI&+dbs(9vT`P@><0h&h8&!f_PNS z(}CG=G{B1(%24eZjK!&uy5EDGgvu|nGkH`i#|k>LpU}1VCs4bJU~Ql>sSb*|EPr3J zxcr0m<=E<_L`~~vMO(c5Slr#d(64#>;;y5s{j2BW1)W>$nH1Yc8*M2IReoT*%$q3l zCMynZS2QImnm(&IyIzs3ZrHAFPgJ*mR^7Ee_t0!CaRRKnYTI=%;X0VI;7x9v3vd(BGwXZAzM;;K6@-hT1B{>7dL z4%d5EZ(WU-A6Y%S+PvZX__dFx;=|YDo@cil&;7z(oh;>&rB%r)Z?f_*_;re4p2j>U zm|%JC_xf-3CtYB(e&p{8#UnPEWCn#PG|+Y z$0aKdY*)4>DqE9fZR-vHsp&s9t=Gg$eLF_R4SfStssk4E%aj3D2L>SK$?8b*5I528KOD^;>A5%mR^0Ba|zG6WO3u#@Ot6h zXIC%B-6yh!Ntt1UUrYZSSm(|2{ZG-K(7sc>1=Odf>7Em&PdgaQpRMgV!`!Q)drq0| zRkN6{vqSm4Cc3ACz1Qq(g^$0grlIv;)%Z%`8zd6f5kFlDJg{&U#Ew81e z1qh^hFD(i%d2)*EGM3SC`xg8jU4Ue-u`H)uhq+wLJiTVfsQ<+A%fwzL3PrB~Cp_pJ z>Rh8>1~#7}reCTyGDz1B0)CbXMEMn!5MgwBU3A4zA>IvoLr_ z?@Zvuw_#-;G2~z;p5XK`-sT?|9P#&Ed%C~-S~ncPmtdFhn3d3~<g57c7^p9}+z+()>{Z&p^*vBC_RK*b|> zme7t+?+h8k5SEgIa5AT#-!*o^Y3!tz#Ay>D7rp{SyF+bBQ|`VMmh>J#+Z02>tAVhF zr-KzM_ht6gMyXb4e_5rfrNVjl1xQK@orS;8~8ndDcg+;&u(0t%)*8ET>w`Y z;bw4GfY}VCgbh$S?q!2HdNR!jP^~>sis;$%#7Zu!o?28SfN!*0s}hj=1?WRnbI`S_D{*Gfzfk}5nnWB-1wcpN9zQC;W8_i1 z1Brx-*O5?y4q9LcZ6v?yDHg>K0j3eLa1k;pN0=ywCs|Z_XYlsm7sZ}Cqqj$s71cj@ z>iwr4l-2&A=KY$@vLoAN#}j48H_J{wG_!@a9itiDm}X=dHTF^9bJ4l+^j}~Y4Ns6$ zpUG_2(aON1XVmNG=({jOe9%WnVjvn`!S+pKNJb5iJh>ua5Q8vgeE7M1a5}~&3-x|7 zZ@rE;0XEJ0;i+NM5-ZcNISzp_5wzd*HcPSM>k)OU;V%~S&F|z!;0mh+zWo?tjuq!r zn?pOWXz_iRg!nHY0i}{?&KI!Y1SW(KP`?%5#RPXg<4_krfX`hNP^Qf;2NC=!z;+-J zdrLSBLLdk;=)}I?4{($>_`-Kd{Nv8l%iW4kl=>G6&iT>1ukDi5=oA=Akzv@`&QA;JB1 zSfpwP6=QDqJs%}c8`KY%&f$vRp8?2aMCt{|WFZye|M7yBEw=T6dKmz8?yWEGCw=U094g66+vTYe?}OXx(gNE0(MAH5OTg*W70f;V)(d^_IG;BD5R zDIgWOc2OHHRyPbeV&~)NT&}$wfSb&jP1>_HPJQCq-k`L22KNT{aqr$7|6BINuOuS- zOUr;y;gWFxE=T)gaJ3ol`gQ`MV`jJ#kN-j!z`ab&S;Yo;^AC0R5Be@C#xc0cE`Vt& z&fs4uuoGaxq8%&B5NU&(9NOJ~Q9!HE%M1*BiHz($M=o_jKu%K3cnbw|Ct^7(CV6%m zKX9+rOUcMBY82c&27kB#SCK_L3RQIptw~j9D%2{Qb&r8W&JCqhFP}>P0Yq~!3XTg0 zG|gE%9t%y27zI@<)3ba4?t}+Lyo;n>K}C~Yv6IL@xJux?XFymb2=m^z-76p$SkKB|QUp1EY|p9~z7_`^ZVrw$CZ&=al{Ls6z?r(9fx= z|4yCRq|W@DI`VVs82tY`>fq;;1Fo~sMGNL_%AKIxNt=7I?mh1<@6y$kl1*Df((e18 zMus+keQv8*EZw%2C#>bmr#Gz!7EI8vxO(X#Ue$?Lx5eA9Y!+ROQ-$}Pg-hP0x;x(6 z-sNkn$2VQAo6fccd(u^*7RD=^R;5i>`?jki;p*6^-*lZ{$bV>}%+92ZOBP&AI^4;; zBDjaMV>g)%3rxyI(d;(mN>HvO>q@$6lO^RzcgfO0lBDbu?cSkGFhy9f2Dfy6osCnj zE$YOM$wW`kOJxW#LGQRs^Z>o&#jgSQGSWTt((^kMzIO^(+D9+-?NInm6;K6bi}HK3 zw`TvzYYProzQOoT^&8borcJsuW$@6>74ugVydE)>!9xmCrxB*~jnc)kmGc{(4ckKL zCf)VWV5dz8bcs%3HdU!Zc~+X%Z>~=wl<%RTQiZw*P>@YsfWJaCzTv zc^b5GrFW$RL7Tp)s!ehSl7;SvW+!b*<`v_g3#1sxe^CH;Yj5YJjF1PEJMzCm2<)yo zWx%on(A_p&lAueL<~QksDT5UTJq&Ia+O(1P6}}Mo5GhZ2GQhK&bgeqW=TbPsjgIt0 z>(q&oy{depaxt_?dmb8Cx)pmWOJNQ$1PjK>$xXWKp}|0(#LDs%=HTKN>sX9^#*}@| zy1roZ7kcqVTGCRK%qvOe6((ICxOTHBB6^oAS8i=68.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "rag7" +version = "1.0.0" +description = "RAG7 - Production-ready LangGraph-based RAG system" +readme = "README.md" +requires-python = ">=3.11" +license = {text = "MIT"} +authors = [ + {name = "RAG7 Team", email = "team@example.com"} +] +keywords = ["rag", "langgraph", "langchain", "llm", "ai"] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Scientific/Engineering :: Artificial Intelligence", +] + +dependencies = [ + "langgraph>=0.0.26", + "langchain>=0.1.0", + "langchain-core>=0.1.0", + "langchain-community>=0.0.13", + "openai>=1.6.1", + "anthropic>=0.8.1", + "chromadb>=0.4.22", + "psycopg2-binary>=2.9.9", + "sqlalchemy>=2.0.23", + "redis>=5.0.1", + "fastapi>=0.104.1", + "uvicorn[standard]>=0.24.0", + "httpx>=0.25.1", + "numpy>=1.26.2", + "pandas>=2.1.4", + "pydantic>=2.5.0", + "pydantic-settings>=2.1.0", + "tiktoken>=0.5.2", + "sentence-transformers>=2.2.2", + "python-dotenv>=1.0.0", + "tenacity>=8.2.3", + "aiofiles>=23.2.1", + "prometheus-client>=0.19.0", + "opentelemetry-api>=1.21.0", + "opentelemetry-sdk>=1.21.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.4.3", + "pytest-asyncio>=0.21.1", + "pytest-cov>=4.1.0", + "black>=23.12.1", + "flake8>=7.0.0", + "isort>=5.13.2", + "mypy>=1.7.1", +] + +[project.urls] +Homepage = "https://github.com/Stacey77/rag7" +Documentation = "https://github.com/Stacey77/rag7/blob/main/docs" +Repository = "https://github.com/Stacey77/rag7" +Issues = "https://github.com/Stacey77/rag7/issues" + +[tool.setuptools] +packages = ["rag7"] + +[tool.black] +line-length = 100 +target-version = ['py311'] +include = '\.pyi?$' +extend-exclude = ''' +/( + # directories + \.eggs + | \.git + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | build + | dist +)/ +''' + +[tool.isort] +profile = "black" +line_length = 100 +skip_gitignore = true + +[tool.pytest.ini_options] +minversion = "7.0" +addopts = "-ra -q --strict-markers" +testpaths = ["tests"] +pythonpath = ["."] + +[tool.mypy] +python_version = "3.11" +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = false +ignore_missing_imports = true + +[tool.coverage.run] +source = ["rag7"] +omit = ["*/tests/*", "*/test_*.py"] + +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "def __repr__", + "raise AssertionError", + "raise NotImplementedError", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", +] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..63b2edd --- /dev/null +++ b/requirements.txt @@ -0,0 +1,56 @@ +# RAG7 LangGraph Application Requirements +# Python dependencies for the RAG7 project + +# Core Framework +langgraph==0.0.26 +langchain==0.1.0 +langchain-core==0.1.0 +langchain-community==0.0.13 + +# LLM Providers +openai==1.6.1 +anthropic==0.8.1 + +# Vector Stores +chromadb==0.4.22 +faiss-cpu==1.7.4 + +# Database & Storage +psycopg2-binary==2.9.9 +sqlalchemy==2.0.23 +redis==5.0.1 + +# Web Framework (for API) +fastapi==0.104.1 +uvicorn[standard]==0.24.0 +httpx==0.25.1 + +# Data Processing +numpy==1.26.2 +pandas==2.1.4 +pydantic==2.5.0 +pydantic-settings==2.1.0 + +# Text Processing +tiktoken==0.5.2 +sentence-transformers==2.2.2 + +# Utilities +python-dotenv==1.0.0 +tenacity==8.2.3 +aiofiles==23.2.1 + +# Observability & Monitoring +prometheus-client==0.19.0 +opentelemetry-api==1.21.0 +opentelemetry-sdk==1.21.0 +opentelemetry-instrumentation==0.42b0 + +# Development & Testing (optional, uncomment if needed) +# pytest==7.4.3 +# pytest-asyncio==0.21.1 +# pytest-cov==4.1.0 +# black==23.12.1 +# flake8==7.0.0 +# isort==5.13.2 +# mypy==1.7.1 From fea76af5d6ab01ea1eeedef07d557492730c2a00 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 8 Dec 2025 02:00:27 +0000 Subject: [PATCH 3/3] chore: add .gitignore and remove __pycache__ --- .gitignore | 22 ++++++++++++++++++ .../api/__pycache__/server.cpython-312.pyc | Bin 15431 -> 0 bytes 2 files changed, 22 insertions(+) create mode 100644 .gitignore delete mode 100644 integration/api/__pycache__/server.cpython-312.pyc diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..79e1579 --- /dev/null +++ b/.gitignore @@ -0,0 +1,22 @@ +*.pyc +__pycache__/ +*.pyo +*.pyd +.Python +*.so +*.egg +*.egg-info/ +dist/ +build/ +.eggs/ +*.log +.env +.venv +venv/ +ENV/ +.DS_Store +.idea/ +.vscode/ +*.swp +*.swo +*~ diff --git a/integration/api/__pycache__/server.cpython-312.pyc b/integration/api/__pycache__/server.cpython-312.pyc deleted file mode 100644 index 038f944fc11d59d025c84e569decdfc2bfc16f02..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 15431 zcmch8Yg8Opnpjm=^}A`Hn@965Ko45tDf9w8tQG=;)7jTH6Y@I!w{4SYQ6F;dhW zil;^?p5_fED0nNbGbOqiWAjvA*-qoyhIsF_AsHf)))j9N(E7`9H?Mr~8} zQTvo*)G?Jenn%h_Vdqr-Xg-}m zjaC6v^}dh=e4!4R?zd)NXczC+p(lNrIq5mRDBVXb^kL(R`4YaA_wZ$F<#z#J_>9&G zb(6hnK06f(g<8IXcaPKfTdUOJX=)#MH(#|@tH)-mA# z%(Fdfo-O-~c$~C6q3=F_QfP&ir?OhM?bGt~?v_>x|MA!O4#6z6mccan&UEgKP(5bi z&%SL;@4zwWb^OV-2=?r~=FbWCF_u4{HJju6v^%k<-3e&-R93r_&@TGqc>INQJ73mL zpWX-hVj8+D3%Wz7;4i(>KvCj(a$j_9Qd^f!u zkOZC!WkH1lvx3NtMMbVZ5E<_k1Je^+R18iClC1XAY{5g9gG)AvlABHgXd}AS_RC&_VDOsgVmrcy1~h3CU406d7-}eTi!u@G=T}DHN0y z#uu4YEQ8b7HW2pGiiHozf*hI>ysTo@X07C19vOMM`({ueaEf_I`287}pJI~afIK5X zGsl(T!GR$`nvO;!0luAGgG0mDLOdT9ZUjV9(mNWO&MGk|){C$i*P^@-R*XF%A8P~K{1j_ z#RS_r6_Ay}e&0ZE?~w25%YHzD|N2mWtaw6}r#o9(!m#ON4?9n{9cx$YSU=Ktt$XnL zh+;U|s^t6n`v;%vzT_Vq>g(+r7*^;bkLku(Q7&x9TC>rg1F;(Vp)DuPO0qEZ*x1(G z+T0p5w(!xQ6f?Dm0%W{K#VCj5u%Orh>p?N3Z97ns1b6}*0kD5Y3@cUwkJ%BxS6(0- zj^6MCrpH4ONhwr6g8(B$DiM z!G`+7LS$T?P>cj{AeK*B0apWN#zKPl5v{_SXP`HKh>y93fu8Xg$-pQ!NiD;Aw|Z0r zqmi-D_>6i~0OjZN!dPG?Ec?d-LE!mWwO5l07IXV}KBV$G2SgA6W)CP9J?AY@EPg+- zl;5w|{QjvZKNH5h!|(t7nE+un_%i$bK!AR~m=B8+kKhU!IINU>zdsO(MBzw)X-iNK znkOY9&4^71#PV?Bh%NYy+J(qsf|}k()FVi#$7NaPVoP8;)B;-pG~YZuD;`EzOisX0 zdJB?2r0%mv?iZFUN%4mBKW*D8ytdH2*uKeLyI)neay)*bXRE4rp?C2+OTk+&ZnC}i zxq~Zw{B-{ocWvRyl4aR_I}c$GH*So?2l=hT!a~pD<)xA3`ekXk;r2@j$AL{&xbJP= zkmJvcZF$ERE-y|j$;(|U?&Zt3Ur#s>ZL#BGhd!%aT!QPu=h0t7vLBbMv1#g6dBmtw zD{tb>S+&R`HlS_$F50$aL0hw+krSTeC=+kbg2@9Ir^X{`j)GD1vS5%$^nRR#Gfdum z#ZG8$a7ISfs8!#;kv9sub-~1`qCz|hdm^F73??_8H zr2V=K46l(s$W(4B6b^?Z0cai0HoIQqpORvw-J%G*sfh?KG{%huLScbdi~?483&b{@ z7n&DhJ0@s0sPv6<6_M)1lbD>tgwS+`hRG^Sm_2N zNWVw#yGoatxcAJK>nzY;*Cu-wC~|dV133tC;C|iVRW{!F^j6(73;jzC%k8(D zHrZ$H*B)At;~g(<)xNZFb;+|_f4gdved&G$w^9^u@87CG&N#i)b?e*~drh4aPA50b z{$r^5zo1zIZ_1(`3pw@8X&N=ash^>Dpe#7=ty#6UtXi93hgv(<&N{q#it{p2Xms5o zR;+%j3JwQELsX9DSe{nCm(Cs+cFl+)9J3s%5l}ldtyt697z%t&6Rctt3>quE4&ru7 zK!Va~$vH!c{Mz?`*YU0Cr5m~={fa%9Lz8bWZFFH7CTP=1S0O>#s3EkmTC?TqP)VcX zesTG7d%W?ft>Oy{eT&j2d*Qy@v+Q1N+H#+UlWusEJ$=8tW`&8jUD_(|UKm*F+G4v^ zd}Hsq@r}Cj?*QL;M%mO83b3~y=IsEbVSd);wJUi;pn1%uQR~6aPy}S$6WA@tRAnJp zF^bqhth5VcZZv{O23;nj{wgL_4HhrKDn&%aZ$w=m04VIs(P&uZ^()wl!G!QXLFFF4 zzKpe)48l*sZARSHW%G6S;&J3_kV@_GYMV>hWZ-~Kx zrdwFwr#>{yGqNtks7bR9nR)Zf9P@pOXXmI%OQuxML5&e=aDIE9NynZ7_=NFH`q(sB zvIc67o=eZ>Es8h226O!$bBFmOqmi0tzug+=3^5pAUD>?xTdl@C$4)wP&5yUt8F}mX zS>E=cIWyBac9=Rvy?dpek_+@!2Pm;`-t_-CL$}^r=B@3wT`tzk=F)oxyPb1>Z^F*a zn_f5Fpl?t&ndhk+^uK3r&|XKZvOhY`foT9|C$Zur?nVd7RZw9TmsRb)f2S-b`>#TtY)_#?ti z8PsAqAj*CmMX_nC0yROg=+Xf!CQVIHXhA}P=F~?-;jB1dQht3m!~hh%06(b|xasey zUoKEPCdyTEr|fpwcPkdIq)b$i=L2T>%4$)fvT4)P94~E&7q%{3`2}lx=jt0*K`M7H zH?F+ATKXTVKdO#5oZg@}4*$i`KR>$Nc_q<#CEjs0UeFJMIK}20o%hS?R!(h}9o=>x zU9>zX^4#gW-S^$AD@PJVjq7Z}eSFdK3ybqT`z?FYS+W#c3a2Q#z9d;vy8I8`I{Xc;I9k(1> z|0~|Lu66ou&+6&*&W(wWrHz+XQ2AAxkWvz6_YBt z!FX}%2t(5Y+ybl|7?*-DRQj<(jvRVxklB0aWglyk=>sg?Hthz?b)@NsBX5v#4D!f! zJ(ue+(_6^tLk*7{{eW0w?rJj_rU>J)u{j$O9XLHg!Chi1?a|>H!l73MBLBdOd|B`| zlNKZy666^%B5~kAg63!#g@S^{gM@)WjaH>5a52XV)8Hl&BEgU#shuchb+>25a#)m? zB|10}i5cw#SgQ*vMfBAwrWrXHiQZ7GAt|c5QB>sx9kq&saNNu^;kiBfjN(v7@)OxB zz5?(!;U}T$qq38ea+D|YJr7*vcUo?@Y`W^VT}=sB)26E>Sy1|*wETnPKj?VB)uI zF6YYMhTU_fDH%<4@}N+a3-zx_m!8YK zb$Xsm3z;YDAiTn;r9}9>3j#0#zmxS;8%0J3n*SbI@hQqkN%Rc^O+62)2~EA*qY5Dq zmubZS#AVey%RA&eeWW>7-#=7CvRC>%?B_iDI{WUV`raefoUs{=O?GRLy0bKkle&m! z6m39x0ys$HB=nPpz^9tcIUIN{K}rIRadMQK0FW?#t5Q?#ab#SbdeMcCXvK^Y2>5V7 zd>Kl={3#@gA^M79O**K+j_`iV0CtQEMa&EKXo91c}$Ec8488DtAq<>yQqF>fo}{B-p*v-=8A~3xJ=^XB3HO23;VpOD z*ObXxox)`O*<@Ay4@Tb~-8yl69)E7E8AN0Nt* zESZ*G`>de;i-yjP)7xhT6K4j&(2YO+O#ILgRwWARlb)(2Afb5|)Z&LGd3OzeyRP{@2nU!XCV7>}sbM%e!pUzoIX;nEo{bxqn-E@dWdUfxg&g z`h;o5e48DLKRHWZY-c|?$3p&7x*6Ji+CpQ#jRu%c+e!X}smp9eX?fo9=~*-6?=fb8 zxo5L>wKDhI?yjTEy+b(Wy`x4fX=Nb)IIrc^ww7`9{o+z$nHcx4d9H)HWnaI{DiS!_^s--0l8V{Y*g}d8xFaRhMTkm zrwB1N(vmhaSG5+57~=_^h?l7Nm*VlP*n+5cYKRlT2k3kPah`ht>bx)&KOMk3g%6JV4SSX0@kv8K9N~x=8TgyPuA)^&YOP= zT@=g#UGRl-CP-Z$8ud|O?zMWaynD_(XP7gs6=k%ktX}JNxXhbZdo6~!<(N5)R01_< z-t2jHo1yYyy*hV4eCfWu7}h*GR^FsRHvSrZ_u=<5O-9;Y+h0b0o|chm`8?U6&m{8( zx%tBH3d7%OH_LM1D7{1S5q)Npje0Kg=F7A9L(Z(LIRm5TU|rI@<#$85-)dbIGP-@q zGiTB14=5q6g}^T7S||F6_dI;%K7H+?2i`VsdEIh@mebc#{=cueYM*)jwl$;od(XpH z@6%V#HScmm)C88`n6#nZq5AR#E|3)fzY!H*8G~46mfAKl5t1^Nk_<6U@FPgjipaoB z1YGlGfNBRe6o$<)+zv94jD=dBs992|{eis+F^8;CeB!v&NkDOG8DZMVslf_T0_DN* z3@AzCYS^!l(}H%5piYvY9e5uaIYAC)^{g3ina)XvY7C`Q+XM{vsZ+`tLRAwRIW`K1mZQC`=@Ley(+lKy*sQ} z1CeNCb}9;?e})7A)leUa@;TBG$(JF3ZIvfbpM?Fus65#{H3B#Vv@{^1VHk6CMd3a~ zL~a_HofcvRf$8aRC`f{cEt66-0u5Pw6AKpO)M6A{6{rYF$A!3B}08BAU zGr=GrMYniK{3ac+i4~;%RO&XPJ4<&$oQrw(j-f`4I=PPwMDJEiJRl+zjun!fLL9>RALt=soT)slDvb7+PVj~jSk zz`-V35VBxIwQe##IU5#xkkI@mT95mCeIwn^`JPW-pKxgY!)^?*5$i;$)PURl!`(yA z_H}jpuMha1_4W1pF7|g5D@Li-KwABM10&r-1HOI&+dbs(9vT`P@><0h&h8&!f_PNS z(}CG=G{B1(%24eZjK!&uy5EDGgvu|nGkH`i#|k>LpU}1VCs4bJU~Ql>sSb*|EPr3J zxcr0m<=E<_L`~~vMO(c5Slr#d(64#>;;y5s{j2BW1)W>$nH1Yc8*M2IReoT*%$q3l zCMynZS2QImnm(&IyIzs3ZrHAFPgJ*mR^7Ee_t0!CaRRKnYTI=%;X0VI;7x9v3vd(BGwXZAzM;;K6@-hT1B{>7dL z4%d5EZ(WU-A6Y%S+PvZX__dFx;=|YDo@cil&;7z(oh;>&rB%r)Z?f_*_;re4p2j>U zm|%JC_xf-3CtYB(e&p{8#UnPEWCn#PG|+Y z$0aKdY*)4>DqE9fZR-vHsp&s9t=Gg$eLF_R4SfStssk4E%aj3D2L>SK$?8b*5I528KOD^;>A5%mR^0Ba|zG6WO3u#@Ot6h zXIC%B-6yh!Ntt1UUrYZSSm(|2{ZG-K(7sc>1=Odf>7Em&PdgaQpRMgV!`!Q)drq0| zRkN6{vqSm4Cc3ACz1Qq(g^$0grlIv;)%Z%`8zd6f5kFlDJg{&U#Ew81e z1qh^hFD(i%d2)*EGM3SC`xg8jU4Ue-u`H)uhq+wLJiTVfsQ<+A%fwzL3PrB~Cp_pJ z>Rh8>1~#7}reCTyGDz1B0)CbXMEMn!5MgwBU3A4zA>IvoLr_ z?@Zvuw_#-;G2~z;p5XK`-sT?|9P#&Ed%C~-S~ncPmtdFhn3d3~<g57c7^p9}+z+()>{Z&p^*vBC_RK*b|> zme7t+?+h8k5SEgIa5AT#-!*o^Y3!tz#Ay>D7rp{SyF+bBQ|`VMmh>J#+Z02>tAVhF zr-KzM_ht6gMyXb4e_5rfrNVjl1xQK@orS;8~8ndDcg+;&u(0t%)*8ET>w`Y z;bw4GfY}VCgbh$S?q!2HdNR!jP^~>sis;$%#7Zu!o?28SfN!*0s}hj=1?WRnbI`S_D{*Gfzfk}5nnWB-1wcpN9zQC;W8_i1 z1Brx-*O5?y4q9LcZ6v?yDHg>K0j3eLa1k;pN0=ywCs|Z_XYlsm7sZ}Cqqj$s71cj@ z>iwr4l-2&A=KY$@vLoAN#}j48H_J{wG_!@a9itiDm}X=dHTF^9bJ4l+^j}~Y4Ns6$ zpUG_2(aON1XVmNG=({jOe9%WnVjvn`!S+pKNJb5iJh>ua5Q8vgeE7M1a5}~&3-x|7 zZ@rE;0XEJ0;i+NM5-ZcNISzp_5wzd*HcPSM>k)OU;V%~S&F|z!;0mh+zWo?tjuq!r zn?pOWXz_iRg!nHY0i}{?&KI!Y1SW(KP`?%5#RPXg<4_krfX`hNP^Qf;2NC=!z;+-J zdrLSBLLdk;=)}I?4{($>_`-Kd{Nv8l%iW4kl=>G6&iT>1ukDi5=oA=Akzv@`&QA;JB1 zSfpwP6=QDqJs%}c8`KY%&f$vRp8?2aMCt{|WFZye|M7yBEw=T6dKmz8?yWEGCw=U094g66+vTYe?}OXx(gNE0(MAH5OTg*W70f;V)(d^_IG;BD5R zDIgWOc2OHHRyPbeV&~)NT&}$wfSb&jP1>_HPJQCq-k`L22KNT{aqr$7|6BINuOuS- zOUr;y;gWFxE=T)gaJ3ol`gQ`MV`jJ#kN-j!z`ab&S;Yo;^AC0R5Be@C#xc0cE`Vt& z&fs4uuoGaxq8%&B5NU&(9NOJ~Q9!HE%M1*BiHz($M=o_jKu%K3cnbw|Ct^7(CV6%m zKX9+rOUcMBY82c&27kB#SCK_L3RQIptw~j9D%2{Qb&r8W&JCqhFP}>P0Yq~!3XTg0 zG|gE%9t%y27zI@<)3ba4?t}+Lyo;n>K}C~Yv6IL@xJux?XFymb2=m^z-76p$SkKB|QUp1EY|p9~z7_`^ZVrw$CZ&=al{Ls6z?r(9fx= z|4yCRq|W@DI`VVs82tY`>fq;;1Fo~sMGNL_%AKIxNt=7I?mh1<@6y$kl1*Df((e18 zMus+keQv8*EZw%2C#>bmr#Gz!7EI8vxO(X#Ue$?Lx5eA9Y!+ROQ-$}Pg-hP0x;x(6 z-sNkn$2VQAo6fccd(u^*7RD=^R;5i>`?jki;p*6^-*lZ{$bV>}%+92ZOBP&AI^4;; zBDjaMV>g)%3rxyI(d;(mN>HvO>q@$6lO^RzcgfO0lBDbu?cSkGFhy9f2Dfy6osCnj zE$YOM$wW`kOJxW#LGQRs^Z>o&#jgSQGSWTt((^kMzIO^(+D9+-?NInm6;K6bi}HK3 zw`TvzYYProzQOoT^&8borcJsuW$@6>74ugVydE)>!9xmCrxB*~jnc)kmGc{(4ckKL zCf)VWV5dz8bcs%3HdU!Zc~+X%Z>~=wl<%RTQiZw*P>@YsfWJaCzTv zc^b5GrFW$RL7Tp)s!ehSl7;SvW+!b*<`v_g3#1sxe^CH;Yj5YJjF1PEJMzCm2<)yo zWx%on(A_p&lAueL<~QksDT5UTJq&Ia+O(1P6}}Mo5GhZ2GQhK&bgeqW=TbPsjgIt0 z>(q&oy{depaxt_?dmb8Cx)pmWOJNQ$1PjK>$xXWKp}|0(#LDs%=HTKN>sX9^#*}@| zy1roZ7kcqVTGCRK%qvOe6((ICxOTHBB6^oAS8i