diff --git a/.env.example b/.env.example index bcf2e5a..abb5386 100644 --- a/.env.example +++ b/.env.example @@ -4,34 +4,21 @@ # ========================================== # Database Configuration # ========================================== -POSTGRES_DB=genesis -POSTGRES_USER=genesis -POSTGRES_PASSWORD=changeme_secure_password -POSTGRES_PORT=5432 +DATABASE_URL=postgresql://genesis_app:changeme@localhost:5432/genesis # ========================================== # LLM API Keys # ========================================== -# OpenAI API Key (required for OpenAI models) OPENAI_API_KEY=sk-your-openai-api-key-here - -# Anthropic API Key (required for Claude models) ANTHROPIC_API_KEY=sk-ant-your-anthropic-api-key-here # ========================================== # E2B Cloud Sandbox (Optional) # ========================================== -# E2B API Key (required only if using E2B execution backend) E2B_API_KEY=your-e2b-api-key-here # ========================================== # Application Settings # ========================================== -# Port for the Genesis Web UI -GENESIS_WEBUI_PORT=8000 - -# ========================================== -# Advanced Database Configuration (Optional) -# ========================================== -# Full database URL (alternative to individual postgres settings) -# DATABASE_URL=postgresql://genesis:password@postgres:5432/genesis +PORT=8080 +RUST_LOG=info diff --git a/.factory/settings.json b/.factory/settings.json new file mode 100644 index 0000000..565f14a --- /dev/null +++ b/.factory/settings.json @@ -0,0 +1,5 @@ +{ + "enabledPlugins": { + "core@factory-plugins": true + } +} \ No newline at end of file diff --git a/.gitguardian.yaml b/.gitguardian.yaml new file mode 100644 index 0000000..02d6988 --- /dev/null +++ b/.gitguardian.yaml @@ -0,0 +1,13 @@ +# GitGuardian configuration +# See: https://docs.gitguardian.com/ggshield-docs/reference/gitguardian-yaml + +paths-ignore: + # Example env file with placeholder values, not real secrets + - ".env.example" + + # Terraform files use variable interpolation, not literal secrets + - "terraform/*.tf" + + # Test fixtures use ephemeral local containers + - "genesis_rust_backend/tests/**" + - "**/tests/**" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..59fe074 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,126 @@ +name: CI + +on: + pull_request: + branches: [main] + push: + branches: [main] + +env: + CARGO_TERM_COLOR: always + +jobs: + rust-check: + name: Rust lint & test + runs-on: ubuntu-latest + defaults: + run: + working-directory: genesis_rust_backend + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt, clippy + + - uses: Swatinem/rust-cache@v2 + with: + workspaces: genesis_rust_backend + + - name: Check formatting + run: cargo fmt --check + + - name: Clippy + run: cargo clippy -- -D warnings + + - name: Unit tests + run: cargo test --lib + + - name: Integration tests (testcontainers) + run: cargo test --test db_integration --test memory_tests + + liquibase-validate: + name: Validate migrations + runs-on: ubuntu-latest + services: + postgres: + image: postgres:15 + env: + POSTGRES_DB: genesis_test + POSTGRES_USER: genesis + POSTGRES_PASSWORD: genesis_test + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - uses: actions/checkout@v4 + + - name: Install Liquibase + run: | + wget -q https://github.com/liquibase/liquibase/releases/download/v4.29.2/liquibase-4.29.2.tar.gz + mkdir -p /opt/liquibase + tar xzf liquibase-4.29.2.tar.gz -C /opt/liquibase + echo "/opt/liquibase" >> $GITHUB_PATH + + - name: Validate changelogs + working-directory: migrations + run: | + liquibase \ + --url="jdbc:postgresql://localhost:5432/genesis_test" \ + --username=genesis \ + --password=genesis_test \ + --changeLogFile=changelogs/db.changelog-master.yaml \ + validate + + - name: Run migrations + working-directory: migrations + run: | + liquibase \ + --url="jdbc:postgresql://localhost:5432/genesis_test" \ + --username=genesis \ + --password=genesis_test \ + --changeLogFile=changelogs/db.changelog-master.yaml \ + update + + - name: Verify full_ddl.sql is in sync + run: | + # Normalize function: strip version comments and other pg_dump metadata + normalize() { + sed '/^-- Dumped from database version/d; /^-- Dumped by pg_dump version/d; /^\\restrict/d; /^\\unrestrict/d; /^SELECT pg_catalog/d; /^SET default_table_access_method/d' "$1" + } + PGPASSWORD=genesis_test pg_dump \ + -h localhost -p 5432 -U genesis -d genesis_test \ + --schema-only --no-owner --no-privileges \ + --exclude-table='databasechangelog*' \ + > /tmp/full_ddl_generated.sql + normalize migrations/full_ddl.sql > /tmp/full_ddl_repo.sql + normalize /tmp/full_ddl_generated.sql > /tmp/full_ddl_ci.sql + diff -u /tmp/full_ddl_repo.sql /tmp/full_ddl_ci.sql || \ + (echo "ERROR: migrations/full_ddl.sql is out of sync. Run scripts/export_ddl.sh and commit the result." && exit 1) + + terraform-plan: + name: Terraform plan + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + defaults: + run: + working-directory: terraform + steps: + - uses: actions/checkout@v4 + + - uses: hashicorp/setup-terraform@v3 + with: + terraform_version: "1.9" + + - name: Terraform init + run: terraform init -backend=false + + - name: Terraform validate + run: terraform validate + + - name: Terraform format check + run: terraform fmt -check -recursive diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..fa200fb --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,179 @@ +name: Deploy + +on: + push: + branches: [main] + paths: + - 'genesis_rust_backend/**' + - 'genesis/webui/frontend/**' + - 'migrations/**' + - 'terraform/**' + - '.github/workflows/deploy.yml' + +env: + PROJECT_ID: openloop-491716 + REGION: europe-west2 + REGISTRY: europe-west2-docker.pkg.dev/openloop-491716/genesis + SERVICE_NAME: genesis-backend + +jobs: + build-and-push-backend: + name: Build & push backend + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - id: auth + uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + + - uses: google-github-actions/setup-gcloud@v2 + + - name: Configure Docker + run: gcloud auth configure-docker ${{ env.REGION }}-docker.pkg.dev --quiet + + - name: Build and push + working-directory: genesis_rust_backend + run: | + IMAGE="${{ env.REGISTRY }}/backend:${{ github.sha }}" + docker build -t "${IMAGE}" -t "${{ env.REGISTRY }}/backend:latest" . + docker push "${IMAGE}" + docker push "${{ env.REGISTRY }}/backend:latest" + + build-and-push-frontend: + name: Build & push frontend + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - id: auth + uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + + - uses: google-github-actions/setup-gcloud@v2 + + - name: Configure Docker + run: gcloud auth configure-docker ${{ env.REGION }}-docker.pkg.dev --quiet + + - name: Build and push + working-directory: genesis/webui/frontend + run: | + IMAGE="${{ env.REGISTRY }}/frontend:${{ github.sha }}" + docker build -t "${IMAGE}" -t "${{ env.REGISTRY }}/frontend:latest" . + docker push "${IMAGE}" + docker push "${{ env.REGISTRY }}/frontend:latest" + + migrate: + name: Run database migrations + runs-on: ubuntu-latest + needs: build-and-push-backend + steps: + - uses: actions/checkout@v4 + + - id: auth + uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + + - uses: google-github-actions/setup-gcloud@v2 + + - name: Install Cloud SQL Proxy + run: | + curl -o cloud-sql-proxy https://storage.googleapis.com/cloud-sql-connectors/cloud-sql-proxy/v2.14.1/cloud-sql-proxy.linux.amd64 + chmod +x cloud-sql-proxy + + - name: Install Liquibase + run: | + wget -q https://github.com/liquibase/liquibase/releases/download/v4.29.2/liquibase-4.29.2.tar.gz + mkdir -p /opt/liquibase + tar xzf liquibase-4.29.2.tar.gz -C /opt/liquibase + echo "/opt/liquibase" >> $GITHUB_PATH + + - name: Start Cloud SQL Proxy + run: | + ./cloud-sql-proxy "${{ secrets.CLOUD_SQL_CONNECTION_NAME }}" \ + --port 5432 & + sleep 5 + + - name: Run migrations + working-directory: migrations + run: | + liquibase \ + --url="jdbc:postgresql://localhost:5432/${{ secrets.DB_NAME }}" \ + --username="${{ secrets.DB_USER }}" \ + --password="${{ secrets.DB_PASSWORD }}" \ + --changeLogFile=changelogs/db.changelog-master.yaml \ + update + + deploy-backend: + name: Deploy backend to Cloud Run + runs-on: ubuntu-latest + needs: [build-and-push-backend, migrate] + steps: + - uses: actions/checkout@v4 + + - id: auth + uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + + - uses: google-github-actions/setup-gcloud@v2 + + - name: Deploy backend + run: | + gcloud run deploy ${{ env.SERVICE_NAME }} \ + --project=${{ env.PROJECT_ID }} \ + --region=${{ env.REGION }} \ + --image="${{ env.REGISTRY }}/backend:${{ github.sha }}" \ + --platform=managed \ + --allow-unauthenticated \ + --service-account=genesis-cloud-run@${{ env.PROJECT_ID }}.iam.gserviceaccount.com \ + --set-secrets="DATABASE_URL=genesis-database-url:latest,OPENAI_API_KEY=genesis-openai-api-key:latest,ANTHROPIC_API_KEY=genesis-anthropic-api-key:latest" \ + --set-env-vars="RUST_LOG=info" \ + --vpc-connector=genesis-connector \ + --vpc-egress=private-ranges-only \ + --port=8080 \ + --cpu=1 \ + --memory=512Mi \ + --min-instances=0 \ + --max-instances=3 + + deploy-frontend: + name: Deploy frontend to Cloud Run + runs-on: ubuntu-latest + needs: [build-and-push-frontend, deploy-backend] + steps: + - uses: actions/checkout@v4 + + - id: auth + uses: google-github-actions/auth@v2 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + + - uses: google-github-actions/setup-gcloud@v2 + + - name: Get backend URL + id: backend + run: | + URL=$(gcloud run services describe ${{ env.SERVICE_NAME }} \ + --project=${{ env.PROJECT_ID }} \ + --region=${{ env.REGION }} \ + --format='value(status.url)') + echo "url=${URL}" >> "$GITHUB_OUTPUT" + + - name: Deploy frontend + run: | + gcloud run deploy genesis-frontend \ + --project=${{ env.PROJECT_ID }} \ + --region=${{ env.REGION }} \ + --image="${{ env.REGISTRY }}/frontend:${{ github.sha }}" \ + --platform=managed \ + --allow-unauthenticated \ + --set-env-vars="BACKEND_URL=${{ steps.backend.outputs.url }}" \ + --port=8080 \ + --cpu=1 \ + --memory=256Mi \ + --min-instances=0 \ + --max-instances=2 diff --git a/.gitignore b/.gitignore index c9b542f..f1cdf57 100644 --- a/.gitignore +++ b/.gitignore @@ -176,3 +176,16 @@ cython_debug/ .pre-commit-cache/ results/ *.log + +# Terraform +terraform/.terraform/ +terraform/*.tfstate +terraform/*.tfstate.backup +terraform/.terraform.lock.hcl +terraform/terraform.tfvars + +# Rust +genesis_rust_backend/target/ + +# SQLx offline cache (committed intentionally, but gitignore target) +# genesis_rust_backend/.sqlx/ diff --git a/.mcp.json b/.mcp.json index d284355..8b77d25 100644 --- a/.mcp.json +++ b/.mcp.json @@ -4,7 +4,7 @@ "type": "stdio", "command": "python3", "args": ["-m", "genesis.mcp_server"], - "cwd": "/Users/georgepearse/Genesis" + "cwd": "." } } } diff --git a/CLICKHOUSE_INTEGRATION_COMPLETE.md b/CLICKHOUSE_INTEGRATION_COMPLETE.md deleted file mode 100644 index fe83ee5..0000000 --- a/CLICKHOUSE_INTEGRATION_COMPLETE.md +++ /dev/null @@ -1,235 +0,0 @@ -# ClickHouse Integration - ✅ COMPLETE - -## Summary - -Successfully integrated ClickHouse logging into the Genesis evolution runner. All 7 tables are now being populated with real-time evolution data. - -## What Was Added - -### 1. Code Changes in `genesis/core/runner.py` - -#### Instance Variables -- `self.run_id`: Unique identifier for each evolution run -- `self.logged_generations`: Track which generations have been logged to avoid duplicates - -#### Logging Points - -**At Evolution Start (`run()` method):** -```python -ch_logger.log_evolution_run( - run_id=self.run_id, - task_name=task_name, - config=config_dict, - population_size=target_gens, - cluster_type=self.evo_config.job_type, - database_path=str(self.db_config.db_path), - status="running", -) -``` - -**At Evolution End (`run()` method):** -```python -ch_logger.update_evolution_run( - run_id=self.run_id, - status="completed", - total_generations=self.completed_generations, -) -``` - -**After Each Individual Evaluation (`_process_completed_job()`):** -```python -ch_logger.log_individual( - run_id=self.run_id, - individual_id=db_program.id, - generation=job.generation, - parent_id=job.parent_id or "", - mutation_type=mutation_type, - fitness_score=combined_score, - combined_score=combined_score, - metrics={"public": public_metrics, "private": private_metrics}, - is_pareto=is_pareto, - api_cost=..., - embed_cost=..., - novelty_cost=..., - code_hash=code_hash, - code_size=len(evaluated_code), -) - -ch_logger.log_lineage( - run_id=self.run_id, - child_id=db_program.id, - parent_id=job.parent_id, - generation=job.generation, - mutation_type=mutation_type, - fitness_delta=fitness_delta, - edit_summary=edit_summary, -) -``` - -**When Generation Completes (`_update_completed_generations()`):** -```python -# New helper method: _log_generation_to_clickhouse() -ch_logger.log_generation( - run_id=self.run_id, - generation=generation, - num_individuals=len(programs), - best_score=best_score, - avg_score=avg_score, - pareto_size=pareto_size, - total_cost=total_cost, - metadata={...}, -) - -ch_logger.log_pareto_front( - run_id=self.run_id, - generation=generation, - pareto_individuals=pareto_data, -) -``` - -### 2. Files Modified - -- ✅ `genesis/core/runner.py` - Added logging integration -- ✅ `genesis/utils/clickhouse_logger.py` - Already had helper methods -- ✅ `docs/clickhouse.md` - Comprehensive documentation -- ✅ `docs/clickhouse_schema_reference.md` - Quick reference -- ✅ `scripts/test_clickhouse.py` - Connection testing tool - -## Verification - -### Test Results - -```bash -$ python scripts/test_clickhouse.py -``` - -**Table Status (All ✅):** -- `evolution_runs`: 1 row - Tracks each experiment run -- `generations`: 2 rows - Per-generation statistics -- `individuals`: 2 rows - Every code variant evaluated -- `pareto_fronts`: 5 rows - Pareto frontier snapshots -- `code_lineages`: 1 row - Parent-child relationships -- `llm_logs`: 2 rows - LLM API calls (auto-logged) -- `agent_actions`: 3 rows - System events (auto-logged) - -### Sample Data - -**Evolution Run:** -``` -📍 Run: run_20251124_191253_ec91890b - Task: genesis_circle_packing - Status: running - Generations: 0 -``` - -**Generation Progress:** -``` -Gen 0 | Individuals: 5 | Best: 53.24 | Pareto: 5 | Cost: $0.00 -Gen 1 | Individuals: 1 | Best: 0.00 | Pareto: 0 | Cost: $0.04 -``` - -**Individual Variants:** -``` -1bf48f09... | Gen 0 | init | Score: 53.24 | Cost: $0.00 | Size: 13440B -f4b0bf8e... | Gen 1 | full | Score: 0.00 | Cost: $0.04 | Size: 14624B -``` - -**Lineages:** -``` -📉 Gen 1 | f4b0bf8e... ← 821997f8... | full | Δ -53.24 -``` - -## Usage - -### Run an Experiment - -```bash -genesis_launch task@_global_=squeeze_hnsw cluster@_global_=local evolution@_global_=small_budget -``` - -All data will be automatically logged to ClickHouse in real-time. - -### Query the Data - -```python -from genesis.utils.clickhouse_logger import ch_logger - -# Best individual across all runs -result = ch_logger.client.query(""" - SELECT i.run_id, i.individual_id, i.fitness_score, i.generation - FROM individuals i - ORDER BY i.fitness_score DESC - LIMIT 1 -""") - -# Evolution progress -result = ch_logger.client.query(""" - SELECT generation, best_score, avg_score, pareto_size - FROM generations - WHERE run_id = 'YOUR_RUN_ID' - ORDER BY generation -""") -``` - -### Visualize with Test Script - -```bash -python scripts/test_clickhouse.py -``` - -Shows: -- Connection status -- Table schemas -- Row counts -- Example queries -- Sample data - -## Benefits - -1. **Real-time Monitoring** - Watch evolution progress live -2. **Historical Analysis** - Compare runs over time -3. **Cost Tracking** - Monitor API spending per generation -4. **Lineage Tracing** - Understand evolutionary pathways -5. **Pareto Analysis** - Track multi-objective optimization -6. **Debugging** - See exactly what happened and when - -## Next Steps - -### Integration with WebUI - -The ClickHouse data can now be integrated into the Genesis WebUI for: -- Real-time dashboards -- Interactive lineage trees -- Pareto frontier visualization -- Cost analytics -- Run comparison - -### Grafana Dashboards - -Create Grafana dashboards for production monitoring: -- Evolution progress over time -- API cost burn rate -- Model performance comparison -- Success/failure rates - -### Data Analysis - -Use ClickHouse for advanced analytics: -- Mutation type effectiveness -- Island migration patterns -- Novelty search impact -- Meta-recommendation influence - -## Documentation - -- **Full Guide**: `docs/clickhouse.md` (889 lines) -- **Schema Reference**: `docs/clickhouse_schema_reference.md` -- **Test Script**: `scripts/test_clickhouse.py` -- **Setup Summary**: `CLICKHOUSE_SETUP.md` - ---- - -**Status**: ✅ Complete and verified -**Date**: November 24, 2025 -**Tables**: 7/7 operational -**Test Runs**: Successful with `squeeze_hnsw` and `circle_packing` tasks diff --git a/CLICKHOUSE_SETUP.md b/CLICKHOUSE_SETUP.md deleted file mode 100644 index cfa03ab..0000000 --- a/CLICKHOUSE_SETUP.md +++ /dev/null @@ -1,142 +0,0 @@ -# ClickHouse Setup - Summary - -✅ **All tables successfully created and verified!** - -## What Was Added - -### 1. Database Schema (7 Tables) -All tables automatically created via `genesis/utils/clickhouse_logger.py`: - -| Table | Rows | Purpose | -|-------|------|---------| -| `evolution_runs` | 0 | Track each experiment run | -| `generations` | 0 | Per-generation statistics | -| `individuals` | 0 | Every code variant evaluated | -| `pareto_fronts` | 0 | Pareto frontier snapshots | -| `code_lineages` | 0 | Parent-child relationships | -| `llm_logs` | 0 | All LLM API calls (auto-logged) | -| `agent_actions` | 0 | System events (auto-logged) | - -### 2. Helper Methods -Added to `genesis/utils/clickhouse_logger.py`: -- `log_evolution_run()` - Start of experiment -- `log_generation()` - Generation stats -- `log_individual()` - Individual evaluation -- `log_pareto_front()` - Pareto frontier -- `log_lineage()` - Parent-child relationship -- `update_evolution_run()` - Mark run complete - -### 3. Documentation -- **`docs/clickhouse.md`** (889 lines) - Complete guide with: - - Detailed explanation of every table and column - - Example queries for common analyses - - Usage examples in code - - Troubleshooting guide - - Data volume estimates - - Visualization setup - -- **`docs/clickhouse_schema_reference.md`** - Quick reference: - - All table schemas in one place - - SQL DDL for recreation - - Entity relationship diagram - - Index optimization tips - -### 4. Testing Script -- **`scripts/test_clickhouse.py`** - Verify connection and show schema - -## Quick Start - -### Test Connection -```bash -python scripts/test_clickhouse.py -``` - -### View Tables -All tables are created automatically. Current state: -- ✅ 7 tables exist -- ✅ Schemas validated -- ⏳ Waiting for data (need to integrate logging in runner.py) - -## Next Steps - -### To Start Logging Data - -The tables exist but are empty. To populate them: - -1. **Update `genesis/core/runner.py`** - Add logging calls: -```python -from genesis.utils.clickhouse_logger import ch_logger - -# At start of evolution -ch_logger.log_evolution_run( - run_id=run_id, - task_name=cfg.task_name, - config=OmegaConf.to_container(cfg), - population_size=cfg.evolution.pop_size, - cluster_type=cfg.cluster.type, - database_path=db_path, -) - -# After each generation -ch_logger.log_generation( - run_id=run_id, - generation=gen, - num_individuals=len(population), - best_score=max(scores), - avg_score=np.mean(scores), - pareto_size=len(pareto_front), - total_cost=sum(costs), -) - -# After each individual evaluation -ch_logger.log_individual( - run_id=run_id, - individual_id=ind.id, - generation=gen, - parent_id=ind.parent_id, - mutation_type=ind.mutation_type, - fitness_score=ind.fitness, - combined_score=ind.combined_score, - metrics=ind.metrics, - is_pareto=ind.is_pareto, - api_cost=ind.api_cost, - embed_cost=ind.embed_cost, - novelty_cost=ind.novelty_cost, - code_hash=hash(ind.code), - code_size=len(ind.code), -) -``` - -2. **Run an experiment** - Data will flow automatically: -```bash -genesis_launch variant=circle_packing_example -``` - -3. **Query the data**: -```bash -python scripts/test_clickhouse.py # Will show row counts -``` - -## Documentation - -- **Full Guide**: `docs/clickhouse.md` -- **Schema Reference**: `docs/clickhouse_schema_reference.md` -- **Test Script**: `scripts/test_clickhouse.py` - -## Verify Setup - -```bash -# Check environment -echo $CLICKHOUSE_URL - -# Test connection -python scripts/test_clickhouse.py - -# Should output: -# ✅ ClickHouse connection successful! -# 📊 7 tables created -``` - ---- - -**Status**: ✅ Setup complete, ready for integration! diff --git a/Dockerfile b/Dockerfile index de44e65..d5837b8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,41 +1,15 @@ -# Backend Dockerfile for Genesis Evolution Platform -FROM python:3.12-slim +# Root Dockerfile - builds the Rust backend +# For local Python development, use: uv pip install -e ".[dev]" -# Set working directory +FROM rust:1.83-bookworm AS builder WORKDIR /app - -# Install system dependencies -RUN apt-get update && apt-get install -y \ - build-essential \ - curl \ - git \ - postgresql-client \ - && rm -rf /var/lib/apt/lists/* - -# Copy dependency files -COPY pyproject.toml ./ -COPY README.md ./ - -# Copy application code -COPY genesis/ ./genesis/ -COPY configs/ ./configs/ -COPY examples/ ./examples/ -COPY tests/ ./tests/ - -# Install Python dependencies -RUN pip install --no-cache-dir --upgrade pip && \ - pip install --no-cache-dir -e . - -# Create directory for results and database -RUN mkdir -p /app/results /app/data - -# Expose port for any potential API/web service -EXPOSE 8000 - -# Set environment variables -ENV PYTHONUNBUFFERED=1 -ENV GENESIS_DATA_DIR=/app/data -ENV GENESIS_RESULTS_DIR=/app/results - -# Default command (can be overridden in docker-compose) -CMD ["python", "-m", "genesis.launch_hydra"] +COPY genesis_rust_backend/Cargo.toml genesis_rust_backend/Cargo.lock ./ +RUN mkdir src && echo "fn main() {}" > src/main.rs && cargo build --release && rm -rf src +COPY genesis_rust_backend/src ./src +RUN touch src/main.rs && cargo build --release + +FROM gcr.io/distroless/cc-debian12 +COPY --from=builder /app/target/release/genesis_rust_backend /genesis_rust_backend +ENV PORT=8080 +EXPOSE 8080 +ENTRYPOINT ["/genesis_rust_backend"] diff --git a/SESSION_SUMMARY.md b/SESSION_SUMMARY.md deleted file mode 100644 index 491d7a2..0000000 --- a/SESSION_SUMMARY.md +++ /dev/null @@ -1,294 +0,0 @@ -# Genesis Session Summary - November 24, 2025 - -## 🎯 Main Objectives Completed - -### 1. ✅ ClickHouse Integration - COMPLETE - -**Created 7 operational tables:** -- `evolution_runs` - Experiment tracking -- `generations` - Per-generation statistics -- `individuals` - Code variant evaluations -- `pareto_fronts` - Pareto frontier snapshots -- `code_lineages` - Parent-child relationships -- `llm_logs` - LLM API call tracking (auto-logged) -- `agent_actions` - System event tracking (auto-logged) - -**Integration points added to `genesis/core/runner.py`:** -- Evolution run start/end logging -- Individual code variant logging after each evaluation -- Generation statistics when generation completes -- Pareto frontier snapshots -- Lineage tracking for parent-child relationships -- Cost breakdown (API, embedding, novelty) - -**Documentation created:** -- `docs/clickhouse.md` (889 lines) - Complete integration guide -- `docs/clickhouse_schema_reference.md` - Quick schema reference -- `scripts/test_clickhouse.py` - Connection test & verification tool -- `CLICKHOUSE_SETUP.md` - Initial setup summary -- `CLICKHOUSE_INTEGRATION_COMPLETE.md` - Final completion summary - -**Verified with test runs:** -- `squeeze_hnsw` (Rust HNSW optimization) -- `circle_packing` (Python circle packing) -- All tables populated successfully ✅ - -### 2. ✅ LLM Documentation - COMPLETE - -**Created comprehensive LLM guide:** -- `docs/available_llms.md` - Complete LLM documentation - -**Documented 60+ models across 6 providers:** - -**Anthropic Claude:** -- claude-3-5-haiku, claude-3-5-sonnet, claude-3-opus -- claude-3-7-sonnet (reasoning) -- claude-4-sonnet, claude-sonnet-4-5 (reasoning) - -**OpenAI GPT:** -- gpt-4o-mini, gpt-4o, gpt-4.1 series -- o1, o3-mini, o3, o4-mini (reasoning models) -- gpt-4.5-preview, gpt-5 series (future) - -**DeepSeek:** -- deepseek-chat (ultra cost-effective) -- deepseek-reasoner (reasoning) - -**Google Gemini:** -- gemini-2.5-pro, gemini-2.5-flash, gemini-2.5-flash-lite -- gemini-3-pro-preview (future) - -**AWS Bedrock & Azure:** -- Full Bedrock support for Anthropic models -- Azure OpenAI integration - -**Dynamic Model Selection System:** -- Asymmetric UCB (Upper Confidence Bound) bandit algorithm -- Automatic learning of best-performing models -- Adaptive exploration/exploitation balance -- Cost-performance optimization -- Real-time monitoring and logging - ---- - -## 📊 Current State - -### ClickHouse Tables Status - -``` -✅ evolution_runs : 1 row - Tracking experiment runs -✅ generations : 2 rows - Per-generation statistics -✅ individuals : 2 rows - Code variant evaluations -✅ pareto_fronts : 5 rows - Pareto frontier snapshots -✅ code_lineages : 1 row - Parent-child relationships -✅ llm_logs : 2 rows - LLM API calls -✅ agent_actions : 3 rows - System events -``` - -All 7 tables operational and verified! - -### LLM Support - -- **6 providers** integrated (Anthropic, OpenAI, DeepSeek, Gemini, Bedrock, Azure) -- **60+ models** available -- **3 selection modes**: Static single, Static multi, Dynamic UCB -- **Cost tracking** integrated with ClickHouse -- **Automatic adaptation** via UCB bandit algorithm - ---- - -## 📁 Files Created/Modified - -### Created: -- ✅ `docs/clickhouse.md` (889 lines) -- ✅ `docs/clickhouse_schema_reference.md` -- ✅ `docs/available_llms.md` -- ✅ `scripts/test_clickhouse.py` -- ✅ `CLICKHOUSE_SETUP.md` -- ✅ `CLICKHOUSE_INTEGRATION_COMPLETE.md` -- ✅ `SESSION_SUMMARY.md` (this file) - -### Modified: -- ✅ `genesis/core/runner.py` - Added ClickHouse logging integration -- ✅ `genesis/utils/clickhouse_logger.py` - Enhanced schema and helper methods -- ✅ `mkdocs.yml` - Added new documentation pages -- ✅ `scripts/README.md` - Added test_clickhouse.py documentation - ---- - -## 🚀 Key Features Implemented - -### Real-time Evolution Tracking -- Every code variant logged to ClickHouse -- Generation statistics automatically computed -- Pareto frontier snapshots -- Cost tracking per individual, generation, and run - -### Multi-Provider LLM Support -- Seamless switching between providers -- Unified pricing across all models -- Automatic retry with exponential backoff -- Cost tracking and monitoring - -### Dynamic Model Selection -- UCB bandit algorithm learns best models -- Balances exploration vs exploitation -- Adapts to task-specific performance -- Automatic cost-performance optimization - -### Comprehensive Analytics -- SQL queries for evolution analysis -- Lineage tracing for phylogenetic trees -- Cost breakdown by model and component -- Performance monitoring over time - ---- - -## 📚 Documentation Highlights - -### ClickHouse Integration -- Full table schema explanations with example rows -- 20+ example SQL queries for common analyses -- Data volume estimates and retention policies -- Integration guide for runner.py -- Troubleshooting section -- Grafana/Metabase visualization setup - -### LLM Guide -- Pricing comparison across all models -- Best practices for cost optimization -- Configuration examples for different use cases -- Dynamic selection algorithm explanation -- Environment variable setup -- Model performance monitoring - ---- - -## 🔍 Testing & Verification - -### Tests Performed: -1. ✅ ClickHouse connection test -2. ✅ Table creation verification -3. ✅ HNSW optimization run (2 generations) -4. ✅ Circle packing run (1 generation) -5. ✅ Data verification in all 7 tables -6. ✅ Cost tracking verification -7. ✅ Lineage tracking verification - -### Results: -- All tables populated correctly -- Real-time logging working -- Cost tracking accurate -- Lineage relationships preserved -- Generation statistics computed correctly - ---- - -## 💡 Usage Examples - -### Run Evolution with ClickHouse Logging -```bash -genesis_launch task@_global_=squeeze_hnsw cluster@_global_=local evolution@_global_=small_budget -``` -All data automatically logged to ClickHouse! - -### Test ClickHouse Connection -```bash -python scripts/test_clickhouse.py -``` - -### Dynamic Model Selection -```yaml -evo_config: - llm_models: - - gpt-4.1 - - claude-3-5-sonnet-20241022 - - gemini-2.5-flash - - deepseek-chat - llm_dynamic_selection: "ucb" -``` - -### Query Evolution Data -```python -from genesis.utils.clickhouse_logger import ch_logger - -result = ch_logger.client.query(""" - SELECT generation, best_score, avg_score, pareto_size - FROM generations - WHERE run_id = 'YOUR_RUN_ID' - ORDER BY generation -""") -``` - ---- - -## 🎁 Benefits Delivered - -### For Researchers: -- ✅ Complete evolution tracking for reproducibility -- ✅ Historical analysis across multiple runs -- ✅ Cost optimization via model selection -- ✅ Lineage analysis for evolutionary insights - -### For Developers: -- ✅ Real-time debugging via ClickHouse queries -- ✅ Performance monitoring dashboards -- ✅ Automatic model adaptation -- ✅ Comprehensive logging for troubleshooting - -### For Production: -- ✅ Cost tracking and budgeting -- ✅ Scalable data storage (ClickHouse) -- ✅ Multi-provider redundancy -- ✅ Automatic failover and retry logic - ---- - -## 📈 Next Steps - -### Immediate: -1. Integrate ClickHouse data into WebUI for real-time dashboards -2. Create Grafana dashboards for monitoring -3. Add more example queries to documentation - -### Future: -1. Multi-objective Pareto frontier visualization -2. Interactive lineage tree viewer -3. Cost prediction based on historical data -4. A/B testing framework for model comparison - ---- - -## 📊 Statistics - -- **Lines of Code Added**: ~500 (runner integration + helpers) -- **Documentation Written**: ~2000 lines -- **Tables Created**: 7 -- **Models Documented**: 60+ -- **Providers Integrated**: 6 -- **Test Runs Completed**: 3 - ---- - -## ✅ Completion Checklist - -- [x] ClickHouse tables designed and created -- [x] Integration added to evolution runner -- [x] Test script created and working -- [x] Comprehensive documentation written -- [x] Schema reference created -- [x] Test runs completed successfully -- [x] LLM providers documented -- [x] Dynamic selection explained -- [x] Configuration examples provided -- [x] Cost tracking verified -- [x] Lineage tracking verified -- [x] MkDocs navigation updated - ---- - -**Status**: All objectives completed successfully! ✅ - -**Date**: November 24, 2025 - -**Next Session**: WebUI integration of ClickHouse data diff --git a/docker-compose.yml b/docker-compose.yml index ad1362c..4bf9a63 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,75 +1,80 @@ version: '3.8' services: - # ClickHouse Database - clickhouse: - image: clickhouse/clickhouse-server:latest - container_name: genesis-clickhouse + postgres: + image: postgres:15 + container_name: genesis-postgres + environment: + POSTGRES_DB: genesis + POSTGRES_USER: genesis_app + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-genesis_dev} ports: - - "8123:8123" - - "9000:9000" + - "5432:5432" volumes: - - clickhouse_data:/var/lib/clickhouse - ulimits: - nofile: - soft: 262144 - hard: 262144 + - postgres_data:/var/lib/postgresql/data networks: - genesis-network restart: unless-stopped healthcheck: - test: ["CMD", "wget", "--spider", "-q", "localhost:8123/ping"] + test: ["CMD-SHELL", "pg_isready -U genesis_app -d genesis"] interval: 10s timeout: 5s retries: 5 - # Genesis Backend Application + liquibase: + image: liquibase/liquibase:4.29 + container_name: genesis-liquibase + depends_on: + postgres: + condition: service_healthy + volumes: + - ./migrations:/liquibase/changelog + networks: + - genesis-network + command: > + --url=jdbc:postgresql://postgres:5432/genesis + --username=genesis_app + --password=${POSTGRES_PASSWORD:-genesis_dev} + --changeLogFile=changelog/changelogs/db.changelog-master.yaml + update + backend: build: - context: . + context: genesis_rust_backend dockerfile: Dockerfile container_name: genesis-backend environment: - # Genesis application settings - GENESIS_DATA_DIR: /app/data - GENESIS_RESULTS_DIR: /app/results - PYTHONUNBUFFERED: 1 - - # ClickHouse Configuration - CLICKHOUSE_URL: http://default:@clickhouse:8123/default - CLICKHOUSE_HOST: clickhouse - CLICKHOUSE_PORT: 8123 - - # LLM API Keys (to be provided via .env file) + DATABASE_URL: postgresql://genesis_app:${POSTGRES_PASSWORD:-genesis_dev}@postgres:5432/genesis OPENAI_API_KEY: ${OPENAI_API_KEY:-} ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-} + RUST_LOG: info + PORT: "8080" + ports: + - "${GENESIS_PORT:-8080}:8080" + depends_on: + liquibase: + condition: service_completed_successfully + networks: + - genesis-network + restart: unless-stopped - # Optional E2B configuration - E2B_API_KEY: ${E2B_API_KEY:-} - - # Web UI configuration - GENESIS_WEBUI_PORT: ${GENESIS_WEBUI_PORT:-8000} - volumes: - # Mount directories for persistent data - - ./data:/app/data - - ./results:/app/results - - # Mount source code for development (comment out for production) - - ./genesis:/app/genesis - - ./configs:/app/configs - - ./examples:/app/examples + frontend: + build: + context: genesis/webui/frontend + dockerfile: Dockerfile + container_name: genesis-frontend + environment: + BACKEND_URL: http://backend:8080 ports: - - "${GENESIS_WEBUI_PORT:-8000}:8000" + - "${FRONTEND_PORT:-3000}:8080" depends_on: - clickhouse: - condition: service_healthy + - backend networks: - genesis-network restart: unless-stopped - command: ["python", "-m", "genesis.webui.visualization", "--port", "8000"] volumes: - clickhouse_data: + postgres_data: driver: local networks: diff --git a/genesis/favicon.png b/genesis/favicon.png deleted file mode 100644 index c25e6bd..0000000 Binary files a/genesis/favicon.png and /dev/null differ diff --git a/genesis/llm/query.py b/genesis/llm/query.py deleted file mode 100644 index 17dd231..0000000 --- a/genesis/llm/query.py +++ /dev/null @@ -1,407 +0,0 @@ -from typing import List, Union, Optional, Dict -import random -from pydantic import BaseModel -from .client import get_client_llm -from .models.pricing import ( - CLAUDE_MODELS, - OPENAI_MODELS, - DEEPSEEK_MODELS, - GEMINI_MODELS, - BEDROCK_MODELS, - REASONING_OAI_MODELS, - REASONING_CLAUDE_MODELS, - REASONING_DEEPSEEK_MODELS, - REASONING_GEMINI_MODELS, - REASONING_AZURE_MODELS, - REASONING_BEDROCK_MODELS, - OPENROUTER_MODELS, -) -from .models import ( - query_anthropic, - query_openai, - query_deepseek, - query_gemini, - QueryResult, -) -import logging -import time - -logger = logging.getLogger(__name__) - - -THINKING_TOKENS = { - "auto": 4096, - "low": 2048, - "medium": 4096, - "high": 8192, - "max": 16384, -} - - -def sample_batch_kwargs( - num_samples: int, - model_names: Union[List[str], str] = "gpt-4o-mini-2024-07-18", - temperatures: Union[List[float], float] = 0.0, - max_tokens: Union[List[int], int] = 4096, - reasoning_efforts: Union[List[str], str] = "", - model_sample_probs: Optional[List[float]] = None, - unique_filter: bool = False, -): - """Sample a dictionary of kwargs for a given model.""" - all_kwargs = [] - attempts = 0 - max_attempts = num_samples * 10 # Prevent infinite loops - - while len(all_kwargs) < num_samples and attempts < max_attempts: - kwargs_dict = sample_model_kwargs( - model_names=model_names, - temperatures=temperatures, - max_tokens=max_tokens, - reasoning_efforts=reasoning_efforts, - model_sample_probs=model_sample_probs, - ) - - if unique_filter: - if kwargs_dict not in all_kwargs: - all_kwargs.append(kwargs_dict) - else: - all_kwargs.append(kwargs_dict) - - attempts += 1 - - if len(all_kwargs) < num_samples: - logger.info( - f"Could not generate {num_samples} unique kwargs combinations " - f"after {max_attempts} attempts" - ) - logger.info(f"Returning {len(all_kwargs)} unique kwargs combinations.") - - return all_kwargs - - -def sample_model_kwargs( - model_names: Union[List[str], str] = "gpt-4o-mini-2024-07-18", - temperatures: Union[List[float], float] = 0.0, - max_tokens: Union[List[int], int] = 4096, - reasoning_efforts: Union[List[str], str] = "", - model_sample_probs: Optional[List[float]] = None, -): - """Sample a dictionary of kwargs for a given model.""" - # Make all inputs lists - if isinstance(model_names, str): - model_names = [model_names] - if isinstance(temperatures, float): - temperatures = [temperatures] - if isinstance(max_tokens, int): - max_tokens = [max_tokens] - if isinstance(reasoning_efforts, str): - reasoning_efforts = [reasoning_efforts] - - kwargs_dict = {} - # perform model sampling if list provided - if model_sample_probs is not None: - if len(model_sample_probs) != len(model_names): - raise ValueError( - "model_sample_probs must have the same length as model_names" - ) - if not abs(sum(model_sample_probs) - 1.0) < 1e-9: - raise ValueError("model_sample_probs must sum to 1") - kwargs_dict["model_name"] = random.choices( - model_names, weights=model_sample_probs, k=1 - )[0] - else: - kwargs_dict["model_name"] = random.choice(model_names) - - # perform temperature sampling if list provided - # set temperature to 1.0 for reasoning models - if kwargs_dict["model_name"] in ( - REASONING_OAI_MODELS - + REASONING_CLAUDE_MODELS - + REASONING_DEEPSEEK_MODELS - + REASONING_GEMINI_MODELS - + REASONING_AZURE_MODELS - + REASONING_BEDROCK_MODELS - ): - kwargs_dict["temperature"] = 1.0 - else: - kwargs_dict["temperature"] = random.choice(temperatures) - - # perform reasoning effort sampling if list provided - # set max_completion_tokens for OAI reasoning models - if kwargs_dict["model_name"] in (REASONING_OAI_MODELS + REASONING_AZURE_MODELS): - kwargs_dict["max_output_tokens"] = random.choice(max_tokens) - r_effort = random.choice(reasoning_efforts) - if r_effort != "auto": - kwargs_dict["reasoning"] = {"effort": r_effort} - - if kwargs_dict["model_name"] in (REASONING_GEMINI_MODELS): - kwargs_dict["max_tokens"] = random.choice(max_tokens) - r_effort = random.choice(reasoning_efforts) - # Always enable thinking if effort is auto or specified - t = THINKING_TOKENS.get(r_effort, 4096) - thinking_tokens = t if t < kwargs_dict["max_tokens"] else 1024 - kwargs_dict["extra_body"] = { - "extra_body": { - "google": { - "thinking_config": { - "thinking_budget": thinking_tokens, - "include_thoughts": True, - } - } - } - } - - elif kwargs_dict["model_name"] in ( - REASONING_CLAUDE_MODELS + REASONING_BEDROCK_MODELS - ): - kwargs_dict["max_tokens"] = min(random.choice(max_tokens), 16384) - r_effort = random.choice(reasoning_efforts) - - # Enable thinking tokens - t = THINKING_TOKENS.get(r_effort, 4096) - thinking_tokens = t if t < kwargs_dict["max_tokens"] else 1024 - - # sample only from thinking tokens that are valid - kwargs_dict["thinking"] = { - "type": "enabled", - "budget_tokens": thinking_tokens, - } - - else: - if ( - kwargs_dict["model_name"] in CLAUDE_MODELS - or kwargs_dict["model_name"] in BEDROCK_MODELS - or kwargs_dict["model_name"] in REASONING_CLAUDE_MODELS - or kwargs_dict["model_name"] in REASONING_BEDROCK_MODELS - or kwargs_dict["model_name"] in DEEPSEEK_MODELS - or kwargs_dict["model_name"] in REASONING_DEEPSEEK_MODELS - or kwargs_dict["model_name"].startswith("openrouter/") - ): - kwargs_dict["max_tokens"] = random.choice(max_tokens) - else: - kwargs_dict["max_output_tokens"] = random.choice(max_tokens) - - return kwargs_dict - - -def query( - model_name: str, - msg: str, - system_msg: str, - msg_history: List = [], - output_model: Optional[BaseModel] = None, - model_posteriors: Optional[Dict[str, float]] = None, - tools: Optional[List[Dict]] = None, - tool_map: Optional[Dict[str, callable]] = None, - **kwargs, -) -> QueryResult: - """Query the LLM.""" - client, model_name_processed = get_client_llm( - model_name, structured_output=output_model is not None - ) - if model_name.startswith("openrouter/"): - # OpenRouter uses the OpenAI-compatible client - query_fn = query_openai - # We need to pass the processed model name (e.g., anthropic/claude-3-5-sonnet) - # to the query function, but get_client_llm already returns it. - # However, query_openai expects the model_name argument to be passed to client.chat.completions.create - # so we should use the one returned by get_client_llm. - model_name = model_name_processed - elif model_name in CLAUDE_MODELS.keys() or "anthropic" in model_name: - query_fn = query_anthropic - elif model_name in OPENAI_MODELS.keys(): - query_fn = query_openai - elif model_name in DEEPSEEK_MODELS.keys(): - query_fn = query_deepseek - elif model_name in GEMINI_MODELS.keys(): - query_fn = query_gemini - else: - raise ValueError(f"Model {model_name} not supported.") - - start_time = time.time() - - # Loop for tool calling - max_tool_iterations = 5 - current_msg_history = list(msg_history) - current_msg = msg - total_cost = 0.0 - total_input_tokens = 0 - total_output_tokens = 0 - - for i in range(max_tool_iterations): - result = query_fn( - client, - model_name, - current_msg, - system_msg, - current_msg_history, - output_model, - model_posteriors=model_posteriors, - tools=tools, - **kwargs, - ) - - # Accumulate costs and tokens - total_cost += result.cost - total_input_tokens += result.input_tokens - total_output_tokens += result.output_tokens - - # Check for tool calls - if result.tool_calls and tool_map: - # Execute tools - tool_outputs = [] - for tool_call in result.tool_calls: - function_name = tool_call.get("name") or tool_call.get( - "function", {} - ).get("name") - function_args = tool_call.get("input") or tool_call.get( - "function", {} - ).get("arguments") - call_id = tool_call.get("id") - - if isinstance(function_args, str): - try: - import json - - function_args = json.loads(function_args) - except: - pass - - if function_name in tool_map: - try: - logger.info( - f"Executing tool {function_name} with args {function_args}" - ) - output = tool_map[function_name](**function_args) - except Exception as e: - output = {"error": str(e)} - else: - output = {"error": f"Tool {function_name} not found"} - - tool_outputs.append( - { - "tool_call_id": call_id, - "output": str(output), - "name": function_name, - } - ) - - # Prepare history for next iteration - # The query_fn already appended the assistant message with tool calls to new_msg_history in result - current_msg_history = result.new_msg_history - - # Add tool outputs to history - for tool_output in tool_outputs: - if "anthropic" in model_name or model_name in CLAUDE_MODELS: - current_msg_history.append( - { - "role": "user", - "content": [ - { - "type": "tool_result", - "tool_use_id": tool_output["tool_call_id"], - "content": tool_output["output"], - } - ], - } - ) - else: - # OpenAI format - current_msg_history.append( - { - "role": "tool", - "tool_call_id": tool_output["tool_call_id"], - "name": tool_output["name"], - "content": tool_output["output"], - } - ) - - # For the next iteration, msg is empty because context is in history - # But query_fn signature expects a msg. - # For Anthropic, we just continue conversation. - # For OpenAI, we send new history. - # We need to adjust query_fn to handle empty msg if history has it? - # actually query_fn appends msg to history. If msg is empty string, it appends empty user message which might be weird. - # But we can just pass "Continue" or similar if needed, or rely on the fact that we updated history. - # Actually, let's look at query_fn again. It does `new_msg_history = msg_history + [{"role": "user", "content": msg}]` - # We don't want to add a user message if we are just returning tool outputs. - # We need to pass the *updated* history as msg_history, and an empty msg? - # Or better, we modify query logic to just use the updated history. - - # Since we can't easily modify query_fn signature behaviour without breaking things, - # we will pass a dummy prompt like "continue" or rely on the fact that for tool use loops, - # the "user" response IS the tool output. - # Wait, `query_fn` adds a user message with `msg`. - # If we pass `msg=""`, it adds an empty user message. - # Correct flow for tool use: - # 1. User: msg - # 2. Assistant: tool_call - # 3. Tool: tool_result - # 4. Assistant: final response - - # In our loop: - # Iter 0: msg passed as `msg`. `msg_history` is initial history. - # `query_fn` constructs `[...history, user:msg]`. Returns `assistant:tool_call`. - # We get `result.new_msg_history` which is `[...history, user:msg, assistant:tool_call]`. - # We append `tool:tool_result` to this. - # Iter 1: We want to get the next assistant response. - # If we call `query_fn` again, we need to pass the updated history. - # But `query_fn` enforces adding a user message `msg`. - # We cannot use `query_fn` as is for the loop step if it forces a new user message. - # We need to change `query_fn` to allow `msg` to be None/skip if history is sufficient? - # Or we hack it by passing the tool output as the `msg` for the next turn? - - # For OpenAI: - # messages = [system] + history + [user:msg] - # If we want to send tool outputs, they are "tool" role messages. - # So `msg` is not appropriate for tool outputs. - - # We need to modify `query_fn` to handle this scenario or implement the loop logic inside `query_fn`. - # But `query` wraps `query_fn`. - - # Let's modify `query` to NOT use `query_fn` for subsequent iterations, but call client directly? - # That duplicates logic. - - # Better: Modify `query_fn` to accept `msg=None` and skip adding it if so. - - current_msg = None - # We will modify query_fn to handle msg=None - - else: - # No tool calls, we are done - break - - end_time = time.time() - - # Update result costs/tokens to reflect total - result.cost = total_cost - result.input_tokens = total_input_tokens - result.output_tokens = total_output_tokens - - # Log to ClickHouse - try: - from genesis.utils.clickhouse_logger import ch_logger - - log_messages = [] - if system_msg: - log_messages.append({"role": "system", "content": system_msg}) - if msg_history: - log_messages.extend(msg_history) - if msg: - log_messages.append({"role": "user", "content": msg}) - - ch_logger.log_llm_interaction( - model=model_name, - messages=log_messages, - response=result.content if result else "None", - cost=result.cost - if result and hasattr(result, "cost") and result.cost - else 0.0, - execution_time=end_time - start_time, - metadata=kwargs, - thought=result.thought if result and hasattr(result, "thought") else "", - ) - except Exception as e: - logger.warning(f"Failed to log to ClickHouse: {e}") - - return result diff --git a/genesis/plots/__init__.py b/genesis/plots/__init__.py deleted file mode 100644 index efbacf4..0000000 --- a/genesis/plots/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -from .plot_lineage_tree import plot_lineage_tree -from .plot_improvement import plot_improvement -from .plot_pareto import plot_pareto -from .plot_similarity import plot_embed_similarity - -__all__ = [ - "plot_lineage_tree", - "plot_improvement", - "plot_pareto", - "plot_embed_similarity", -] diff --git a/genesis/plots/code_path_anim.py b/genesis/plots/code_path_anim.py deleted file mode 100644 index d5d36d9..0000000 --- a/genesis/plots/code_path_anim.py +++ /dev/null @@ -1,1384 +0,0 @@ -import os -import glob -import shutil -import subprocess -import tempfile -from PIL import Image, ImageDraw, ImageFont -from pygments.lexers import get_lexer_for_filename, guess_lexer - -from pygments.styles import get_style_by_name -import difflib - -from moviepy import VideoClip -import numpy as np -from genesis.utils import load_programs_to_df, get_path_to_best_node, store_best_path -import argparse - -parser = argparse.ArgumentParser() -parser.add_argument( - "--results_dir", - type=str, - default="examples/agent_design/results_20250620_133347", -) -args = parser.parse_args() - -results_dir = args.results_dir - -df = load_programs_to_df(f"{results_dir}/evolution_db.sqlite") -print(df.head()) -best_path = get_path_to_best_node(df) -store_best_path(best_path, results_dir) -best_path_dir = os.path.join(results_dir, "best_path") - -# copy main.py to original.py -shutil.copy(f"{best_path_dir}/main.py", f"{best_path_dir}/original.py") -# --- Configuration --- -BASE_CODE_FILE = f"{best_path_dir}/original.py" # Replace with your base file -PATCH_DIR = f"{best_path_dir}/patches" # Directory containing your .patch files - -# Custom patch labels (optional) - if provided, these will be used instead -# of patch file names. Set to None to use patch file names, or provide a -# list of custom descriptions -INIT_LABEL_STATE = "Initial" -patch_labels = best_path.patch_name.iloc[1:].tolist() - - -OUTPUT_VIDEO = f"{results_dir}/code_evolution.mp4" -# 2560×1440 (2K / QHD) -# 3840×2160 (4K / UHD) -VIDEO_WIDTH = 3840 # Increased from 1920 to 3840 -VIDEO_HEIGHT = 2160 # Increased from 1080 to 2160 -FPS = 25 - -# Center content in frame -CENTER_CONTENT = True # Set to True to center content in frame - -# Background setting -USE_WHITE_BACKGROUND = ( - True # Set to True for white background, False for dark background -) - -# Appearance -# Try these common monospace fonts in order of preference -FONT_PATH = ( - "/System/Library/Fonts/Menlo.ttc" # Default - if os.path.exists("/System/Library/Fonts/Menlo.ttc") - else "/System/Library/Fonts/SFMono-Regular.otf" # SF Mono - if os.path.exists("/System/Library/Fonts/SFMono-Regular.otf") - else "/System/Library/Fonts/Monaco.ttf" # Monaco - if os.path.exists("/System/Library/Fonts/Monaco.ttf") - else "/System/Library/Fonts/Courier.ttc" # Courier - if os.path.exists("/System/Library/Fonts/Courier.ttc") - else "/System/Library/Fonts/Andale Mono.ttf" # Andale Mono - if os.path.exists("/System/Library/Fonts/Andale Mono.ttf") - else "/System/Library/Fonts/Menlo.ttc" # Fallback to Menlo -) - -FONT_PATH = "/System/Library/Fonts/Andale Mono.ttf" -try: - FONT_SIZE = 30 - LINE_HEIGHT_RATIO = 1.3 - LINE_HEIGHT = int(FONT_SIZE * LINE_HEIGHT_RATIO) - FONT = ImageFont.truetype(FONT_PATH, FONT_SIZE) -except IOError: - print(f"Warning: Font {FONT_PATH} not found. Using PIL default font.") - FONT = ImageFont.load_default() - try: - bbox = FONT.getbbox("M") - FONT_SIZE = bbox[3] - bbox[1] if bbox else 10 - LINE_HEIGHT_RATIO = 1.5 - LINE_HEIGHT = int(FONT_SIZE * LINE_HEIGHT_RATIO) - except AttributeError: - FONT_SIZE = 10 - LINE_HEIGHT_RATIO = 1.5 - LINE_HEIGHT = 15 - print("Using fallback font size and line height.") - - -# Apply color scheme based on background choice -if USE_WHITE_BACKGROUND: - # Light theme colors - BG_COLOR = (250, 250, 250) - TEXT_COLOR = (30, 30, 30) - ADDED_LINE_BG_COLOR = (200, 255, 200, 180) # Light green with alpha - HISTORY_ADDED_LINE_BG_COLOR = (180, 240, 180, 160) - HISTORY_PANE_BG_COLOR = (240, 240, 240) - HISTORY_PANE_BORDER_COLOR = (180, 180, 180) - HISTORY_PANE_TEXT_COLOR = (30, 30, 30) - MINI_DIFF_BG_COLOR = (230, 230, 230) - MINI_DIFF_ADDED_COLOR = (0, 120, 0) - MINI_DIFF_REMOVED_COLOR = (150, 0, 0) - MINI_DIFF_CONTEXT_COLOR = (120, 120, 120) - MINI_DIFF_ACTIVE_BORDER_COLOR = (180, 160, 0) - PYGMENTS_STYLE = "default" # Light syntax highlighting -else: - # Dark theme colors (default) - BG_COLOR = (30, 30, 30) - TEXT_COLOR = (220, 220, 220) - ADDED_LINE_BG_COLOR = (30, 70, 30, 200) # Dark green with alpha - HISTORY_ADDED_LINE_BG_COLOR = (25, 55, 25, 180) - HISTORY_PANE_BG_COLOR = (38, 38, 38) - HISTORY_PANE_BORDER_COLOR = (60, 60, 60) - HISTORY_PANE_TEXT_COLOR = (200, 200, 200) - MINI_DIFF_BG_COLOR = (45, 45, 45) - MINI_DIFF_ADDED_COLOR = (0, 180, 0) - MINI_DIFF_REMOVED_COLOR = (180, 0, 0) - MINI_DIFF_CONTEXT_COLOR = (100, 100, 100) - MINI_DIFF_ACTIVE_BORDER_COLOR = (220, 220, 0) - PYGMENTS_STYLE = "monokai" # Dark syntax highlighting - -style = get_style_by_name(PYGMENTS_STYLE) - -# --- History Panes (like first video) --- -SHOW_HISTORY_PANES = True -NUM_HISTORY_PANES_TO_SHOW = 3 -HISTORY_PANE_Y_START = 94 -HISTORY_PANE_X_START_OFFSET_FROM_RIGHT = 5 -HISTORY_PANE_SPACING = 10 -HISTORY_FONT_SIZE = 8 -HISTORY_LABEL_FONT_SIZE = 30 # Match the main font size for consistency -TITLE_FONT_SIZE = 55 # Font size for the main iteration title -HISTORY_LINE_HEIGHT_RATIO = 1.2 -HISTORY_MAX_LINES_TO_DRAW = 1000 -MAIN_PANE_X_OFFSET = 20 -# Align main pane top with history panes -MAIN_PANE_Y_OFFSET = HISTORY_PANE_Y_START -MAIN_PANE_RIGHT_MARGIN_IF_HISTORY = 5 - -# --- Mini-diff settings (fallback if SHOW_HISTORY_PANES is False) --- -MINI_DIFF_PANE_WIDTH = 150 -MINI_DIFF_WIDTH = 12 -MINI_DIFF_HEIGHT_PER_LINE = 2 -MINI_DIFF_SPACING = 7 -MINI_DIFF_MAX_LINES = 1000 -MINI_DIFF_TEXT_SIZE = 30 - - -# Animation timing -CHARS_PER_SECOND = 150 -HOLD_DURATION_PER_ITERATION = ( - 1.0 # Increased duration since we're just holding each frame -) -# SCROLL_DURATION_PER_ITERATION = 1.5 # Will be made dynamic -SCROLL_SPEED_LINES_PER_SECOND = 40 # Lines to scroll per second -MIN_SCROLL_DURATION = 0.75 # Minimum time for a scroll animation -SCROLL_PAUSE_AT_TOP = 0.1 # Pause at top before scrolling -SCROLL_PAUSE_AT_BOTTOM = 1.5 # Pause at bottom before next iteration - -# Smooth transition settings for history panels -HISTORY_TRANSITION_DURATION = 0.8 # Duration for history panels to fade in/out -# Duration for main pane to slide in from left -MAIN_PANE_SLIDE_IN_DURATION = 0.6 - -# --- Helper Functions --- - - -def get_file_content(filepath): - try: - with open(filepath, "r", encoding="utf-8") as f: - return f.read() - except FileNotFoundError: - return "" - except Exception as e: - print(f"Error reading {filepath}: {e}") - return "" - - -def apply_patch(base_file_path, patch_file_path, target_dir): - temp_file_name = os.path.basename(base_file_path) - temp_file_path = os.path.join(target_dir, temp_file_name) - - if not os.path.exists(temp_file_path) and os.path.exists(base_file_path): - shutil.copy2(base_file_path, temp_file_path) - elif not os.path.exists(temp_file_path) and not os.path.exists(base_file_path): - with open(temp_file_path, "w", encoding="utf-8") as f: # Ensure utf-8 - pass - - cmd = [ - "git", - "apply", - "--ignore-whitespace", - "--recount", - ] # --recount helps with line numbers - # Simple p-level detection - patch_preview_content = get_file_content(patch_file_path) - if ( - f"--- a/{temp_file_name}" in patch_preview_content - or f"--- a\\{temp_file_name}" in patch_preview_content - ): - cmd.append("-p1") - - cmd.append(os.path.abspath(patch_file_path)) - - try: - subprocess.run( - cmd, - cwd=target_dir, - check=True, - capture_output=True, - text=True, - encoding="utf-8", - errors="ignore", - ) - except subprocess.CalledProcessError as e: - print(f"Error applying patch {patch_file_path} with git apply:") - print("stdout:", e.stdout) - print("stderr:", e.stderr) - print("Attempting manual patch application (VERY basic)...") - # Extremely simplified manual patch for additions/removals only - current_content_lines = get_file_content(temp_file_path).splitlines(True) - patch_content_lines = patch_preview_content.splitlines(True) - - output_lines = [] - c_idx = 0 - for p_line in patch_content_lines: - if ( - p_line.startswith("---") - or p_line.startswith("+++") - or p_line.startswith("@@") - or p_line.startswith("diff") - ): - continue - if p_line.startswith("+"): - output_lines.append(p_line[1:]) - elif p_line.startswith("-"): - c_idx += 1 # Skip corresponding line in current - else: # Context - if c_idx < len(current_content_lines): - output_lines.append(current_content_lines[c_idx]) - c_idx += 1 - if c_idx < len(current_content_lines): # Append remaining - output_lines.extend(current_content_lines[c_idx:]) - - with open(temp_file_path, "w", encoding="utf-8") as f: - f.writelines(output_lines) - print( - f"Manual patch attempt for {patch_file_path} completed. " - f"Result may be imperfect." - ) - - return get_file_content(temp_file_path) - - -def get_diff_details(old_code, new_code): - added_lines_indices = set() - new_code_lines = new_code.splitlines() - old_code_lines = old_code.splitlines() - - s = difflib.SequenceMatcher(None, old_code_lines, new_code_lines, autojunk=False) - for tag, i1, i2, j1, j2 in s.get_opcodes(): - if tag == "insert": - for i in range(j1, j2): - added_lines_indices.add(i) - elif tag == "replace": - for i in range(j1, j2): - added_lines_indices.add(i) - return added_lines_indices - - -def draw_mini_diff( - patch_content, is_active=False, font_mini_diff=None -): # For fallback mode - img_height = MINI_DIFF_HEIGHT_PER_LINE * MINI_DIFF_MAX_LINES - img_width = MINI_DIFF_WIDTH - - final_img_width = img_width + (4 if is_active else 0) - final_img_height = img_height + (4 if is_active else 0) - - base_img = Image.new("RGB", (img_width, img_height), MINI_DIFF_BG_COLOR) - draw = ImageDraw.Draw(base_img) - y = 0 - lines_drawn = 0 - for line in patch_content.splitlines(): - if lines_drawn >= MINI_DIFF_MAX_LINES: - break - if ( - line.startswith("+++") - or line.startswith("---") - or line.startswith("diff") - or line.startswith("index") - or line.startswith("@@") - ): - continue - color = MINI_DIFF_CONTEXT_COLOR - if line.startswith("+"): - color = MINI_DIFF_ADDED_COLOR - elif line.startswith("-"): - color = MINI_DIFF_REMOVED_COLOR - draw.rectangle((0, y, img_width, y + MINI_DIFF_HEIGHT_PER_LINE), fill=color) - y += MINI_DIFF_HEIGHT_PER_LINE - lines_drawn += 1 - - if is_active: - border_img = Image.new( - "RGB", (final_img_width, final_img_height), MINI_DIFF_ACTIVE_BORDER_COLOR - ) - border_img.paste(base_img, (2, 2)) - return border_img - return base_img - - -# --- Prepare Fonts --- -try: - HISTORY_FONT = ImageFont.truetype(FONT_PATH, HISTORY_FONT_SIZE) - HISTORY_LABEL_FONT = ImageFont.truetype(FONT_PATH, HISTORY_LABEL_FONT_SIZE) - TITLE_FONT = ImageFont.truetype(FONT_PATH, TITLE_FONT_SIZE) -except IOError: - HISTORY_FONT = FONT # Fallback - HISTORY_LABEL_FONT = FONT # Fallback - TITLE_FONT = FONT # Fallback for title font -HISTORY_LINE_HEIGHT = int(HISTORY_FONT_SIZE * HISTORY_LINE_HEIGHT_RATIO) - -try: - MINI_DIFF_FONT = ImageFont.truetype(FONT_PATH, MINI_DIFF_TEXT_SIZE) -except IOError: - MINI_DIFF_FONT = ImageFont.load_default() - - -# --- Prepare States --- -print("Preparing code states and diffs...") -patch_files = sorted(glob.glob(os.path.join(PATCH_DIR, "*.patch"))) -if not patch_files: - print(f"No patch files found in {PATCH_DIR}. Exiting.") - exit() - -if not os.path.exists(BASE_CODE_FILE): - print(f"Base code file {BASE_CODE_FILE} not found. Assuming empty base.") - with open(BASE_CODE_FILE, "w", encoding="utf-8") as f: - pass - -code_states = [] -raw_patch_contents_for_minidiff = [] # Only for fallback mini-diffs - -base_content = get_file_content(BASE_CODE_FILE) -code_states.append( - {"content": base_content, "added_lines": set(), "patch_name": INIT_LABEL_STATE} -) - -temp_dir = tempfile.mkdtemp() -temp_base_path = os.path.join(temp_dir, os.path.basename(BASE_CODE_FILE)) -if os.path.exists(BASE_CODE_FILE): - shutil.copy2(BASE_CODE_FILE, temp_base_path) -else: - with open(temp_base_path, "w", encoding="utf-8") as f: - pass - -previous_content = base_content -for i, patch_file in enumerate(patch_files): - print(f"Processing patch {i + 1}/{len(patch_files)}: {patch_file}") - patch_name = os.path.basename(patch_file) - - # Use custom label if provided, otherwise use patch file name - if patch_labels and i < len(patch_labels): - display_name = patch_labels[i] - else: - display_name = patch_name - - current_content = apply_patch(temp_base_path, patch_file, temp_dir) - - added_lines_indices = get_diff_details(previous_content, current_content) - code_states.append( - { - "content": current_content, - "added_lines": added_lines_indices, - "patch_name": display_name, - } - ) - - # Only load raw patches if needed for mini-diffs - if not SHOW_HISTORY_PANES: - raw_patch_contents_for_minidiff.append(get_file_content(patch_file)) - - previous_content = current_content - -# Pre-calculate mini_diff_images if that mode is selected (for fallback) -mini_diff_images = [] -if not SHOW_HISTORY_PANES: - for i, raw_patch_text in enumerate(raw_patch_contents_for_minidiff): - mini_diff_images.append( - draw_mini_diff( - raw_patch_text, is_active=False, font_mini_diff=MINI_DIFF_FONT - ) - ) - - -# Calculate total duration -total_duration = 0 -for i, state in enumerate(code_states): - lines_in_state = len(state["content"].splitlines()) - max_visible_lines = (VIDEO_HEIGHT - MAIN_PANE_Y_OFFSET - 50) // LINE_HEIGHT - - # Early calculation of estimated main pane width for scrolling - # This is a rough estimate - the exact width is calculated later - if SHOW_HISTORY_PANES and i > 0: - # Rough estimate when history panes are visible - estimated_main_pane_width = int(VIDEO_WIDTH * 0.4) - else: - # No history panes - use most of the width - estimated_main_pane_width = VIDEO_WIDTH - MAIN_PANE_X_OFFSET * 2 - - scroll_duration_for_state = 0 - if lines_in_state > max_visible_lines: - lines_to_scroll = lines_in_state - max_visible_lines - scroll_duration_for_state = max( - MIN_SCROLL_DURATION, lines_to_scroll / SCROLL_SPEED_LINES_PER_SECOND - ) - # Need scrolling for this state - duration_for_this_state = ( - HOLD_DURATION_PER_ITERATION - + scroll_duration_for_state - + SCROLL_PAUSE_AT_TOP - + SCROLL_PAUSE_AT_BOTTOM - ) - else: - # No scrolling needed - duration_for_this_state = HOLD_DURATION_PER_ITERATION - - # Add slide-in time for iterations after the first one - if i > 0: # Only add slide-in time for iterations after the first - duration_for_this_state += MAIN_PANE_SLIDE_IN_DURATION - total_duration += duration_for_this_state - -print(f"Total estimated duration: {total_duration:.2f}s") - -try: - lexer = get_lexer_for_filename(BASE_CODE_FILE, stripall=False) -except Exception: # Specify Exception - try: - lexer = guess_lexer(code_states[0]["content"] if code_states else "") - except Exception: # Specify Exception - from pygments.lexers.special import TextLexer - - lexer = TextLexer() -print(f"Using Pygments lexer: {lexer.name}") - - -# --- Animation Function --- -def make_frame(t): - # Calculate which code state to display based on time with scrolling - current_time = 0 - state_index = 0 - scroll_offset = 0 - history_transition_alpha = ( - 1.0 # Alpha for history panels (0.0 = invisible, 1.0 = fully visible) - ) - main_pane_transition_progress = ( - 1.0 # 0.0 = no history layout, 1.0 = full history layout - ) - main_pane_slide_progress = 1.0 # 0.0 = off-screen left, 1.0 = fully slid in - - for i, state in enumerate(code_states): - lines_in_state = len(state["content"].splitlines()) - max_visible_lines = (VIDEO_HEIGHT - MAIN_PANE_Y_OFFSET - 50) // LINE_HEIGHT - - # Early calculation of estimated main pane width for scrolling - # This is a rough estimate - the exact width is calculated later - if SHOW_HISTORY_PANES and i > 0: - # Rough estimate when history panes are visible - estimated_main_pane_width = int(VIDEO_WIDTH * 0.4) - else: - # No history panes - use most of the width - estimated_main_pane_width = VIDEO_WIDTH - MAIN_PANE_X_OFFSET * 2 - - scroll_duration_for_state = 0 - if lines_in_state > max_visible_lines: - lines_to_scroll = lines_in_state - max_visible_lines - scroll_duration_for_state = max( - MIN_SCROLL_DURATION, lines_to_scroll / SCROLL_SPEED_LINES_PER_SECOND - ) - # This state needs scrolling - base_state_duration = ( - HOLD_DURATION_PER_ITERATION - + scroll_duration_for_state - + SCROLL_PAUSE_AT_TOP - + SCROLL_PAUSE_AT_BOTTOM - ) - else: - # No scrolling needed for this state - base_state_duration = HOLD_DURATION_PER_ITERATION - - # Add slide-in time for iterations after the first one - if i > 0: # Only add slide-in time for iterations after the first - state_duration = base_state_duration + MAIN_PANE_SLIDE_IN_DURATION - else: - state_duration = base_state_duration - - if t >= current_time and t < current_time + state_duration: - state_index = i - time_in_state = t - current_time - - # Handle slide-in animation (only for iterations after the first) - if i > 0 and time_in_state < MAIN_PANE_SLIDE_IN_DURATION: - # Main pane is sliding in from the left - main_pane_slide_progress = time_in_state / MAIN_PANE_SLIDE_IN_DURATION - # During slide-in, don't scroll and stay at beginning - # of content - scroll_offset = 0 - content_time_progress = 0 # Don't start content scrolling yet - else: - # Slide-in is complete (or not needed for first iteration) - main_pane_slide_progress = 1.0 - if i > 0: - content_time_progress = time_in_state - MAIN_PANE_SLIDE_IN_DURATION - else: - content_time_progress = time_in_state - - # Set history panel visibility based on iteration - # (no special transitions) - if i == 0: - # First iteration - no history panels - history_transition_alpha = 0.0 - main_pane_transition_progress = 0.0 - else: - # All other iterations - history panels fully visible - history_transition_alpha = 1.0 - main_pane_transition_progress = 1.0 - - # Handle scrolling logic based on content time progress - if lines_in_state > max_visible_lines and content_time_progress >= 0: - if content_time_progress < HOLD_DURATION_PER_ITERATION: - # Initial hold at top - scroll_offset = 0 - elif ( - content_time_progress - < HOLD_DURATION_PER_ITERATION + SCROLL_PAUSE_AT_TOP - ): - # Pause at top before scrolling - scroll_offset = 0 - elif ( - content_time_progress - < HOLD_DURATION_PER_ITERATION - + SCROLL_PAUSE_AT_TOP - + scroll_duration_for_state - ): - # Scrolling phase - scroll_progress = ( - content_time_progress - - HOLD_DURATION_PER_ITERATION - - SCROLL_PAUSE_AT_TOP - ) / scroll_duration_for_state - - # Calculate accurate max scroll by working backwards from the last line - # to find how many lines fit in the viewport when accounting for wrapping - state_lines = state["content"].splitlines() - available_height = VIDEO_HEIGHT - MAIN_PANE_Y_OFFSET - 50 - max_width = estimated_main_pane_width - 20 - - # Work backwards from the last line to find the optimal scroll offset - current_height = 0 - optimal_scroll_offset = len(state_lines) - - for line_idx in range(len(state_lines) - 1, -1, -1): - line_text = state_lines[line_idx].rstrip("\r\n") - - # Estimate line height considering wrapping (simplified) - if not line_text: - line_height = LINE_HEIGHT - else: - # Quick wrapping estimation - line_width = len(line_text) * ( - FONT_SIZE * 0.6 - ) # Rough estimate - if line_width > max_width: - # Estimate how many visual lines this logical line will take - estimated_wrap_lines = max( - 1, int(line_width / max_width) + 1 - ) - line_height = LINE_HEIGHT * estimated_wrap_lines - else: - line_height = LINE_HEIGHT - - if current_height + line_height > available_height: - optimal_scroll_offset = line_idx + 1 - break - current_height += line_height - - optimal_scroll_offset = max(0, optimal_scroll_offset) - scroll_offset = int(scroll_progress * optimal_scroll_offset) - else: - # Pause at bottom - use the calculated optimal offset - state_lines = state["content"].splitlines() - available_height = VIDEO_HEIGHT - MAIN_PANE_Y_OFFSET - 50 - max_width = estimated_main_pane_width - 20 - - current_height = 0 - optimal_scroll_offset = len(state_lines) - - for line_idx in range(len(state_lines) - 1, -1, -1): - line_text = state_lines[line_idx].rstrip("\r\n") - - if not line_text: - line_height = LINE_HEIGHT - else: - line_width = len(line_text) * (FONT_SIZE * 0.6) - if line_width > max_width: - estimated_wrap_lines = max( - 1, int(line_width / max_width) + 1 - ) - line_height = LINE_HEIGHT * estimated_wrap_lines - else: - line_height = LINE_HEIGHT - - if current_height + line_height > available_height: - optimal_scroll_offset = line_idx + 1 - break - current_height += line_height - - scroll_offset = max(0, optimal_scroll_offset) - else: - scroll_offset = 0 - break - current_time += state_duration - - if state_index >= len(code_states): - state_index = len(code_states) - 1 - - current_state_data = code_states[state_index] - code_to_display_full = current_state_data["content"] - code_to_display_typed = code_to_display_full # Always show full content - added_lines_for_this_main_state = current_state_data["added_lines"] - iter_idx = state_index # For compatibility with rest of code - - img = Image.new("RGB", (VIDEO_WIDTH, VIDEO_HEIGHT), BG_COLOR) - draw = ImageDraw.Draw(img, "RGBA") - - # --- Calculate Pane Dimensions with Smooth Transitions --- - actual_num_history_panes_to_render = 0 - history_pane_individual_width = 0 - - # Calculate dimensions for both no-history and with-history layouts - main_code_pane_width_no_history = VIDEO_WIDTH - MAIN_PANE_X_OFFSET * 2 - main_pane_x_offset_no_history = MAIN_PANE_X_OFFSET - - main_code_pane_width_with_history = main_code_pane_width_no_history - main_pane_x_offset_with_history = MAIN_PANE_X_OFFSET - total_width_for_all_history_panes = 0 - - if SHOW_HISTORY_PANES and iter_idx > 0: - actual_num_history_panes_to_render = min(NUM_HISTORY_PANES_TO_SHOW, iter_idx) - - # Calculate width for history panes first - total_history_spacing = ( - (actual_num_history_panes_to_render - 1) * HISTORY_PANE_SPACING - if actual_num_history_panes_to_render > 0 - else 0 - ) - # Let's try to give history panes a fixed relative width - # or a portion of remaining space - # Example: allocate ~40% of width to all history panes together - total_width_for_all_history_panes = ( - int(VIDEO_WIDTH * 0.6) - HISTORY_PANE_X_START_OFFSET_FROM_RIGHT - ) - - if actual_num_history_panes_to_render > 0: - history_pane_individual_width = int( - (total_width_for_all_history_panes - total_history_spacing) - / actual_num_history_panes_to_render - ) - history_pane_individual_width = max( - 100, history_pane_individual_width - ) # Min width - - main_code_pane_width_with_history = VIDEO_WIDTH - ( - total_width_for_all_history_panes - + MAIN_PANE_X_OFFSET - + MAIN_PANE_RIGHT_MARGIN_IF_HISTORY - + HISTORY_PANE_X_START_OFFSET_FROM_RIGHT - ) - main_code_pane_width_with_history = max( - int(VIDEO_WIDTH * 0.35), main_code_pane_width_with_history - ) # Ensure main pane has some decent width - elif not SHOW_HISTORY_PANES: - main_code_pane_width_no_history = ( - VIDEO_WIDTH - MAIN_PANE_X_OFFSET - MINI_DIFF_PANE_WIDTH - 10 - ) # Space for minidiffs - - # Interpolate between no-history and with-history layouts - main_code_pane_width = int( - main_code_pane_width_no_history * (1 - main_pane_transition_progress) - + main_code_pane_width_with_history * main_pane_transition_progress - ) - - # Calculate the centered position if enabled - if CENTER_CONTENT: - content_total_width_no_history = main_code_pane_width_no_history - content_total_width_with_history = main_code_pane_width_with_history - if SHOW_HISTORY_PANES and actual_num_history_panes_to_render > 0: - content_total_width_with_history += ( - total_width_for_all_history_panes - + HISTORY_PANE_X_START_OFFSET_FROM_RIGHT - + MAIN_PANE_RIGHT_MARGIN_IF_HISTORY - ) - - main_pane_x_offset_no_history = max( - MAIN_PANE_X_OFFSET, (VIDEO_WIDTH - content_total_width_no_history) // 2 - ) - main_pane_x_offset_with_history = max( - MAIN_PANE_X_OFFSET, (VIDEO_WIDTH - content_total_width_with_history) // 2 - ) - - # Interpolate main pane position based on history layout - main_pane_x_offset_final = int( - main_pane_x_offset_no_history * (1 - main_pane_transition_progress) - + main_pane_x_offset_with_history * main_pane_transition_progress - ) - - # Apply slide-in animation - start from off-screen left - slide_in_start_x = -main_code_pane_width # Start completely off-screen to the left - main_pane_x_offset = int( - slide_in_start_x * (1 - main_pane_slide_progress) - + main_pane_x_offset_final * main_pane_slide_progress - ) - - # Draw iteration title at the top of the main pane - iter_text_content = f"{current_state_data['patch_name']}" - title_margin_top = 67 # Adjusted for larger TITLE_FONT_SIZE - title_padding = 12 # Match the history label padding - - # Measure text to create background - iter_text_w = ( - TITLE_FONT.getlength(iter_text_content) - if hasattr(TITLE_FONT, "getlength") - else draw.textlength(iter_text_content, font=TITLE_FONT) - if hasattr(draw, "textlength") - else TITLE_FONT.getbbox(iter_text_content)[2] - if hasattr(TITLE_FONT, "getbbox") - else 400 - ) - - # Calculate center position for title - title_center_x = main_pane_x_offset + (main_code_pane_width // 2) - title_x_start = title_center_x - (iter_text_w // 2) - title_padding - title_x_end = title_center_x + (iter_text_w // 2) + title_padding - title_text_x = title_center_x - (iter_text_w // 2) - - # Draw background with same style as patch labels - # Ensure title_bg_color assignment is on multiple lines if too long - title_bg_color = (60, 60, 60) if not USE_WHITE_BACKGROUND else (220, 220, 220) - - title_box_y1 = MAIN_PANE_Y_OFFSET - title_margin_top - title_padding - title_box_y2 = ( - MAIN_PANE_Y_OFFSET - title_margin_top + TITLE_FONT_SIZE + title_padding - ) - - draw.rectangle( - ( - title_x_start, - title_box_y1, - title_x_end, - title_box_y2, - ), - fill=title_bg_color, - ) - - # Draw text - draw.text( - (title_text_x, MAIN_PANE_Y_OFFSET - title_margin_top), - iter_text_content, - font=TITLE_FONT, - fill=TEXT_COLOR, - ) - - # 1. Draw Main Code Pane (Left) with scrolling - y_offset = MAIN_PANE_Y_OFFSET - main_lines = code_to_display_typed.splitlines(True) - - # Calculate dynamic height based on number of lines - total_code_lines = len(main_lines) - max_displayable_lines = min( - total_code_lines, (VIDEO_HEIGHT - MAIN_PANE_Y_OFFSET - 50) // LINE_HEIGHT - ) - # Add some padding - main_pane_height = max_displayable_lines * LINE_HEIGHT + 20 - - # Apply scrolling offset - start_line = scroll_offset - end_line = min(start_line + max_displayable_lines, total_code_lines) - - for line_idx in range(start_line, end_line): - if line_idx >= len(main_lines): - break - if y_offset + LINE_HEIGHT > MAIN_PANE_Y_OFFSET + main_pane_height: - break - - line_text_orig = main_lines[line_idx] - line_text = line_text_orig.rstrip("\r\n") - - # Wrap long lines instead of truncating - max_width = main_code_pane_width - 20 - line_width = ( - FONT.getlength(line_text) - if hasattr(FONT, "getlength") - else draw.textlength(line_text, font=FONT) - if hasattr(draw, "textlength") - else FONT.getbbox(line_text)[2] - if hasattr(FONT, "getbbox") - else len(line_text) * 8 - ) - - if line_width > max_width: - # Calculate approximate characters per line - avg_char_w = ( - FONT.getlength("M") - if hasattr(FONT, "getlength") - else draw.textlength("M", font=FONT) - if hasattr(draw, "textlength") - else FONT.getbbox("M")[2] - if hasattr(FONT, "getbbox") - else 8 - ) - chars_per_line = ( - int(max_width / avg_char_w) if avg_char_w > 0 else int(max_width / 8) - ) - - # Split into multiple lines - lines_to_draw = [] - remaining = line_text - - while remaining: - if len(remaining) <= chars_per_line: - lines_to_draw.append(remaining) - break - - # Try to break at a space if possible - split_pos = chars_per_line - if " " in remaining[:chars_per_line]: - # Find the last space in the allowed width - last_space = remaining[:chars_per_line].rstrip().rfind(" ") - if last_space > 0: - split_pos = last_space + 1 - - lines_to_draw.append(remaining[:split_pos]) - remaining = remaining[split_pos:] - - # Draw wrapped lines - first_line = True - for wrapped_line in lines_to_draw: - # Draw background for added lines on each wrapped line segment - if line_idx in added_lines_for_this_main_state: - draw.rectangle( - ( - main_pane_x_offset - 5, - y_offset - 2, - main_pane_x_offset + main_code_pane_width - 10, - y_offset + LINE_HEIGHT - 2, - ), - fill=ADDED_LINE_BG_COLOR, - ) - - line_x_cursor = main_pane_x_offset - if not first_line: - # Add indentation for wrapped lines - line_x_cursor += 20 - - try: - tokens_on_line = lexer.get_tokens(wrapped_line) - for ttype, tvalue in tokens_on_line: - style_for_token = style.style_for_token(ttype) - color = style_for_token["color"] - token_color = TEXT_COLOR - if color: - try: - token_color = ( - int(color[0:2], 16), - int(color[2:4], 16), - int(color[4:6], 16), - ) - except ValueError: - pass - - draw.text( - (line_x_cursor, y_offset), - tvalue, - font=FONT, - fill=token_color, - ) - - token_width = ( - FONT.getlength(tvalue) - if hasattr(FONT, "getlength") - else draw.textlength(tvalue, font=FONT) - if hasattr(draw, "textlength") - else FONT.getbbox(tvalue)[2] - if hasattr(FONT, "getbbox") - else len(tvalue) * 8 - ) - line_x_cursor += token_width - except Exception: - # Fallback to drawing the entire line with spaces preserved - draw.text( - ( - main_pane_x_offset - if first_line - else main_pane_x_offset + 20, - y_offset, - ), - wrapped_line, - font=FONT, - fill=TEXT_COLOR, - ) - - y_offset += LINE_HEIGHT - first_line = False - - # Stop if we've reached the bottom of the main pane - if y_offset + LINE_HEIGHT > MAIN_PANE_Y_OFFSET + main_pane_height: - break - else: - # Single line rendering (no wrapping needed) - # Draw background for added lines - if line_idx in added_lines_for_this_main_state: - draw.rectangle( - ( - main_pane_x_offset - 5, - y_offset - 2, - main_pane_x_offset + main_code_pane_width - 10, - y_offset + LINE_HEIGHT - 2, - ), - fill=ADDED_LINE_BG_COLOR, - ) - - line_x_cursor = main_pane_x_offset - try: - tokens_on_line = lexer.get_tokens(line_text) - for ttype, tvalue in tokens_on_line: - style_for_token = style.style_for_token(ttype) - color = style_for_token["color"] - token_color = TEXT_COLOR - if color: - try: - token_color = ( - int(color[0:2], 16), - int(color[2:4], 16), - int(color[4:6], 16), - ) - except ValueError: - pass - - draw.text( - (line_x_cursor, y_offset), tvalue, font=FONT, fill=token_color - ) - - token_width = ( - FONT.getlength(tvalue) - if hasattr(FONT, "getlength") - else draw.textlength(tvalue, font=FONT) - if hasattr(draw, "textlength") - else FONT.getbbox(tvalue)[2] - if hasattr(FONT, "getbbox") - else len(tvalue) * 8 - ) - line_x_cursor += token_width - except Exception: - # Fallback to drawing the entire line with spaces preserved - draw.text( - (main_pane_x_offset, y_offset), - line_text, - font=FONT, - fill=TEXT_COLOR, - ) - y_offset += LINE_HEIGHT - - # 2. Draw History Panes (Right) with smooth transitions - if ( - SHOW_HISTORY_PANES - and actual_num_history_panes_to_render > 0 - and history_transition_alpha > 0 - ): - # Create a separate image for history panes to apply alpha blending - history_img = Image.new("RGBA", (VIDEO_WIDTH, VIDEO_HEIGHT), (0, 0, 0, 0)) - history_draw = ImageDraw.Draw(history_img, "RGBA") - - # Position history panes after the main pane - current_history_pane_x_start_coord = ( - main_pane_x_offset - + main_code_pane_width - + MAIN_PANE_RIGHT_MARGIN_IF_HISTORY - ) - - for i in range(actual_num_history_panes_to_render): - # Display panes from left to right: most recent edits first - history_iter_index_to_display = iter_idx - 1 - i - - state_to_draw = code_states[history_iter_index_to_display] - history_code_content = state_to_draw["content"] - added_lines_in_this_history_version = state_to_draw["added_lines"] - - # Calculate dynamic height for this history pane based on its content - hist_lines = history_code_content.splitlines() - max_hist_lines = min(len(hist_lines), HISTORY_MAX_LINES_TO_DRAW) - history_pane_height = ( - max_hist_lines * HISTORY_LINE_HEIGHT + 30 - ) # Add padding - - pane_x1 = current_history_pane_x_start_coord - pane_y1 = HISTORY_PANE_Y_START - pane_x2 = current_history_pane_x_start_coord + history_pane_individual_width - pane_y2 = pane_y1 + history_pane_height # Use calculated height - - history_draw.rectangle( - (pane_x1, pane_y1, pane_x2, pane_y2), fill=HISTORY_PANE_BG_COLOR - ) - history_draw.rectangle( - (pane_x1, pane_y1, pane_x2, pane_y2), - outline=HISTORY_PANE_BORDER_COLOR, - width=1, - ) - - hist_text_x = pane_x1 + 5 - hist_text_y = pane_y1 + 5 - - for line_num_in_hist, hist_line_text_orig in enumerate(hist_lines): - if line_num_in_hist >= max_hist_lines: - break - if hist_text_y + HISTORY_LINE_HEIGHT > pane_y2 - 5: - break - - hist_line_text = hist_line_text_orig.rstrip("\r\n") - - drawable_hist_line = hist_line_text - max_text_width_in_pane = history_pane_individual_width - 10 - - # Check if line needs wrapping - current_line_width_px = ( - HISTORY_FONT.getlength(drawable_hist_line) - if hasattr(HISTORY_FONT, "getlength") - else draw.textlength(drawable_hist_line, font=HISTORY_FONT) - if hasattr(draw, "textlength") - else HISTORY_FONT.getbbox(drawable_hist_line)[2] - if hasattr(HISTORY_FONT, "getbbox") - else len(drawable_hist_line) * HISTORY_FONT_SIZE - ) - - # If line is too long, wrap it rather than truncating - if current_line_width_px > max_text_width_in_pane: - # Calculate approximately how many characters fit per line - avg_char_w = ( - HISTORY_FONT.getlength("M") - if hasattr(HISTORY_FONT, "getlength") - else draw.textlength("M", font=HISTORY_FONT) - if hasattr(draw, "textlength") - else HISTORY_FONT.getbbox("M")[2] - if hasattr(HISTORY_FONT, "getbbox") - else HISTORY_FONT_SIZE - ) - chars_per_line = ( - int(max_text_width_in_pane / avg_char_w) - if avg_char_w > 0 - else int(max_text_width_in_pane / HISTORY_FONT_SIZE) - ) - - # Split into multiple lines - wrapped_lines = [] - remaining = drawable_hist_line - - while ( - remaining and hist_text_y + HISTORY_LINE_HEIGHT <= pane_y2 - 5 - ): - if len(remaining) <= chars_per_line: - wrapped_lines.append(remaining) - break - - # Try to break at a space if possible - split_pos = chars_per_line - if " " in remaining[:chars_per_line]: - # Find the last space in the allowed width - last_space = remaining[:chars_per_line].rstrip().rfind(" ") - if last_space > 0: - split_pos = last_space + 1 - - wrapped_lines.append(remaining[:split_pos]) - remaining = remaining[split_pos:] - - # Draw wrapped lines - for wrapped_line in wrapped_lines: - # Draw background for added lines on each wrapped line - # segment - if line_num_in_hist in added_lines_in_this_history_version: - history_draw.rectangle( - ( - hist_text_x - 2, - hist_text_y, - pane_x2 - 3, - hist_text_y + HISTORY_LINE_HEIGHT - 1, - ), - fill=HISTORY_ADDED_LINE_BG_COLOR, - ) - hist_line_x_cursor = hist_text_x - try: - tokens_on_hist_line = lexer.get_tokens(wrapped_line) - for ttype, tvalue in tokens_on_hist_line: - style_for_token = style.style_for_token(ttype) - color = style_for_token["color"] - token_color = HISTORY_PANE_TEXT_COLOR - if color: - try: - token_color = ( - int(color[0:2], 16), - int(color[2:4], 16), - int(color[4:6], 16), - ) - except ValueError: - pass - - history_draw.text( - (hist_line_x_cursor, hist_text_y), - tvalue, - font=HISTORY_FONT, - fill=token_color, - ) - token_w = ( - HISTORY_FONT.getlength(tvalue) - if hasattr(HISTORY_FONT, "getlength") - else draw.textlength(tvalue, font=HISTORY_FONT) - if hasattr(draw, "textlength") - else HISTORY_FONT.getbbox(tvalue)[2] - if hasattr(HISTORY_FONT, "getbbox") - else len(tvalue) * HISTORY_FONT_SIZE - ) - hist_line_x_cursor += token_w - if hist_line_x_cursor > pane_x2 - 7: - break - except Exception: - history_draw.text( - (hist_text_x, hist_text_y), - wrapped_line, - font=HISTORY_FONT, - fill=HISTORY_PANE_TEXT_COLOR, - ) - hist_text_y += HISTORY_LINE_HEIGHT - - # Stop if we've reached the bottom of the pane - if hist_text_y + HISTORY_LINE_HEIGHT > pane_y2 - 5: - break - else: - # Draw single line since it fits - # Draw background for added lines - if line_num_in_hist in added_lines_in_this_history_version: - history_draw.rectangle( - ( - hist_text_x - 2, - hist_text_y, - pane_x2 - 3, - hist_text_y + HISTORY_LINE_HEIGHT - 1, - ), - fill=HISTORY_ADDED_LINE_BG_COLOR, - ) - - hist_line_x_cursor = hist_text_x - try: - tokens_on_hist_line = lexer.get_tokens(drawable_hist_line) - for ttype, tvalue in tokens_on_hist_line: - style_for_token = style.style_for_token(ttype) - color = style_for_token["color"] - token_color = HISTORY_PANE_TEXT_COLOR - if color: - try: - token_color = ( - int(color[0:2], 16), - int(color[2:4], 16), - int(color[4:6], 16), - ) - except ValueError: - pass - - history_draw.text( - (hist_line_x_cursor, hist_text_y), - tvalue, - font=HISTORY_FONT, - fill=token_color, - ) - token_w = ( - HISTORY_FONT.getlength(tvalue) - if hasattr(HISTORY_FONT, "getlength") - else draw.textlength(tvalue, font=HISTORY_FONT) - if hasattr(draw, "textlength") - else HISTORY_FONT.getbbox(tvalue)[2] - if hasattr(HISTORY_FONT, "getbbox") - else len(tvalue) * HISTORY_FONT_SIZE - ) - hist_line_x_cursor += token_w - if hist_line_x_cursor > pane_x2 - 7: - break - except Exception: - history_draw.text( - (hist_text_x, hist_text_y), - drawable_hist_line, - font=HISTORY_FONT, - fill=HISTORY_PANE_TEXT_COLOR, - ) - hist_text_y += HISTORY_LINE_HEIGHT - - history_pane_label = f"{state_to_draw['patch_name']}" - # if len(state_to_draw["patch_name"]) > 33: - # history_pane_label += "..." - label_w = ( - HISTORY_LABEL_FONT.getlength(history_pane_label) - if hasattr(HISTORY_LABEL_FONT, "getlength") - else draw.textlength(history_pane_label, font=HISTORY_LABEL_FONT) - if hasattr(draw, "textlength") - else HISTORY_LABEL_FONT.getbbox(history_pane_label)[2] - if hasattr(HISTORY_LABEL_FONT, "getbbox") - else len(history_pane_label) * HISTORY_LABEL_FONT_SIZE - ) - label_x = pane_x1 + (history_pane_individual_width - label_w) // 2 - label_y = ( - pane_y1 - HISTORY_LABEL_FONT_SIZE - 40 - ) # Position much higher above the pane to avoid overlap - if label_y < 5: - label_y = 5 # Ensure it's visible - - # Draw background rectangle for history pane label - # (similar to iteration title) - label_padding = 2 - label_bg_color = ( - (60, 60, 60) if not USE_WHITE_BACKGROUND else (220, 220, 220) - ) - history_draw.rectangle( - ( - label_x - label_padding, - label_y - label_padding, - label_x + label_w + label_padding, - label_y + HISTORY_LABEL_FONT_SIZE + label_padding, - ), - fill=label_bg_color, - ) - - # Draw label text - history_draw.text( - (label_x, label_y), - history_pane_label, - font=HISTORY_LABEL_FONT, - fill=TEXT_COLOR, - ) - - current_history_pane_x_start_coord += ( - history_pane_individual_width + HISTORY_PANE_SPACING - ) - - # Apply alpha blending for smooth transition - if history_transition_alpha < 1.0: - # Apply alpha to the entire history image - history_img = history_img.convert("RGBA") - alpha = int(255 * history_transition_alpha) - # Create an alpha channel based on transition progress - history_alpha = Image.new("L", history_img.size, alpha) - history_img.putalpha(history_alpha) - - # Composite the history image onto the main image - img = img.convert("RGBA") - img = Image.alpha_composite(img, history_img) - img = img.convert("RGB") - - elif not SHOW_HISTORY_PANES: # Fallback to original mini-diffs - mini_diff_x_start = VIDEO_WIDTH - MINI_DIFF_PANE_WIDTH + 15 - mini_diff_y_start = MAIN_PANE_Y_OFFSET # Align with top of main code - current_patch_display_idx = iter_idx - 1 - - for i in range(len(mini_diff_images)): - if ( - mini_diff_y_start - + (MINI_DIFF_HEIGHT_PER_LINE * MINI_DIFF_MAX_LINES) - + MINI_DIFF_SPACING - > VIDEO_HEIGHT - 30 - ): - break - - md_img_base = mini_diff_images[i] - is_this_one_active = i == current_patch_display_idx - md_img_to_paste = md_img_base - - if ( - is_this_one_active - ): # Redraw with border if active (original mini_diff was pre-rendered) - md_img_to_paste = draw_mini_diff( - raw_patch_contents_for_minidiff[i], - is_active=True, - font_mini_diff=MINI_DIFF_FONT, - ) - - img.paste(md_img_to_paste, (mini_diff_x_start, mini_diff_y_start)) - - patch_label = code_states[i + 1]["patch_name"] - if len(patch_label) > 33: - patch_label = patch_label[:30] + "..." - - text_w_md = ( - MINI_DIFF_FONT.getlength(patch_label) - if hasattr(MINI_DIFF_FONT, "getlength") - else draw.textlength(patch_label, font=MINI_DIFF_FONT) - if hasattr(draw, "textlength") - else MINI_DIFF_FONT.getbbox(patch_label)[2] - if hasattr(MINI_DIFF_FONT, "getbbox") - else len(patch_label) * MINI_DIFF_TEXT_SIZE - ) - text_x_md = ( - mini_diff_x_start + (md_img_to_paste.width // 2) - (text_w_md // 2) - ) - # Moved higher - text_y_md = mini_diff_y_start + md_img_to_paste.height - 2 - if text_y_md + MINI_DIFF_TEXT_SIZE < VIDEO_HEIGHT - 10: - # Draw background rectangle for mini-diff label - label_padding = 4 - label_bg_color = ( - (60, 60, 60) if not USE_WHITE_BACKGROUND else (220, 220, 220) - ) - draw.rectangle( - ( - text_x_md - label_padding, - text_y_md - label_padding, - text_x_md + text_w_md + label_padding, - text_y_md + MINI_DIFF_TEXT_SIZE + label_padding, - ), - fill=label_bg_color, - ) - - # Draw text on top of background - draw.text( - (text_x_md, text_y_md), - patch_label, - font=MINI_DIFF_FONT, - fill=((30, 30, 30) if USE_WHITE_BACKGROUND else (220, 220, 220)), - ) - mini_diff_y_start += ( - md_img_to_paste.height + MINI_DIFF_SPACING + MINI_DIFF_TEXT_SIZE + 5 - ) - - return np.array(img) - - -# --- Create Video --- -print("Creating video clip...") -animation = VideoClip(make_frame, duration=total_duration) - -print(f"Writing video to {OUTPUT_VIDEO}...") -try: - animation.write_videofile( - OUTPUT_VIDEO, fps=FPS, codec="libx264", preset="medium", threads=4, logger="bar" - ) -except Exception as e: - print(f"Error during video writing with libx264: {e}") - print("Trying with mpeg4 codec as fallback...") - try: - animation.write_videofile( - OUTPUT_VIDEO, - fps=FPS, - codec="mpeg4", - preset="medium", - threads=4, - logger="bar", - ) - except Exception as e2: - print(f"Error during video writing with mpeg4: {e2}") - print("Video writing failed with both codecs.") - - -# --- Cleanup --- -print("Cleaning up temporary directory...") -shutil.rmtree(temp_dir) -if os.path.exists(BASE_CODE_FILE) and get_file_content(BASE_CODE_FILE) == "": - if "your_base_code_file.py" in BASE_CODE_FILE: - print(f"Removing placeholder empty base file: {BASE_CODE_FILE}") - # os.remove(BASE_CODE_FILE) # Comment out if you want to keep it - -print("Done!") diff --git a/genesis/plots/plot_improvement.py b/genesis/plots/plot_improvement.py deleted file mode 100644 index 05cb893..0000000 --- a/genesis/plots/plot_improvement.py +++ /dev/null @@ -1,335 +0,0 @@ -import matplotlib.pyplot as plt -import pandas as pd -from typing import Optional, Tuple, List -from matplotlib.figure import Figure -from matplotlib.axes import Axes -from genesis.utils import get_path_to_best_node -import matplotlib.transforms as transforms - - -def plot_improvement( - df: pd.DataFrame, - title: str = "Best Combined Score Over Time", - fig: Optional[Figure] = None, - ax: Optional[Axes] = None, - xlabel: str = "Number of Evaluated LLM Program Proposals", - ylabel: str = "Evolved Performance Score", - ylim: Optional[Tuple[float, float]] = None, - plot_path_to_best_node: bool = True, -): - """ - Plots the improvement of a program over generations. - """ - if fig is None or ax is None: - fig, ax = plt.subplots(figsize=(20, 10)) - - # Plot best score line - # Calculate cumulative maximum and back-fill leading NaNs - # to ensure the line is continuous from the start. - df = df.sort_values(by="generation") - df_filtered = df[df["correct"]].copy() - - line1 = ax.plot( - df_filtered["generation"], - df_filtered["combined_score"].cummax(), - linewidth=3, - color="red", - label="Best Score", - ) - - # Plot individual evaluations as scatter points - scatter1 = ax.scatter( - df_filtered["generation"], - df_filtered["combined_score"], - alpha=1.0, - s=40, - color="black", - label="Individual Evals", - ) - - if ylim is not None: - ax.set_ylim(*ylim) - - # Get the path to the best node - if plot_path_to_best_node: - best_path_df = get_path_to_best_node(df_filtered, score_column="combined_score") - else: - best_path_df = pd.DataFrame() - line_best_path_plot = [] # Initialize to empty list - - if not best_path_df.empty: - # Plot the path to the best node - line_best_path_plot = ax.plot( - best_path_df["generation"], # Use generation for x-axis - best_path_df["combined_score"], - linestyle="-.", - marker="o", - color="blue", - label="Path to Best Node", - markersize=5, - linewidth=2, - ) - # Add annotations if 'patch_name' column exists - if "patch_name" in best_path_df.columns: - _place_non_overlapping_annotations( - ax, best_path_df, "generation", "combined_score", "patch_name" - ) - - # Create a second y-axis for cumulative API cost - ax2 = ax.twinx() - handles = line1 + [scatter1] - if line_best_path_plot: # If the best path was plotted - handles.extend(line_best_path_plot) - - labels = [h.get_label() for h in handles] - - if "api_costs" in df_filtered.columns: - cumulative_api_cost = df["api_costs"].cumsum().bfill() - line2 = ax2.plot( - df["generation"], - cumulative_api_cost, - linewidth=2, - color="orange", - linestyle="--", - label="Cumulative Cost", - ) - ax2.set_ylabel( - "Cumulative API Cost ($)", - fontsize=22, - weight="bold", - color="orange", - labelpad=15, - ) - ax2.tick_params(axis="y", which="major", labelsize=25) - handles.extend(line2) - labels = [h.get_label() for h in handles] # Recreate labels - - ax.legend(handles, labels, fontsize=25, loc="lower right") - - # Customize plot - ax.set_xlabel(xlabel, fontsize=30, weight="bold") - ax.set_ylabel(ylabel, fontsize=30, weight="bold", labelpad=25) - ax.set_title(title, fontsize=40, weight="bold") - ax.tick_params(axis="both", which="major", labelsize=20) - ax.grid(True, alpha=0.3) - - # Remove top and right spines for the primary axis - ax.spines["top"].set_visible(False) - ax.spines["right"].set_visible( - False - ) # Keep right spine if ax2 is present, or manage ax2 spines - - if "api_cost" in df_filtered.columns and ax2: - # Ensure ax2 spine is visible if it exists - ax2.spines["top"].set_visible(False) # Match primary axis top spine - ax2.tick_params(axis="y", which="major", labelsize=30) - - fig.tight_layout() # Adjust layout to prevent overlapping labels - - return fig, ax - - -def _place_non_overlapping_annotations( - ax: Axes, df: pd.DataFrame, x_col: str, y_col: str, text_col: str -): - """ - Places annotations with minimal overlap using a systematic approach. - """ - # Define multiple offset positions to try (in order of preference) - offset_positions = [ - (40, -30), # bottom-right - (40, 30), # top-right - (-40, 30), # top-left - (-40, -30), # bottom-left - (60, 0), # right - (-60, 0), # left - (0, 40), # top - (0, -40), # bottom - (70, -50), # far bottom-right - (-70, 50), # far top-left - ] - - placed_boxes = [] # Store bounding boxes of placed annotations - - for _, row in df.iterrows(): - patch_name_val = str(row.get(text_col, "")) - if pd.notna(patch_name_val) and patch_name_val != "": - if patch_name_val == "nan" or patch_name_val == "none": - patch_name_val = "Base" - - # Wrap long patch names - patch_name_to_plot = _wrap_text(patch_name_val, max_length=15) - - x_pos = float(row[x_col]) - y_pos = float(row[y_col]) - - # Find the best position with minimal overlap - best_offset, best_ha, best_va = _find_best_position( - ax, x_pos, y_pos, patch_name_to_plot, offset_positions, placed_boxes - ) - - # Place the annotation - annotation = ax.annotate( - patch_name_to_plot, - (x_pos, y_pos), - textcoords="offset points", - xytext=best_offset, - ha=best_ha, - va=best_va, - fontsize=11, - fontweight="bold", - color="darkgreen", - bbox=dict( - boxstyle="round,pad=0.3", - fc="lightyellow", - ec="black", - alpha=0.7, - ), - arrowprops=dict( - arrowstyle="-", - shrinkA=5, - shrinkB=5, - connectionstyle="arc3,rad=0.2", - color="black", - ), - zorder=10, - ) - - # Store the bounding box for future collision detection - try: - # Get the bounding box in data coordinates - bbox = annotation.get_window_extent() - inv_transform = ax.transData.inverted() - bbox_data = inv_transform.transform_bbox(bbox) - placed_boxes.append(bbox_data) - except Exception: - # Fallback: approximate bounding box - approx_width = len(patch_name_to_plot) * 0.01 # rough estimate - approx_height = patch_name_to_plot.count("\n") * 0.02 + 0.02 - placed_boxes.append( - transforms.Bbox.from_bounds( - x_pos - approx_width / 2, - y_pos - approx_height / 2, - approx_width, - approx_height, - ) - ) - - -def _wrap_text(text: str, max_length: int = 15) -> str: - """ - Wraps text at word boundaries for better readability. - """ - if len(text) <= max_length: - return text - - # Try to find a good breaking point - mid_point = len(text) // 2 - - # Look for a space near the middle - for offset in range(min(5, mid_point)): - # Check before midpoint - if mid_point - offset > 0 and text[mid_point - offset] == " ": - break_point = mid_point - offset - part1 = text[:break_point].strip() - part2 = text[break_point + 1 :].strip() - return f"{part1}\n{part2}" - - # Check after midpoint - if mid_point + offset < len(text) and text[mid_point + offset] == " ": - break_point = mid_point + offset - part1 = text[:break_point].strip() - part2 = text[break_point + 1 :].strip() - return f"{part1}\n{part2}" - - # No good space found, break at midpoint - return f"{text[:mid_point]}\n{text[mid_point:]}" - - -def _find_best_position( - ax: Axes, - x_pos: float, - y_pos: float, - text: str, - offset_positions: List[Tuple[int, int]], - placed_boxes: List[transforms.Bbox], -) -> Tuple[Tuple[int, int], str, str]: - """ - Finds the best annotation position with minimal overlap. - """ - best_offset = offset_positions[0] - best_overlap_count = float("inf") - - for offset in offset_positions: - # Determine alignment based on offset - ha = "left" if offset[0] >= 0 else "right" - va = "bottom" if offset[1] >= 0 else "top" - - # Estimate the bounding box for this position - estimated_bbox = _estimate_annotation_bbox( - ax, x_pos, y_pos, text, offset, ha, va - ) - - # Count overlaps with existing annotations - overlap_count = sum(1 for bbox in placed_boxes if estimated_bbox.overlaps(bbox)) - - # If no overlaps, use this position - if overlap_count == 0: - return offset, ha, va - - # Track the position with minimum overlaps - if overlap_count < best_overlap_count: - best_overlap_count = overlap_count - best_offset = offset - - # Return the alignment for the best offset - ha = "left" if best_offset[0] >= 0 else "right" - va = "bottom" if best_offset[1] >= 0 else "top" - - return best_offset, ha, va - - -def _estimate_annotation_bbox( - ax: Axes, - x_pos: float, - y_pos: float, - text: str, - offset: Tuple[int, int], - ha: str, - va: str, -) -> transforms.Bbox: - """ - Estimates the bounding box of an annotation in data coordinates. - """ - # Rough estimation based on text length and number of lines - lines = text.split("\n") - max_line_length = max(len(line) for line in lines) - num_lines = len(lines) - - # Approximate dimensions (these are rough estimates) - char_width_data = (ax.get_xlim()[1] - ax.get_xlim()[0]) / 100 - line_height_data = (ax.get_ylim()[1] - ax.get_ylim()[0]) / 50 - - width = max_line_length * char_width_data - height = num_lines * line_height_data - - # Convert offset from points to data coordinates (approximate) - x_offset_data = offset[0] * char_width_data / 8 # rough conversion - y_offset_data = offset[1] * line_height_data / 12 # rough conversion - - # Calculate annotation position based on alignment - if ha == "left": - left = x_pos + x_offset_data - right = left + width - else: # ha == "right" - right = x_pos + x_offset_data - left = right - width - - if va == "bottom": - bottom = y_pos + y_offset_data - top = bottom + height - else: # va == "top" - top = y_pos + y_offset_data - bottom = top - height - - return transforms.Bbox.from_bounds(left, bottom, width, height) diff --git a/genesis/plots/plot_lineage_tree.py b/genesis/plots/plot_lineage_tree.py deleted file mode 100644 index 5d31695..0000000 --- a/genesis/plots/plot_lineage_tree.py +++ /dev/null @@ -1,589 +0,0 @@ -from typing import Optional -import pandas as pd -import matplotlib.pyplot as plt -import matplotlib.colors as mcolors -import networkx as nx -import matplotlib.cm as cm_module -from matplotlib.lines import Line2D -from matplotlib.figure import Figure -from matplotlib.axes import Axes - - -def plot_lineage_tree( - df: pd.DataFrame, - title="Program Lineage Tree", - fig: Figure | None = None, - ax: Axes | None = None, - vmin: Optional[float] = None, - vmax: Optional[float] = None, -): - """ - Generates a tree visualization of program lineage using matplotlib and - NetworkX. - - Args: - df: Pandas DataFrame containing program data. Must include 'id' and - 'parent_id'. - figsize: Size of the figure (width, height) in inches. - """ - if df is None or df.empty: - print("DataFrame is empty or None. Cannot draw tree.") - return - - # set combined score to 0 for incorrect programs - df.loc[~df["correct"], "combined_score"] = 0 - - # Create directed graph - G = nx.DiGraph() - - # Add nodes with attributes for labels - for idx, row in df.iterrows(): - node_id = str(row["id"]) - node_attrs = {} - - # Add available metrics as node attributes - for col in df.columns: - if col in row: - # Skip the code column as it's usually too long - if col != "code": - node_attrs[col] = row[col] - - G.add_node(node_id, **node_attrs) - - # Add edges - for idx, row in df.iterrows(): - child_id = str(row["id"]) - if "parent_id" in row and pd.notna(row["parent_id"]): - parent_id = str(row["parent_id"]) - # Check if parent exists and is not self-referential - if parent_id in G.nodes() and parent_id != child_id: - G.add_edge(parent_id, child_id) - - # Create figure with a specific axes for the graph and colorbar - # Ensure both fig and ax are created together - if fig is None or ax is None: - fig, ax = plt.subplots(figsize=(20, 16)) - - # Group nodes by generation to ensure proper ordering - generation_groups = {} - for node in G.nodes(): - attrs = G.nodes[node] - gen = attrs.get("generation", 0) # Default to 0 if no generation found - if gen not in generation_groups: - generation_groups[gen] = [] - generation_groups[gen].append(node) - - # Identify the roots (earliest generation nodes) - roots = [n for n, d in G.in_degree() if d == 0] - - if not roots: - # If no root is found, use nodes from earliest generation - min_gen = min(generation_groups.keys()) if generation_groups else 0 - roots = generation_groups.get(min_gen, [list(G.nodes())[0]]) - - root = roots[0] - - # Try to use a hierarchical layout that respects parent-child relationships - # Focus on creating a clean tree structure like the reference image - try: - # Use dot layout for hierarchical tree structure with edge crossing - # minimization - pos = nx.nx_agraph.graphviz_layout( - G, - prog="dot", - root=root, - args="-Grankdir=TB -Gsplines=true -Goverlap=false -Gsep=1.0", - ) - except ImportError: - try: - # If pygraphviz not available, try pydot (no args parameter) - pos = nx.drawing.nx_pydot.graphviz_layout(G, prog="dot", root=root) - except ImportError: - print("GraphViz not available, using hierarchical layout") - # Create a clean hierarchical layout manually - pos = {} - - # Find node depths based on distance from root - depths = {} - for node in G.nodes(): - try: - path_len = len(nx.shortest_path(G, root, node)) - 1 - depths[node] = path_len - except nx.NetworkXNoPath: - # If no path, use generation if available - if "generation" in G.nodes[node]: - depths[node] = G.nodes[node]["generation"] - else: - depths[node] = 0 - - # Group nodes by depth/generation - levels = {} - for node, depth in depths.items(): - if depth not in levels: - levels[depth] = [] - levels[depth].append(node) - - # Create clean hierarchical positioning - max_depth = max(levels.keys()) if levels else 0 - # Total nodes in graph for base spacing - total_nodes = len(G.nodes()) - for depth in sorted(levels.keys()): - nodes_at_level = levels[depth] - num_nodes_at_level = len(nodes_at_level) - - if depth == 0: - # Root node at center top - if num_nodes_at_level == 1: - pos[nodes_at_level[0]] = (0, 0) - else: - # Multiple roots - space them out horizontally - spacing = 15.0 - total_width = (num_nodes_at_level - 1) * spacing - start_x = -total_width / 2 - for i, node in enumerate(nodes_at_level): - pos[node] = (start_x + i * spacing, 0) - else: - # For non-root levels, try to position children near - # parents - # to minimize crossings - - # First, collect parent positions for each node - node_parent_info = {} - for node in nodes_at_level: - parent_x_positions = [] - for parent in G.predecessors(node): - if parent in pos: - parent_x_positions.append(pos[parent][0]) - - if parent_x_positions: - avg_parent_x = sum(parent_x_positions) / len( - parent_x_positions - ) - node_parent_info[node] = avg_parent_x - else: - node_parent_info[node] = 0 - - # Sort nodes by their parent positions to reduce crossings - sorted_nodes = sorted( - nodes_at_level, key=lambda n: node_parent_info[n] - ) - - # Position nodes with adequate spacing - more aggressive - # for early levels - # Base spacing should prevent overlapping at all levels - base_spacing = max(15.0, 10.0 * (total_nodes**0.5)) - # Extra spacing for early levels where nodes tend to - # cluster - depth_multiplier = max(1.0, 3.0 / (depth + 1)) - - spacing = base_spacing * depth_multiplier - - y_pos = -depth * 3.0 # Increased vertical spacing - - if num_nodes_at_level == 1: - pos[sorted_nodes[0]] = (0, y_pos) - else: - total_width = (num_nodes_at_level - 1) * spacing - start_x = -total_width / 2 - - for i, node in enumerate(sorted_nodes): - x_pos = start_x + i * spacing - pos[node] = (x_pos, y_pos) - - # Fine-tune position to be closer to parent if possible - if node in node_parent_info: - preferred_x = node_parent_info[node] - # Check if we can move closer to parent without - # overlapping other nodes - use stricter minimum distance - min_distance = spacing * 0.8 - can_move = True - - for other_node in sorted_nodes: - if other_node != node and other_node in pos: - other_x = pos[other_node][0] - if abs(preferred_x - other_x) < min_distance: - can_move = False - break - - if can_move: - # Move towards parent but very conservatively - adjustment = (preferred_x - x_pos) * 0.1 - pos[node] = (x_pos + adjustment, y_pos) - - # Additional fine-tuning to reduce crossings - if pos and len(pos) > 1: - # Try to reduce crossings by adjusting positions within each level - for depth in sorted(levels.keys()): - if depth == 0: # Skip root - continue - - nodes_at_level = levels[depth] - if len(nodes_at_level) <= 1: - continue - - # Calculate crossing score for current arrangement - def count_crossings(node_positions): - crossings = 0 - for i, node1 in enumerate(nodes_at_level): - for j, node2 in enumerate(nodes_at_level): - if i >= j: - continue - - # Get parents of both nodes - parents1 = list(G.predecessors(node1)) - parents2 = list(G.predecessors(node2)) - - for p1 in parents1: - for p2 in parents2: - if p1 in pos and p2 in pos: - # Check if edges cross - p1_x, p1_y = pos[p1] - p2_x, p2_y = pos[p2] - n1_x = node_positions[node1][0] - n2_x = node_positions[node2][0] - - # Simple crossing check - if (p1_x < p2_x and n1_x > n2_x) or ( - p1_x > p2_x and n1_x < n2_x - ): - crossings += 1 - return crossings - - # Try to improve by swapping adjacent nodes - improved = True - max_iterations = 10 - iteration = 0 - - while improved and iteration < max_iterations: - improved = False - iteration += 1 - - for i in range(len(nodes_at_level) - 1): - node1 = nodes_at_level[i] - node2 = nodes_at_level[i + 1] - - # Create temporary positions with swapped nodes - temp_positions = dict(pos) - temp_positions[node1] = pos[node2] - temp_positions[node2] = pos[node1] - - # Check if this reduces crossings - original_crossings = count_crossings(pos) - new_crossings = count_crossings(temp_positions) - - if new_crossings < original_crossings: - # Apply the swap - pos[node1] = temp_positions[node1] - pos[node2] = temp_positions[node2] - # Also swap in the nodes list - nodes_at_level[i], nodes_at_level[i + 1] = ( - nodes_at_level[i + 1], - nodes_at_level[i], - ) - improved = True - - # Calculate base node sizes based on number of nodes - num_nodes = len(G.nodes()) - # Scale node sizes more like the reference image - size_factor = max(0.3, min(1.0, 20 / (num_nodes**0.4))) - - best_node_size = int(1500 * size_factor) - path_node_size = int(800 * size_factor) - regular_node_size = int(600 * size_factor) - - # Find min and max combined_score to create color map - score_values = [] - score_field = "combined_score" # As per user's request - - # Find the best node (highest score) - best_node = None - best_score = float("-inf") - - for node in G.nodes(): - if score_field in G.nodes[node]: - score = G.nodes[node][score_field] - if isinstance(score, (int, float)): - score_values.append(score) - if score > best_score: - best_score = score - best_node = node - - # Find the path from root to the best node (if it exists) - path_to_best = [] - best_path_edges = [] - if best_node: - try: - # Find shortest path from root to best node - path_to_best = nx.shortest_path(G, root, best_node) - # Create list of edge tuples in the path - best_path_edges = list(zip(path_to_best[:-1], path_to_best[1:])) - except nx.NetworkXNoPath: - # No path exists, keep the lists empty - pass - - # Draw regular edges first (thinner, black) - regular_edges = [(u, v) for u, v in G.edges() if (u, v) not in best_path_edges] - nx.draw_networkx_edges( - G, - pos, - edgelist=regular_edges, - arrows=False, - arrowsize=12, - width=1.5, - edge_color="black", - alpha=0.6, - ax=ax, - ) - - # Draw the edges in the path to the best node (thicker, black like reference) - if best_path_edges: - nx.draw_networkx_edges( - G, - pos, - edgelist=best_path_edges, - arrows=False, - arrowsize=20, - width=3.5, - edge_color="black", - alpha=0.9, - ax=ax, - ) - - if score_values: - if vmin is None: - vmin = min(score_values) - if vmax is None: - vmax = max(score_values) - norm = mcolors.Normalize(vmin=vmin, vmax=vmax) - - # Create colormap using proper method - viridis has good contrast with black text - color_map = cm_module.get_cmap("viridis") - - # Draw nodes with colors based on combined_score - for node in G.nodes(): - node_attrs = G.nodes[node] - current_node_size = regular_node_size - # Default for nodes without valid score - current_node_color = "lightgray" - current_edge_color = "black" - current_linewidth = 1.5 - current_node_shape = "o" # Default shape, circle for "diff" - - # Check if node is incorrect first (overrides other shape logic) - is_correct = node_attrs.get("correct", True) # Default to True - if not is_correct: - current_node_shape = "x" # X shape for incorrect nodes - current_node_color = "red" - current_edge_color = "darkred" - current_linewidth = 4.0 # Thicker line for X - current_node_size = int(current_node_size * 1.5) # Larger size - else: - # Determine shape based on patch_type (only for correct nodes) - patch_type = node_attrs.get("patch_type") - if patch_type == "full": - current_node_shape = "s" # Square - elif patch_type == "init": - current_node_shape = "^" # Triangle up - # elif patch_type == "paper": - # current_node_shape = "d" # Diamond - elif patch_type == "cross": - current_node_shape = "P" # Plus (filled) - - if score_field in node_attrs: - score = node_attrs[score_field] - if pd.isna(score): # Check for NaN - current_node_color = "purple" # Highlight NaN scores - elif isinstance(score, (int, float)): - color = color_map(norm(score)) - current_node_color = mcolors.to_hex(color) - - # Check if this is the best node (overrides all except incorrect) - if node == best_node and is_correct: # Only if correct - current_node_size = best_node_size - current_node_color = "gold" - current_edge_color = "black" - current_linewidth = 2.5 - current_node_shape = "*" # Star shape for best node - elif node in path_to_best and is_correct: # Only if correct - current_node_size = path_node_size - # Color for path nodes: - # - Score color if valid - # - Purple if NaN - # - Lightgray otherwise - node_score = node_attrs.get(score_field) - if node_score is not None and not pd.isna(node_score): - # Valid numeric score? - if isinstance(node_score, (int, float)): - color = color_map(norm(node_score)) - current_node_color = mcolors.to_hex(color) - current_edge_color = "black" - current_linewidth = 2.0 - # Keep shape determined by patch_type unless best node - - nx.draw_networkx_nodes( - G, - pos, - nodelist=[node], - node_size=current_node_size, - node_color=current_node_color, - edgecolors=current_edge_color, - linewidths=current_linewidth, - ax=ax, - node_shape=current_node_shape, - ) - - # Add colorbar with proper axes reference - sm = cm_module.ScalarMappable(cmap=color_map, norm=norm) - sm.set_array([]) - cb = plt.colorbar( - sm, - ax=ax, # type: ignore[arg-type] - pad=-0.05, - shrink=0.6, - ) - cb.set_label(label="Combined Fitness Score", size=20, weight="bold") - cb.ax.tick_params(labelsize=16) - else: - # Draw all nodes with default color if no scores available - nx.draw_networkx_nodes( - G, - pos, - node_size=regular_node_size, - node_color="lightblue", - edgecolors="black", - linewidths=1.5, - ax=ax, - ) - - # Prepare simple node labels with generation and combined_score - node_labels = {} - for node in G.nodes(): - attrs = G.nodes[node] - label_parts = [] - - # Add generation if available - if "generation" in attrs: - label_parts.append(f"{attrs['generation']}") - - # # Add combined_score if available - # if score_field in attrs: - # value = attrs[score_field] - # if isinstance(value, float): - # label_parts.append(f"{value:.1f}") - # else: - # label_parts.append(f"{value}") - - # # Join with newline - if label_parts: - # node_labels[node] = "\n".join(label_parts) - node_labels[node] = label_parts[0] - # else: - # # Just use a short version of the ID if no other info available - # node_labels[node] = f"{node[:8]}" - - # Create a new position dictionary for labels with adjusted y-coordinates - # to place labels above nodes - label_pos = {} - for node, (x, y) in pos.items(): - # Move labels slightly above the nodes - label_pos[node] = (x, y + 0.0) - - # Draw the labels with better font options at adjusted positions - nx.draw_networkx_labels( - G, - label_pos, - labels=node_labels, - font_size=12, - font_weight="bold", - font_color="white", - ax=ax, - ) - - # Add legend for the star shape and paths - if best_node: - star_patch = Line2D( - [0], - [0], - marker="*", - color="w", - markerfacecolor="gold", - markersize=20, - label="Best Score", - ) - # Create line legend with appropriate width - path_line = Line2D([0], [0], color="red", linewidth=4, label=r"Path$\to$Best") - # Legend for patch types - diff_patch = Line2D( - [0], - [0], - marker="o", - color="w", - label="Diff Edit", - markerfacecolor="gray", - markersize=10, - ) - full_patch = Line2D( - [0], - [0], - marker="s", - color="w", - label="Full Edit", - markerfacecolor="gray", - markersize=10, - ) - init_patch = Line2D( - [0], - [0], - marker="^", - color="w", - label="Initial", - markerfacecolor="gray", - markersize=10, - ) - # paper_patch = Line2D( - # [0], - # [0], - # marker="d", # Diamond - # color="w", - # label="Paper Edit", - # markerfacecolor="gray", - # markersize=10, - # ) - crossover_patch = Line2D( - [0], - [0], - marker="P", # Plus (filled) - color="w", - label="Cross-Over", - markerfacecolor="gray", - markersize=10, - ) - incorrect_patch = Line2D( - [0], - [0], - marker="x", # X shape - color="w", - label="Incorrect", - markerfacecolor="red", - markeredgecolor="darkred", - markersize=15, - markeredgewidth=3, - ) - - legend_handles = [ - star_patch, - # path_line, - diff_patch, - full_patch, - init_patch, - # paper_patch, - crossover_patch, - incorrect_patch, - ] - ax.legend(handles=legend_handles, loc="upper right", fontsize=25, ncol=2) - - ax.set_title(title, fontsize=40, fontweight="bold") - ax.axis("off") - # Use subplots_adjust for more control over margins, reduce left padding - fig.subplots_adjust(left=0.02, right=0.85, top=0.95, bottom=0.05) - return fig, ax diff --git a/genesis/plots/plot_pareto.py b/genesis/plots/plot_pareto.py deleted file mode 100644 index b930f9a..0000000 --- a/genesis/plots/plot_pareto.py +++ /dev/null @@ -1,377 +0,0 @@ -import matplotlib.pyplot as plt -import pandas as pd -from typing import Optional, Tuple -from matplotlib.figure import Figure -from matplotlib.axes import Axes -import numpy as np - -from .plot_improvement import _wrap_text -from adjustText import adjust_text - - -# Helper function to identify Pareto-optimal points -# Assumes higher values are better for all metrics -def get_pareto_mask(points: np.ndarray) -> np.ndarray: - """ - Finds the Pareto-efficient points. Assumes all objectives are - to be maximized. A point is Pareto efficient if it is not - dominated by any other point. Point A dominates point B if A is - at least as good as B on all objectives, and strictly better - than B on at least one objective. - - :param points: An (n_points, n_objectives) NumPy array. - :return: A (n_points, ) boolean NumPy array indicating Pareto optimality. - """ - num_points = points.shape[0] - is_pareto = np.ones(num_points, dtype=bool) - for i in range(num_points): - if not is_pareto[i]: # If already marked as dominated, skip - continue - for j in range(num_points): - if i == j: - continue - - # Check if point j dominates point i - if np.all(points[j] >= points[i]) and np.any(points[j] > points[i]): - is_pareto[i] = False # Point i is dominated by point j - break - return is_pareto - - -def _place_pareto_annotations_with_connections( - ax, pareto_df, x_col, y_col, x_maximize=True -): - """ - Place patch name annotations for Pareto points using adjustText for - optimal positioning. Deduplicates based on coordinates and patch_name - to avoid multiple annotations for the same point. - """ - # Force axis limits to be updated after any inversion - ax.figure.canvas.draw_idle() - - annotations = [] - - # Deduplicate based on coordinates and patch_name to avoid multiple - # annotations for the same program point (e.g., island copies) - unique_points = {} - for _, row in pareto_df.iterrows(): - patch_name_val = str(row.get("patch_name", "")) - if ( - pd.notna(patch_name_val) - and patch_name_val != "" - and patch_name_val not in ["nan", "none"] - ): - x_pos = float(row[x_col]) - y_pos = float(row[y_col]) - - # Create a key based on coordinates and patch name - key = (x_pos, y_pos, patch_name_val) - - # Only keep the first occurrence of each unique point - if key not in unique_points: - unique_points[key] = row - - # Now create annotations for unique points only - for (x_pos, y_pos, patch_name_val), row in unique_points.items(): - # Wrap long patch names - patch_name_to_plot = _wrap_text(patch_name_val, max_length=12) - - # Calculate initial offset position based on x_maximize - # Use smaller, more conservative offsets to stay within bounds - x_range = abs(ax.get_xlim()[1] - ax.get_xlim()[0]) - y_range = abs(ax.get_ylim()[1] - ax.get_ylim()[0]) - - # Get axis limits for constraint checking - x_min, x_max = ax.get_xlim() - y_min, y_max = ax.get_ylim() - - if x_maximize: - # For maximization, place labels to the left (worse direction) - x_offset = -x_range * 0.15 # Reduced from 0.3 to 0.15 - else: - # For minimization, place labels to the right (worse direction) - x_offset = x_range * 0.15 # Reduced from 0.3 to 0.15 - - # Calculate proposed text position - text_x = x_pos + x_offset - text_y = y_pos - - # Ensure text position stays within bounds with margin - margin_x = x_range * 0.05 # 5% margin - margin_y = y_range * 0.05 # 5% margin - - text_x = max(x_min + margin_x, min(x_max - margin_x, text_x)) - text_y = max(y_min + margin_y, min(y_max - margin_y, text_y)) - - # Create annotation with constrained position - annotation = ax.annotate( - patch_name_to_plot, - xy=(x_pos, y_pos), # Point to connect arrow to - xytext=(text_x, text_y), # Constrained text position - fontsize=20, - fontweight="bold", - color="darkgreen", - bbox=dict( - boxstyle="round,pad=0.3", - fc="lightyellow", - ec="black", - alpha=0.7, - ), - zorder=4.0, - arrowprops=dict( - arrowstyle="->", - shrinkA=5, - shrinkB=5, - # connectionstyle="arc3,rad=0.4", - color="black", - linewidth=3, # Make arrow thick - ), - ) - annotations.append(annotation) - - # Simple grid-based positioning to avoid overlaps completely - if annotations: - # Set clipping on annotations to ensure they don't extend beyond axes - for annotation in annotations: - annotation.set_clip_on(True) - - # Sort annotations by x-coordinate of their data points for consistent ordering - annotations_with_points = [] - for annotation in annotations: - # Get the xy position (data point) from the annotation - xy_pos = annotation.xy - annotations_with_points.append((xy_pos[0], annotation)) - - # Sort by x-coordinate - annotations_with_points.sort(key=lambda x: x[0]) - - # Position annotations close to points but with smart vertical spacing - x_min, x_max = ax.get_xlim() - y_min, y_max = ax.get_ylim() - x_range = x_max - x_min - y_range = y_max - y_min - - # Create vertical slots to prevent overlaps - n_annotations = len(annotations_with_points) - - # Calculate vertical spacing for labels - label_zone_height = y_range * 0.6 # Use 60% of plot height for labels - label_zone_bottom = y_min + y_range * 0.15 # Start 15% from bottom - - if n_annotations > 1: - y_spacing = label_zone_height / (n_annotations - 1) - else: - y_spacing = 0 - - # Position each annotation - for i, (data_x, annotation) in enumerate(annotations_with_points): - # Get the original data point position - data_point_x, data_point_y = annotation.xy - - # Calculate y position in the vertical slot system - if n_annotations == 1: - label_y = label_zone_bottom + label_zone_height / 2 - else: - label_y = label_zone_bottom + i * y_spacing - - # Position horizontally close to the data point but with some offset - if x_maximize: - # For maximization, place labels to the left - label_x = data_point_x - x_range * 0.03 - else: - # For minimization, place labels to the right - label_x = data_point_x + x_range * 0.03 - - # Ensure labels stay within bounds - margin_x = x_range * 0.05 - margin_y = y_range * 0.05 - - label_x = data_point_x - min(x_range * 0.03, margin_x) - # + max(x_min + margin_x, min(x_max - margin_x, label_x)) - label_y = max(y_min + margin_y, min(y_max - margin_y, label_y)) - - # Set the new position - annotation.set_position((label_x, label_y)) - - -def plot_pareto( - df: pd.DataFrame, - x_variable: str, - y_variable: str, - x_maximize: bool = True, - y_maximize: bool = True, - x_lim: Optional[Tuple[float, float]] = None, - y_lim: Optional[Tuple[float, float]] = None, - title: str = "Pareto Front Analysis", - xlabel: Optional[str] = None, - ylabel: Optional[str] = None, - fig: Optional[Figure] = None, - ax: Optional[Axes] = None, -): - """ - Plots a 2D Pareto front with lineage connections, aiming for - clarity and aesthetics. Axes are inverted as needed so that better - is always higher and to the right. - """ - x_metric_col_name, y_metric_col_name = x_variable, y_variable - - # Determine axis labels - final_xlabel = xlabel if xlabel is not None else x_metric_col_name - final_ylabel = ylabel if ylabel is not None else y_metric_col_name - - required_plotting_cols = [x_metric_col_name, y_metric_col_name] - missing_metrics = [col for col in required_plotting_cols if col not in df.columns] - if missing_metrics: - raise ValueError( - f"DataFrame missing required metric columns: {missing_metrics}" - ) - - if fig is None or ax is None: - fig, ax = plt.subplots(figsize=(12, 9)) - - if x_lim is not None: - ax.set_xlim(x_lim) - if y_lim is not None: - ax.set_ylim(*y_lim) - - df_plot = df.copy() - - if "correct" in df_plot.columns: - try: - df_plot["correct"] = df_plot["correct"].astype(bool) - except Exception as e: - print( - f"Warning: Could not convert 'correct' column to boolean: " - f"{e}. Using as is." - ) - - original_row_count = len(df_plot) - df_plot = df_plot[df_plot["correct"]] - if len(df_plot) < original_row_count: - print( - f"Filtered to {len(df_plot)} 'correct' rows from " - f"{original_row_count} total." - ) - if df_plot.empty: - print("No 'correct' points found to plot.") - ax.set_title(title, fontsize=32, fontweight="bold", pad=15) - ax.set_xlabel(final_xlabel, fontsize=25, fontweight="bold", labelpad=15) - ax.set_ylabel(final_ylabel, fontsize=25, fontweight="bold", labelpad=15) - ax.grid( - True, linestyle=":", alpha=0.9, color="lightgray" - ) # User preference for grid alpha - ax.spines["top"].set_visible(False) - ax.spines["right"].set_visible(False) - if fig: - fig.tight_layout() - return fig, ax - else: - print("Warning: 'correct' column not found. Plotting all points.") - - for col in [x_variable, y_variable]: - df_plot[col] = pd.to_numeric(df_plot[col], errors="coerce") - df_plot = df_plot.dropna(subset=[x_variable, y_variable]) - ax.tick_params(axis="both", which="major", labelsize=20) - if df_plot.empty: - print("No data to plot after processing metric columns.") - ax.set_title(title, fontsize=32, fontweight="bold", pad=15) - ax.set_xlabel(final_xlabel, fontsize=25, fontweight="bold", labelpad=15) - ax.set_ylabel(final_ylabel, fontsize=25, fontweight="bold", labelpad=15) - ax.grid( - True, linestyle=":", alpha=0.9, color="lightgray" - ) # User preference for grid alpha - ax.spines["top"].set_visible(False) - ax.spines["right"].set_visible(False) - if fig: - fig.tight_layout() - return fig, ax - - # Prepare metric values for Pareto calculation - # Invert values for minimization objectives so higher is always better - metric_values = df_plot[[x_variable, y_variable]].values.copy() - if not x_maximize: - metric_values[:, 0] = -metric_values[:, 0] - if not y_maximize: - metric_values[:, 1] = -metric_values[:, 1] - - pareto_mask = get_pareto_mask(metric_values) - df_plot["is_pareto"] = pareto_mask - - pareto_df = df_plot[df_plot["is_pareto"]].copy() - non_pareto_df = df_plot[~df_plot["is_pareto"]].copy() - - # Plot non-Pareto points - if not non_pareto_df.empty: - ax.scatter( - non_pareto_df[x_metric_col_name], - non_pareto_df[y_metric_col_name], - color="dimgray", - s=100, - alpha=1.0, - zorder=1, - label="Dominated/Other", - ) - - # Plot Pareto points on top - if not pareto_df.empty: - ax.scatter( - pareto_df[x_metric_col_name], - pareto_df[y_metric_col_name], - color="orangered", - s=200, - alpha=1.0, - marker="o", - edgecolor="black", - linewidth=1, - zorder=3, - label="Pareto Optimal", - ) - # Draw connections for Pareto frontier - if not pareto_df.empty and len(pareto_df) > 1: - # Sort Pareto points by x-coordinate to form proper frontier - pareto_sorted = pareto_df.sort_values(x_metric_col_name) - - # Connect consecutive points along the sorted frontier - x_coords = pareto_sorted[x_metric_col_name].values - y_coords = pareto_sorted[y_metric_col_name].values - - ax.plot( - x_coords, - y_coords, - color="red", - linewidth=4, - alpha=0.7, - zorder=2, - ) - - # Invert axes BEFORE annotations if needed so better is always higher - # and to the right - if not x_maximize: - ax.invert_xaxis() - if not y_maximize: - ax.invert_yaxis() - - # Annotate Pareto points with patch names using optimization - if not pareto_df.empty and "patch_name" in pareto_df.columns: - _place_pareto_annotations_with_connections( - ax, pareto_df, x_metric_col_name, y_metric_col_name, x_maximize - ) - - ax.set_xlabel(final_xlabel, fontsize=25, fontweight="bold", labelpad=15) - ax.set_ylabel(final_ylabel, fontsize=25, fontweight="bold", labelpad=15) - ax.set_title(title, fontsize=32, fontweight="bold", pad=15) - - handles, labels = ax.get_legend_handles_labels() - by_label = dict(zip(labels, handles)) - if by_label: - ax.legend(by_label.values(), by_label.keys(), loc="best", fontsize=25) - - ax.grid( - True, linestyle=":", alpha=0.9, color="lightgray" - ) # User preference for grid alpha - ax.spines["top"].set_visible(False) - ax.spines["right"].set_visible(False) - - if fig: - fig.tight_layout() - return fig, ax diff --git a/genesis/plots/plot_similarity.py b/genesis/plots/plot_similarity.py deleted file mode 100644 index 2d2cc53..0000000 --- a/genesis/plots/plot_similarity.py +++ /dev/null @@ -1,65 +0,0 @@ -import matplotlib.pyplot as plt -import seaborn as sns -import numpy as np - - -def plot_embed_similarity( - embeds, - perfs, - ordered=True, - title="Code Embedding Cosine Similarity", - fig=None, - axs=None, - vmin=None, - vmax=None, -): - """ - Plot the similarity of embeddings and the performance of programs. - """ - from sklearn.metrics.pairwise import cosine_similarity - - similarity_matrix = cosine_similarity(embeds) - - if ordered: - from scipy.cluster.hierarchy import linkage, leaves_list - - # Perform hierarchical clustering - linkage_matrix = linkage(embeds, method="ward") - ordered_indices = leaves_list(linkage_matrix) - - # Reorder matrix - similarity_matrix = similarity_matrix[ordered_indices][:, ordered_indices] - perfs = perfs[ordered_indices] - title += " (Clustered)" - - # Plot similarity matrix - fig, axs = plt.subplots( - 1, 2, figsize=(12, 8), gridspec_kw={"width_ratios": [20, 1]} - ) - sns.heatmap(similarity_matrix, cmap="viridis", ax=axs[0]) - axs[0].set_title(title, fontsize=25) - axs[0].set_xlabel("Program Index") - axs[0].set_ylabel("Program Index") - - if ordered: - # set xticks to be the program ids using ordered_indices - axs[0].set_xticks(np.arange(len(ordered_indices))[::3]) - axs[0].set_xticklabels(ordered_indices[::3]) - axs[0].set_yticks(np.arange(len(ordered_indices))[::3]) - axs[0].set_yticklabels(ordered_indices[::3]) - - # Plot performance heatmap - sns.heatmap( - perfs.reshape(-1, 1), - cmap="Reds_r", - ax=axs[1], - vmin=vmin, - vmax=vmax, - xticklabels=False, - yticklabels=False, - ) - axs[1].set_title("Score", fontsize=14) - axs[1].set_xticks([]) - axs[1].set_yticks([]) - fig.tight_layout() - return fig, axs, similarity_matrix diff --git a/genesis/webui/frontend/.dockerignore b/genesis/webui/frontend/.dockerignore new file mode 100644 index 0000000..655562d --- /dev/null +++ b/genesis/webui/frontend/.dockerignore @@ -0,0 +1,5 @@ +node_modules +dist +test-results +playwright-report +.env* diff --git a/genesis/webui/frontend/Dockerfile b/genesis/webui/frontend/Dockerfile new file mode 100644 index 0000000..d7d476c --- /dev/null +++ b/genesis/webui/frontend/Dockerfile @@ -0,0 +1,12 @@ +FROM node:20-alpine AS build +WORKDIR /app +COPY package.json package-lock.json ./ +RUN npm ci +COPY . . +RUN npm run build + +FROM nginx:alpine +COPY --from=build /app/dist /usr/share/nginx/html +COPY nginx.conf /etc/nginx/templates/default.conf.template +EXPOSE 8080 +ENV BACKEND_URL=http://localhost:8080 diff --git a/genesis/webui/frontend/nginx.conf b/genesis/webui/frontend/nginx.conf new file mode 100644 index 0000000..08f50b1 --- /dev/null +++ b/genesis/webui/frontend/nginx.conf @@ -0,0 +1,22 @@ +server { + listen 8080; + root /usr/share/nginx/html; + index index.html; + + location /api/ { + proxy_pass ${BACKEND_URL}/api/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location / { + try_files $uri $uri/ /index.html; + } + + location ~* \.(js|css|png|jpg|jpeg|gif|ico|svg|woff|woff2|ttf|eot)$ { + expires 1y; + add_header Cache-Control "public, immutable"; + } +} diff --git a/genesis/webui/frontend/package-lock.json b/genesis/webui/frontend/package-lock.json index 438fc32..865ec4d 100644 --- a/genesis/webui/frontend/package-lock.json +++ b/genesis/webui/frontend/package-lock.json @@ -103,7 +103,6 @@ "integrity": "sha512-2BCOP7TN8M+gVDj7/ht3hsaO/B/n5oDbiAyyvnRlNOs+u1o+JWNYTQrmpuNp1/Wq2gcFrI01JAW+paEKDMx/CA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.27.1", "@babel/generator": "^7.28.3", @@ -3111,7 +3110,6 @@ "integrity": "sha512-6mDvHUFSjyT2B2yeNx2nUgMxh9LtOWvkhIU3uePn2I2oyNymUAX1NIsdgviM4CH+JSrp2D2hsMvJOkxY+0wNRA==", "devOptional": true, "license": "MIT", - "peer": true, "dependencies": { "csstype": "^3.0.2" } @@ -3122,7 +3120,6 @@ "integrity": "sha512-/EEvYBdT3BflCWvTMO7YkYBHVE9Ci6XdqZciZANQgKpaiDRGOLIlRo91jbTNRQjgPFWVaRxcYc0luVNFitz57A==", "devOptional": true, "license": "MIT", - "peer": true, "peerDependencies": { "@types/react": "^19.2.0" } @@ -3212,7 +3209,6 @@ "integrity": "sha512-n1H6IcDhmmUEG7TNVSspGmiHHutt7iVKtZwRppD7e04wha5MrkV1h3pti9xQLcCMt6YWsncpoT0HMjkH1FNwWQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.46.0", "@typescript-eslint/types": "8.46.0", @@ -3484,7 +3480,6 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -3804,7 +3799,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "baseline-browser-mapping": "^2.8.25", "caniuse-lite": "^1.0.30001754", @@ -3948,7 +3942,6 @@ "resolved": "https://registry.npmjs.org/chart.js/-/chart.js-4.5.1.tgz", "integrity": "sha512-GIjfiT9dbmHRiYi6Nl2yFCq7kkwdkp1W/lp2J99rX0yo9tgJGn3lKQATztIjb5tVtevcBtIdICNWqlq5+E8/Pw==", "license": "MIT", - "peer": true, "dependencies": { "@kurkle/color": "^0.3.0" }, @@ -4713,7 +4706,6 @@ "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", "license": "ISC", - "peer": true, "engines": { "node": ">=12" } @@ -5203,7 +5195,6 @@ "integrity": "sha512-XyLmROnACWqSxiGYArdef1fItQd47weqB7iwtfr9JHwRrqIXZdcFMvvEcL9xHCmL0SNsOvF0c42lWyM1U5dgig==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -8096,7 +8087,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", @@ -8364,7 +8354,6 @@ "resolved": "https://registry.npmjs.org/react/-/react-19.2.0.tgz", "integrity": "sha512-tmbWg6W31tQLeB5cdIBOicJDJRR2KzXsV7uSK9iNfLWQ5bIZfxuPEHp7M8wiHyHnn0DD1i7w3Zmin0FtkrwoCQ==", "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -8384,7 +8373,6 @@ "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.0.tgz", "integrity": "sha512-UlbRu4cAiGaIewkPyiRGJk0imDN2T3JjieT6spoL2UeSf5od4n5LB/mQ4ejmxhCFT1tYe8IvaFulzynWovsEFQ==", "license": "MIT", - "peer": true, "dependencies": { "scheduler": "^0.27.0" }, @@ -9456,7 +9444,6 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -9603,7 +9590,6 @@ "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.20.6.tgz", "integrity": "sha512-ytQKuwgmrrkDTFP4LjR0ToE2nqgy886GpvRSpU0JAnrdBYppuY5rLkRUYPU1yCryb24SsKBTL/hlDQAEFVwtZg==", "license": "MIT", - "peer": true, "dependencies": { "esbuild": "~0.25.0", "get-tsconfig": "^4.7.5" @@ -9685,7 +9671,6 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -9850,7 +9835,6 @@ "integrity": "sha512-4nVGliEpxmhCL8DslSAUdxlB6+SMrhB0a1v5ijlh1xB1nEPuy1mxaHxysVucLHuWryAxLWg6a5ei+U4TLn/rFg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "esbuild": "^0.25.0", "fdir": "^6.5.0", @@ -9944,7 +9928,6 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, diff --git a/genesis/webui/frontend/src/components/ErrorBoundary.tsx b/genesis/webui/frontend/src/components/ErrorBoundary.tsx index f8990e4..58bfd85 100644 --- a/genesis/webui/frontend/src/components/ErrorBoundary.tsx +++ b/genesis/webui/frontend/src/components/ErrorBoundary.tsx @@ -1,4 +1,4 @@ -import React, { Component, ErrorInfo, ReactNode } from 'react'; +import { Component, type ErrorInfo, type ReactNode } from 'react'; interface Props { children: ReactNode; diff --git a/genesis/webui/frontend/src/context/GenesisContext.tsx b/genesis/webui/frontend/src/context/GenesisContext.tsx index 832e2f2..6739037 100644 --- a/genesis/webui/frontend/src/context/GenesisContext.tsx +++ b/genesis/webui/frontend/src/context/GenesisContext.tsx @@ -183,7 +183,7 @@ export function GenesisProvider({ children }: { children: ReactNode }) { const stats = computeStats(state.programs); - const loadDatabases = useCallback(async (force = false) => { + const loadDatabases = useCallback(async (_force = false) => { dispatch({ type: 'SET_LOADING', payload: true }); dispatch({ type: 'SET_ERROR', payload: null }); diff --git a/genesis/webui/frontend/src/lib/utils.ts b/genesis/webui/frontend/src/lib/utils.ts index d084cca..ec0d75c 100644 --- a/genesis/webui/frontend/src/lib/utils.ts +++ b/genesis/webui/frontend/src/lib/utils.ts @@ -1,6 +1,5 @@ import { type ClassValue, clsx } from "clsx" -import { twMerge } from "tailwind-merge" export function cn(...inputs: ClassValue[]) { - return twMerge(clsx(inputs)) + return clsx(inputs) } diff --git a/genesis/webui/frontend/src/services/api.ts b/genesis/webui/frontend/src/services/api.ts index 4d0a3bb..716aed7 100644 --- a/genesis/webui/frontend/src/services/api.ts +++ b/genesis/webui/frontend/src/services/api.ts @@ -1,19 +1,35 @@ import type { DatabaseInfo, Program, MetaFile, MetaContent } from '../types'; -const API_BASE = 'http://localhost:8000'; +const API_BASE = import.meta.env.VITE_API_URL || '/api'; export async function listDatabases(): Promise { - const response = await fetch(`${API_BASE}/list_databases`); + const response = await fetch(`${API_BASE}/runs`); if (!response.ok) { - throw new Error(`Failed to load database list (HTTP ${response.status})`); + throw new Error(`Failed to load runs (HTTP ${response.status})`); } - return response.json(); + const data = await response.json(); + return (data.runs ?? []).map( + (r: { + run_id: string; + task_name: string; + status: string; + start_time: string; + population_size: number; + total_generations: number; + }) => ({ + path: r.run_id, + name: `${r.task_name} (${r.status})`, + sort_key: r.start_time, + stats: { + total: r.population_size, + working: r.total_generations, + }, + }) + ); } -export async function getPrograms(dbPath: string): Promise { - const response = await fetch( - `${API_BASE}/get_programs?db_path=${encodeURIComponent(dbPath)}` - ); +export async function getPrograms(runId: string): Promise { + const response = await fetch(`${API_BASE}/runs/${runId}/individuals`); if (!response.ok) { if (response.status === 503) { throw new Error( @@ -22,35 +38,103 @@ export async function getPrograms(dbPath: string): Promise { } throw new Error(`Failed to load data (HTTP ${response.status})`); } - return response.json(); + const data = await response.json(); + return (data.individuals ?? []).map( + (r: { + id: string; + parent_id: string | null; + code: string; + language: string; + generation: number; + timestamp: string; + agent_name: string; + combined_score: number; + fitness_score: number; + metrics: Record; + text_feedback: string; + metadata: { + patch_name: string; + patch_type: string; + api_cost: number; + embed_cost: number; + novelty_cost: number; + }; + correct: boolean; + is_pareto: boolean; + code_size: number; + }) => ({ + id: r.id, + parent_id: r.parent_id, + code: r.code, + language: r.language, + generation: r.generation, + timestamp: new Date(r.timestamp).getTime() / 1000, + agent_name: r.agent_name, + combined_score: r.combined_score, + public_metrics: r.metrics, + private_metrics: null, + text_feedback: r.text_feedback || null, + metadata: r.metadata ?? { patch_name: 'unknown', patch_type: 'unknown' }, + complexity: r.code_size ?? null, + embedding: null, + embedding_pca_2d: null, + embedding_pca_3d: null, + island_idx: null, + correct: r.correct, + }) + ); } -export async function getMetaFiles(dbPath: string): Promise { - const response = await fetch( - `${API_BASE}/get_meta_files?db_path=${encodeURIComponent(dbPath)}` - ); +export async function getMetaFiles(runId: string): Promise { + const response = await fetch(`${API_BASE}/runs/${runId}/generations`); if (!response.ok) { if (response.status === 404) { return []; } - throw new Error(`Failed to load meta files (HTTP ${response.status})`); + throw new Error(`Failed to load generations (HTTP ${response.status})`); } - return response.json(); + const data = await response.json(); + return (data.generations ?? []).map( + (g: { generation: number; timestamp: string }) => ({ + generation: g.generation, + filename: `generation_${g.generation}`, + path: `${runId}/generations/${g.generation}`, + }) + ); } export async function getMetaContent( - dbPath: string, + runId: string, generation: number ): Promise { const response = await fetch( - `${API_BASE}/get_meta_content?db_path=${encodeURIComponent(dbPath)}&generation=${generation}` + `${API_BASE}/runs/${runId}/generations/${generation}` ); if (!response.ok) { - throw new Error(`Failed to load meta content (HTTP ${response.status})`); + throw new Error( + `Failed to load generation details (HTTP ${response.status})` + ); } - return response.json(); + const data = await response.json(); + const content = [ + `# Generation ${data.generation}`, + ``, + `- **Individuals**: ${data.num_individuals}`, + `- **Best Score**: ${data.best_score?.toFixed(4) ?? 'N/A'}`, + `- **Avg Score**: ${data.avg_score?.toFixed(4) ?? 'N/A'}`, + `- **Pareto Size**: ${data.pareto_size}`, + `- **Total Cost**: $${data.total_cost?.toFixed(4) ?? '0'}`, + ``, + data.metadata ? `## Metadata\n\`\`\`json\n${JSON.stringify(data.metadata, null, 2)}\n\`\`\`` : '', + ].join('\n'); + + return { + generation: data.generation, + filename: `generation_${data.generation}`, + content, + }; } -export function getMetaPdfUrl(dbPath: string, generation: number): string { - return `${API_BASE}/download_meta_pdf?db_path=${encodeURIComponent(dbPath)}&generation=${generation}`; +export function getMetaPdfUrl(runId: string, generation: number): string { + return `${API_BASE}/runs/${runId}/generations/${generation}`; } diff --git a/genesis/webui/frontend/vite.config.ts b/genesis/webui/frontend/vite.config.ts index 688367a..1b8579c 100644 --- a/genesis/webui/frontend/vite.config.ts +++ b/genesis/webui/frontend/vite.config.ts @@ -12,11 +12,7 @@ export default defineConfig({ }, server: { proxy: { - '/list_databases': 'http://localhost:8000', - '/get_programs': 'http://localhost:8000', - '/get_meta_files': 'http://localhost:8000', - '/get_meta_content': 'http://localhost:8000', - '/download_meta_pdf': 'http://localhost:8000', + '/api': 'http://localhost:8080', }, }, }); diff --git a/genesis_rust_backend/.dockerignore b/genesis_rust_backend/.dockerignore new file mode 100644 index 0000000..eacfe58 --- /dev/null +++ b/genesis_rust_backend/.dockerignore @@ -0,0 +1,4 @@ +target/ +.git/ +*.md +tests/ diff --git a/genesis_rust_backend/Cargo.lock b/genesis_rust_backend/Cargo.lock index 4acc381..9f5888d 100644 --- a/genesis_rust_backend/Cargo.lock +++ b/genesis_rust_backend/Cargo.lock @@ -2,6 +2,21 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -17,6 +32,26 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -29,17 +64,149 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "axum" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8" +dependencies = [ + "axum-core", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + [[package]] name = "base64" version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" +dependencies = [ + "serde_core", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bollard" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97ccca1260af6a459d75994ad5acc1651bcabcbdbc41467cc9786519ab854c30" +dependencies = [ + "base64 0.22.1", + "bollard-stubs", + "bytes", + "futures-core", + "futures-util", + "hex", + "home", + "http", + "http-body-util", + "hyper", + "hyper-named-pipe", + "hyper-rustls", + "hyper-util", + "hyperlocal", + "log", + "pin-project-lite", + "rustls", + "rustls-native-certs", + "rustls-pemfile", + "rustls-pki-types", + "serde", + "serde_derive", + "serde_json", + "serde_repr", + "serde_urlencoded", + "thiserror 2.0.18", + "tokio", + "tokio-util", + "tower-service", + "url", + "winapi", +] + +[[package]] +name = "bollard-stubs" +version = "1.47.1-rc.27.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f179cfbddb6e77a5472703d4b30436bff32929c0aa8a9008ecf23d1d3cdd0da" +dependencies = [ + "serde", + "serde_repr", + "serde_with", +] [[package]] name = "bumpalo" @@ -47,6 +214,12 @@ version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.11.1" @@ -89,18 +262,159 @@ dependencies = [ "windows-link", ] +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crossbeam-queue" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "powerfmt", + "serde_core", +] + [[package]] name = "diff" version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", + "subtle", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -112,6 +426,38 @@ dependencies = [ "syn", ] +[[package]] +name = "docker_credential" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d89dfcba45b4afad7450a99b39e751590463e45c04728cf555d36bb66940de8" +dependencies = [ + "base64 0.21.7", + "serde", + "serde_json", +] + +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +dependencies = [ + "serde", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -128,18 +474,62 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", +] + +[[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + [[package]] name = "fastrand" version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "filetime" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f98844151eee8917efc50bd9e8318cb963ae8b297431495d3f758616ea5c57db" +dependencies = [ + "cfg-if", + "libc", + "libredox", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "flume" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" +dependencies = [ + "futures-core", + "futures-sink", + "spin", +] + [[package]] name = "foldhash" version = "0.1.5" @@ -156,20 +546,57 @@ dependencies = [ ] [[package]] -name = "futures-channel" +name = "futures" version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" dependencies = [ + "futures-channel", "futures-core", + "futures-executor", + "futures-io", "futures-sink", + "futures-task", + "futures-util", ] [[package]] -name = "futures-core" +name = "futures-channel" version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] [[package]] name = "futures-io" @@ -177,6 +604,17 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "futures-sink" version = "0.3.32" @@ -195,8 +633,10 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ + "futures-channel", "futures-core", "futures-io", + "futures-macro", "futures-sink", "futures-task", "memchr", @@ -204,20 +644,40 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "genesis_rust_backend" version = "0.1.0" dependencies = [ "anyhow", + "axum", "chrono", + "dotenvy", "pretty_assertions", "rand 0.8.5", "reqwest", "serde", "serde_json", "serde_yaml", + "sqlx", "tempfile", + "testcontainers", + "testcontainers-modules", "thiserror 1.0.69", + "tokio", + "tower-http", + "tracing", + "tracing-subscriber", + "uuid", ] [[package]] @@ -260,12 +720,20 @@ dependencies = [ "wasip3", ] +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ + "allocator-api2", + "equivalent", "foldhash", ] @@ -275,12 +743,54 @@ version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown 0.15.5", +] + [[package]] name = "heck" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "http" version = "1.4.0" @@ -320,6 +830,12 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + [[package]] name = "hyper" version = "1.8.1" @@ -333,6 +849,7 @@ dependencies = [ "http", "http-body", "httparse", + "httpdate", "itoa", "pin-project-lite", "pin-utils", @@ -341,6 +858,21 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-named-pipe" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b7d8abf35697b81a825e386fc151e0d503e8cb5fcb93cc8669c376dfd6f278" +dependencies = [ + "hex", + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", + "winapi", +] + [[package]] name = "hyper-rustls" version = "0.27.7" @@ -355,7 +887,7 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", - "webpki-roots", + "webpki-roots 1.0.6", ] [[package]] @@ -364,7 +896,7 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-util", @@ -381,6 +913,21 @@ dependencies = [ "tracing", ] +[[package]] +name = "hyperlocal" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" +dependencies = [ + "hex", + "http-body-util", + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "iana-time-zone" version = "0.1.65" @@ -492,6 +1039,12 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "1.1.0" @@ -513,6 +1066,17 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + [[package]] name = "indexmap" version = "2.13.0" @@ -557,6 +1121,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin", +] + [[package]] name = "leb128fmt" version = "0.1.0" @@ -569,6 +1142,34 @@ version = "0.2.182" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "libredox" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ddbf48fd451246b1f8c2610bd3b4ac0cc6e149d89832867093ab69a17194f08" +dependencies = [ + "bitflags 2.11.0", + "libc", + "plain", + "redox_syscall 0.7.3", +] + +[[package]] +name = "libsqlite3-sys" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +dependencies = [ + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -581,6 +1182,15 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.29" @@ -593,12 +1203,43 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "memchr" version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + [[package]] name = "mio" version = "1.1.1" @@ -610,6 +1251,57 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.5", + "smallvec", + "zeroize", +] + +[[package]] +name = "num-conv" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -617,6 +1309,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -625,6 +1318,75 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.18", + "smallvec", + "windows-link", +] + +[[package]] +name = "parse-display" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914a1c2265c98e2446911282c6ac86d8524f495792c38c5bd884f80499c7538a" +dependencies = [ + "parse-display-derive", + "regex", + "regex-syntax", +] + +[[package]] +name = "parse-display-derive" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ae7800a4c974efd12df917266338e79a7a74415173caf7e70aa0a0707345281" +dependencies = [ + "proc-macro2", + "quote", + "regex", + "regex-syntax", + "structmeta", + "syn", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -643,6 +1405,39 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + [[package]] name = "potential_utf" version = "0.1.4" @@ -652,6 +1447,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -820,17 +1621,91 @@ dependencies = [ ] [[package]] -name = "reqwest" -version = "0.12.28" +name = "redox_syscall" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" dependencies = [ - "base64", - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "http", + "bitflags 1.3.2", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.11.0", +] + +[[package]] +name = "redox_syscall" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce70a74e890531977d37e532c34d45e9055d2409ed08ddba14529471ed0be16" +dependencies = [ + "bitflags 2.11.0", +] + +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-core", + "http", "http-body", "http-body-util", "hyper", @@ -856,7 +1731,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots", + "webpki-roots 1.0.6", ] [[package]] @@ -873,6 +1748,26 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rsa" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" +dependencies = [ + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "signature", + "spki", + "subtle", + "zeroize", +] + [[package]] name = "rustc-hash" version = "2.1.1" @@ -885,7 +1780,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags", + "bitflags 2.11.0", "errno", "libc", "linux-raw-sys", @@ -906,6 +1801,27 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "rustls-pki-types" version = "1.14.0" @@ -939,6 +1855,68 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "schemars" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "schemars" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags 2.11.0", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.27" @@ -988,6 +1966,28 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_repr" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -1000,52 +2000,394 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "3.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd5414fad8e6907dbdd5bc441a50ae8d6e26151a03b1de04d89a5576de61d01f" +dependencies = [ + "base64 0.22.1", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.13.0", + "schemars 0.9.0", + "schemars 1.2.1", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3db8978e608f1fe7357e211969fd9abdcae80bac1ba7a3369bb7eb6b404eb65" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "serde_yaml" version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap", + "indexmap 2.13.0", "itoa", "ryu", "serde", - "unsafe-libyaml", + "unsafe-libyaml", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core 0.6.4", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +dependencies = [ + "serde", +] + +[[package]] +name = "socket2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "sqlx" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fefb893899429669dcdd979aff487bd78f4064e5e7907e4269081e0ef7d97dc" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" +dependencies = [ + "base64 0.22.1", + "bytes", + "chrono", + "crc", + "crossbeam-queue", + "either", + "event-listener", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashbrown 0.15.5", + "hashlink", + "indexmap 2.13.0", + "log", + "memchr", + "once_cell", + "percent-encoding", + "rustls", + "serde", + "serde_json", + "sha2", + "smallvec", + "thiserror 2.0.18", + "tokio", + "tokio-stream", + "tracing", + "url", + "uuid", + "webpki-roots 0.26.11", +] + +[[package]] +name = "sqlx-macros" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2d452988ccaacfbf5e0bdbc348fb91d7c8af5bee192173ac3636b5fb6e6715d" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19a9c1841124ac5a61741f96e1d9e2ec77424bf323962dd894bdb93f37d5219b" +dependencies = [ + "dotenvy", + "either", + "heck", + "hex", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" +dependencies = [ + "atoi", + "base64 0.22.1", + "bitflags 2.11.0", + "byteorder", + "bytes", + "chrono", + "crc", + "digest", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "percent-encoding", + "rand 0.8.5", + "rsa", + "serde", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 2.0.18", + "tracing", + "uuid", + "whoami", +] + +[[package]] +name = "sqlx-postgres" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" +dependencies = [ + "atoi", + "base64 0.22.1", + "bitflags 2.11.0", + "byteorder", + "chrono", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "rand 0.8.5", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 2.0.18", + "tracing", + "uuid", + "whoami", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2d12fe70b2c1b4401038055f90f151b78208de1f9f89a7dbfd41587a10c3eea" +dependencies = [ + "atoi", + "chrono", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "serde_urlencoded", + "sqlx-core", + "thiserror 2.0.18", + "tracing", + "url", + "uuid", ] [[package]] -name = "shlex" -version = "1.3.0" +name = "stable_deref_trait" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] -name = "slab" -version = "0.4.12" +name = "stringprep" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] [[package]] -name = "smallvec" -version = "1.15.1" +name = "strsim" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] -name = "socket2" -version = "0.6.2" +name = "structmeta" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0" +checksum = "2e1575d8d40908d70f6fd05537266b90ae71b15dbbe7a8b7dffa2b759306d329" dependencies = [ - "libc", - "windows-sys 0.60.2", + "proc-macro2", + "quote", + "structmeta-derive", + "syn", ] [[package]] -name = "stable_deref_trait" -version = "1.2.1" +name = "structmeta-derive" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] [[package]] name = "subtle" @@ -1097,6 +2439,44 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "testcontainers" +version = "0.23.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59a4f01f39bb10fc2a5ab23eb0d888b1e2bb168c157f61a1b98e6c501c639c74" +dependencies = [ + "async-trait", + "bollard", + "bollard-stubs", + "bytes", + "docker_credential", + "either", + "etcetera", + "futures", + "log", + "memchr", + "parse-display", + "pin-project-lite", + "serde", + "serde_json", + "serde_with", + "thiserror 2.0.18", + "tokio", + "tokio-stream", + "tokio-tar", + "tokio-util", + "url", +] + +[[package]] +name = "testcontainers-modules" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d43ed4e8f58424c3a2c6c56dbea6643c3c23e8666a34df13c54f0a184e6c707" +dependencies = [ + "testcontainers", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -1137,6 +2517,46 @@ dependencies = [ "syn", ] +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + +[[package]] +name = "time-macros" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinystr" version = "0.8.2" @@ -1171,11 +2591,25 @@ dependencies = [ "bytes", "libc", "mio", + "parking_lot", "pin-project-lite", + "signal-hook-registry", "socket2", + "tokio-macros", "windows-sys 0.61.2", ] +[[package]] +name = "tokio-macros" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tokio-rustls" version = "0.26.4" @@ -1186,6 +2620,45 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-tar" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5714c010ca3e5c27114c1cdeb9d14641ace49874aa5626d7149e47aedace75" +dependencies = [ + "filetime", + "futures-core", + "libc", + "redox_syscall 0.3.5", + "tokio", + "tokio-stream", + "xattr", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + [[package]] name = "tower" version = "0.5.3" @@ -1199,6 +2672,7 @@ dependencies = [ "tokio", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -1207,7 +2681,7 @@ version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" dependencies = [ - "bitflags", + "bitflags 2.11.0", "bytes", "futures-util", "http", @@ -1217,6 +2691,7 @@ dependencies = [ "tower", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -1237,10 +2712,23 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ + "log", "pin-project-lite", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tracing-core" version = "0.1.36" @@ -1248,6 +2736,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", ] [[package]] @@ -1256,12 +2774,39 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "typenum" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" + +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + [[package]] name = "unicode-xid" version = "0.2.6" @@ -1290,6 +2835,7 @@ dependencies = [ "idna", "percent-encoding", "serde", + "serde_derive", ] [[package]] @@ -1298,6 +2844,36 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "uuid" +version = "1.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9" +dependencies = [ + "getrandom 0.4.1", + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "want" version = "0.3.1" @@ -1331,6 +2907,12 @@ dependencies = [ "wit-bindgen", ] +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + [[package]] name = "wasm-bindgen" version = "0.2.113" @@ -1407,7 +2989,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" dependencies = [ "anyhow", - "indexmap", + "indexmap 2.13.0", "wasm-encoder", "wasmparser", ] @@ -1418,9 +3000,9 @@ version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ - "bitflags", + "bitflags 2.11.0", "hashbrown 0.15.5", - "indexmap", + "indexmap 2.13.0", "semver", ] @@ -1444,6 +3026,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.6", +] + [[package]] name = "webpki-roots" version = "1.0.6" @@ -1453,6 +3044,38 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "whoami" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" +dependencies = [ + "libredox", + "wasite", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-core" version = "0.62.2" @@ -1512,6 +3135,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -1539,6 +3171,21 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -1572,6 +3219,12 @@ dependencies = [ "windows_x86_64_msvc 0.53.1", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" @@ -1584,6 +3237,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" @@ -1596,6 +3255,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -1620,6 +3285,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + [[package]] name = "windows_i686_msvc" version = "0.52.6" @@ -1632,6 +3303,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" @@ -1644,6 +3321,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" @@ -1656,6 +3339,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" @@ -1696,7 +3385,7 @@ checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" dependencies = [ "anyhow", "heck", - "indexmap", + "indexmap 2.13.0", "prettyplease", "syn", "wasm-metadata", @@ -1726,8 +3415,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", - "bitflags", - "indexmap", + "bitflags 2.11.0", + "indexmap 2.13.0", "log", "serde", "serde_derive", @@ -1746,7 +3435,7 @@ checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" dependencies = [ "anyhow", "id-arena", - "indexmap", + "indexmap 2.13.0", "log", "semver", "serde", @@ -1762,6 +3451,16 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + [[package]] name = "yansi" version = "1.0.1" diff --git a/genesis_rust_backend/Cargo.toml b/genesis_rust_backend/Cargo.toml index daa64d1..8f4d4d4 100644 --- a/genesis_rust_backend/Cargo.toml +++ b/genesis_rust_backend/Cargo.toml @@ -2,7 +2,7 @@ name = "genesis_rust_backend" version = "0.1.0" edition = "2021" -description = "Rust backend implementation scaffold for Genesis" +description = "Rust backend for the Genesis evolution platform" license = "MIT" [dependencies] @@ -13,8 +13,18 @@ serde_yaml = "0.9" rand = "0.8" chrono = { version = "0.4", features = ["serde"] } thiserror = "1" -reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls"] } +reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } tempfile = "3" +tokio = { version = "1", features = ["full"] } +sqlx = { version = "0.8", features = ["runtime-tokio", "tls-rustls", "postgres", "uuid", "chrono", "json"] } +axum = "0.8" +uuid = { version = "1", features = ["v4", "serde"] } +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } +tower-http = { version = "0.6", features = ["cors", "trace"] } +dotenvy = "0.15" [dev-dependencies] pretty_assertions = "1" +testcontainers = "0.23" +testcontainers-modules = { version = "0.11", features = ["postgres"] } diff --git a/genesis_rust_backend/Dockerfile b/genesis_rust_backend/Dockerfile new file mode 100644 index 0000000..a0cfb1b --- /dev/null +++ b/genesis_rust_backend/Dockerfile @@ -0,0 +1,20 @@ +FROM rust:1.83-bookworm AS builder + +WORKDIR /app + +# Cache dependencies by building a dummy project first +COPY Cargo.toml Cargo.lock ./ +RUN mkdir src && echo "fn main() {}" > src/main.rs && cargo build --release && rm -rf src + +# Build real application +COPY src ./src +RUN touch src/main.rs && cargo build --release + +FROM gcr.io/distroless/cc-debian12 + +COPY --from=builder /app/target/release/genesis_rust_backend /genesis_rust_backend + +ENV PORT=8080 +EXPOSE 8080 + +ENTRYPOINT ["/genesis_rust_backend"] diff --git a/genesis_rust_backend/src/config.rs b/genesis_rust_backend/src/config.rs index c12bc73..a29cd88 100644 --- a/genesis_rust_backend/src/config.rs +++ b/genesis_rust_backend/src/config.rs @@ -14,11 +14,7 @@ pub struct EvolutionConfig { pub language: String, pub use_text_feedback: bool, - pub db_backend: String, - pub clickhouse_url: Option, - pub clickhouse_user: Option, - pub clickhouse_password: Option, - pub clickhouse_database: String, + pub database_url: Option, pub llm_backend: String, pub openai_base_url: String, @@ -39,6 +35,8 @@ pub struct EvolutionConfig { pub gepa_min_improvement: f64, pub gepa_exploration_weight: f64, pub gepa_candidate_instructions: Option>, + + pub server_port: u16, } impl Default for EvolutionConfig { @@ -54,12 +52,7 @@ impl Default for EvolutionConfig { language: "python".to_string(), use_text_feedback: false, - db_backend: "in_memory".to_string(), - clickhouse_url: std::env::var("CLICKHOUSE_URL").ok(), - clickhouse_user: std::env::var("CLICKHOUSE_USER").ok(), - clickhouse_password: std::env::var("CLICKHOUSE_PASSWORD").ok(), - clickhouse_database: std::env::var("CLICKHOUSE_DB") - .unwrap_or_else(|_| "default".to_string()), + database_url: std::env::var("DATABASE_URL").ok(), llm_backend: "mock".to_string(), openai_base_url: "https://api.openai.com".to_string(), @@ -79,6 +72,11 @@ impl Default for EvolutionConfig { gepa_min_improvement: 0.0, gepa_exploration_weight: 1.1, gepa_candidate_instructions: None, + + server_port: std::env::var("PORT") + .ok() + .and_then(|p| p.parse().ok()) + .unwrap_or(8080), } } } @@ -108,14 +106,8 @@ impl EvolutionConfig { if self.openai_api_key.is_none() { self.openai_api_key = std::env::var("OPENAI_API_KEY").ok(); } - if self.clickhouse_url.is_none() { - self.clickhouse_url = std::env::var("CLICKHOUSE_URL").ok(); - } - if self.clickhouse_user.is_none() { - self.clickhouse_user = std::env::var("CLICKHOUSE_USER").ok(); - } - if self.clickhouse_password.is_none() { - self.clickhouse_password = std::env::var("CLICKHOUSE_PASSWORD").ok(); + if self.database_url.is_none() { + self.database_url = std::env::var("DATABASE_URL").ok(); } self } diff --git a/genesis_rust_backend/src/core/alma_memory.rs b/genesis_rust_backend/src/core/alma_memory.rs index fb415b0..ef96726 100644 --- a/genesis_rust_backend/src/core/alma_memory.rs +++ b/genesis_rust_backend/src/core/alma_memory.rs @@ -24,7 +24,12 @@ pub struct AlmaMemorySystem { } impl AlmaMemorySystem { - pub fn new(enabled: bool, max_entries: usize, max_retrievals: usize, min_success_delta: f64) -> Self { + pub fn new( + enabled: bool, + max_entries: usize, + max_retrievals: usize, + min_success_delta: f64, + ) -> Self { Self { enabled, max_entries: max_entries.max(1), @@ -60,12 +65,17 @@ impl AlmaMemorySystem { } .to_string(); - let summary = [patch_description, diff_summary, text_feedback, error_message] - .iter() - .filter(|s| !s.is_empty()) - .copied() - .collect::>() - .join(" | "); + let summary = [ + patch_description, + diff_summary, + text_feedback, + error_message, + ] + .iter() + .filter(|s| !s.is_empty()) + .copied() + .collect::>() + .join(" | "); let keywords = extract_keywords(&format!( "{} {} {} {} {} {}", @@ -113,7 +123,8 @@ impl AlmaMemorySystem { .iter() .filter(|k| context_set.contains(*k)) .count() as f64; - let recency = 1.0 / (1.0 + current_generation.saturating_sub(entry.generation) as f64); + let recency = + 1.0 / (1.0 + current_generation.saturating_sub(entry.generation) as f64); let impact = entry.score_delta.abs(); (2.0 * overlap + recency + impact, entry) }) @@ -131,8 +142,14 @@ impl AlmaMemorySystem { } let mut out = String::from("# ALMA Long-Term Memory\n"); - let successes = selected.iter().filter(|e| e.memory_type == "success").collect::>(); - let failures = selected.iter().filter(|e| e.memory_type == "failure").collect::>(); + let successes = selected + .iter() + .filter(|e| e.memory_type == "success") + .collect::>(); + let failures = selected + .iter() + .filter(|e| e.memory_type == "failure") + .collect::>(); if !successes.is_empty() { out.push_str("## Reuse These Successful Patterns\n"); @@ -183,8 +200,21 @@ impl AlmaMemorySystem { fn extract_keywords(text: &str) -> Vec { let stopwords: HashSet<&str> = [ - "the", "and", "for", "with", "that", "this", "from", "into", "code", "patch", - "generation", "score", "delta", "result", "error", + "the", + "and", + "for", + "with", + "that", + "this", + "from", + "into", + "code", + "patch", + "generation", + "score", + "delta", + "result", + "error", ] .iter() .copied() diff --git a/genesis_rust_backend/src/core/gepa_optimizer.rs b/genesis_rust_backend/src/core/gepa_optimizer.rs index 2ba35d3..932b9a1 100644 --- a/genesis_rust_backend/src/core/gepa_optimizer.rs +++ b/genesis_rust_backend/src/core/gepa_optimizer.rs @@ -106,8 +106,8 @@ impl GepaStyleOptimizer { return; } - let instruction = candidate_id - .and_then(|idx| self.candidate_instructions.get(idx).cloned()); + let instruction = + candidate_id.and_then(|idx| self.candidate_instructions.get(idx).cloned()); self.traces.push(GepaTrace { generation, parent_score, @@ -121,7 +121,8 @@ impl GepaStyleOptimizer { candidate_instruction: instruction, }); - self.traces.sort_by(|a, b| b.score_delta.total_cmp(&a.score_delta)); + self.traces + .sort_by(|a, b| b.score_delta.total_cmp(&a.score_delta)); if self.traces.len() > self.max_traces { self.traces.truncate(self.max_traces); } @@ -178,12 +179,7 @@ impl GepaStyleOptimizer { return None; } let mut lines = Vec::new(); - for (idx, t) in self - .traces - .iter() - .take(self.num_fewshot_traces) - .enumerate() - { + for (idx, t) in self.traces.iter().take(self.num_fewshot_traces).enumerate() { lines.push(format!("## Successful Trace {}", idx + 1)); lines.push(format!("- Generation: {}", t.generation)); lines.push(format!("- Score delta: {:+.4}", t.score_delta)); diff --git a/genesis_rust_backend/src/core/novelty_judge.rs b/genesis_rust_backend/src/core/novelty_judge.rs index 85d29ec..c1a402c 100644 --- a/genesis_rust_backend/src/core/novelty_judge.rs +++ b/genesis_rust_backend/src/core/novelty_judge.rs @@ -5,7 +5,9 @@ pub struct NoveltyJudge { impl NoveltyJudge { pub fn new(similarity_threshold: f64) -> Self { - Self { similarity_threshold } + Self { + similarity_threshold, + } } pub fn should_accept(&self, similarity: f64) -> bool { diff --git a/genesis_rust_backend/src/core/runner.rs b/genesis_rust_backend/src/core/runner.rs index 74d891a..201913a 100644 --- a/genesis_rust_backend/src/core/runner.rs +++ b/genesis_rust_backend/src/core/runner.rs @@ -1,6 +1,7 @@ use anyhow::Result; use chrono::Utc; use std::collections::HashMap; +use uuid::Uuid; use crate::config::EvolutionConfig; use crate::core::alma_memory::AlmaMemorySystem; @@ -8,21 +9,23 @@ use crate::core::gepa_optimizer::GepaStyleOptimizer; use crate::core::novelty_judge::NoveltyJudge; use crate::core::sampler::PromptSampler; use crate::core::summarizer::MetaSummarizer; -use crate::database::{build_database, ProgramDatabase}; +use crate::database::PgProgramDatabase; use crate::launch::{build_scheduler, JobScheduler}; -use crate::llm::{build_llm_client, LlmClient}; +use crate::llm::{build_llm_client, LlmClientDyn}; use crate::types::{PatchRequest, Program}; pub struct EvolutionRunner { pub cfg: EvolutionConfig, - pub db: Box, - pub llm: Box, + pub db: Option, + pub llm: Box, pub scheduler: Box, pub prompt_sampler: PromptSampler, pub meta_summarizer: MetaSummarizer, pub novelty_judge: NoveltyJudge, pub alma_memory: AlmaMemorySystem, pub gepa_optimizer: GepaStyleOptimizer, + pub run_id: Option, + programs: Vec, } impl EvolutionRunner { @@ -35,11 +38,12 @@ impl EvolutionRunner { cfg.use_text_feedback, ); - let db = build_database(&cfg).unwrap_or_else(|_| build_database(&EvolutionConfig::default()).expect("default db")); - let llm = build_llm_client(&cfg).unwrap_or_else(|_| build_llm_client(&EvolutionConfig::default()).expect("default llm")); + let llm = build_llm_client(&cfg).unwrap_or_else(|_| { + build_llm_client(&EvolutionConfig::default()).expect("default llm") + }); let scheduler = build_scheduler(&cfg); - let mut runner = Self { + Self { novelty_judge: NoveltyJudge::new(1.0), alma_memory: AlmaMemorySystem::new( cfg.alma_enabled, @@ -56,41 +60,66 @@ impl EvolutionRunner { cfg.gepa_candidate_instructions.clone(), ), cfg, - db, + db: None, llm, scheduler, prompt_sampler, meta_summarizer: MetaSummarizer::default(), - }; + run_id: None, + programs: Vec::new(), + } + } - runner.restore_state(); - runner + pub async fn init_db(&mut self) -> Result<()> { + if let Some(url) = &self.cfg.database_url { + let db = PgProgramDatabase::new(url).await?; + self.db = Some(db); + } + Ok(()) } - pub fn run(&mut self) -> Result<()> { - self.run_generation_0()?; + pub async fn run(&mut self) -> Result<()> { + if let Some(db) = &self.db { + let run_id = db + .create_evolution_run( + self.cfg.task_sys_msg.as_deref().unwrap_or("unknown"), + &serde_json::json!({}), + self.cfg.num_generations as i32, + None, + None, + ) + .await?; + self.run_id = Some(run_id); + } + + self.run_generation_0().await?; for generation in 1..self.cfg.num_generations { - self.run_generation(generation)?; + self.run_generation(generation).await?; + } + + if let (Some(db), Some(run_id)) = (&self.db, self.run_id) { + db.update_evolution_run_status(run_id, "completed", self.cfg.num_generations as i32) + .await?; } self.save_state()?; - if let Some(best) = self.db.get_best_program()? { + if let Some(best) = self.get_best_program() { println!( - "[rust-backend] best program gen={} score={:.4}", + "[genesis] best program gen={} score={:.4}", best.generation, best.combined_score ); } Ok(()) } - fn run_generation_0(&mut self) -> Result<()> { + async fn run_generation_0(&mut self) -> Result<()> { let (sys, user) = self.prompt_sampler.initial_program_prompt(); - let resp = self.llm.query(&user, &sys)?; + let resp = self.llm.query_dyn(&user, &sys).await?; let eval = self.scheduler.run(&resp.content, 0)?; let program = Program { - id: uuid(0), + id: Uuid::new_v4(), code: resp.content, language: self.cfg.language.clone(), parent_id: None, @@ -104,15 +133,37 @@ impl EvolutionRunner { timestamp: Utc::now(), }; - self.db.add(program.clone())?; - self.meta_summarizer.add_evaluated_program(program); + if let (Some(db), Some(run_id)) = (&self.db, self.run_id) { + db.add_individual( + run_id, + program.id, + program.generation, + program.parent_id, + "init", + program.combined_score, + program.combined_score, + &serde_json::json!({}), + false, + 0.0, + 0.0, + 0.0, + "", + program.code.len() as i32, + &program.code, + &program.language, + &program.text_feedback, + ) + .await?; + } + + self.meta_summarizer.add_evaluated_program(program.clone()); + self.programs.push(program); Ok(()) } - fn run_generation(&mut self, generation: usize) -> Result<()> { - let (parent, archive, top_k) = self - .db - .sample(generation)? + async fn run_generation(&mut self, generation: usize) -> Result<()> { + let parent = self + .get_best_program() .ok_or_else(|| anyhow::anyhow!("no parent available for generation {generation}"))?; let meta_recommendations = if self @@ -124,15 +175,26 @@ impl EvolutionRunner { self.meta_summarizer.get_current() }; - let alma_context = self - .alma_memory - .build_prompt_context(generation, &parent.code, &parent.text_feedback); + let alma_context = + self.alma_memory + .build_prompt_context(generation, &parent.code, &parent.text_feedback); let gepa_ctx = self.gepa_optimizer.build_prompt_context(); + let top_k: Vec = { + let mut sorted = self.programs.clone(); + sorted.sort_by(|a, b| b.combined_score.total_cmp(&a.combined_score)); + sorted + .into_iter() + .take(6) + .filter(|p| p.id != parent.id) + .collect() + }; + let archive = top_k.iter().take(4).cloned().collect(); + let req = PatchRequest { parent: parent.clone(), archive_inspirations: archive, - top_k_inspirations: top_k, + top_k_inspirations: top_k.into_iter().take(2).collect(), meta_recommendations, alma_memory_context: alma_context, gepa_instruction: gepa_ctx.candidate_instruction.clone(), @@ -140,7 +202,7 @@ impl EvolutionRunner { }; let (patch_sys, patch_msg, patch_type) = self.prompt_sampler.sample(&req); - let resp = self.llm.query(&patch_msg, &patch_sys)?; + let resp = self.llm.query_dyn(&patch_msg, &patch_sys).await?; let eval = self.scheduler.run(&resp.content, generation)?; if !self.novelty_judge.should_accept(0.0) { @@ -160,11 +222,11 @@ impl EvolutionRunner { } let program = Program { - id: uuid(generation), + id: Uuid::new_v4(), code: resp.content, language: self.cfg.language.clone(), - parent_id: Some(parent.id.clone()), - generation, + parent_id: Some(parent.id), + generation: generation as i32, combined_score: eval.combined_score, correct: eval.correct, public_metrics: HashMap::new(), @@ -174,7 +236,41 @@ impl EvolutionRunner { timestamp: Utc::now(), }; - self.db.add(program.clone())?; + if let (Some(db), Some(run_id)) = (&self.db, self.run_id) { + db.add_individual( + run_id, + program.id, + program.generation, + program.parent_id, + &patch_type, + program.combined_score, + program.combined_score, + &serde_json::json!({}), + false, + 0.0, + 0.0, + 0.0, + "", + program.code.len() as i32, + &program.code, + &program.language, + &program.text_feedback, + ) + .await?; + + let delta = program.combined_score - parent.combined_score; + db.log_lineage( + run_id, + program.id, + Some(parent.id), + program.generation, + &patch_type, + delta, + "", + ) + .await?; + } + self.meta_summarizer.add_evaluated_program(program.clone()); let delta = program.combined_score - parent.combined_score; @@ -183,8 +279,8 @@ impl EvolutionRunner { parent.combined_score, program.combined_score, &patch_type, - "rust-backend-patch", - "Rust backend generated mutation", + "genesis-patch", + "Generated mutation", "diff summary unavailable", gepa_ctx.candidate_id, program.correct, @@ -195,38 +291,40 @@ impl EvolutionRunner { program.combined_score, program.correct, &patch_type, - "rust-backend-patch", - "Rust backend generated mutation", + "genesis-patch", + "Generated mutation", "diff summary unavailable", &program.text_feedback, "", ); println!( - "[rust-backend] gen={} parent={:.4} child={:.4} delta={:+.4}", + "[genesis] gen={} parent={:.4} child={:.4} delta={:+.4}", generation, parent.combined_score, program.combined_score, delta ); + self.programs.push(program); self.save_state()?; Ok(()) } + fn get_best_program(&self) -> Option { + self.programs + .iter() + .filter(|p| p.correct) + .max_by(|a, b| a.combined_score.total_cmp(&b.combined_score)) + .cloned() + .or_else(|| self.programs.last().cloned()) + } + fn save_state(&self) -> Result<()> { self.alma_memory.save_state("alma_memory.json")?; self.gepa_optimizer.save_state("gepa_state.json")?; Ok(()) } - fn restore_state(&mut self) { + pub fn restore_state(&mut self) { let _ = self.alma_memory.load_state("alma_memory.json"); let _ = self.gepa_optimizer.load_state("gepa_state.json"); } } - -fn uuid(generation: usize) -> String { - format!( - "rust-gen-{}-{}", - generation, - Utc::now().timestamp_nanos_opt().unwrap_or_default() - ) -} diff --git a/genesis_rust_backend/src/core/sampler.rs b/genesis_rust_backend/src/core/sampler.rs index f5f7aca..2cd113b 100644 --- a/genesis_rust_backend/src/core/sampler.rs +++ b/genesis_rust_backend/src/core/sampler.rs @@ -57,7 +57,11 @@ impl PromptSampler { let patch_type = self.sample_patch_type(&req.archive_inspirations, &req.top_k_inspirations); let mut history = String::new(); - append_program_list(&mut history, "Archive Inspirations", &req.archive_inspirations); + append_program_list( + &mut history, + "Archive Inspirations", + &req.archive_inspirations, + ); append_program_list(&mut history, "Top-K Inspirations", &req.top_k_inspirations); if let Some(alma_memory_context) = &req.alma_memory_context { diff --git a/genesis_rust_backend/src/database/mod.rs b/genesis_rust_backend/src/database/mod.rs index 6d22d97..3b70d26 100644 --- a/genesis_rust_backend/src/database/mod.rs +++ b/genesis_rust_backend/src/database/mod.rs @@ -1,264 +1,332 @@ +#![allow(clippy::too_many_arguments)] + use anyhow::{Context, Result}; use chrono::{DateTime, Utc}; -use reqwest::blocking::Client; -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; - -use crate::config::EvolutionConfig; -use crate::types::Program; - -pub trait ProgramDatabase { - fn add(&mut self, program: Program) -> Result<()>; - fn get(&self, id: &str) -> Result>; - fn get_best_program(&self) -> Result>; - fn get_top_programs(&self, n: usize) -> Result>; - fn sample(&self, target_generation: usize) -> Result, Vec)>>; -} - -pub fn build_database(cfg: &EvolutionConfig) -> Result> { - match cfg.db_backend.as_str() { - "clickhouse" => Ok(Box::new(ClickHouseProgramDatabase::new(cfg)?)), - _ => Ok(Box::new(InMemoryProgramDatabase::new())), - } -} +use serde_json::Value as JsonValue; +use sqlx::postgres::PgPoolOptions; +use sqlx::PgPool; +use uuid::Uuid; -#[derive(Debug, Default, Clone)] -pub struct InMemoryProgramDatabase { - programs: Vec, +#[derive(Clone)] +pub struct PgProgramDatabase { + pool: PgPool, } -impl InMemoryProgramDatabase { - pub fn new() -> Self { - Self::default() +impl PgProgramDatabase { + pub async fn new(database_url: &str) -> Result { + let pool = PgPoolOptions::new() + .max_connections(5) + .connect(database_url) + .await + .with_context(|| "failed to connect to postgres")?; + Ok(Self { pool }) } -} -impl ProgramDatabase for InMemoryProgramDatabase { - fn add(&mut self, program: Program) -> Result<()> { - self.programs.push(program); - Ok(()) + pub fn from_pool(pool: PgPool) -> Self { + Self { pool } } - fn get(&self, id: &str) -> Result> { - Ok(self.programs.iter().find(|p| p.id == id).cloned()) + pub fn pool(&self) -> &PgPool { + &self.pool } - fn get_best_program(&self) -> Result> { - Ok(self - .programs - .iter() - .filter(|p| p.correct) - .max_by(|a, b| a.combined_score.total_cmp(&b.combined_score)) - .cloned()) - } + // -- evolution_runs -- - fn get_top_programs(&self, n: usize) -> Result> { - let mut ps = self.programs.clone(); - ps.sort_by(|a, b| b.combined_score.total_cmp(&a.combined_score)); - Ok(ps.into_iter().take(n).collect()) - } + pub async fn create_evolution_run( + &self, + task_name: &str, + config: &JsonValue, + population_size: i32, + cluster_type: Option<&str>, + database_path: Option<&str>, + ) -> Result { + let row = sqlx::query_scalar::<_, Uuid>( + r#"INSERT INTO evolution_runs (task_name, config, population_size, cluster_type, database_path) + VALUES ($1, $2, $3, $4, $5) + RETURNING run_id"#, + ) + .bind(task_name) + .bind(config) + .bind(population_size) + .bind(cluster_type) + .bind(database_path) + .fetch_one(&self.pool) + .await + .with_context(|| "failed to insert evolution_run")?; - fn sample(&self, _target_generation: usize) -> Result, Vec)>> { - let parent = self.get_best_program()?.or_else(|| self.programs.last().cloned()); - let Some(parent) = parent else { - return Ok(None); - }; - let mut inspirations = self.get_top_programs(6)?; - inspirations.retain(|p| p.id != parent.id); - let archive = inspirations.iter().take(4).cloned().collect::>(); - let top_k = inspirations.into_iter().take(2).collect::>(); - Ok(Some((parent, archive, top_k))) + Ok(row) } -} -#[derive(Debug)] -pub struct ClickHouseProgramDatabase { - client: Client, - base_url: String, - user: Option, - password: Option, - database: String, -} + pub async fn update_evolution_run_status( + &self, + run_id: Uuid, + status: &str, + total_generations: i32, + ) -> Result<()> { + sqlx::query( + r#"UPDATE evolution_runs + SET status = $1, total_generations = $2, end_time = now() + WHERE run_id = $3"#, + ) + .bind(status) + .bind(total_generations) + .bind(run_id) + .execute(&self.pool) + .await + .with_context(|| "failed to update evolution_run")?; -impl ClickHouseProgramDatabase { - pub fn new(cfg: &EvolutionConfig) -> Result { - let base_url = cfg - .clickhouse_url - .clone() - .unwrap_or_else(|| "http://localhost:8123".to_string()); - - let db = Self { - client: Client::new(), - base_url, - user: cfg.clickhouse_user.clone(), - password: cfg.clickhouse_password.clone(), - database: cfg.clickhouse_database.clone(), - }; - db.init_schema()?; - Ok(db) + Ok(()) } - fn init_schema(&self) -> Result<()> { - let sql = format!( - "CREATE TABLE IF NOT EXISTS {}.programs (\ - id String,\ - code String,\ - language String,\ - parent_id Nullable(String),\ - generation UInt32,\ - combined_score Float64,\ - correct UInt8,\ - public_metrics String,\ - private_metrics String,\ - text_feedback String,\ - metadata String,\ - timestamp DateTime64(3)\ - ) ENGINE = ReplacingMergeTree(timestamp) ORDER BY id", - self.database - ); - self.exec_sql(&sql)?; + // -- generations -- + + pub async fn log_generation( + &self, + run_id: Uuid, + generation: i32, + num_individuals: i32, + best_score: f64, + avg_score: f64, + pareto_size: i32, + total_cost: f64, + metadata: &JsonValue, + ) -> Result<()> { + sqlx::query( + r#"INSERT INTO generations (run_id, generation, num_individuals, best_score, avg_score, pareto_size, total_cost, metadata) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8)"#, + ) + .bind(run_id) + .bind(generation) + .bind(num_individuals) + .bind(best_score) + .bind(avg_score) + .bind(pareto_size) + .bind(total_cost) + .bind(metadata) + .execute(&self.pool) + .await + .with_context(|| "failed to insert generation")?; + Ok(()) } - fn exec_sql(&self, sql: &str) -> Result { - let url = format!("{}/", self.base_url.trim_end_matches('/')); - let mut req = self.client.post(url).query(&[("query", sql)]); - if let Some(user) = &self.user { - req = req.basic_auth(user, self.password.clone()); - } - let resp = req - .send() - .with_context(|| "clickhouse request failed")? - .error_for_status() - .with_context(|| "clickhouse non-success response")?; - Ok(resp.text().unwrap_or_default()) - } + // -- individuals -- - fn select_rows(&self, sql: &str) -> Result> { - let out = self.exec_sql(sql)?; - let mut programs = Vec::new(); - for line in out.lines() { - if line.trim().is_empty() { - continue; - } - let row: DbProgramRow = serde_json::from_str(line) - .with_context(|| "failed to parse clickhouse JSONEachRow line")?; - programs.push(row.try_into_program()?); - } - Ok(programs) - } -} + pub async fn add_individual( + &self, + run_id: Uuid, + individual_id: Uuid, + generation: i32, + parent_id: Option, + mutation_type: &str, + fitness_score: f64, + combined_score: f64, + metrics: &JsonValue, + is_pareto: bool, + api_cost: f64, + embed_cost: f64, + novelty_cost: f64, + code_hash: &str, + code_size: i32, + code: &str, + language: &str, + text_feedback: &str, + ) -> Result<()> { + sqlx::query( + r#"INSERT INTO individuals + (run_id, individual_id, generation, parent_id, mutation_type, + fitness_score, combined_score, metrics, is_pareto, + api_cost, embed_cost, novelty_cost, code_hash, code_size, + code, language, text_feedback) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)"#, + ) + .bind(run_id) + .bind(individual_id) + .bind(generation) + .bind(parent_id) + .bind(mutation_type) + .bind(fitness_score) + .bind(combined_score) + .bind(metrics) + .bind(is_pareto) + .bind(api_cost) + .bind(embed_cost) + .bind(novelty_cost) + .bind(code_hash) + .bind(code_size) + .bind(code) + .bind(language) + .bind(text_feedback) + .execute(&self.pool) + .await + .with_context(|| "failed to insert individual")?; -impl ProgramDatabase for ClickHouseProgramDatabase { - fn add(&mut self, program: Program) -> Result<()> { - let row = DbProgramRow::from_program(&program)?; - let payload = serde_json::to_string(&row)?; - let sql = format!( - "INSERT INTO {}.programs FORMAT JSONEachRow\n{}", - self.database, payload - ); - self.exec_sql(&sql)?; Ok(()) } - fn get(&self, id: &str) -> Result> { - let sql = format!( - "SELECT * FROM {}.programs WHERE id = '{}' ORDER BY timestamp DESC LIMIT 1 FORMAT JSONEachRow", - self.database, - id.replace('\'', "''") - ); - Ok(self.select_rows(&sql)?.into_iter().next()) + pub async fn get_best_individual(&self, run_id: Uuid) -> Result> { + let row = sqlx::query_as::<_, IndividualRow>( + r#"SELECT run_id, individual_id, generation, timestamp, parent_id, + mutation_type, fitness_score, combined_score, metrics, + is_pareto, api_cost, embed_cost, novelty_cost, code_hash, code_size + FROM individuals + WHERE run_id = $1 + ORDER BY combined_score DESC + LIMIT 1"#, + ) + .bind(run_id) + .fetch_optional(&self.pool) + .await + .with_context(|| "failed to fetch best individual")?; + + Ok(row) } - fn get_best_program(&self) -> Result> { - let sql = format!( - "SELECT * FROM {}.programs WHERE correct = 1 ORDER BY combined_score DESC LIMIT 1 FORMAT JSONEachRow", - self.database - ); - Ok(self.select_rows(&sql)?.into_iter().next()) + pub async fn get_top_individuals(&self, run_id: Uuid, n: i64) -> Result> { + let rows = sqlx::query_as::<_, IndividualRow>( + r#"SELECT run_id, individual_id, generation, timestamp, parent_id, + mutation_type, fitness_score, combined_score, metrics, + is_pareto, api_cost, embed_cost, novelty_cost, code_hash, code_size + FROM individuals + WHERE run_id = $1 + ORDER BY combined_score DESC + LIMIT $2"#, + ) + .bind(run_id) + .bind(n) + .fetch_all(&self.pool) + .await + .with_context(|| "failed to fetch top individuals")?; + + Ok(rows) } - fn get_top_programs(&self, n: usize) -> Result> { - let sql = format!( - "SELECT * FROM {}.programs ORDER BY combined_score DESC LIMIT {} FORMAT JSONEachRow", - self.database, - n.max(1) - ); - self.select_rows(&sql) + // -- pareto_fronts -- + + pub async fn log_pareto_front( + &self, + run_id: Uuid, + generation: i32, + individual_id: Uuid, + fitness_score: f64, + combined_score: f64, + metrics: &JsonValue, + ) -> Result<()> { + sqlx::query( + r#"INSERT INTO pareto_fronts (run_id, generation, individual_id, fitness_score, combined_score, metrics) + VALUES ($1, $2, $3, $4, $5, $6)"#, + ) + .bind(run_id) + .bind(generation) + .bind(individual_id) + .bind(fitness_score) + .bind(combined_score) + .bind(metrics) + .execute(&self.pool) + .await + .with_context(|| "failed to insert pareto_front")?; + + Ok(()) } - fn sample(&self, _target_generation: usize) -> Result, Vec)>> { - let parent = self.get_best_program()?; - let Some(parent) = parent else { - return Ok(None); - }; - - let mut inspirations = self.get_top_programs(6)?; - inspirations.retain(|p| p.id != parent.id); - let archive = inspirations.iter().take(4).cloned().collect::>(); - let top_k = inspirations.into_iter().take(2).collect::>(); - Ok(Some((parent, archive, top_k))) + // -- code_lineages -- + + pub async fn log_lineage( + &self, + run_id: Uuid, + child_id: Uuid, + parent_id: Option, + generation: i32, + mutation_type: &str, + fitness_delta: f64, + edit_summary: &str, + ) -> Result<()> { + sqlx::query( + r#"INSERT INTO code_lineages (run_id, child_id, parent_id, generation, mutation_type, fitness_delta, edit_summary) + VALUES ($1, $2, $3, $4, $5, $6, $7)"#, + ) + .bind(run_id) + .bind(child_id) + .bind(parent_id) + .bind(generation) + .bind(mutation_type) + .bind(fitness_delta) + .bind(edit_summary) + .execute(&self.pool) + .await + .with_context(|| "failed to insert code_lineage")?; + + Ok(()) } -} -#[derive(Debug, Clone, Serialize, Deserialize)] -struct DbProgramRow { - id: String, - code: String, - language: String, - parent_id: Option, - generation: u32, - combined_score: f64, - correct: u8, - public_metrics: String, - private_metrics: String, - text_feedback: String, - metadata: String, - timestamp: String, -} + // -- llm_logs -- + + pub async fn log_llm_interaction( + &self, + model: &str, + messages: &JsonValue, + response: &str, + thought: &str, + cost: f64, + execution_time: f64, + metadata: &JsonValue, + ) -> Result<()> { + sqlx::query( + r#"INSERT INTO llm_logs (model, messages, response, thought, cost, execution_time, metadata) + VALUES ($1, $2, $3, $4, $5, $6, $7)"#, + ) + .bind(model) + .bind(messages) + .bind(response) + .bind(thought) + .bind(cost) + .bind(execution_time) + .bind(metadata) + .execute(&self.pool) + .await + .with_context(|| "failed to insert llm_log")?; -impl DbProgramRow { - fn from_program(p: &Program) -> Result { - Ok(Self { - id: p.id.clone(), - code: p.code.clone(), - language: p.language.clone(), - parent_id: p.parent_id.clone(), - generation: p.generation as u32, - combined_score: p.combined_score, - correct: if p.correct { 1 } else { 0 }, - public_metrics: serde_json::to_string(&p.public_metrics)?, - private_metrics: serde_json::to_string(&p.private_metrics)?, - text_feedback: p.text_feedback.clone(), - metadata: serde_json::to_string(&p.metadata)?, - timestamp: p.timestamp.to_rfc3339(), - }) + Ok(()) } - fn try_into_program(self) -> Result { - let public_metrics = serde_json::from_str::>(&self.public_metrics) - .unwrap_or_default(); - let private_metrics = serde_json::from_str::>(&self.private_metrics) - .unwrap_or_default(); - let metadata = serde_json::from_str::>(&self.metadata) - .unwrap_or_default(); - let timestamp = DateTime::parse_from_rfc3339(&self.timestamp) - .map(|dt| dt.with_timezone(&Utc)) - .unwrap_or_else(|_| Utc::now()); - - Ok(Program { - id: self.id, - code: self.code, - language: self.language, - parent_id: self.parent_id, - generation: self.generation as usize, - combined_score: self.combined_score, - correct: self.correct == 1, - public_metrics, - private_metrics, - text_feedback: self.text_feedback, - metadata, - timestamp, - }) + // -- agent_actions -- + + pub async fn log_agent_action( + &self, + action_type: &str, + details: &JsonValue, + metadata: &JsonValue, + ) -> Result<()> { + sqlx::query( + r#"INSERT INTO agent_actions (action_type, details, metadata) + VALUES ($1, $2, $3)"#, + ) + .bind(action_type) + .bind(details) + .bind(metadata) + .execute(&self.pool) + .await + .with_context(|| "failed to insert agent_action")?; + + Ok(()) } } + +#[derive(Debug, Clone, sqlx::FromRow)] +pub struct IndividualRow { + pub run_id: Uuid, + pub individual_id: Uuid, + pub generation: i32, + pub timestamp: DateTime, + pub parent_id: Option, + pub mutation_type: String, + pub fitness_score: f64, + pub combined_score: f64, + pub metrics: JsonValue, + pub is_pareto: bool, + pub api_cost: f64, + pub embed_cost: f64, + pub novelty_cost: f64, + pub code_hash: String, + pub code_size: i32, +} diff --git a/genesis_rust_backend/src/launch/mod.rs b/genesis_rust_backend/src/launch/mod.rs index eb4999a..049ccd6 100644 --- a/genesis_rust_backend/src/launch/mod.rs +++ b/genesis_rust_backend/src/launch/mod.rs @@ -77,7 +77,10 @@ impl JobScheduler for LocalCommandScheduler { let stderr = String::from_utf8_lossy(&output.stderr).to_string(); if let Ok(json) = serde_json::from_str::(&stdout) { - let correct = json.get("correct").and_then(|v| v.as_bool()).unwrap_or(output.status.success()); + let correct = json + .get("correct") + .and_then(|v| v.as_bool()) + .unwrap_or(output.status.success()); let combined_score = json .get("combined_score") .and_then(|v| v.as_f64()) diff --git a/genesis_rust_backend/src/lib.rs b/genesis_rust_backend/src/lib.rs index 5768867..5f94f28 100644 --- a/genesis_rust_backend/src/lib.rs +++ b/genesis_rust_backend/src/lib.rs @@ -7,4 +7,5 @@ pub mod types; pub use config::EvolutionConfig; pub use core::runner::EvolutionRunner; +pub use database::PgProgramDatabase; pub use types::{Program, RunningJob}; diff --git a/genesis_rust_backend/src/llm/mod.rs b/genesis_rust_backend/src/llm/mod.rs index 24ab5ed..5905617 100644 --- a/genesis_rust_backend/src/llm/mod.rs +++ b/genesis_rust_backend/src/llm/mod.rs @@ -1,5 +1,5 @@ use anyhow::{anyhow, Context, Result}; -use reqwest::blocking::Client; +use reqwest::Client; use serde::{Deserialize, Serialize}; use crate::config::EvolutionConfig; @@ -11,30 +11,48 @@ pub struct LlmResponse { pub thought: String, } -pub trait LlmClient { - fn query(&self, user_msg: &str, system_msg: &str) -> Result; +pub trait LlmClient: Send + Sync { + fn query( + &self, + user_msg: &str, + system_msg: &str, + ) -> impl std::future::Future> + Send; } -pub fn build_llm_client(cfg: &EvolutionConfig) -> Result> { +pub fn build_llm_client(cfg: &EvolutionConfig) -> Result> { match cfg.llm_backend.as_str() { "openai" => Ok(Box::new(OpenAiClient::new(cfg)?)), _ => Ok(Box::new(MockLlmClient)), } } +pub trait LlmClientDyn: Send + Sync { + fn query_dyn( + &self, + user_msg: &str, + system_msg: &str, + ) -> std::pin::Pin> + Send + '_>>; +} + #[derive(Debug, Clone, Default)] pub struct MockLlmClient; -impl LlmClient for MockLlmClient { - fn query(&self, user_msg: &str, _system_msg: &str) -> Result { +impl LlmClientDyn for MockLlmClient { + fn query_dyn( + &self, + user_msg: &str, + _system_msg: &str, + ) -> std::pin::Pin> + Send + '_>> { let code = format!( - "mock-patch\nMock mutation from Rust backend\n```python\n# mutated\n{}\n```", + "mock-patch\nMock mutation\n```python\n# mutated\n{}\n```", user_msg.lines().take(3).collect::>().join("\n") ); - Ok(LlmResponse { - content: code, - cost: 0.0, - thought: "mock-thought".to_string(), + Box::pin(async move { + Ok(LlmResponse { + content: code, + cost: 0.0, + thought: "mock-thought".to_string(), + }) }) } } @@ -62,9 +80,16 @@ impl OpenAiClient { } } -impl LlmClient for OpenAiClient { - fn query(&self, user_msg: &str, system_msg: &str) -> Result { - let url = format!("{}/v1/chat/completions", self.base_url.trim_end_matches('/')); +impl LlmClientDyn for OpenAiClient { + fn query_dyn( + &self, + user_msg: &str, + system_msg: &str, + ) -> std::pin::Pin> + Send + '_>> { + let url = format!( + "{}/v1/chat/completions", + self.base_url.trim_end_matches('/') + ); let req = OpenAiChatRequest { model: self.model.clone(), messages: vec![ @@ -80,29 +105,34 @@ impl LlmClient for OpenAiClient { temperature: Some(0.7), }; - let resp = self - .http - .post(url) - .bearer_auth(&self.api_key) - .json(&req) - .send() - .with_context(|| "openai request failed")? - .error_for_status() - .with_context(|| "openai non-success response")? - .json::() - .with_context(|| "failed to decode openai response")?; - - let content = resp - .choices - .first() - .map(|c| c.message.content.clone()) - .unwrap_or_default(); - - let cost = 0.0; - Ok(LlmResponse { - content, - cost, - thought: String::new(), + let http = self.http.clone(); + let api_key = self.api_key.clone(); + + Box::pin(async move { + let resp = http + .post(url) + .bearer_auth(&api_key) + .json(&req) + .send() + .await + .with_context(|| "openai request failed")? + .error_for_status() + .with_context(|| "openai non-success response")? + .json::() + .await + .with_context(|| "failed to decode openai response")?; + + let content = resp + .choices + .first() + .map(|c| c.message.content.clone()) + .unwrap_or_default(); + + Ok(LlmResponse { + content, + cost: 0.0, + thought: String::new(), + }) }) } } diff --git a/genesis_rust_backend/src/main.rs b/genesis_rust_backend/src/main.rs index a9067d0..a89d255 100644 --- a/genesis_rust_backend/src/main.rs +++ b/genesis_rust_backend/src/main.rs @@ -1,8 +1,35 @@ +use std::net::SocketAddr; +use std::sync::Arc; + use anyhow::Result; +use axum::{ + extract::{Path, State}, + http::StatusCode, + routing::get, + Json, Router, +}; +use serde_json::json; +use tower_http::cors::CorsLayer; +use tower_http::trace::TraceLayer; +use tracing_subscriber::EnvFilter; + use genesis_rust_backend::config::EvolutionConfig; -use genesis_rust_backend::core::runner::EvolutionRunner; +use genesis_rust_backend::database::PgProgramDatabase; + +struct AppState { + db: PgProgramDatabase, + #[allow(dead_code)] + config: EvolutionConfig, +} + +#[tokio::main] +async fn main() -> Result<()> { + dotenvy::dotenv().ok(); + + tracing_subscriber::fmt() + .with_env_filter(EnvFilter::from_default_env()) + .init(); -fn main() -> Result<()> { let args: Vec = std::env::args().collect(); let cfg = if let Some(idx) = args.iter().position(|a| a == "--config") { if let Some(path) = args.get(idx + 1) { @@ -14,6 +41,335 @@ fn main() -> Result<()> { EvolutionConfig::default() }; - let mut runner = EvolutionRunner::new(cfg); - runner.run() + let database_url = cfg + .database_url + .as_deref() + .unwrap_or("postgresql://localhost:5432/genesis"); + + let db = PgProgramDatabase::new(database_url).await?; + tracing::info!("connected to postgres"); + + let state = Arc::new(AppState { + db, + config: cfg.clone(), + }); + + let app = Router::new() + .route("/health", get(health)) + .route("/api/runs", get(list_runs)) + .route("/api/runs/{run_id}", get(get_run)) + .route("/api/runs/{run_id}/individuals", get(list_individuals)) + .route("/api/runs/{run_id}/generations", get(list_generations)) + .route( + "/api/runs/{run_id}/generations/{generation}", + get(get_generation), + ) + .route("/api/runs/{run_id}/lineage", get(get_lineage)) + .layer(CorsLayer::permissive()) + .layer(TraceLayer::new_for_http()) + .with_state(state); + + let addr = SocketAddr::from(([0, 0, 0, 0], cfg.server_port)); + tracing::info!("listening on {}", addr); + + let listener = tokio::net::TcpListener::bind(addr).await?; + axum::serve(listener, app).await?; + + Ok(()) +} + +async fn health() -> (StatusCode, Json) { + (StatusCode::OK, Json(json!({"status": "ok"}))) +} + +// --- Runs --- + +async fn list_runs( + State(state): State>, +) -> Result, StatusCode> { + let rows = sqlx::query_as::<_, EvolutionRunRow>( + r#"SELECT run_id, start_time, end_time, task_name, status, + total_generations, population_size, config + FROM evolution_runs + ORDER BY start_time DESC + LIMIT 50"#, + ) + .fetch_all(state.db.pool()) + .await + .map_err(|e| { + tracing::error!("failed to list runs: {e}"); + StatusCode::INTERNAL_SERVER_ERROR + })?; + + let runs: Vec = rows.into_iter().map(|r| run_to_json(&r)).collect(); + Ok(Json(json!({"runs": runs}))) +} + +async fn get_run( + State(state): State>, + Path(run_id): Path, +) -> Result, StatusCode> { + let row = sqlx::query_as::<_, EvolutionRunRow>( + r#"SELECT run_id, start_time, end_time, task_name, status, + total_generations, population_size, config + FROM evolution_runs + WHERE run_id = $1"#, + ) + .bind(run_id) + .fetch_optional(state.db.pool()) + .await + .map_err(|e| { + tracing::error!("failed to get run: {e}"); + StatusCode::INTERNAL_SERVER_ERROR + })? + .ok_or(StatusCode::NOT_FOUND)?; + + Ok(Json(run_to_json(&row))) +} + +fn run_to_json(r: &EvolutionRunRow) -> serde_json::Value { + json!({ + "run_id": r.run_id, + "start_time": r.start_time, + "end_time": r.end_time, + "task_name": r.task_name, + "status": r.status, + "total_generations": r.total_generations, + "population_size": r.population_size, + "config": r.config, + }) +} + +// --- Individuals --- + +async fn list_individuals( + State(state): State>, + Path(run_id): Path, +) -> Result, StatusCode> { + let rows = sqlx::query_as::<_, IndividualApiRow>( + r#"SELECT i.run_id, i.individual_id, i.generation, i.timestamp, + i.parent_id, i.mutation_type, i.fitness_score, + i.combined_score, i.metrics, i.is_pareto, + i.api_cost, i.embed_cost, i.novelty_cost, + i.code_hash, i.code_size, i.code, i.language, i.text_feedback + FROM individuals i + WHERE i.run_id = $1 + ORDER BY i.generation ASC, i.combined_score DESC"#, + ) + .bind(run_id) + .fetch_all(state.db.pool()) + .await + .map_err(|e| { + tracing::error!("failed to list individuals: {e}"); + StatusCode::INTERNAL_SERVER_ERROR + })?; + + let individuals: Vec = rows + .into_iter() + .map(|r| { + json!({ + "id": r.individual_id, + "parent_id": r.parent_id, + "code": r.code, + "language": r.language, + "generation": r.generation, + "timestamp": r.timestamp, + "agent_name": r.mutation_type, + "combined_score": r.combined_score, + "fitness_score": r.fitness_score, + "metrics": r.metrics, + "text_feedback": r.text_feedback, + "metadata": { + "patch_name": r.mutation_type, + "patch_type": r.mutation_type, + "api_cost": r.api_cost, + "embed_cost": r.embed_cost, + "novelty_cost": r.novelty_cost, + }, + "correct": r.is_pareto, + "is_pareto": r.is_pareto, + "code_hash": r.code_hash, + "code_size": r.code_size, + }) + }) + .collect(); + + Ok(Json(json!({"individuals": individuals}))) +} + +// --- Generations --- + +async fn list_generations( + State(state): State>, + Path(run_id): Path, +) -> Result, StatusCode> { + let rows = sqlx::query_as::<_, GenerationRow>( + r#"SELECT run_id, generation, timestamp, num_individuals, + best_score, avg_score, pareto_size, total_cost, metadata + FROM generations + WHERE run_id = $1 + ORDER BY generation ASC"#, + ) + .bind(run_id) + .fetch_all(state.db.pool()) + .await + .map_err(|e| { + tracing::error!("failed to list generations: {e}"); + StatusCode::INTERNAL_SERVER_ERROR + })?; + + let generations: Vec = rows + .into_iter() + .map(|r| { + json!({ + "generation": r.generation, + "timestamp": r.timestamp, + "num_individuals": r.num_individuals, + "best_score": r.best_score, + "avg_score": r.avg_score, + "pareto_size": r.pareto_size, + "total_cost": r.total_cost, + "metadata": r.metadata, + }) + }) + .collect(); + + Ok(Json(json!({"generations": generations}))) +} + +async fn get_generation( + State(state): State>, + Path((run_id, generation)): Path<(uuid::Uuid, i32)>, +) -> Result, StatusCode> { + let row = sqlx::query_as::<_, GenerationRow>( + r#"SELECT run_id, generation, timestamp, num_individuals, + best_score, avg_score, pareto_size, total_cost, metadata + FROM generations + WHERE run_id = $1 AND generation = $2"#, + ) + .bind(run_id) + .bind(generation) + .fetch_optional(state.db.pool()) + .await + .map_err(|e| { + tracing::error!("failed to get generation: {e}"); + StatusCode::INTERNAL_SERVER_ERROR + })? + .ok_or(StatusCode::NOT_FOUND)?; + + Ok(Json(json!({ + "generation": row.generation, + "timestamp": row.timestamp, + "num_individuals": row.num_individuals, + "best_score": row.best_score, + "avg_score": row.avg_score, + "pareto_size": row.pareto_size, + "total_cost": row.total_cost, + "metadata": row.metadata, + }))) +} + +// --- Lineage --- + +async fn get_lineage( + State(state): State>, + Path(run_id): Path, +) -> Result, StatusCode> { + let rows = sqlx::query_as::<_, LineageRow>( + r#"SELECT id, run_id, child_id, parent_id, generation, + mutation_type, timestamp, fitness_delta, edit_summary + FROM code_lineages + WHERE run_id = $1 + ORDER BY generation ASC"#, + ) + .bind(run_id) + .fetch_all(state.db.pool()) + .await + .map_err(|e| { + tracing::error!("failed to get lineage: {e}"); + StatusCode::INTERNAL_SERVER_ERROR + })?; + + let edges: Vec = rows + .into_iter() + .map(|r| { + json!({ + "id": r.id, + "child_id": r.child_id, + "parent_id": r.parent_id, + "generation": r.generation, + "mutation_type": r.mutation_type, + "fitness_delta": r.fitness_delta, + "edit_summary": r.edit_summary, + }) + }) + .collect(); + + Ok(Json(json!({"edges": edges}))) +} + +// --- Row types --- + +#[derive(sqlx::FromRow)] +struct EvolutionRunRow { + run_id: uuid::Uuid, + start_time: chrono::DateTime, + end_time: Option>, + task_name: String, + status: String, + total_generations: i32, + population_size: i32, + config: serde_json::Value, +} + +#[derive(sqlx::FromRow)] +struct IndividualApiRow { + #[allow(dead_code)] + run_id: uuid::Uuid, + individual_id: uuid::Uuid, + generation: i32, + timestamp: chrono::DateTime, + parent_id: Option, + mutation_type: String, + fitness_score: f64, + combined_score: f64, + metrics: serde_json::Value, + is_pareto: bool, + api_cost: f64, + embed_cost: f64, + novelty_cost: f64, + code_hash: String, + code_size: i32, + code: String, + language: String, + text_feedback: String, +} + +#[derive(sqlx::FromRow)] +struct GenerationRow { + #[allow(dead_code)] + run_id: uuid::Uuid, + generation: i32, + timestamp: chrono::DateTime, + num_individuals: i32, + best_score: f64, + avg_score: f64, + pareto_size: i32, + total_cost: f64, + metadata: serde_json::Value, +} + +#[derive(sqlx::FromRow)] +struct LineageRow { + id: uuid::Uuid, + #[allow(dead_code)] + run_id: uuid::Uuid, + child_id: uuid::Uuid, + parent_id: Option, + generation: i32, + mutation_type: String, + #[allow(dead_code)] + timestamp: chrono::DateTime, + fitness_delta: f64, + edit_summary: String, } diff --git a/genesis_rust_backend/src/types.rs b/genesis_rust_backend/src/types.rs index 533e074..de973a5 100644 --- a/genesis_rust_backend/src/types.rs +++ b/genesis_rust_backend/src/types.rs @@ -1,14 +1,15 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; +use uuid::Uuid; #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct Program { - pub id: String, + pub id: Uuid, pub code: String, pub language: String, - pub parent_id: Option, - pub generation: usize, + pub parent_id: Option, + pub generation: i32, pub combined_score: f64, pub correct: bool, pub public_metrics: HashMap, @@ -21,8 +22,8 @@ pub struct Program { #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct RunningJob { pub job_id: String, - pub generation: usize, - pub parent_id: Option, + pub generation: i32, + pub parent_id: Option, pub patch_type: String, pub patch_name: Option, pub patch_description: Option, diff --git a/genesis_rust_backend/tests/common/mod.rs b/genesis_rust_backend/tests/common/mod.rs new file mode 100644 index 0000000..daef1b5 --- /dev/null +++ b/genesis_rust_backend/tests/common/mod.rs @@ -0,0 +1,45 @@ +use sqlx::PgPool; +use testcontainers::runners::AsyncRunner; +use testcontainers::ContainerAsync; +use testcontainers::ImageExt; +use testcontainers_modules::postgres::Postgres; + +pub struct TestDb { + pub pool: PgPool, + _container: ContainerAsync, +} + +impl TestDb { + pub async fn new() -> Self { + let container = Postgres::default() + .with_tag("15") + .start() + .await + .expect("failed to start postgres container"); + + let host_port = container + .get_host_port_ipv4(5432) + .await + .expect("failed to get postgres port"); + + let url = format!("postgresql://postgres:postgres@127.0.0.1:{host_port}/postgres"); + let pool = PgPool::connect(&url) + .await + .expect("failed to connect to test postgres"); + + let ddl = std::fs::read_to_string( + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../migrations/full_ddl.sql"), + ) + .expect("failed to read full_ddl.sql"); + + sqlx::raw_sql(&ddl) + .execute(&pool) + .await + .expect("failed to apply DDL"); + + Self { + pool, + _container: container, + } + } +} diff --git a/genesis_rust_backend/tests/db_integration.rs b/genesis_rust_backend/tests/db_integration.rs new file mode 100644 index 0000000..48b8bf0 --- /dev/null +++ b/genesis_rust_backend/tests/db_integration.rs @@ -0,0 +1,275 @@ +mod common; + +use genesis_rust_backend::database::PgProgramDatabase; +use serde_json::json; +use uuid::Uuid; + +async fn db_from_test() -> PgProgramDatabase { + let test_db = common::TestDb::new().await; + // Leak the container handle so it stays alive for the test duration. + // Each test gets its own container so there is no cross-contamination. + let pool = test_db.pool.clone(); + std::mem::forget(test_db); + PgProgramDatabase::from_pool(pool) +} + +#[tokio::test] +async fn create_and_list_evolution_run() { + let db = db_from_test().await; + + let run_id = db + .create_evolution_run( + "circle_packing", + &json!({"lr": 0.01}), + 20, + Some("local"), + None, + ) + .await + .expect("create_evolution_run failed"); + + let row = + sqlx::query_scalar::<_, String>("SELECT task_name FROM evolution_runs WHERE run_id = $1") + .bind(run_id) + .fetch_one(db.pool()) + .await + .expect("fetch failed"); + + assert_eq!(row, "circle_packing"); +} + +#[tokio::test] +async fn update_evolution_run_status() { + let db = db_from_test().await; + + let run_id = db + .create_evolution_run("test_task", &json!({}), 10, None, None) + .await + .unwrap(); + + db.update_evolution_run_status(run_id, "completed", 5) + .await + .unwrap(); + + let (status, gens) = sqlx::query_as::<_, (String, i32)>( + "SELECT status, total_generations FROM evolution_runs WHERE run_id = $1", + ) + .bind(run_id) + .fetch_one(db.pool()) + .await + .unwrap(); + + assert_eq!(status, "completed"); + assert_eq!(gens, 5); +} + +#[tokio::test] +async fn add_and_get_best_individual() { + let db = db_from_test().await; + + let run_id = db + .create_evolution_run("test_task", &json!({}), 10, None, None) + .await + .unwrap(); + + let id_a = Uuid::new_v4(); + let id_b = Uuid::new_v4(); + + db.add_individual( + run_id, + id_a, + 0, + None, + "diff", + 0.5, + 0.5, + &json!({"acc": 0.5}), + false, + 0.01, + 0.0, + 0.0, + "aaa", + 100, + "print('hello')", + "python", + "", + ) + .await + .unwrap(); + + db.add_individual( + run_id, + id_b, + 1, + Some(id_a), + "full", + 0.9, + 0.9, + &json!({"acc": 0.9}), + true, + 0.02, + 0.0, + 0.0, + "bbb", + 120, + "print('world')", + "python", + "looks good", + ) + .await + .unwrap(); + + let best = db.get_best_individual(run_id).await.unwrap().unwrap(); + assert_eq!(best.individual_id, id_b); + assert!((best.combined_score - 0.9).abs() < 1e-9); +} + +#[tokio::test] +async fn get_top_individuals_ordering() { + let db = db_from_test().await; + + let run_id = db + .create_evolution_run("test_task", &json!({}), 10, None, None) + .await + .unwrap(); + + for i in 0..5 { + db.add_individual( + run_id, + Uuid::new_v4(), + i, + None, + "diff", + i as f64 * 0.1, + i as f64 * 0.1, + &json!({}), + false, + 0.0, + 0.0, + 0.0, + &format!("hash_{i}"), + 100, + &format!("code_{i}"), + "python", + "", + ) + .await + .unwrap(); + } + + let top = db.get_top_individuals(run_id, 3).await.unwrap(); + assert_eq!(top.len(), 3); + assert!(top[0].combined_score >= top[1].combined_score); + assert!(top[1].combined_score >= top[2].combined_score); +} + +#[tokio::test] +async fn log_generation() { + let db = db_from_test().await; + + let run_id = db + .create_evolution_run("test_task", &json!({}), 10, None, None) + .await + .unwrap(); + + db.log_generation(run_id, 0, 10, 0.8, 0.5, 3, 0.1, &json!({"note": "gen0"})) + .await + .unwrap(); + + let count = sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM generations WHERE run_id = $1") + .bind(run_id) + .fetch_one(db.pool()) + .await + .unwrap(); + + assert_eq!(count, 1); +} + +#[tokio::test] +async fn log_pareto_front() { + let db = db_from_test().await; + + let run_id = db + .create_evolution_run("test_task", &json!({}), 10, None, None) + .await + .unwrap(); + + let ind_id = Uuid::new_v4(); + db.log_pareto_front(run_id, 0, ind_id, 0.8, 0.75, &json!({})) + .await + .unwrap(); + + let count = + sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM pareto_fronts WHERE run_id = $1") + .bind(run_id) + .fetch_one(db.pool()) + .await + .unwrap(); + + assert_eq!(count, 1); +} + +#[tokio::test] +async fn log_lineage() { + let db = db_from_test().await; + + let run_id = db + .create_evolution_run("test_task", &json!({}), 10, None, None) + .await + .unwrap(); + + let child = Uuid::new_v4(); + let parent = Uuid::new_v4(); + db.log_lineage(run_id, child, Some(parent), 1, "diff", 0.1, "improved loop") + .await + .unwrap(); + + let count = + sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM code_lineages WHERE run_id = $1") + .bind(run_id) + .fetch_one(db.pool()) + .await + .unwrap(); + + assert_eq!(count, 1); +} + +#[tokio::test] +async fn log_llm_interaction() { + let db = db_from_test().await; + + db.log_llm_interaction( + "gpt-4", + &json!([{"role": "user", "content": "hello"}]), + "response text", + "thought text", + 0.005, + 1.2, + &json!({}), + ) + .await + .unwrap(); + + let count = sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM llm_logs") + .fetch_one(db.pool()) + .await + .unwrap(); + + assert_eq!(count, 1); +} + +#[tokio::test] +async fn log_agent_action() { + let db = db_from_test().await; + + db.log_agent_action("mutate", &json!({"gen": 3}), &json!({})) + .await + .unwrap(); + + let count = sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM agent_actions") + .fetch_one(db.pool()) + .await + .unwrap(); + + assert_eq!(count, 1); +} diff --git a/genesis_rust_backend/tests/memory_tests.rs b/genesis_rust_backend/tests/memory_tests.rs index 5890cdb..131f33b 100644 --- a/genesis_rust_backend/tests/memory_tests.rs +++ b/genesis_rust_backend/tests/memory_tests.rs @@ -42,5 +42,8 @@ fn gepa_returns_candidate_and_fewshot() { ); let ctx2 = gepa.build_prompt_context(); - assert!(ctx2.fewshot_examples.unwrap_or_default().contains("Successful Trace")); + assert!(ctx2 + .fewshot_examples + .unwrap_or_default() + .contains("Successful Trace")); } diff --git a/migrations/changelogs/001-evolution-runs.sql b/migrations/changelogs/001-evolution-runs.sql new file mode 100644 index 0000000..174968f --- /dev/null +++ b/migrations/changelogs/001-evolution-runs.sql @@ -0,0 +1,20 @@ +--liquibase formatted sql + +--changeset genesis:001-create-evolution-runs +CREATE TABLE evolution_runs ( + run_id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + start_time TIMESTAMPTZ NOT NULL DEFAULT now(), + end_time TIMESTAMPTZ, + task_name TEXT NOT NULL, + config JSONB NOT NULL DEFAULT '{}', + status TEXT NOT NULL DEFAULT 'running', + total_generations INTEGER NOT NULL DEFAULT 0, + population_size INTEGER NOT NULL DEFAULT 0, + cluster_type TEXT, + database_path TEXT +); + +CREATE INDEX idx_evolution_runs_task ON evolution_runs(task_name); +CREATE INDEX idx_evolution_runs_status ON evolution_runs(status); +CREATE INDEX idx_evolution_runs_start_time ON evolution_runs(start_time DESC); +--rollback DROP TABLE evolution_runs; diff --git a/migrations/changelogs/002-generations.sql b/migrations/changelogs/002-generations.sql new file mode 100644 index 0000000..50a775f --- /dev/null +++ b/migrations/changelogs/002-generations.sql @@ -0,0 +1,19 @@ +--liquibase formatted sql + +--changeset genesis:002-create-generations +CREATE TABLE generations ( + run_id UUID NOT NULL REFERENCES evolution_runs(run_id) ON DELETE CASCADE, + generation INTEGER NOT NULL, + timestamp TIMESTAMPTZ NOT NULL DEFAULT now(), + num_individuals INTEGER NOT NULL DEFAULT 0, + best_score DOUBLE PRECISION NOT NULL DEFAULT 0, + avg_score DOUBLE PRECISION NOT NULL DEFAULT 0, + pareto_size INTEGER NOT NULL DEFAULT 0, + total_cost DOUBLE PRECISION NOT NULL DEFAULT 0, + metadata JSONB NOT NULL DEFAULT '{}', + + PRIMARY KEY (run_id, generation) +); + +CREATE INDEX idx_generations_run ON generations(run_id); +--rollback DROP TABLE generations; diff --git a/migrations/changelogs/003-individuals.sql b/migrations/changelogs/003-individuals.sql new file mode 100644 index 0000000..9b8d804 --- /dev/null +++ b/migrations/changelogs/003-individuals.sql @@ -0,0 +1,27 @@ +--liquibase formatted sql + +--changeset genesis:003-create-individuals +CREATE TABLE individuals ( + run_id UUID NOT NULL REFERENCES evolution_runs(run_id) ON DELETE CASCADE, + individual_id UUID NOT NULL DEFAULT gen_random_uuid(), + generation INTEGER NOT NULL, + timestamp TIMESTAMPTZ NOT NULL DEFAULT now(), + parent_id UUID, + mutation_type TEXT NOT NULL, + fitness_score DOUBLE PRECISION NOT NULL DEFAULT 0, + combined_score DOUBLE PRECISION NOT NULL DEFAULT 0, + metrics JSONB NOT NULL DEFAULT '{}', + is_pareto BOOLEAN NOT NULL DEFAULT false, + api_cost DOUBLE PRECISION NOT NULL DEFAULT 0, + embed_cost DOUBLE PRECISION NOT NULL DEFAULT 0, + novelty_cost DOUBLE PRECISION NOT NULL DEFAULT 0, + code_hash TEXT NOT NULL DEFAULT '', + code_size INTEGER NOT NULL DEFAULT 0, + + PRIMARY KEY (run_id, individual_id) +); + +CREATE INDEX idx_individuals_run_gen ON individuals(run_id, generation); +CREATE INDEX idx_individuals_score ON individuals(combined_score DESC); +CREATE INDEX idx_individuals_parent ON individuals(parent_id); +--rollback DROP TABLE individuals; diff --git a/migrations/changelogs/004-pareto-fronts.sql b/migrations/changelogs/004-pareto-fronts.sql new file mode 100644 index 0000000..8ffbd3b --- /dev/null +++ b/migrations/changelogs/004-pareto-fronts.sql @@ -0,0 +1,17 @@ +--liquibase formatted sql + +--changeset genesis:004-create-pareto-fronts +CREATE TABLE pareto_fronts ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + run_id UUID NOT NULL REFERENCES evolution_runs(run_id) ON DELETE CASCADE, + generation INTEGER NOT NULL, + timestamp TIMESTAMPTZ NOT NULL DEFAULT now(), + individual_id UUID NOT NULL, + fitness_score DOUBLE PRECISION NOT NULL DEFAULT 0, + combined_score DOUBLE PRECISION NOT NULL DEFAULT 0, + metrics JSONB NOT NULL DEFAULT '{}' +); + +CREATE INDEX idx_pareto_fronts_run_gen ON pareto_fronts(run_id, generation); +CREATE INDEX idx_pareto_fronts_score ON pareto_fronts(fitness_score DESC); +--rollback DROP TABLE pareto_fronts; diff --git a/migrations/changelogs/005-code-lineages.sql b/migrations/changelogs/005-code-lineages.sql new file mode 100644 index 0000000..b4e37f6 --- /dev/null +++ b/migrations/changelogs/005-code-lineages.sql @@ -0,0 +1,19 @@ +--liquibase formatted sql + +--changeset genesis:005-create-code-lineages +CREATE TABLE code_lineages ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + run_id UUID NOT NULL REFERENCES evolution_runs(run_id) ON DELETE CASCADE, + child_id UUID NOT NULL, + parent_id UUID, + generation INTEGER NOT NULL, + mutation_type TEXT NOT NULL, + timestamp TIMESTAMPTZ NOT NULL DEFAULT now(), + fitness_delta DOUBLE PRECISION NOT NULL DEFAULT 0, + edit_summary TEXT NOT NULL DEFAULT '' +); + +CREATE INDEX idx_code_lineages_run_gen ON code_lineages(run_id, generation); +CREATE INDEX idx_code_lineages_child ON code_lineages(child_id); +CREATE INDEX idx_code_lineages_parent ON code_lineages(parent_id); +--rollback DROP TABLE code_lineages; diff --git a/migrations/changelogs/006-llm-logs.sql b/migrations/changelogs/006-llm-logs.sql new file mode 100644 index 0000000..c90d824 --- /dev/null +++ b/migrations/changelogs/006-llm-logs.sql @@ -0,0 +1,18 @@ +--liquibase formatted sql + +--changeset genesis:006-create-llm-logs +CREATE TABLE llm_logs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + timestamp TIMESTAMPTZ NOT NULL DEFAULT now(), + model TEXT NOT NULL, + messages JSONB NOT NULL DEFAULT '[]', + response TEXT NOT NULL DEFAULT '', + thought TEXT NOT NULL DEFAULT '', + cost DOUBLE PRECISION NOT NULL DEFAULT 0, + execution_time DOUBLE PRECISION NOT NULL DEFAULT 0, + metadata JSONB NOT NULL DEFAULT '{}' +); + +CREATE INDEX idx_llm_logs_timestamp ON llm_logs(timestamp DESC); +CREATE INDEX idx_llm_logs_model ON llm_logs(model); +--rollback DROP TABLE llm_logs; diff --git a/migrations/changelogs/007-agent-actions.sql b/migrations/changelogs/007-agent-actions.sql new file mode 100644 index 0000000..199eb76 --- /dev/null +++ b/migrations/changelogs/007-agent-actions.sql @@ -0,0 +1,14 @@ +--liquibase formatted sql + +--changeset genesis:007-create-agent-actions +CREATE TABLE agent_actions ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + timestamp TIMESTAMPTZ NOT NULL DEFAULT now(), + action_type TEXT NOT NULL, + details JSONB NOT NULL DEFAULT '{}', + metadata JSONB NOT NULL DEFAULT '{}' +); + +CREATE INDEX idx_agent_actions_timestamp ON agent_actions(timestamp DESC); +CREATE INDEX idx_agent_actions_type ON agent_actions(action_type); +--rollback DROP TABLE agent_actions; diff --git a/migrations/changelogs/008-add-code-to-individuals.sql b/migrations/changelogs/008-add-code-to-individuals.sql new file mode 100644 index 0000000..66146d2 --- /dev/null +++ b/migrations/changelogs/008-add-code-to-individuals.sql @@ -0,0 +1,7 @@ +--liquibase formatted sql + +--changeset genesis:008-add-code-to-individuals +ALTER TABLE individuals ADD COLUMN code TEXT NOT NULL DEFAULT ''; +ALTER TABLE individuals ADD COLUMN language TEXT NOT NULL DEFAULT 'python'; +ALTER TABLE individuals ADD COLUMN text_feedback TEXT NOT NULL DEFAULT ''; +--rollback ALTER TABLE individuals DROP COLUMN text_feedback; ALTER TABLE individuals DROP COLUMN language; ALTER TABLE individuals DROP COLUMN code; diff --git a/migrations/changelogs/db.changelog-master.yaml b/migrations/changelogs/db.changelog-master.yaml new file mode 100644 index 0000000..02ce552 --- /dev/null +++ b/migrations/changelogs/db.changelog-master.yaml @@ -0,0 +1,25 @@ +databaseChangeLog: + - include: + file: changelogs/001-evolution-runs.sql + relativeToChangelogFile: false + - include: + file: changelogs/002-generations.sql + relativeToChangelogFile: false + - include: + file: changelogs/003-individuals.sql + relativeToChangelogFile: false + - include: + file: changelogs/004-pareto-fronts.sql + relativeToChangelogFile: false + - include: + file: changelogs/005-code-lineages.sql + relativeToChangelogFile: false + - include: + file: changelogs/006-llm-logs.sql + relativeToChangelogFile: false + - include: + file: changelogs/007-agent-actions.sql + relativeToChangelogFile: false + - include: + file: changelogs/008-add-code-to-individuals.sql + relativeToChangelogFile: false diff --git a/migrations/full_ddl.sql b/migrations/full_ddl.sql new file mode 100644 index 0000000..33f5a55 --- /dev/null +++ b/migrations/full_ddl.sql @@ -0,0 +1,350 @@ +-- +-- PostgreSQL database dump +-- + + +-- Dumped from database version 15.15 (Debian 15.15-1.pgdg13+1) +-- Dumped by pg_dump version 15.15 (Debian 15.15-1.pgdg13+1) + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +SET default_tablespace = ''; + + +-- +-- Name: agent_actions; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.agent_actions ( + id uuid DEFAULT gen_random_uuid() NOT NULL, + "timestamp" timestamp with time zone DEFAULT now() NOT NULL, + action_type text NOT NULL, + details jsonb DEFAULT '{}'::jsonb NOT NULL, + metadata jsonb DEFAULT '{}'::jsonb NOT NULL +); + + +-- +-- Name: code_lineages; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.code_lineages ( + id uuid DEFAULT gen_random_uuid() NOT NULL, + run_id uuid NOT NULL, + child_id uuid NOT NULL, + parent_id uuid, + generation integer NOT NULL, + mutation_type text NOT NULL, + "timestamp" timestamp with time zone DEFAULT now() NOT NULL, + fitness_delta double precision DEFAULT 0 NOT NULL, + edit_summary text DEFAULT ''::text NOT NULL +); + + +-- +-- Name: evolution_runs; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.evolution_runs ( + run_id uuid DEFAULT gen_random_uuid() NOT NULL, + start_time timestamp with time zone DEFAULT now() NOT NULL, + end_time timestamp with time zone, + task_name text NOT NULL, + config jsonb DEFAULT '{}'::jsonb NOT NULL, + status text DEFAULT 'running'::text NOT NULL, + total_generations integer DEFAULT 0 NOT NULL, + population_size integer DEFAULT 0 NOT NULL, + cluster_type text, + database_path text +); + + +-- +-- Name: generations; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.generations ( + run_id uuid NOT NULL, + generation integer NOT NULL, + "timestamp" timestamp with time zone DEFAULT now() NOT NULL, + num_individuals integer DEFAULT 0 NOT NULL, + best_score double precision DEFAULT 0 NOT NULL, + avg_score double precision DEFAULT 0 NOT NULL, + pareto_size integer DEFAULT 0 NOT NULL, + total_cost double precision DEFAULT 0 NOT NULL, + metadata jsonb DEFAULT '{}'::jsonb NOT NULL +); + + +-- +-- Name: individuals; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.individuals ( + run_id uuid NOT NULL, + individual_id uuid DEFAULT gen_random_uuid() NOT NULL, + generation integer NOT NULL, + "timestamp" timestamp with time zone DEFAULT now() NOT NULL, + parent_id uuid, + mutation_type text NOT NULL, + fitness_score double precision DEFAULT 0 NOT NULL, + combined_score double precision DEFAULT 0 NOT NULL, + metrics jsonb DEFAULT '{}'::jsonb NOT NULL, + is_pareto boolean DEFAULT false NOT NULL, + api_cost double precision DEFAULT 0 NOT NULL, + embed_cost double precision DEFAULT 0 NOT NULL, + novelty_cost double precision DEFAULT 0 NOT NULL, + code_hash text DEFAULT ''::text NOT NULL, + code_size integer DEFAULT 0 NOT NULL, + code text DEFAULT ''::text NOT NULL, + language text DEFAULT 'python'::text NOT NULL, + text_feedback text DEFAULT ''::text NOT NULL +); + + +-- +-- Name: llm_logs; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.llm_logs ( + id uuid DEFAULT gen_random_uuid() NOT NULL, + "timestamp" timestamp with time zone DEFAULT now() NOT NULL, + model text NOT NULL, + messages jsonb DEFAULT '[]'::jsonb NOT NULL, + response text DEFAULT ''::text NOT NULL, + thought text DEFAULT ''::text NOT NULL, + cost double precision DEFAULT 0 NOT NULL, + execution_time double precision DEFAULT 0 NOT NULL, + metadata jsonb DEFAULT '{}'::jsonb NOT NULL +); + + +-- +-- Name: pareto_fronts; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.pareto_fronts ( + id uuid DEFAULT gen_random_uuid() NOT NULL, + run_id uuid NOT NULL, + generation integer NOT NULL, + "timestamp" timestamp with time zone DEFAULT now() NOT NULL, + individual_id uuid NOT NULL, + fitness_score double precision DEFAULT 0 NOT NULL, + combined_score double precision DEFAULT 0 NOT NULL, + metrics jsonb DEFAULT '{}'::jsonb NOT NULL +); + + +-- +-- Name: agent_actions agent_actions_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.agent_actions + ADD CONSTRAINT agent_actions_pkey PRIMARY KEY (id); + + +-- +-- Name: code_lineages code_lineages_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.code_lineages + ADD CONSTRAINT code_lineages_pkey PRIMARY KEY (id); + + +-- +-- Name: evolution_runs evolution_runs_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.evolution_runs + ADD CONSTRAINT evolution_runs_pkey PRIMARY KEY (run_id); + + +-- +-- Name: generations generations_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.generations + ADD CONSTRAINT generations_pkey PRIMARY KEY (run_id, generation); + + +-- +-- Name: individuals individuals_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.individuals + ADD CONSTRAINT individuals_pkey PRIMARY KEY (run_id, individual_id); + + +-- +-- Name: llm_logs llm_logs_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.llm_logs + ADD CONSTRAINT llm_logs_pkey PRIMARY KEY (id); + + +-- +-- Name: pareto_fronts pareto_fronts_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.pareto_fronts + ADD CONSTRAINT pareto_fronts_pkey PRIMARY KEY (id); + + +-- +-- Name: idx_agent_actions_timestamp; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_agent_actions_timestamp ON public.agent_actions USING btree ("timestamp" DESC); + + +-- +-- Name: idx_agent_actions_type; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_agent_actions_type ON public.agent_actions USING btree (action_type); + + +-- +-- Name: idx_code_lineages_child; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_code_lineages_child ON public.code_lineages USING btree (child_id); + + +-- +-- Name: idx_code_lineages_parent; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_code_lineages_parent ON public.code_lineages USING btree (parent_id); + + +-- +-- Name: idx_code_lineages_run_gen; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_code_lineages_run_gen ON public.code_lineages USING btree (run_id, generation); + + +-- +-- Name: idx_evolution_runs_start_time; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_evolution_runs_start_time ON public.evolution_runs USING btree (start_time DESC); + + +-- +-- Name: idx_evolution_runs_status; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_evolution_runs_status ON public.evolution_runs USING btree (status); + + +-- +-- Name: idx_evolution_runs_task; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_evolution_runs_task ON public.evolution_runs USING btree (task_name); + + +-- +-- Name: idx_generations_run; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_generations_run ON public.generations USING btree (run_id); + + +-- +-- Name: idx_individuals_parent; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_individuals_parent ON public.individuals USING btree (parent_id); + + +-- +-- Name: idx_individuals_run_gen; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_individuals_run_gen ON public.individuals USING btree (run_id, generation); + + +-- +-- Name: idx_individuals_score; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_individuals_score ON public.individuals USING btree (combined_score DESC); + + +-- +-- Name: idx_llm_logs_model; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_llm_logs_model ON public.llm_logs USING btree (model); + + +-- +-- Name: idx_llm_logs_timestamp; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_llm_logs_timestamp ON public.llm_logs USING btree ("timestamp" DESC); + + +-- +-- Name: idx_pareto_fronts_run_gen; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_pareto_fronts_run_gen ON public.pareto_fronts USING btree (run_id, generation); + + +-- +-- Name: idx_pareto_fronts_score; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX idx_pareto_fronts_score ON public.pareto_fronts USING btree (fitness_score DESC); + + +-- +-- Name: code_lineages code_lineages_run_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.code_lineages + ADD CONSTRAINT code_lineages_run_id_fkey FOREIGN KEY (run_id) REFERENCES public.evolution_runs(run_id) ON DELETE CASCADE; + + +-- +-- Name: generations generations_run_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.generations + ADD CONSTRAINT generations_run_id_fkey FOREIGN KEY (run_id) REFERENCES public.evolution_runs(run_id) ON DELETE CASCADE; + + +-- +-- Name: individuals individuals_run_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.individuals + ADD CONSTRAINT individuals_run_id_fkey FOREIGN KEY (run_id) REFERENCES public.evolution_runs(run_id) ON DELETE CASCADE; + + +-- +-- Name: pareto_fronts pareto_fronts_run_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.pareto_fronts + ADD CONSTRAINT pareto_fronts_run_id_fkey FOREIGN KEY (run_id) REFERENCES public.evolution_runs(run_id) ON DELETE CASCADE; + + +-- +-- PostgreSQL database dump complete +-- + + diff --git a/migrations/liquibase.properties b/migrations/liquibase.properties new file mode 100644 index 0000000..3f6f9d7 --- /dev/null +++ b/migrations/liquibase.properties @@ -0,0 +1,6 @@ +changeLogFile=changelogs/db.changelog-master.yaml +url=${LIQUIBASE_URL} +username=${LIQUIBASE_USERNAME} +password=${LIQUIBASE_PASSWORD} +driver=org.postgresql.Driver +liquibase.hub.mode=off diff --git a/reproduce_issue.py b/reproduce_issue.py deleted file mode 100644 index 85ea366..0000000 --- a/reproduce_issue.py +++ /dev/null @@ -1,120 +0,0 @@ -import time -from playwright.sync_api import sync_playwright - - -def run(): - with sync_playwright() as p: - browser = p.chromium.launch(headless=True) - page = browser.new_page() - - # Capture console logs - page.on("console", lambda msg: print(f"Browser Console: {msg.text}")) - page.on("pageerror", lambda exc: print(f"Browser Error: {exc}")) - - print("Navigating to WebUI...") - page.goto("http://localhost:5173") - - # Wait for loading - page.wait_for_load_state("networkidle") - print("Page loaded.") - - # Take initial screenshot - page.screenshot(path="screenshot_initial.png") - - # Look for the Task dropdown. - # In the code Sidebar.tsx, the button has text "Select a task..." or the current task name. - # We can find the button that precedes the "Result" label or just search by text. - - # Click task dropdown trigger - print("Opening task dropdown...") - try: - # Try to find the button that toggles the task list - # It has text "Select a task..." initially - page.get_by_text("Select a task...").click() - except: - # Maybe a task is already selected? - print( - "Could not find 'Select a task...', checking if a task is pre-selected." - ) - # This might happen if state is persisted or default. - - # Wait for dropdown to appear - time.sleep(1) - - # Try to click "squeeze_hnsw" - try: - print("Selecting 'squeeze_hnsw'...") - page.get_by_text("squeeze_hnsw").click() - except Exception as e: - print(f"Could not find 'squeeze_hnsw' option: {e}") - # If not found, maybe try any available task? - # Let's list what we see - page.screenshot(path="screenshot_dropdown.png") - browser.close() - return - - # Wait for results to load and select the first one - time.sleep(1) - print("Selecting first result...") - try: - # Click the "Select a result..." button - page.get_by_text("Select a result...").click() - time.sleep(0.5) - # Click the first result button in the dropdown (assuming it exists) - # The result buttons are in the dropdown container - # We can just pick the first one that looks like a result - # The sidebar logic renders buttons in the dropdown. - # Let's just try to find any button in the result dropdown. - # Or we can wait for the database to load if it auto-selects? - # The code says: "Auto-select first result if available" in handleTaskSelect - # So maybe we don't need to manually select a result if the task switch triggers it. - pass - except: - print("Could not interact with result dropdown, maybe already selected.") - - # Wait for the table to populate - print("Waiting for programs table...") - try: - # Wait for at least one row - page.wait_for_selector( - "tr.model-cell", timeout=5000 - ) # Attempting to find a row - except: - # The table rows have class 'model-cell' on one td, but the row itself doesn't have a specific class other than maybe selected/incorrect - # Let's look for a td. - try: - page.wait_for_selector("table.programs-table tbody tr", timeout=5000) - except Exception as e: - print(f"Table did not populate: {e}") - page.screenshot(path="screenshot_table_fail.png") - browser.close() - return - - print("Table populated. Clicking first program row...") - # Click the first row - rows = page.locator("table.programs-table tbody tr") - if rows.count() > 0: - rows.first.click() - print("Clicked first row.") - else: - print("No rows found.") - - # Wait a bit for the "White Screen" or Code View - time.sleep(2) - - # Screenshot result - page.screenshot(path="screenshot_after_click.png") - print("Screenshot saved to 'screenshot_after_click.png'") - - # Check if Error Boundary triggered - if page.get_by_text("Something went wrong").is_visible(): - print("!!! Error Boundary Caught an Error !!!") - # Extract error text - error_text = page.locator("pre").text_content() - print(f"Error content: {error_text}") - - browser.close() - - -if __name__ == "__main__": - run() diff --git a/screenshot_initial.png b/screenshot_initial.png deleted file mode 100644 index d8438e0..0000000 Binary files a/screenshot_initial.png and /dev/null differ diff --git a/screenshot_table_fail.png b/screenshot_table_fail.png deleted file mode 100644 index 7293bde..0000000 Binary files a/screenshot_table_fail.png and /dev/null differ diff --git a/scripts/bootstrap-gcp.sh b/scripts/bootstrap-gcp.sh new file mode 100755 index 0000000..180613e --- /dev/null +++ b/scripts/bootstrap-gcp.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +# +# Bootstrap Genesis infrastructure on GCP. +# +# Prerequisites: +# - gcloud CLI installed and authenticated (gcloud auth login) +# - terraform >= 1.5 installed +# - You have Owner or Editor role on the project +# +# Usage: +# ./scripts/bootstrap-gcp.sh +# +# After this script completes, it will print the GitHub secrets +# you need to configure at: +# https://github.com/GeorgePearse/Genesis/settings/secrets/actions +# +set -euo pipefail + +PROJECT_ID="visdet-482415" +REGION="europe-west2" +SA_NAME="genesis-bootstrap" +SA_EMAIL="${SA_NAME}@${PROJECT_ID}.iam.gserviceaccount.com" +STATE_BUCKET="genesis-tf-state-visdet-482415" +KEY_FILE="/tmp/genesis-gcp-sa-key.json" + +echo "=========================================" +echo " Genesis GCP Bootstrap" +echo " Project: ${PROJECT_ID}" +echo " Region: ${REGION}" +echo "=========================================" +echo "" + +# ------------------------------------------- +# Step 1: Set project +# ------------------------------------------- +echo "[1/7] Setting project..." +gcloud config set project "${PROJECT_ID}" + +# ------------------------------------------- +# Step 2: Enable required APIs +# ------------------------------------------- +echo "[2/7] Enabling GCP APIs (this may take a minute)..." +gcloud services enable \ + sqladmin.googleapis.com \ + run.googleapis.com \ + artifactregistry.googleapis.com \ + secretmanager.googleapis.com \ + vpcaccess.googleapis.com \ + servicenetworking.googleapis.com \ + compute.googleapis.com \ + iam.googleapis.com \ + cloudresourcemanager.googleapis.com \ + containerregistry.googleapis.com + +# ------------------------------------------- +# Step 3: Create Terraform state bucket +# ------------------------------------------- +echo "[3/7] Creating Terraform state bucket..." +if gcloud storage buckets describe "gs://${STATE_BUCKET}" &>/dev/null; then + echo " Bucket already exists, skipping." +else + gcloud storage buckets create "gs://${STATE_BUCKET}" \ + --location="${REGION}" \ + --uniform-bucket-level-access +fi + +# ------------------------------------------- +# Step 4: Create bootstrap service account +# ------------------------------------------- +echo "[4/7] Creating bootstrap service account..." +if gcloud iam service-accounts describe "${SA_EMAIL}" &>/dev/null 2>&1; then + echo " Service account already exists, skipping creation." +else + gcloud iam service-accounts create "${SA_NAME}" \ + --display-name="Genesis Bootstrap SA" +fi + +ROLES=( + "roles/editor" + "roles/iam.serviceAccountAdmin" + "roles/iam.serviceAccountKeyAdmin" + "roles/resourcemanager.projectIamAdmin" + "roles/secretmanager.admin" + "roles/storage.admin" + "roles/servicenetworking.networksAdmin" +) + +for ROLE in "${ROLES[@]}"; do + echo " Granting ${ROLE}..." + gcloud projects add-iam-policy-binding "${PROJECT_ID}" \ + --member="serviceAccount:${SA_EMAIL}" \ + --role="${ROLE}" \ + --quiet \ + --condition=None 2>/dev/null || true +done + +echo " Creating key file..." +if [ -f "${KEY_FILE}" ]; then + echo " Key file already exists at ${KEY_FILE}, reusing." +else + gcloud iam service-accounts keys create "${KEY_FILE}" \ + --iam-account="${SA_EMAIL}" +fi + +# ------------------------------------------- +# Step 5: Terraform init and apply +# ------------------------------------------- +echo "[5/7] Running Terraform..." +export GOOGLE_APPLICATION_CREDENTIALS="${KEY_FILE}" + +cd "$(dirname "$0")/../terraform" + +echo " terraform init..." +terraform init + +echo "" +echo " terraform plan..." +terraform plan -out=tfplan \ + -var="openai_api_key=PLACEHOLDER_REPLACE_ME" \ + -var="anthropic_api_key=PLACEHOLDER_REPLACE_ME" + +echo "" +echo " About to apply. Review the plan above." +read -rp " Proceed with terraform apply? [y/N] " confirm +if [[ "${confirm}" != "y" && "${confirm}" != "Y" ]]; then + echo " Aborted." + exit 1 +fi + +terraform apply tfplan +rm -f tfplan + +# ------------------------------------------- +# Step 6: Extract outputs +# ------------------------------------------- +echo "[6/7] Extracting outputs..." +CLOUD_SQL_CONN=$(terraform output -raw cloud_sql_instance) +DB_PASSWORD=$(terraform output -raw db_password) +CLOUD_RUN_URL=$(terraform output -raw cloud_run_url) +GITHUB_SA=$(terraform output -raw github_actions_service_account) + +cd - > /dev/null + +# ------------------------------------------- +# Step 7: Create key for GitHub Actions SA +# ------------------------------------------- +echo "[7/7] Creating GitHub Actions SA key..." +GH_SA_KEY_FILE="/tmp/genesis-github-actions-key.json" +gcloud iam service-accounts keys create "${GH_SA_KEY_FILE}" \ + --iam-account="${GITHUB_SA}" 2>/dev/null || echo " (key may already exist)" + +echo "" +echo "=========================================" +echo " Bootstrap complete!" +echo "=========================================" +echo "" +echo " Cloud Run URL: ${CLOUD_RUN_URL}" +echo "" +echo " Next steps:" +echo "" +echo " 1. Seed your real API keys:" +echo " echo -n 'sk-your-real-openai-key' | gcloud secrets versions add genesis-openai-api-key --data-file=-" +echo " echo -n 'sk-ant-your-real-key' | gcloud secrets versions add genesis-anthropic-api-key --data-file=-" +echo "" +echo " 2. Add these GitHub repo secrets at:" +echo " https://github.com/GeorgePearse/Genesis/settings/secrets/actions" +echo "" +echo " GCP_SA_KEY = contents of ${GH_SA_KEY_FILE}" +echo " CLOUD_SQL_CONNECTION_NAME = ${CLOUD_SQL_CONN}" +echo " DB_NAME = genesis" +echo " DB_USER = genesis_app" +echo " DB_PASSWORD = ${DB_PASSWORD}" +echo "" +echo " 3. Push to main -- the CI/CD pipeline will build, migrate, and deploy." +echo "" +echo " 4. Clean up key files when done:" +echo " rm -f ${KEY_FILE} ${GH_SA_KEY_FILE}" +echo "" diff --git a/scripts/debug_embedding_length.py b/scripts/debug_embedding_length.py deleted file mode 100644 index 4196634..0000000 --- a/scripts/debug_embedding_length.py +++ /dev/null @@ -1,29 +0,0 @@ -import os -import clickhouse_connect -from dotenv import load_dotenv - -load_dotenv() - - -def get_client(): - url = os.getenv("CLICKHOUSE_URL") - if url: - import re - - match = re.match(r"https?://([^:]+):([^@]+)@([^:]+):(\d+)", url) - if match: - return clickhouse_connect.get_client( - host=match.group(3), - port=int(match.group(4)), - username=match.group(1), - password=match.group(2), - secure=url.startswith("https"), - ) - return clickhouse_connect.get_client(host="localhost") - - -client = get_client() - -pid = "f190e52a-47e8-4bf4-ab62-2434c35c0853" -res = client.query(f"SELECT length(embedding) FROM programs WHERE id = '{pid}'") -print(res.result_rows) diff --git a/scripts/debug_list_db.py b/scripts/debug_list_db.py deleted file mode 100644 index fa9c3d9..0000000 --- a/scripts/debug_list_db.py +++ /dev/null @@ -1,43 +0,0 @@ -import os -import clickhouse_connect -from dotenv import load_dotenv - -load_dotenv() - - -def get_client(): - url = os.getenv("CLICKHOUSE_URL") - if url: - import re - - match = re.match(r"https?://([^:]+):([^@]+)@([^:]+):(\d+)", url) - if match: - return clickhouse_connect.get_client( - host=match.group(3), - port=int(match.group(4)), - username=match.group(1), - password=match.group(2), - secure=url.startswith("https"), - ) - return clickhouse_connect.get_client(host="localhost") - - -client = get_client() - -query = """ -SELECT - JSONExtractString(metadata, 'original_run_id') as run_id, - JSONExtractString(metadata, 'migration_source') as source_path, - count() as total, - sum(CASE WHEN correct = 1 THEN 1 ELSE 0 END) as working, - max(timestamp) as last_ts -FROM programs -WHERE JSONExtractString(metadata, 'original_run_id') != '' -GROUP BY run_id, source_path -ORDER BY last_ts DESC -LIMIT 5 -""" - -res = client.query(query) -for row in res.result_rows: - print(row) diff --git a/scripts/debug_metadata.py b/scripts/debug_metadata.py deleted file mode 100644 index 4e02b07..0000000 --- a/scripts/debug_metadata.py +++ /dev/null @@ -1,29 +0,0 @@ -import os -import clickhouse_connect -from dotenv import load_dotenv - -load_dotenv() - - -def get_client(): - url = os.getenv("CLICKHOUSE_URL") - if url: - import re - - match = re.match(r"https?://([^:]+):([^@]+)@([^:]+):(\d+)", url) - if match: - return clickhouse_connect.get_client( - host=match.group(3), - port=int(match.group(4)), - username=match.group(1), - password=match.group(2), - secure=url.startswith("https"), - ) - return clickhouse_connect.get_client(host="localhost") - - -client = get_client() - -pid = "812f0ecd-e476-439c-8a02-b4359888388e" -res = client.query(f"SELECT metadata FROM programs WHERE id = '{pid}'") -print(res.result_rows) diff --git a/scripts/export_ddl.sh b/scripts/export_ddl.sh new file mode 100755 index 0000000..cdb9896 --- /dev/null +++ b/scripts/export_ddl.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(dirname "$SCRIPT_DIR")" +OUTPUT_FILE="${REPO_ROOT}/migrations/full_ddl.sql" + +DB_NAME="genesis_ddl_export" +DB_USER="postgres" +DB_PASSWORD="postgres" +CONTAINER_NAME="genesis-ddl-export-$$" +PG_PORT=54399 + +cleanup() { + echo "Cleaning up..." + docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +echo "Starting temporary Postgres container..." +docker run -d \ + --name "$CONTAINER_NAME" \ + -e POSTGRES_DB="$DB_NAME" \ + -e POSTGRES_USER="$DB_USER" \ + -e POSTGRES_PASSWORD="$DB_PASSWORD" \ + -p "${PG_PORT}:5432" \ + postgres:15 >/dev/null + +echo "Waiting for Postgres to be ready..." +for i in $(seq 1 30); do + if docker exec "$CONTAINER_NAME" pg_isready -U "$DB_USER" -d "$DB_NAME" >/dev/null 2>&1; then + break + fi + if [ "$i" -eq 30 ]; then + echo "ERROR: Postgres did not become ready in time" >&2 + exit 1 + fi + sleep 1 +done + +echo "Running Liquibase migrations..." +cd "${REPO_ROOT}/migrations" +liquibase \ + --url="jdbc:postgresql://localhost:${PG_PORT}/${DB_NAME}" \ + --username="$DB_USER" \ + --password="$DB_PASSWORD" \ + --changeLogFile=changelogs/db.changelog-master.yaml \ + update +cd "${REPO_ROOT}" + +echo "Exporting schema via pg_dump..." +docker exec "$CONTAINER_NAME" pg_dump \ + -U "$DB_USER" \ + -d "$DB_NAME" \ + --schema-only \ + --no-owner \ + --no-privileges \ + --exclude-table='databasechangelog*' \ + | sed '/^\\restrict/d; /^\\unrestrict/d; /^SELECT pg_catalog/d; /^SET default_table_access_method/d' \ + > "$OUTPUT_FILE" + +echo "DDL exported to ${OUTPUT_FILE}" diff --git a/scripts/fix_metadata.py b/scripts/fix_metadata.py deleted file mode 100644 index d8c661c..0000000 --- a/scripts/fix_metadata.py +++ /dev/null @@ -1,93 +0,0 @@ -import os -import json -import clickhouse_connect -from dotenv import load_dotenv - -load_dotenv() - - -def get_client(): - url = os.getenv("CLICKHOUSE_URL") - if url: - import re - - match = re.match(r"https?://([^:]+):([^@]+)@([^:]+):(\d+)", url) - if match: - return clickhouse_connect.get_client( - host=match.group(3), - port=int(match.group(4)), - username=match.group(1), - password=match.group(2), - secure=url.startswith("https"), - ) - return clickhouse_connect.get_client(host="localhost") - - -client = get_client() - -# 1. Get latest run_id and path -res = client.query( - "SELECT run_id, database_path FROM evolution_runs ORDER BY start_time DESC LIMIT 1" -) -if not res.result_rows: - print("No run_id found") - exit() - -run_id, source_path = res.result_rows[0] -print(f"Latest run_id: {run_id}") -print(f"Source path: {source_path}") - -# 2. Get programs without original_run_id in metadata -# We assume they are recent programs -query = """ -SELECT id, metadata -FROM programs -WHERE JSONExtractString(metadata, 'original_run_id') = '' -ORDER BY timestamp DESC -LIMIT 1000 -""" - -programs = client.query(query) -print(f"Found {len(programs.result_rows)} programs without run_id") - -count = 0 -for row in programs.result_rows: - pid, meta_str = row - try: - meta = None - try: - # strict=False allows control characters like newlines in strings - meta = json.loads(meta_str, strict=False) - except json.JSONDecodeError: - try: - # Fallback: try to fix newlines manually if strict=False fails - fixed_str = ( - meta_str.replace("\n", "\\n") - .replace("\r", "\\r") - .replace("\t", "\\t") - ) - meta = json.loads(fixed_str, strict=False) - except: - pass - - if meta is None: - print(f"Skipping {pid}: Could not parse metadata even with strict=False") - continue - - # Check if we need to update - if "original_run_id" not in meta or meta.get("original_run_id") != run_id: - meta["original_run_id"] = run_id - meta["migration_source"] = source_path - - # Update DB using parameters - new_meta_str = json.dumps(meta) - - client.command( - "ALTER TABLE programs UPDATE metadata = {meta:String} WHERE id = {pid:String}", - parameters={"meta": new_meta_str, "pid": pid}, - ) - count += 1 - except Exception as e: - print(f"Error updating {pid}: {e}") - -print(f"Updated {count} programs") diff --git a/scripts/fix_metadata_insert.py b/scripts/fix_metadata_insert.py deleted file mode 100644 index f05bc09..0000000 --- a/scripts/fix_metadata_insert.py +++ /dev/null @@ -1,148 +0,0 @@ -import os -import time -import json -import clickhouse_connect -from dotenv import load_dotenv - -load_dotenv() - - -def get_client(): - url = os.getenv("CLICKHOUSE_URL") - if url: - import re - - match = re.match(r"https?://([^:]+):([^@]+)@([^:]+):(\d+)", url) - if match: - return clickhouse_connect.get_client( - host=match.group(3), - port=int(match.group(4)), - username=match.group(1), - password=match.group(2), - secure=url.startswith("https"), - ) - return clickhouse_connect.get_client(host="localhost") - - -client = get_client() - -# 1. Get latest run_id and path -res = client.query( - "SELECT run_id, database_path FROM evolution_runs ORDER BY start_time DESC LIMIT 1" -) -if not res.result_rows: - print("No run_id found") - exit() - -run_id, source_path = res.result_rows[0] -print(f"Latest run_id: {run_id}") -print(f"Source path: {source_path}") - -# 2. Fetch programs without run_id -print("Fetching programs without run_id...") -query = """ -SELECT * -FROM programs -WHERE JSONExtractString(metadata, 'original_run_id') = '' -ORDER BY timestamp DESC -LIMIT 1000 -""" -res = client.query(query) - -if not res.result_rows: - print("No programs without run_id found.") - exit() - -print(f"Found {len(res.result_rows)} programs.") - -programs = [] - - -class Program: - def __init__(self, data, cols): - for k, v in zip(cols, data): - setattr(self, k, v) - - -for row in res.result_rows: - programs.append(Program(row, res.column_names)) - -# 3. Update metadata string and timestamp -print("Updating metadata via INSERT...") -updated_rows = [] -for i, program in enumerate(programs): - meta_str = program.metadata - - # Simple string injection to avoid JSON parsing errors - # Note: This assumes meta_str starts with '{' - injection = f'"original_run_id": "{run_id}", "migration_source": "{source_path}", ' - - if meta_str and meta_str.strip().startswith("{"): - new_meta_str = meta_str.strip().replace("{", "{" + injection, 1) - else: - # Fallback if metadata is empty or broken - new_meta_str = "{" + injection[:-2] + "}" # Remove trailing comma - - program.metadata = new_meta_str - program.timestamp = time.time() - - updated_rows.append( - [ - program.id, - program.code, - program.language, - program.parent_id, - program.archive_inspiration_ids, - program.top_k_inspiration_ids, - program.generation, - program.timestamp, - program.code_diff, - program.combined_score, - program.public_metrics, - program.private_metrics, - program.text_feedback, - program.complexity, - program.embedding, - program.embedding_pca_2d, - program.embedding_pca_3d, - program.embedding_cluster_id, - 1 if program.correct else 0, - program.children_count, - program.metadata, - program.island_idx if program.island_idx is not None else -1, - program.migration_history, - 1 if program.in_archive else 0, - ] - ) - -client.insert( - "programs", - updated_rows, - column_names=[ - "id", - "code", - "language", - "parent_id", - "archive_inspiration_ids", - "top_k_inspiration_ids", - "generation", - "timestamp", - "code_diff", - "combined_score", - "public_metrics", - "private_metrics", - "text_feedback", - "complexity", - "embedding", - "embedding_pca_2d", - "embedding_pca_3d", - "embedding_cluster_id", - "correct", - "children_count", - "metadata", - "island_idx", - "migration_history", - "in_archive", - ], -) -print(f"Updated {len(programs)} programs.") diff --git a/terraform/apis.tf b/terraform/apis.tf new file mode 100644 index 0000000..1272c26 --- /dev/null +++ b/terraform/apis.tf @@ -0,0 +1,21 @@ +locals { + required_apis = [ + "artifactregistry.googleapis.com", + "run.googleapis.com", + "sqladmin.googleapis.com", + "servicenetworking.googleapis.com", + "vpcaccess.googleapis.com", + "secretmanager.googleapis.com", + "iam.googleapis.com", + "compute.googleapis.com", + ] +} + +resource "google_project_service" "apis" { + for_each = toset(local.required_apis) + + project = var.project_id + service = each.value + + disable_on_destroy = false +} diff --git a/terraform/artifact_registry.tf b/terraform/artifact_registry.tf new file mode 100644 index 0000000..39b5f28 --- /dev/null +++ b/terraform/artifact_registry.tf @@ -0,0 +1,17 @@ +resource "google_artifact_registry_repository" "genesis" { + location = var.region + repository_id = "genesis" + format = "DOCKER" + description = "Genesis backend container images" + + cleanup_policies { + id = "keep-recent" + action = "KEEP" + + most_recent_versions { + keep_count = 10 + } + } + + depends_on = [google_project_service.apis["artifactregistry.googleapis.com"]] +} diff --git a/terraform/cloud_run.tf b/terraform/cloud_run.tf new file mode 100644 index 0000000..aec9125 --- /dev/null +++ b/terraform/cloud_run.tf @@ -0,0 +1,98 @@ +resource "google_cloud_run_v2_service" "genesis" { + name = "genesis-backend" + location = var.region + ingress = "INGRESS_TRAFFIC_ALL" + + template { + service_account = google_service_account.cloud_run.email + + scaling { + min_instance_count = var.cloud_run_min_instances + max_instance_count = var.cloud_run_max_instances + } + + vpc_access { + connector = google_vpc_access_connector.genesis.id + egress = "PRIVATE_RANGES_ONLY" + } + + containers { + image = var.cloud_run_image + + ports { + container_port = 8080 + } + + resources { + limits = { + cpu = "1" + memory = "512Mi" + } + } + + env { + name = "DATABASE_URL" + value_source { + secret_key_ref { + secret = google_secret_manager_secret.database_url.secret_id + version = "latest" + } + } + } + + env { + name = "OPENAI_API_KEY" + value_source { + secret_key_ref { + secret = google_secret_manager_secret.openai_api_key.secret_id + version = "latest" + } + } + } + + env { + name = "ANTHROPIC_API_KEY" + value_source { + secret_key_ref { + secret = google_secret_manager_secret.anthropic_api_key.secret_id + version = "latest" + } + } + } + + env { + name = "RUST_LOG" + value = "info" + } + + startup_probe { + http_get { + path = "/health" + } + initial_delay_seconds = 5 + period_seconds = 5 + failure_threshold = 3 + } + + liveness_probe { + http_get { + path = "/health" + } + period_seconds = 30 + } + } + } + + depends_on = [ + google_secret_manager_secret_version.database_url, + google_project_service.apis["run.googleapis.com"], + ] +} + +resource "google_cloud_run_v2_service_iam_member" "public" { + project = var.project_id + location = var.region + name = google_cloud_run_v2_service.genesis.name + role = "roles/run.invoker" + member = "allUsers" +} diff --git a/terraform/cloud_run_frontend.tf b/terraform/cloud_run_frontend.tf new file mode 100644 index 0000000..7930123 --- /dev/null +++ b/terraform/cloud_run_frontend.tf @@ -0,0 +1,47 @@ +resource "google_cloud_run_v2_service" "genesis_frontend" { + name = "genesis-frontend" + location = var.region + ingress = "INGRESS_TRAFFIC_ALL" + + template { + service_account = google_service_account.cloud_run.email + + scaling { + min_instance_count = 0 + max_instance_count = 2 + } + + containers { + image = var.frontend_image + + ports { + container_port = 8080 + } + + resources { + limits = { + cpu = "1" + memory = "256Mi" + } + } + + env { + name = "BACKEND_URL" + value = google_cloud_run_v2_service.genesis.uri + } + } + } + + depends_on = [ + google_cloud_run_v2_service.genesis, + google_project_service.apis["run.googleapis.com"], + ] +} + +resource "google_cloud_run_v2_service_iam_member" "frontend_public" { + project = var.project_id + location = var.region + name = google_cloud_run_v2_service.genesis_frontend.name + role = "roles/run.invoker" + member = "allUsers" +} diff --git a/terraform/cloud_sql.tf b/terraform/cloud_sql.tf new file mode 100644 index 0000000..407130b --- /dev/null +++ b/terraform/cloud_sql.tf @@ -0,0 +1,63 @@ +resource "google_sql_database_instance" "genesis" { + name = "genesis-postgres-${var.environment}" + database_version = "POSTGRES_15" + region = var.region + + depends_on = [ + google_service_networking_connection.private_vpc, + google_project_service.apis["sqladmin.googleapis.com"], + ] + + settings { + tier = var.db_tier + availability_type = "ZONAL" + disk_size = 10 + disk_autoresize = true + + ip_configuration { + ipv4_enabled = false + private_network = google_compute_network.genesis.id + enable_private_path_for_google_cloud_services = true + } + + backup_configuration { + enabled = true + start_time = "03:00" + point_in_time_recovery_enabled = true + transaction_log_retention_days = 7 + + backup_retention_settings { + retained_backups = 7 + } + } + + maintenance_window { + day = 7 + hour = 4 + update_track = "stable" + } + + database_flags { + name = "log_min_duration_statement" + value = "1000" + } + } + + deletion_protection = true +} + +resource "google_sql_database" "genesis" { + name = var.db_name + instance = google_sql_database_instance.genesis.name +} + +resource "random_password" "db_password" { + length = 32 + special = false +} + +resource "google_sql_user" "genesis_app" { + name = var.db_user + instance = google_sql_database_instance.genesis.name + password = random_password.db_password.result +} diff --git a/terraform/iam.tf b/terraform/iam.tf new file mode 100644 index 0000000..7331a54 --- /dev/null +++ b/terraform/iam.tf @@ -0,0 +1,63 @@ +resource "google_service_account" "cloud_run" { + account_id = "genesis-cloud-run" + display_name = "Genesis Cloud Run Service Account" +} + +resource "google_project_iam_member" "cloud_run_sql" { + project = var.project_id + role = "roles/cloudsql.client" + member = "serviceAccount:${google_service_account.cloud_run.email}" +} + +resource "google_secret_manager_secret_iam_member" "database_url" { + secret_id = google_secret_manager_secret.database_url.id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.cloud_run.email}" +} + +resource "google_secret_manager_secret_iam_member" "openai_key" { + secret_id = google_secret_manager_secret.openai_api_key.id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.cloud_run.email}" +} + +resource "google_secret_manager_secret_iam_member" "anthropic_key" { + secret_id = google_secret_manager_secret.anthropic_api_key.id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.cloud_run.email}" +} + +resource "google_service_account" "github_actions" { + account_id = "genesis-github-actions" + display_name = "Genesis GitHub Actions CI/CD" +} + +resource "google_project_iam_member" "github_actions_ar" { + project = var.project_id + role = "roles/artifactregistry.writer" + member = "serviceAccount:${google_service_account.github_actions.email}" +} + +resource "google_project_iam_member" "github_actions_run" { + project = var.project_id + role = "roles/run.admin" + member = "serviceAccount:${google_service_account.github_actions.email}" +} + +resource "google_project_iam_member" "github_actions_sa_user" { + project = var.project_id + role = "roles/iam.serviceAccountUser" + member = "serviceAccount:${google_service_account.github_actions.email}" +} + +resource "google_project_iam_member" "github_actions_sql" { + project = var.project_id + role = "roles/cloudsql.client" + member = "serviceAccount:${google_service_account.github_actions.email}" +} + +resource "google_secret_manager_secret_iam_member" "github_actions_db_url" { + secret_id = google_secret_manager_secret.database_url.id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.github_actions.email}" +} diff --git a/terraform/main.tf b/terraform/main.tf new file mode 100644 index 0000000..8f77402 --- /dev/null +++ b/terraform/main.tf @@ -0,0 +1,24 @@ +terraform { + required_version = ">= 1.5" + + required_providers { + google = { + source = "hashicorp/google" + version = "~> 5.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } + + backend "gcs" { + bucket = "genesis-tf-state-visdet-482415" + prefix = "terraform/state" + } +} + +provider "google" { + project = var.project_id + region = var.region +} diff --git a/terraform/networking.tf b/terraform/networking.tf new file mode 100644 index 0000000..39942d2 --- /dev/null +++ b/terraform/networking.tf @@ -0,0 +1,40 @@ +resource "google_compute_network" "genesis" { + name = "genesis-vpc" + auto_create_subnetworks = false + + depends_on = [google_project_service.apis["compute.googleapis.com"]] +} + +resource "google_compute_subnetwork" "genesis" { + name = "genesis-subnet" + ip_cidr_range = "10.0.0.0/24" + region = var.region + network = google_compute_network.genesis.id +} + +resource "google_compute_global_address" "private_ip" { + name = "genesis-db-private-ip" + purpose = "VPC_PEERING" + address_type = "INTERNAL" + prefix_length = 16 + network = google_compute_network.genesis.id +} + +resource "google_service_networking_connection" "private_vpc" { + network = google_compute_network.genesis.id + service = "servicenetworking.googleapis.com" + reserved_peering_ranges = [google_compute_global_address.private_ip.name] + + depends_on = [google_project_service.apis["servicenetworking.googleapis.com"]] +} + +resource "google_vpc_access_connector" "genesis" { + name = "genesis-connector" + region = var.region + network = google_compute_network.genesis.name + ip_cidr_range = "10.8.0.0/28" + min_instances = 2 + max_instances = 3 + + depends_on = [google_project_service.apis["vpcaccess.googleapis.com"]] +} diff --git a/terraform/outputs.tf b/terraform/outputs.tf new file mode 100644 index 0000000..afc297c --- /dev/null +++ b/terraform/outputs.tf @@ -0,0 +1,47 @@ +output "cloud_run_url" { + description = "Genesis backend URL" + value = google_cloud_run_v2_service.genesis.uri +} + +output "frontend_url" { + description = "Genesis frontend URL" + value = google_cloud_run_v2_service.genesis_frontend.uri +} + +output "artifact_registry" { + description = "Docker registry URL" + value = "${var.region}-docker.pkg.dev/${var.project_id}/${google_artifact_registry_repository.genesis.repository_id}" +} + +output "cloud_sql_instance" { + description = "Cloud SQL instance connection name" + value = google_sql_database_instance.genesis.connection_name +} + +output "cloud_sql_private_ip" { + description = "Cloud SQL private IP" + value = google_sql_database_instance.genesis.private_ip_address + sensitive = true +} + +output "db_password" { + description = "Database password for genesis_app user" + value = random_password.db_password.result + sensitive = true +} + +output "database_url" { + description = "Full Postgres connection string (via private IP)" + value = "postgresql://${var.db_user}:${random_password.db_password.result}@${google_sql_database_instance.genesis.private_ip_address}:5432/${var.db_name}" + sensitive = true +} + +output "cloud_run_service_account" { + description = "Cloud Run service account email" + value = google_service_account.cloud_run.email +} + +output "github_actions_service_account" { + description = "GitHub Actions service account email" + value = google_service_account.github_actions.email +} diff --git a/terraform/secrets.tf b/terraform/secrets.tf new file mode 100644 index 0000000..1d67bc9 --- /dev/null +++ b/terraform/secrets.tf @@ -0,0 +1,56 @@ +resource "google_secret_manager_secret" "database_url" { + secret_id = "genesis-database-url" + + replication { + auto {} + } + + depends_on = [google_project_service.apis["secretmanager.googleapis.com"]] +} + +resource "google_secret_manager_secret_version" "database_url" { + secret = google_secret_manager_secret.database_url.id + secret_data = "postgresql://${google_sql_user.genesis_app.name}:${random_password.db_password.result}@${google_sql_database_instance.genesis.private_ip_address}:5432/${google_sql_database.genesis.name}" +} + +resource "google_secret_manager_secret" "openai_api_key" { + secret_id = "genesis-openai-api-key" + + replication { + auto {} + } + + lifecycle { + ignore_changes = [labels] + } +} + +resource "google_secret_manager_secret_version" "openai_api_key" { + secret = google_secret_manager_secret.openai_api_key.id + secret_data = var.openai_api_key + + lifecycle { + ignore_changes = [secret_data] + } +} + +resource "google_secret_manager_secret" "anthropic_api_key" { + secret_id = "genesis-anthropic-api-key" + + replication { + auto {} + } + + lifecycle { + ignore_changes = [labels] + } +} + +resource "google_secret_manager_secret_version" "anthropic_api_key" { + secret = google_secret_manager_secret.anthropic_api_key.id + secret_data = var.anthropic_api_key + + lifecycle { + ignore_changes = [secret_data] + } +} diff --git a/terraform/variables.tf b/terraform/variables.tf new file mode 100644 index 0000000..c43a23e --- /dev/null +++ b/terraform/variables.tf @@ -0,0 +1,73 @@ +variable "project_id" { + description = "GCP project ID" + type = string + default = "visdet-482415" +} + +variable "region" { + description = "GCP region for all resources" + type = string + default = "europe-west2" +} + +variable "environment" { + description = "Deployment environment (dev, staging, prod)" + type = string + default = "dev" +} + +variable "db_tier" { + description = "Cloud SQL machine tier" + type = string + default = "db-f1-micro" +} + +variable "db_name" { + description = "PostgreSQL database name" + type = string + default = "genesis" +} + +variable "db_user" { + description = "PostgreSQL application user" + type = string + default = "genesis_app" +} + +variable "cloud_run_image" { + description = "Container image for backend Cloud Run (set by CI/CD; uses placeholder for initial bootstrap)" + type = string + default = "us-docker.pkg.dev/cloudrun/container/hello" +} + +variable "frontend_image" { + description = "Container image for frontend Cloud Run (set by CI/CD; uses placeholder for initial bootstrap)" + type = string + default = "us-docker.pkg.dev/cloudrun/container/hello" +} + +variable "cloud_run_min_instances" { + description = "Minimum Cloud Run instances" + type = number + default = 0 +} + +variable "cloud_run_max_instances" { + description = "Maximum Cloud Run instances" + type = number + default = 3 +} + +variable "openai_api_key" { + description = "OpenAI API key (initial seed; update via gcloud secrets)" + type = string + sensitive = true + default = "PLACEHOLDER_REPLACE_ME" +} + +variable "anthropic_api_key" { + description = "Anthropic API key (initial seed; update via gcloud secrets)" + type = string + sensitive = true + default = "PLACEHOLDER_REPLACE_ME" +}