feat(runtime)!: persist response chains on neon postgresql

.env.example

@@ -1,10 +1,7 @@
-# Database
-CODEX_LB_DATABASE_URL=sqlite+aiosqlite:///~/.codex-lb/store.db
-# Optional PostgreSQL example (SQLite stays default if not set):
-# CODEX_LB_DATABASE_URL=postgresql+asyncpg://codex_lb:codex_lb@127.0.0.1:5432/codex_lb
+# Database (Neon PostgreSQL)
+CODEX_LB_DATABASE_URL=postgresql+asyncpg://USER:PASSWORD@EP-POOLER.REGION.aws.neon.tech/codex_lb?sslmode=require
+CODEX_LB_DATABASE_MIGRATION_URL=postgresql+asyncpg://USER:PASSWORD@EP-DIRECT.REGION.aws.neon.tech/codex_lb?sslmode=require
 CODEX_LB_DATABASE_MIGRATE_ON_STARTUP=true
-CODEX_LB_DATABASE_SQLITE_PRE_MIGRATE_BACKUP_ENABLED=true
-CODEX_LB_DATABASE_SQLITE_PRE_MIGRATE_BACKUP_MAX_FILES=5
 
 # Upstream ChatGPT base URL (no /codex suffix)
 CODEX_LB_UPSTREAM_BASE_URL=https://chatgpt.com/backend-api
@@ -26,8 +23,8 @@ CODEX_LB_OAUTH_CALLBACK_PORT=1455
 CODEX_LB_TOKEN_REFRESH_TIMEOUT_SECONDS=30
 CODEX_LB_TOKEN_REFRESH_INTERVAL_DAYS=8
 
-# Encryption key file (optional override; recommended for Docker volumes)
-# CODEX_LB_ENCRYPTION_KEY_FILE=/var/lib/codex-lb/encryption.key
+# Encryption key file (recommended for Docker volumes)
+CODEX_LB_ENCRYPTION_KEY_FILE=/var/lib/codex-lb/encryption.key
 
 # Upstream usage fetch
 CODEX_LB_USAGE_FETCH_TIMEOUT_SECONDS=10

.github/workflows/branch-image.yml

@@ -0,0 +1,93 @@
+name: Branch Docker Image
+
+on:
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: "Git ref to build (branch, tag, or SHA)"
+        required: true
+      image_tag:
+        description: "Optional image tag override"
+        required: false
+      platforms:
+        description: "Comma-separated target platforms"
+        required: false
+        default: "linux/amd64"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ inputs.ref }}
+  cancel-in-progress: true
+
+jobs:
+  docker-image:
+    name: Build and push branch image
+    runs-on: ubuntu-24.04
+
+    permissions:
+      contents: read
+      packages: write
+
+    outputs:
+      image: ${{ steps.image.outputs.image }}
+      tag: ${{ steps.image.outputs.tag }}
+      sha_tag: ${{ steps.image.outputs.sha_tag }}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6.0.2
+        with:
+          ref: ${{ inputs.ref }}
+
+      - name: Compute image tag
+        id: image
+        shell: bash
+        env:
+          INPUT_IMAGE_TAG: ${{ inputs.image_tag }}
+        run: |
+          set -euo pipefail
+
+          short_sha="$(git rev-parse --short=12 HEAD)"
+          ref_name="$(git rev-parse --abbrev-ref HEAD)"
+          if [ "${ref_name}" = "HEAD" ]; then
+            ref_name="${short_sha}"
+          fi
+
+          slug="$(printf '%s' "${ref_name}"             | tr '[:upper:]' '[:lower:]'             | sed -E 's#[^a-z0-9._-]+#-#g; s#-+#-#g; s#(^[-.]+|[-.]+$)##g')"
+
+          tag="${INPUT_IMAGE_TAG:-${slug}-${short_sha}}"
+          image="ghcr.io/${GITHUB_REPOSITORY}:${tag}"
+
+          echo "tag=${tag}" >> "${GITHUB_OUTPUT}"
+          echo "sha_tag=sha-${short_sha}" >> "${GITHUB_OUTPUT}"
+          echo "image=${image}" >> "${GITHUB_OUTPUT}"
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Log in to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: Dockerfile
+          push: true
+          platforms: ${{ inputs.platforms }}
+          tags: |
+            ${{ steps.image.outputs.image }}
+            ghcr.io/${{ github.repository }}:${{ steps.image.outputs.sha_tag }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Summarize image
+        run: |
+          echo "Built image: ${{ steps.image.outputs.image }}" >> "${GITHUB_STEP_SUMMARY}"
+          echo "Extra tag: ghcr.io/${{ github.repository }}:${{ steps.image.outputs.sha_tag }}" >> "${GITHUB_STEP_SUMMARY}"

README.md

@@ -51,9 +51,13 @@ docker volume create codex-lb-data
 docker run -d --name codex-lb \
   -p 2455:2455 -p 1455:1455 \
   -v codex-lb-data:/var/lib/codex-lb \
+  -e CODEX_LB_DATABASE_URL=postgresql+asyncpg://USER:PASSWORD@EP-POOLER.REGION.aws.neon.tech/codex_lb?sslmode=require \
+  -e CODEX_LB_DATABASE_MIGRATION_URL=postgresql+asyncpg://USER:PASSWORD@EP-DIRECT.REGION.aws.neon.tech/codex_lb?sslmode=require \
   ghcr.io/soju06/codex-lb:latest
 
 # or uvx
+CODEX_LB_DATABASE_URL=postgresql+asyncpg://USER:PASSWORD@EP-POOLER.REGION.aws.neon.tech/codex_lb?sslmode=require \
+CODEX_LB_DATABASE_MIGRATION_URL=postgresql+asyncpg://USER:PASSWORD@EP-DIRECT.REGION.aws.neon.tech/codex_lb?sslmode=require \
 uvx codex-lb
 ```
 
@@ -289,16 +293,16 @@ Authorization: Bearer sk-clb-...
 
 Environment variables with `CODEX_LB_` prefix or `.env.local`. See [`.env.example`](.env.example).
 Dashboard auth is configured in Settings.
-SQLite is the default database backend; PostgreSQL is optional via `CODEX_LB_DATABASE_URL` (for example `postgresql+asyncpg://...`).
+Neon PostgreSQL is required for runtime persistence. Set `CODEX_LB_DATABASE_URL` to the pooled Neon DSN and `CODEX_LB_DATABASE_MIGRATION_URL` to the direct Neon DSN used by Alembic/startup migrations.
 
 ## Data
 
-| Environment | Path |
-|-------------|------|
+| Environment | Local files |
+|-------------|-------------|
 | Local / uvx | `~/.codex-lb/` |
 | Docker | `/var/lib/codex-lb/` |
 
-Backup this directory to preserve your data.
+These local paths primarily store the encryption key and other local runtime files. Application data lives in Neon PostgreSQL, so backup and recovery must include your Neon database, not just the local directory.
 
 ## Development
 

app/core/config/settings.py

@@ -5,7 +5,7 @@
 from pathlib import Path
 from typing import Annotated, Literal
 
-from pydantic import Field, field_validator
+from pydantic import Field, field_validator, model_validator
 from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict
 
 BASE_DIR = Path(__file__).resolve().parents[3]
@@ -31,7 +31,6 @@ def _default_oauth_callback_host() -> str:
 
 
 DEFAULT_HOME_DIR = _default_home_dir()
-DEFAULT_DB_PATH = DEFAULT_HOME_DIR / "store.db"
 DEFAULT_ENCRYPTION_KEY_FILE = DEFAULT_HOME_DIR / "encryption.key"
 
 
@@ -43,14 +42,12 @@ class Settings(BaseSettings):
         extra="ignore",
     )
 
-    database_url: str = f"sqlite+aiosqlite:///{DEFAULT_DB_PATH}"
+    database_url: str = Field(min_length=1)
+    database_migration_url: str | None = None
     database_pool_size: int = Field(default=15, gt=0)
     database_max_overflow: int = Field(default=10, ge=0)
     database_pool_timeout_seconds: float = Field(default=30.0, gt=0)
     database_migrate_on_startup: bool = True
-    database_sqlite_pre_migrate_backup_enabled: bool = True
-    database_sqlite_pre_migrate_backup_max_files: int = Field(default=5, ge=1)
-    database_sqlite_startup_check_mode: Literal["quick", "full", "off"] = "quick"
     database_alembic_auto_remap_enabled: bool = True
     upstream_base_url: str = "https://chatgpt.com/backend-api"
     upstream_stream_transport: Literal["http", "websocket", "auto"] = "auto"
@@ -97,16 +94,22 @@ class Settings(BaseSettings):
         default_factory=lambda: ["127.0.0.1/32", "::1/128"]
     )
 
-    @field_validator("database_url")
+    @field_validator("database_url", "database_migration_url")
     @classmethod
-    def _expand_database_url(cls, value: str) -> str:
-        for prefix in ("sqlite+aiosqlite:///", "sqlite:///"):
-            if value.startswith(prefix):
-                path = value[len(prefix) :]
-                if path.startswith("~"):
-                    return f"{prefix}{Path(path).expanduser()}"
+    def _normalize_database_url(cls, value: str | None) -> str | None:
+        if value is None:
+            return None
+        value = value.strip()
+        if not value:
+            raise ValueError("database url must not be empty")
         return value
 
+    @model_validator(mode="after")
+    def _finalize_database_urls(self) -> Settings:
+        if self.database_migration_url is None:
+            self.database_migration_url = self.database_url
+        return self
+
     @field_validator("encryption_key_file", mode="before")
     @classmethod
     def _expand_encryption_key_file(cls, value: str | Path) -> Path:
@@ -171,4 +174,9 @@ def _validate_upstream_compact_timeout_seconds(cls, value: float | None) -> floa
 
 @lru_cache(maxsize=1)
 def get_settings() -> Settings:
-    return Settings()
+    settings = Settings()
+    if settings.database_migrate_on_startup and not settings.database_migration_url:
+        raise RuntimeError(
+            "CODEX_LB_DATABASE_MIGRATION_URL is required when database migrations on startup are enabled"
+        )
+    return settings

app/core/openai/requests.py

@@ -116,7 +116,7 @@ def _is_input_file_with_id(item: Mapping[str, JsonValue]) -> bool:
     return isinstance(file_id, str) and bool(file_id)
 
 
-def _sanitize_input_items(input_items: list[JsonValue]) -> list[JsonValue]:
+def sanitize_input_items(input_items: list[JsonValue]) -> list[JsonValue]:
     sanitized_input: list[JsonValue] = []
     for item in input_items:
         sanitized_item = _sanitize_interleaved_reasoning_input_item(item)
@@ -336,12 +336,12 @@ def _validate_input_type(cls, value: JsonValue) -> JsonValue:
             normalized = _normalize_input_text(value)
             if _has_input_file_id(normalized):
                 raise ValueError("input_file.file_id is not supported")
-            return _sanitize_input_items(normalized)
+            return sanitize_input_items(normalized)
         if is_json_list(value):
             input_items = cast(list[JsonValue], value)
             if _has_input_file_id(input_items):
                 raise ValueError("input_file.file_id is not supported")
-            return _sanitize_input_items(input_items)
+            return sanitize_input_items(input_items)
         raise ValueError("input must be a string or array")
 
     @field_validator("include")
@@ -366,13 +366,6 @@ def _ensure_store_false(cls, value: bool | None) -> bool:
             raise ValueError("store must be false")
         return False if value is None else value
 
-    @field_validator("previous_response_id")
-    @classmethod
-    def _reject_previous_response_id(cls, value: str | None) -> str | None:
-        if value is None:
-            return value
-        raise ValueError("previous_response_id is not supported")
-
     @field_validator("tools")
     @classmethod
     def _validate_tools(cls, value: list[JsonValue]) -> list[JsonValue]:
@@ -418,12 +411,12 @@ def _validate_input_type(cls, value: JsonValue) -> JsonValue:
             normalized = _normalize_input_text(value)
             if _has_input_file_id(normalized):
                 raise ValueError("input_file.file_id is not supported")
-            return _sanitize_input_items(normalized)
+            return sanitize_input_items(normalized)
         if is_json_list(value):
             input_items = cast(list[JsonValue], value)
             if _has_input_file_id(input_items):
                 raise ValueError("input_file.file_id is not supported")
-            return _sanitize_input_items(input_items)
+            return sanitize_input_items(input_items)
         raise ValueError("input must be a string or array")
 
     @model_validator(mode="before")
@@ -478,7 +471,7 @@ def _sanitize_interleaved_reasoning_input(payload: dict[str, JsonValue]) -> None
     input_items = _json_list_or_none(input_value)
     if input_items is None:
         return
-    payload["input"] = _sanitize_input_items(input_items)
+    payload["input"] = sanitize_input_items(input_items)
 
 
 def _normalize_openai_compatible_aliases(payload: dict[str, JsonValue]) -> None:

app/db/alembic/env.py

@@ -21,7 +21,7 @@ def _sync_database_url() -> str:
     configured = config.get_main_option("sqlalchemy.url")
     if configured:
         return configured
-    return to_sync_database_url(get_settings().database_url)
+    return to_sync_database_url(get_settings().database_migration_url or get_settings().database_url)
 
 
 def run_migrations_offline() -> None:
@@ -34,7 +34,6 @@ def run_migrations_offline() -> None:
         literal_binds=True,
         dialect_opts={"paramstyle": "named"},
         compare_type=True,
-        render_as_batch=url.startswith("sqlite"),
     )
 
     with context.begin_transaction():
@@ -57,7 +56,6 @@ def run_migrations_online() -> None:
             connection=connection,
             target_metadata=target_metadata,
             compare_type=True,
-            render_as_batch=connection.dialect.name == "sqlite",
         )
 
         with context.begin_transaction():

app/db/alembic/versions/20260315_120000_add_response_snapshots.py

@@ -0,0 +1,75 @@
+"""add durable response snapshots
+
+Revision ID: 20260315_120000_add_response_snapshots
+Revises: 20260312_120000_add_dashboard_upstream_stream_transport
+Create Date: 2026-03-15 12:00:00.000000
+"""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.engine import Connection
+
+# revision identifiers, used by Alembic.
+revision = "20260315_120000_add_response_snapshots"
+down_revision = "20260312_120000_add_dashboard_upstream_stream_transport"
+branch_labels = None
+depends_on = None
+
+
+def _table_exists(connection: Connection, table_name: str) -> bool:
+    inspector = sa.inspect(connection)
+    return inspector.has_table(table_name)
+
+
+def _columns(connection: Connection, table_name: str) -> set[str]:
+    inspector = sa.inspect(connection)
+    if not inspector.has_table(table_name):
+        return set()
+    return {str(column["name"]) for column in inspector.get_columns(table_name) if column.get("name") is not None}
+
+
+def _indexes(connection: Connection, table_name: str) -> set[str]:
+    inspector = sa.inspect(connection)
+    if not inspector.has_table(table_name):
+        return set()
+    return {str(index["name"]) for index in inspector.get_indexes(table_name) if index.get("name") is not None}
+
+
+def upgrade() -> None:
+    bind = op.get_bind()
+    if not _table_exists(bind, "response_snapshots"):
+        op.create_table(
+            "response_snapshots",
+            sa.Column("response_id", sa.String(), nullable=False),
+            sa.Column("parent_response_id", sa.String(), nullable=True),
+            sa.Column("account_id", sa.String(), nullable=True),
+            sa.Column("api_key_id", sa.String(), nullable=True),
+            sa.Column("model", sa.String(), nullable=False),
+            sa.Column("input_items_json", sa.Text(), nullable=False),
+            sa.Column("response_json", sa.Text(), nullable=False),
+            sa.Column("created_at", sa.DateTime(), nullable=False, server_default=sa.func.now()),
+            sa.PrimaryKeyConstraint("response_id"),
+        )
+    existing_columns = _columns(bind, "response_snapshots")
+    if "api_key_id" not in existing_columns:
+        op.add_column("response_snapshots", sa.Column("api_key_id", sa.String(), nullable=True))
+    existing_indexes = _indexes(bind, "response_snapshots")
+    if "idx_response_snapshots_parent_created_at" not in existing_indexes:
+        op.create_index(
+            "idx_response_snapshots_parent_created_at",
+            "response_snapshots",
+            ["parent_response_id", "created_at"],
+            unique=False,
+        )
+
+
+def downgrade() -> None:
+    bind = op.get_bind()
+    if not _table_exists(bind, "response_snapshots"):
+        return
+    existing_indexes = _indexes(bind, "response_snapshots")
+    if "idx_response_snapshots_parent_created_at" in existing_indexes:
+        op.drop_index("idx_response_snapshots_parent_created_at", table_name="response_snapshots")
+    op.drop_table("response_snapshots")