fix: prevent silent memory loss on consolidation LLM failure

.github/workflows/test.yml

@@ -3,6 +3,7 @@ name: CI
 on:
   pull_request:
     branches: [ main ]
+  workflow_dispatch:
 
 concurrency:
   group: ci-${{ github.workflow }}-${{ github.ref }}
@@ -199,7 +200,6 @@ jobs:
       HINDSIGHT_API_LLM_MODEL: google/gemini-2.5-flash-lite
       HINDSIGHT_API_URL: http://localhost:8888
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu
 
     steps:
     - uses: actions/checkout@v6
@@ -421,7 +421,6 @@ jobs:
       HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       # Prefer CPU-only PyTorch in CI (but keep PyPI for everything else)
-      UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu
 
     steps:
     - uses: actions/checkout@v6
@@ -484,7 +483,6 @@ jobs:
       HINDSIGHT_API_URL: http://localhost:8888
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       # Prefer CPU-only PyTorch in CI (but keep PyPI for everything else)
-      UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu
 
     steps:
     - uses: actions/checkout@v6
@@ -587,7 +585,6 @@ jobs:
       HINDSIGHT_API_URL: http://localhost:8888
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       # Prefer CPU-only PyTorch in CI (but keep PyPI for everything else)
-      UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu
 
     steps:
     - uses: actions/checkout@v6
@@ -719,7 +716,6 @@ jobs:
       HINDSIGHT_API_URL: http://localhost:8888
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       # Prefer CPU-only PyTorch in CI (but keep PyPI for everything else)
-      UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu
 
     steps:
     - uses: actions/checkout@v6
@@ -826,7 +822,6 @@ jobs:
       HINDSIGHT_API_URL: http://localhost:8888
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       # Prefer CPU-only PyTorch in CI (but keep PyPI for everything else)
-      UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu
 
     steps:
     - uses: actions/checkout@v6
@@ -931,7 +926,6 @@ jobs:
       HINDSIGHT_API_URL: http://localhost:8888
       HINDSIGHT_EMBED_PACKAGE_PATH: ${{ github.workspace }}/hindsight-embed
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu
 
     steps:
     - uses: actions/checkout@v6
@@ -1038,7 +1032,6 @@ jobs:
       HINDSIGHT_API_LLM_MODEL: google/gemini-2.5-flash-lite
       HINDSIGHT_API_URL: http://localhost:8888
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu
 
     steps:
     - uses: actions/checkout@v6
@@ -1282,7 +1275,6 @@ jobs:
       HINDSIGHT_API_LLM_VERTEXAI_SERVICE_ACCOUNT_KEY: /tmp/gcp-credentials.json
       HINDSIGHT_API_LLM_MODEL: google/gemini-2.5-flash-lite
       # Prefer CPU-only PyTorch in CI
-      UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu
 
     steps:
     - uses: actions/checkout@v6
@@ -1340,7 +1332,6 @@ jobs:
       HINDSIGHT_LLM_VERTEXAI_SERVICE_ACCOUNT_KEY: /tmp/gcp-credentials.json
       HINDSIGHT_LLM_MODEL: google/gemini-2.5-flash-lite
       # Prefer CPU-only PyTorch in CI
-      UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu
 
     steps:
     - uses: actions/checkout@v6
@@ -1396,7 +1387,6 @@ jobs:
       HINDSIGHT_API_LLM_MODEL: google/gemini-2.5-flash-lite
       HINDSIGHT_API_URL: http://localhost:8888
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu
 
     steps:
     - uses: actions/checkout@v6
@@ -1530,7 +1520,6 @@ jobs:
       HINDSIGHT_API_LLM_VERTEXAI_SERVICE_ACCOUNT_KEY: /tmp/gcp-credentials.json
       HINDSIGHT_API_LLM_MODEL: google/gemini-2.5-flash-lite
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu
 
     steps:
     - uses: actions/checkout@v6
@@ -1603,9 +1592,6 @@ jobs:
 
   verify-generated-files:
     runs-on: ubuntu-latest
-    env:
-      UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu
-
     steps:
     - uses: actions/checkout@v6
 
@@ -1675,9 +1661,6 @@ jobs:
 
   check-openapi-compatibility:
     runs-on: ubuntu-latest
-    env:
-      UV_INDEX: pytorch=https://download.pytorch.org/whl/cpu
-
     steps:
     - uses: actions/checkout@v6
       with:

hindsight-api-slim/hindsight_api/alembic/versions/a3b4c5d6e7f8_add_consolidation_failed_at_to_memory_units.py

@@ -0,0 +1,52 @@
+"""Add consolidation_failed_at column to memory_units for tracking persistent LLM failures.
+
+When all LLM retries are exhausted on a single-memory batch, the memory is marked
+with consolidation_failed_at instead of consolidated_at, so it is not silently lost
+and can be retried later via the API.
+
+Revision ID: a3b4c5d6e7f8
+Revises: g7h8i9j0k1l2
+Create Date: 2026-03-17
+"""
+
+from collections.abc import Sequence
+
+from alembic import context, op
+
+revision: str = "a3b4c5d6e7f8"
+down_revision: str | Sequence[str] | None = "g7h8i9j0k1l2"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def _get_schema_prefix() -> str:
+    """Get schema prefix for table names (required for multi-tenant support)."""
+    schema = context.config.get_main_option("target_schema")
+    return f'"{schema}".' if schema else ""
+
+
+def upgrade() -> None:
+    schema = _get_schema_prefix()
+
+    op.execute(
+        f"""
+        ALTER TABLE {schema}memory_units
+        ADD COLUMN IF NOT EXISTS consolidation_failed_at TIMESTAMPTZ DEFAULT NULL
+        """
+    )
+
+    # Index to efficiently query memories that failed consolidation for a given bank
+    op.execute(
+        f"""
+        CREATE INDEX IF NOT EXISTS idx_memory_units_consolidation_failed
+        ON {schema}memory_units (bank_id, consolidation_failed_at)
+        WHERE consolidation_failed_at IS NOT NULL AND fact_type IN ('experience', 'world')
+        """
+    )
+
+
+def downgrade() -> None:
+    schema = _get_schema_prefix()
+
+    op.execute(f"DROP INDEX IF EXISTS {schema}idx_memory_units_consolidation_failed")
+    op.execute(f"ALTER TABLE {schema}memory_units DROP COLUMN IF EXISTS consolidation_failed_at")

hindsight-api-slim/hindsight_api/api/http.py

@@ -10,7 +10,7 @@
 import logging
 import uuid
 from contextlib import asynccontextmanager
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import Any, Literal
 
 from fastapi import Depends, FastAPI, File, Form, Header, HTTPException, Query, UploadFile
@@ -1328,6 +1328,14 @@ class ClearMemoryObservationsResponse(BaseModel):
     deleted_count: int
 
 
+class RecoverConsolidationResponse(BaseModel):
+    """Response model for recovering failed consolidation."""
+
+    model_config = ConfigDict(json_schema_extra={"example": {"retried_count": 42}})
+
+    retried_count: int
+
+
 class BankStatsResponse(BaseModel):
     """Response model for bank statistics endpoint."""
 
@@ -3902,6 +3910,34 @@ async def api_clear_observations(bank_id: str, request_context: RequestContext =
             logger.error(f"Error in DELETE /v1/default/banks/{bank_id}/observations: {error_detail}")
             raise HTTPException(status_code=500, detail=str(e))
 
+    @app.post(
+        "/v1/default/banks/{bank_id}/consolidation/recover",
+        response_model=RecoverConsolidationResponse,
+        summary="Recover failed consolidation",
+        description=(
+            "Reset all memories that were permanently marked as failed during consolidation "
+            "(after exhausting all LLM retries and adaptive batch splitting) so they are "
+            "picked up again on the next consolidation run. Does not delete any observations."
+        ),
+        operation_id="recover_consolidation",
+        tags=["Banks"],
+    )
+    async def api_recover_consolidation(bank_id: str, request_context: RequestContext = Depends(get_request_context)):
+        """Reset consolidation-failed memories for recovery."""
+        try:
+            result = await app.state.memory.retry_failed_consolidation(bank_id, request_context=request_context)
+            return RecoverConsolidationResponse(retried_count=result["retried_count"])
+        except OperationValidationError as e:
+            raise HTTPException(status_code=e.status_code, detail=e.reason)
+        except (AuthenticationError, HTTPException):
+            raise
+        except Exception as e:
+            import traceback
+
+            error_detail = f"{str(e)}\n\nTraceback:\n{traceback.format_exc()}"
+            logger.error(f"Error in POST /v1/default/banks/{bank_id}/consolidation/recover: {error_detail}")
+            raise HTTPException(status_code=500, detail=str(e))
+
     @app.delete(
         "/v1/default/banks/{bank_id}/memories/{memory_id}/observations",
         response_model=ClearMemoryObservationsResponse,
@@ -4135,7 +4171,7 @@ async def api_create_webhook(
             await bank_utils.get_bank_profile(pool, bank_id)
 
             webhook_id = uuid.uuid4()
-            now = datetime.utcnow().isoformat() + "Z"
+            now = datetime.now(timezone.utc).isoformat()
             row = await pool.fetchrow(
                 f"""
                 INSERT INTO {fq_table("webhooks")}