diff --git a/hindsight-api-slim/hindsight_api/api/http.py b/hindsight-api-slim/hindsight_api/api/http.py index f7a36137c..5d5206bae 100644 --- a/hindsight-api-slim/hindsight_api/api/http.py +++ b/hindsight-api-slim/hindsight_api/api/http.py @@ -425,6 +425,11 @@ class MemoryItem(BaseModel): "A list of tag lists runs one pass per inner list, giving full control over which combinations to use." ), ) + strategy: str | None = Field( + default=None, + description="Named retain strategy for this item. Overrides the bank's default strategy for this item only. " + "Strategies are defined in the bank config under 'retain_strategies'.", + ) @field_validator("timestamp", mode="before") @classmethod @@ -491,6 +496,11 @@ class FileRetainMetadata(BaseModel): description="Parser or ordered fallback chain for this file (overrides request-level parser). " "E.g. 'iris' or ['iris', 'markitdown'].", ) + strategy: str | None = Field( + default=None, + description="Named retain strategy for this file. Overrides the bank's default strategy. " + "Strategies are defined in the bank config under 'retain_strategies'.", + ) class FileRetainRequest(BaseModel): @@ -544,7 +554,11 @@ class RetainResponse(BaseModel): ) operation_id: str | None = Field( default=None, - description="Operation ID for tracking async operations. Use GET /v1/default/banks/{bank_id}/operations to list operations. Only present when async=true.", + description="Operation ID for tracking async operations. Use GET /v1/default/banks/{bank_id}/operations to list operations. Only present when async=true. When items use different per-item strategies, use operation_ids instead.", + ) + operation_ids: list[str] | None = Field( + default=None, + description="Operation IDs when items were submitted as multiple strategy groups (async=true with mixed per-item strategies). operation_id is set to the first entry for backward compatibility.", ) usage: TokenUsage | None = Field( default=None, @@ -4441,10 +4455,13 @@ async def api_retain( metrics = get_metrics_collector() try: - # Prepare contents for processing - contents = [] + # Group items by strategy + strategy_groups: dict[str | None, list[dict]] = {} for item in request.items: - content_dict = {"content": item.content} + effective = item.strategy + if effective not in strategy_groups: + strategy_groups[effective] = [] + content_dict: dict = {"content": item.content} if item.timestamp == "unset": content_dict["event_date"] = None elif item.timestamp: @@ -4461,20 +4478,30 @@ async def api_retain( content_dict["tags"] = item.tags if item.observation_scopes is not None: content_dict["observation_scopes"] = item.observation_scopes - contents.append(content_dict) + strategy_groups[effective].append(content_dict) if request.async_: - # Async processing: queue task and return immediately - result = await app.state.memory.submit_async_retain( - bank_id, contents, document_tags=request.document_tags, request_context=request_context - ) + # Async processing: one submit per strategy group + all_operation_ids = [] + total_items_count = 0 + for group_strategy, contents in strategy_groups.items(): + result = await app.state.memory.submit_async_retain( + bank_id, + contents, + document_tags=request.document_tags, + strategy=group_strategy, + request_context=request_context, + ) + all_operation_ids.append(result["operation_id"]) + total_items_count += result["items_count"] return RetainResponse.model_validate( { "success": True, "bank_id": bank_id, - "items_count": result["items_count"], + "items_count": total_items_count, "async": True, - "operation_id": result["operation_id"], + "operation_id": all_operation_ids[0] if all_operation_ids else None, + "operation_ids": all_operation_ids if len(all_operation_ids) > 1 else None, } ) else: @@ -4493,24 +4520,41 @@ async def api_retain( ), ) - # Synchronous processing: wait for completion (record metrics) + # Synchronous processing: one batch per strategy group, aggregate results + total_items_count = 0 + total_usage = TokenUsage(input_tokens=0, output_tokens=0, total_tokens=0) with metrics.record_operation("retain", bank_id=bank_id, source="api"): - result, usage = await app.state.memory.retain_batch_async( - bank_id=bank_id, - contents=contents, - document_tags=request.document_tags, - request_context=request_context, - return_usage=True, - outbox_callback=app.state.memory._build_retain_outbox_callback( + for group_strategy, contents in strategy_groups.items(): + result, usage = await app.state.memory.retain_batch_async( bank_id=bank_id, contents=contents, - operation_id=None, - schema=_current_schema.get(), - ), - ) + document_tags=request.document_tags, + strategy=group_strategy, + request_context=request_context, + return_usage=True, + outbox_callback=app.state.memory._build_retain_outbox_callback( + bank_id=bank_id, + contents=contents, + operation_id=None, + schema=_current_schema.get(), + ), + ) + total_items_count += len(contents) + if usage: + total_usage = TokenUsage( + input_tokens=total_usage.input_tokens + usage.input_tokens, + output_tokens=total_usage.output_tokens + usage.output_tokens, + total_tokens=total_usage.total_tokens + usage.total_tokens, + ) return RetainResponse.model_validate( - {"success": True, "bank_id": bank_id, "items_count": len(contents), "async": False, "usage": usage} + { + "success": True, + "bank_id": bank_id, + "items_count": total_items_count, + "async": False, + "usage": total_usage, + } ) except OperationValidationError as e: raise HTTPException(status_code=e.status_code, detail=e.reason) @@ -4673,6 +4717,7 @@ async def read(self): "tags": file_meta.tags or [], "timestamp": file_meta.timestamp, "parser": parser_chain, + "strategy": file_meta.strategy, } file_items.append(item) diff --git a/hindsight-api-slim/hindsight_api/config.py b/hindsight-api-slim/hindsight_api/config.py index 624ef6791..612b9a67b 100644 --- a/hindsight-api-slim/hindsight_api/config.py +++ b/hindsight-api-slim/hindsight_api/config.py @@ -267,6 +267,7 @@ def normalize_config_dict(config: dict[str, Any]) -> dict[str, Any]: ENV_RETAIN_EXTRACTION_MODE = "HINDSIGHT_API_RETAIN_EXTRACTION_MODE" ENV_RETAIN_MISSION = "HINDSIGHT_API_RETAIN_MISSION" ENV_RETAIN_CUSTOM_INSTRUCTIONS = "HINDSIGHT_API_RETAIN_CUSTOM_INSTRUCTIONS" +ENV_RETAIN_DEFAULT_STRATEGY = "HINDSIGHT_API_RETAIN_DEFAULT_STRATEGY" ENV_RETAIN_BATCH_TOKENS = "HINDSIGHT_API_RETAIN_BATCH_TOKENS" ENV_RETAIN_ENTITY_LOOKUP = "HINDSIGHT_API_RETAIN_ENTITY_LOOKUP" ENV_RETAIN_BATCH_ENABLED = "HINDSIGHT_API_RETAIN_BATCH_ENABLED" @@ -443,9 +444,11 @@ def normalize_config_dict(config: dict[str, Any]) -> dict[str, Any]: DEFAULT_RETAIN_CHUNK_SIZE = 3000 # Max chars per chunk for fact extraction DEFAULT_RETAIN_EXTRACT_CAUSAL_LINKS = True # Extract causal links between facts DEFAULT_RETAIN_EXTRACTION_MODE = "concise" # Extraction mode: "concise", "verbose", or "custom" -RETAIN_EXTRACTION_MODES = ("concise", "verbose", "custom") # Allowed extraction modes +RETAIN_EXTRACTION_MODES = ("concise", "verbose", "custom", "verbatim", "chunks") # Allowed extraction modes DEFAULT_RETAIN_MISSION = None # Declarative spec of what to retain (injected into any extraction mode) DEFAULT_RETAIN_CUSTOM_INSTRUCTIONS = None # Custom extraction guidelines (only used when mode="custom") +DEFAULT_RETAIN_DEFAULT_STRATEGY = None # Default strategy name (None = no strategy override) +DEFAULT_RETAIN_STRATEGIES: dict | None = None # Named retain strategies (dict of name → config overrides) DEFAULT_RETAIN_BATCH_TOKENS = 10_000 # ~40KB of text # Max chars per sub-batch for async retain auto-splitting DEFAULT_RETAIN_ENTITY_LOOKUP = "trigram" # "full" or "trigram" DEFAULT_RETAIN_BATCH_ENABLED = False # Use LLM Batch API for fact extraction (only when async=True) @@ -719,6 +722,8 @@ class HindsightConfig: retain_extraction_mode: str retain_mission: str | None retain_custom_instructions: str | None + retain_default_strategy: str | None + retain_strategies: dict | None retain_batch_tokens: int retain_batch_enabled: bool retain_batch_poll_interval_seconds: int @@ -849,6 +854,8 @@ class HindsightConfig: "retain_extraction_mode", "retain_mission", "retain_custom_instructions", + "retain_default_strategy", + "retain_strategies", # Entity labels (controlled vocabulary for entity classification) "entity_labels", "entities_allow_free_form", @@ -1101,9 +1108,7 @@ def from_env(cls) -> "HindsightConfig": ENV_RERANKER_LOCAL_TRUST_REMOTE_CODE, str(DEFAULT_RERANKER_LOCAL_TRUST_REMOTE_CODE) ).lower() in ("true", "1"), - reranker_local_fp16=os.getenv( - ENV_RERANKER_LOCAL_FP16, str(DEFAULT_RERANKER_LOCAL_FP16) - ).lower() + reranker_local_fp16=os.getenv(ENV_RERANKER_LOCAL_FP16, str(DEFAULT_RERANKER_LOCAL_FP16)).lower() in ("true", "1"), reranker_local_bucket_batching=os.getenv( ENV_RERANKER_LOCAL_BUCKET_BATCHING, str(DEFAULT_RERANKER_LOCAL_BUCKET_BATCHING) @@ -1177,6 +1182,8 @@ def from_env(cls) -> "HindsightConfig": ), retain_mission=os.getenv(ENV_RETAIN_MISSION) or DEFAULT_RETAIN_MISSION, retain_custom_instructions=os.getenv(ENV_RETAIN_CUSTOM_INSTRUCTIONS) or DEFAULT_RETAIN_CUSTOM_INSTRUCTIONS, + retain_default_strategy=os.getenv(ENV_RETAIN_DEFAULT_STRATEGY) or DEFAULT_RETAIN_DEFAULT_STRATEGY, + retain_strategies=DEFAULT_RETAIN_STRATEGIES, retain_batch_tokens=int(os.getenv(ENV_RETAIN_BATCH_TOKENS, str(DEFAULT_RETAIN_BATCH_TOKENS))), retain_entity_lookup=os.getenv(ENV_RETAIN_ENTITY_LOOKUP, DEFAULT_RETAIN_ENTITY_LOOKUP), retain_batch_enabled=os.getenv(ENV_RETAIN_BATCH_ENABLED, str(DEFAULT_RETAIN_BATCH_ENABLED)).lower() diff --git a/hindsight-api-slim/hindsight_api/config_resolver.py b/hindsight-api-slim/hindsight_api/config_resolver.py index e5de3946a..770952fab 100644 --- a/hindsight-api-slim/hindsight_api/config_resolver.py +++ b/hindsight-api-slim/hindsight_api/config_resolver.py @@ -10,7 +10,7 @@ import json import logging -from dataclasses import asdict +from dataclasses import asdict, replace from typing import Any import asyncpg @@ -239,6 +239,14 @@ async def update_bank_config( logger.warning(f"Failed to check permissions for bank {bank_id}: {e}") # Continue without permission check (fail open for backward compatibility) + # Validate retain_strategies: reject empty string keys + if "retain_strategies" in normalized_updates and normalized_updates["retain_strategies"]: + empty_keys = [k for k in normalized_updates["retain_strategies"] if not str(k).strip()] + if empty_keys: + raise ValueError( + "Strategy names must not be empty strings. Remove entries with empty names before saving." + ) + # Merge with existing config (JSONB || operator) async with self.pool.acquire() as conn: await conn.execute( @@ -273,3 +281,35 @@ async def reset_bank_config(self, bank_id: str) -> None: ) logger.info(f"Reset bank config for {bank_id} to defaults") + + +def apply_strategy(config: HindsightConfig, strategy_name: str) -> HindsightConfig: + """ + Apply a named retain strategy's overrides on top of a resolved config. + + A strategy is a named set of hierarchical field overrides stored in + config.retain_strategies. Any field in _HIERARCHICAL_FIELDS can be + overridden, including retain_extraction_mode, retain_chunk_size, + entity_labels, entities_allow_free_form, etc. + + Unknown strategy names log a warning and return config unchanged. + Unknown or non-hierarchical fields in the strategy are silently ignored. + """ + strategies = config.retain_strategies or {} + if strategy_name not in strategies: + logger.warning(f"Unknown retain strategy '{strategy_name}', using resolved config as-is") + return config + + overrides = strategies[strategy_name] + if not isinstance(overrides, dict): + logger.warning(f"Retain strategy '{strategy_name}' is not a dict, skipping") + return config + + configurable = HindsightConfig.get_configurable_fields() + filtered = {k: v for k, v in overrides.items() if k in configurable} + + if not filtered: + return config + + logger.debug(f"Applying retain strategy '{strategy_name}': {list(filtered.keys())}") + return replace(config, **filtered) diff --git a/hindsight-api-slim/hindsight_api/engine/memory_engine.py b/hindsight-api-slim/hindsight_api/engine/memory_engine.py index d8ae44f76..fb0816688 100644 --- a/hindsight-api-slim/hindsight_api/engine/memory_engine.py +++ b/hindsight-api-slim/hindsight_api/engine/memory_engine.py @@ -561,6 +561,7 @@ async def _handle_batch_retain(self, task_dict: dict[str, Any]): contents = task_dict.get("contents", []) document_tags = task_dict.get("document_tags") operation_id = task_dict.get("operation_id") # For batch API crash recovery + strategy = task_dict.get("strategy") logger.info( f"[BATCH_RETAIN_TASK] Starting background batch retain for bank_id={bank_id}, {len(contents)} items, operation_id={operation_id}" @@ -584,6 +585,7 @@ async def _handle_batch_retain(self, task_dict: dict[str, Any]): document_tags=document_tags, request_context=context, operation_id=operation_id, + strategy=strategy, outbox_callback=self._build_retain_outbox_callback( bank_id=bank_id, contents=contents, @@ -712,6 +714,8 @@ async def _handle_file_convert_retain(self, task_dict: dict[str, Any]): retain_task_payload: dict[str, Any] = {"contents": retain_contents} if document_tags: retain_task_payload["document_tags"] = document_tags + if task_dict.get("strategy"): + retain_task_payload["strategy"] = task_dict["strategy"] # Pass tenant/api_key context through to retain task if task_dict.get("_tenant_id"): @@ -1953,6 +1957,7 @@ async def retain_batch_async( return_usage: bool = False, operation_id: str | None = None, outbox_callback: "Callable[[asyncpg.Connection], Awaitable[None]] | None" = None, + strategy: str | None = None, ): """ Store multiple content items as memory units in ONE batch operation. @@ -2110,6 +2115,7 @@ async def retain_batch_async( confidence_score=confidence_score, document_tags=document_tags, operation_id=operation_id, + strategy=strategy, # Outbox callback runs inside the last sub-batch's transaction so the # webhook delivery row is committed atomically with the final retain data. outbox_callback=outbox_callback if i == len(sub_batches) else None, @@ -2134,6 +2140,7 @@ async def retain_batch_async( confidence_score=confidence_score, document_tags=document_tags, operation_id=operation_id, + strategy=strategy, outbox_callback=outbox_callback, ) @@ -2186,6 +2193,7 @@ async def _retain_batch_async_internal( document_tags: list[str] | None = None, operation_id: str | None = None, outbox_callback: "Callable[[asyncpg.Connection], Awaitable[None]] | None" = None, + strategy: str | None = None, ) -> tuple[list[list[str]], "TokenUsage"]: """ Internal method for batch processing without chunking logic. @@ -2218,6 +2226,13 @@ async def _retain_batch_async_internal( # Resolve bank-specific config for this operation resolved_config = await self._config_resolver.resolve_full_config(bank_id, request_context) + # Apply strategy overrides: explicit strategy > bank default strategy + from hindsight_api.config_resolver import apply_strategy + + effective_strategy = strategy or resolved_config.retain_default_strategy + if effective_strategy: + resolved_config = apply_strategy(resolved_config, effective_strategy) + # Create parent span for retain operation with create_operation_span("retain", bank_id): return await orchestrator.retain_batch( @@ -7377,6 +7392,7 @@ async def submit_async_retain( *, request_context: "RequestContext", document_tags: list[str] | None = None, + strategy: str | None = None, ) -> dict[str, Any]: """Submit a batch retain operation to run asynchronously. @@ -7485,6 +7501,8 @@ async def submit_async_retain( task_payload: dict[str, Any] = {"contents": sub_batch} if document_tags: task_payload["document_tags"] = document_tags + if strategy: + task_payload["strategy"] = strategy # Pass tenant_id and api_key_id through task payload if request_context.tenant_id: task_payload["_tenant_id"] = request_context.tenant_id @@ -7599,6 +7617,8 @@ async def submit_async_file_retain( "document_tags": document_tags or [], "timestamp": item.get("timestamp"), } + if item.get("strategy"): + task_payload["strategy"] = item["strategy"] # Pass tenant_id and api_key_id through task payload if request_context.tenant_id: diff --git a/hindsight-api-slim/hindsight_api/engine/retain/fact_extraction.py b/hindsight-api-slim/hindsight_api/engine/retain/fact_extraction.py index f7c897b53..6a9174ef1 100644 --- a/hindsight-api-slim/hindsight_api/engine/retain/fact_extraction.py +++ b/hindsight-api-slim/hindsight_api/engine/retain/fact_extraction.py @@ -332,6 +332,43 @@ class FactExtractionResponseNoCausal(BaseModel): facts: list[ExtractedFactNoCausal] = Field(description="List of extracted factual statements") +class VerbatimExtractedFact(BaseModel): + """ + Schema for verbatim extraction mode. + + Omits 'what' entirely — the original chunk text is used as fact_text in code. + The LLM only extracts metadata: entities, temporal info, location, people. + """ + + model_config = ConfigDict( + json_schema_mode="validation", + json_schema_extra={"required": ["when", "where", "who", "fact_type"]}, + ) + + when: str = Field(description="When it happened. 'N/A' if unknown.") + where: str = Field(description="Location if relevant. 'N/A' if none.") + who: str = Field(description="People involved with relationships. 'N/A' if general.") + + fact_kind: str = Field(default="conversation", description="'event' or 'conversation'") + occurred_start: str | None = Field(default=None, description="ISO timestamp for events") + occurred_end: str | None = Field(default=None, description="ISO timestamp for event end") + fact_type: Literal["world", "assistant"] = Field(description="'world' or 'assistant'") + entities: list[Entity] | None = Field(default=None, description="People, places, concepts") + + @field_validator("entities", mode="before") + @classmethod + def ensure_entities_list(cls, v): + if v is None: + return [] + return v + + +class VerbatimFactExtractionResponse(BaseModel): + """Response for verbatim extraction mode (one entry per chunk, no fact text).""" + + facts: list[VerbatimExtractedFact] = Field(description="List of metadata entries (one per chunk)") + + def chunk_text(text: str, max_chars: int) -> list[str]: """ Split text into chunks, preserving conversation structure when possible. @@ -552,6 +589,27 @@ def _chunk_conversation(turns: list[dict], max_chars: int) -> list[str]: examples="", # No examples for custom mode ) +# Verbatim mode: preserve the original text exactly, but still extract metadata +_VERBATIM_GUIDELINES = """══════════════════════════════════════════════════════════════════════════ +VERBATIM MODE — Extract metadata only +══════════════════════════════════════════════════════════════════════════ + +The original text will be stored as-is in code. Your ONLY job is to extract metadata. + +RULES: +- Produce EXACTLY ONE entry per input chunk. +- DO NOT include a "what" field — it is not part of the output schema. +- Extract all entities (people, places, organizations, objects, concepts). +- Extract temporal information (occurred_start, occurred_end, fact_kind, when). +- Extract location (where) and people (who). +- fact_type: use "world" unless the content is clearly an interaction with the assistant.""" + +VERBATIM_FACT_EXTRACTION_PROMPT = _BASE_FACT_EXTRACTION_PROMPT.format( + retain_mission_section="{retain_mission_section}", + extraction_guidelines=_VERBATIM_GUIDELINES, + examples="", +) + # Verbose extraction prompt - detailed, comprehensive facts (legacy mode) VERBOSE_FACT_EXTRACTION_PROMPT = """Extract facts from text into structured format with FIVE required dimensions - BE EXTREMELY DETAILED. @@ -770,6 +828,10 @@ def _build_extraction_prompt_and_schema(config) -> tuple[str, type]: ) elif extraction_mode == "verbose": prompt = VERBOSE_FACT_EXTRACTION_PROMPT + elif extraction_mode == "verbatim": + prompt = VERBATIM_FACT_EXTRACTION_PROMPT.format( + retain_mission_section=retain_mission_section, + ) else: base_prompt = CONCISE_FACT_EXTRACTION_PROMPT prompt = base_prompt.format( @@ -777,7 +839,11 @@ def _build_extraction_prompt_and_schema(config) -> tuple[str, type]: ) # Add causal relationships section if enabled - if extract_causal_links: + # Verbatim mode never uses causal relations (no fact text to relate causally) + if extraction_mode == "verbatim": + base_fact_class = VerbatimExtractedFact + base_response_class = VerbatimFactExtractionResponse + elif extract_causal_links: prompt = prompt + CAUSAL_RELATIONSHIPS_SECTION base_fact_class = ExtractedFactVerbose if extraction_mode == "verbose" else ExtractedFact base_response_class = FactExtractionResponseVerbose if extraction_mode == "verbose" else FactExtractionResponse @@ -1012,8 +1078,10 @@ def get_value(field_name): if not what: what = get_value("factual_core") if not what: - logger.warning(f"Skipping fact {i}: missing 'what' field") - continue + # In verbatim mode, 'what' is intentionally absent — text is backfilled from chunk + if extraction_mode != "verbatim": + logger.warning(f"Skipping fact {i}: missing 'what' field") + continue # Critical field: fact_type # LLM uses "assistant" but we convert to "experience" for storage @@ -1046,19 +1114,23 @@ def get_value(field_name): fact_kind = "conversation" # Build combined fact text from the 4 dimensions: what | when | who | why + # In verbatim mode, leave combined_text empty — _collapse_to_verbatim backfills it fact_data = {} - combined_parts = [what] + if extraction_mode == "verbatim": + combined_text = "" + else: + combined_parts = [what] - if when: - combined_parts.append(f"When: {when}") + if when: + combined_parts.append(f"When: {when}") - if who: - combined_parts.append(f"Involving: {who}") + if who: + combined_parts.append(f"Involving: {who}") - if why: - combined_parts.append(why) + if why: + combined_parts.append(why) - combined_text = " | ".join(combined_parts) + combined_text = " | ".join(combined_parts) # Add temporal fields # For events: occurred_start/occurred_end (when the event happened) @@ -1889,6 +1961,52 @@ def get_value(field_name): return extracted_facts, chunks_metadata, total_usage +def _extract_facts_chunks( + contents: list[RetainContent], + config, +) -> tuple[list[ExtractedFactType], list[ChunkMetadata], TokenUsage]: + """ + chunks mode: no LLM call, no entity extraction. + + Each chunk becomes one memory unit with the raw text as fact_text. + User-provided entities from RetainContent.entities are picked up downstream + by entity_processing.py — they are the sole source of entity data in this mode. + """ + extracted_facts: list[ExtractedFactType] = [] + chunks_metadata: list[ChunkMetadata] = [] + global_chunk_idx = 0 + + for content_index, content in enumerate(contents): + chunks = chunk_text(content.content, config.retain_chunk_size) + for chunk in chunks: + chunks_metadata.append( + ChunkMetadata( + chunk_text=chunk, + fact_count=1, + content_index=content_index, + chunk_index=global_chunk_idx, + ) + ) + extracted_facts.append( + ExtractedFactType( + fact_text=chunk, + fact_type="world", + entities=[], + content_index=content_index, + chunk_index=global_chunk_idx, + context=content.context, + mentioned_at=content.event_date, + metadata=content.metadata, + tags=content.tags, + observation_scopes=content.observation_scopes, + ) + ) + global_chunk_idx += 1 + + _add_temporal_offsets(extracted_facts, contents) + return extracted_facts, chunks_metadata, TokenUsage() + + async def extract_facts_from_contents( contents: list[RetainContent], llm_config, @@ -1924,6 +2042,11 @@ async def extract_facts_from_contents( if not contents: return [], [], TokenUsage() + # chunks mode: skip LLM entirely, store each chunk as-is + # Must come before the batch-API check so no LLM queue/locks are acquired + if config.retain_extraction_mode == "chunks": + return _extract_facts_chunks(contents, config) + # Route to batch API if enabled if config.retain_batch_enabled: return await extract_facts_from_contents_batch_api( @@ -2013,15 +2136,46 @@ async def extract_facts_from_contents( global_fact_idx += 1 fact_idx_in_content += 1 - # Step 4: Add time offsets to preserve ordering within each content + # Step 4: For verbatim mode, collapse to one fact per chunk with original text + if config.retain_extraction_mode == "verbatim": + extracted_facts = _collapse_to_verbatim(extracted_facts, chunks_metadata) + + # Step 5: Add time offsets to preserve ordering within each content _add_temporal_offsets(extracted_facts, contents) - # Step 5: Auto-tag facts from label groups with tag=True + # Step 6: Auto-tag facts from label groups with tag=True _inject_label_tags(extracted_facts, config) return extracted_facts, chunks_metadata, total_usage +def _collapse_to_verbatim(facts: list[ExtractedFactType], chunks: list[ChunkMetadata]) -> list[ExtractedFactType]: + """ + For verbatim mode: ensure one fact per chunk with the original chunk text preserved. + + The LLM prompt asks for exactly one fact per chunk, but if it returns more, + this collapses them: keeps the first fact as representative, overrides its + fact_text with the raw chunk text, and merges entities from any extra facts. + """ + chunk_text_map = {c.chunk_index: c.chunk_text for c in chunks} + seen: dict[int, ExtractedFactType] = {} + result: list[ExtractedFactType] = [] + + for fact in facts: + if fact.chunk_index not in seen: + fact.fact_text = chunk_text_map.get(fact.chunk_index, fact.fact_text) + seen[fact.chunk_index] = fact + result.append(fact) + else: + # Merge entities from extra facts into the representative + representative = seen[fact.chunk_index] + for entity in fact.entities: + if entity not in representative.entities: + representative.entities.append(entity) + + return result + + def _parse_datetime(date_str: str): """Parse ISO datetime string.""" from dateutil import parser as date_parser diff --git a/hindsight-api-slim/hindsight_api/main.py b/hindsight-api-slim/hindsight_api/main.py index c0e800660..7a0d536ba 100644 --- a/hindsight-api-slim/hindsight_api/main.py +++ b/hindsight-api-slim/hindsight_api/main.py @@ -257,6 +257,8 @@ def main(): retain_extraction_mode=config.retain_extraction_mode, retain_mission=config.retain_mission, retain_custom_instructions=config.retain_custom_instructions, + retain_default_strategy=config.retain_default_strategy, + retain_strategies=config.retain_strategies, retain_batch_tokens=config.retain_batch_tokens, retain_entity_lookup=config.retain_entity_lookup, retain_batch_enabled=config.retain_batch_enabled, diff --git a/hindsight-api-slim/tests/test_hierarchical_config.py b/hindsight-api-slim/tests/test_hierarchical_config.py index f3c67d324..4b254a795 100644 --- a/hindsight-api-slim/tests/test_hierarchical_config.py +++ b/hindsight-api-slim/tests/test_hierarchical_config.py @@ -89,7 +89,7 @@ async def test_hierarchical_fields_categorization(): assert "entity_labels" in configurable # Verify count is correct - assert len(configurable) == 17 + assert len(configurable) == 19 # Verify credential fields (NEVER exposed) assert "llm_api_key" in credentials diff --git a/hindsight-api-slim/tests/test_retain.py b/hindsight-api-slim/tests/test_retain.py index cfc203abe..1c4481cbb 100644 --- a/hindsight-api-slim/tests/test_retain.py +++ b/hindsight-api-slim/tests/test_retain.py @@ -1,11 +1,13 @@ """ Test retain function and chunk storage. """ -import pytest import logging -from datetime import datetime, timezone, timedelta -from hindsight_api.engine.memory_engine import Budget +from datetime import datetime, timedelta, timezone + +import pytest + from hindsight_api import RequestContext +from hindsight_api.engine.memory_engine import Budget logger = logging.getLogger(__name__) @@ -60,7 +62,7 @@ async def test_retain_with_chunks(memory, request_context): request_context=request_context, ) - print(f"\n=== Recall Results (with chunks) ===") + print("\n=== Recall Results (with chunks) ===") print(f"Found {len(result.results)} results") assert len(result.results) > 0, "Should find facts about Alice" @@ -149,7 +151,7 @@ async def test_chunks_and_entities_follow_fact_order(memory, request_context): request_context=request_context, ) - print(f"\n=== Recall Results ===") + print("\n=== Recall Results ===") print(f"Found {len(result.results)} facts") # Extract the order of entities mentioned in facts @@ -421,7 +423,7 @@ async def test_mentioned_at_vs_occurred(memory, request_context): # Verify it's the historical date, not today assert mentioned_dt.year == 2020, f"mentioned_at should be 2020, got {mentioned_dt.year}" - print(f"✓ Test passed: Historical conversation correctly ingested with event_date=2020") + print("✓ Test passed: Historical conversation correctly ingested with event_date=2020") finally: await memory.delete_bank(bank_id, request_context=request_context) @@ -489,15 +491,15 @@ async def test_occurred_dates_not_defaulted(memory, request_context): # If occurred_start is set, it means the LLM extracted it # In this case, log it but don't fail (LLM behavior can vary) print(f"⚠ LLM extracted occurred_start: {fact.occurred_start}") - print(f" This test expects None for present-tense observations") + print(" This test expects None for present-tense observations") else: - print(f"✓ occurred_start is correctly None (not defaulted to mentioned_at)") + print("✓ occurred_start is correctly None (not defaulted to mentioned_at)") if fact.occurred_end is not None: print(f"⚠ LLM extracted occurred_end: {fact.occurred_end}") - print(f" This test expects None for present-tense observations") + print(" This test expects None for present-tense observations") else: - print(f"✓ occurred_end is correctly None (not defaulted to mentioned_at)") + print("✓ occurred_end is correctly None (not defaulted to mentioned_at)") # At least verify they're not equal to mentioned_at if they are set if fact.occurred_start is not None: @@ -513,7 +515,7 @@ async def test_occurred_dates_not_defaulted(memory, request_context): f"occurred_start={occurred_start_dt}, mentioned_at={mentioned_dt}" ) - print(f"✓ Test passed: occurred dates are not incorrectly defaulted to mentioned_at") + print("✓ Test passed: occurred dates are not incorrectly defaulted to mentioned_at") finally: await memory.delete_bank(bank_id, request_context=request_context) @@ -585,7 +587,7 @@ async def test_mentioned_at_from_context_string(memory, request_context): else: print(f"⚠ LLM did not extract date from context, fell back to now(): {mentioned_dt}") - print(f"✓ mentioned_at is always set (never None)") + print("✓ mentioned_at is always set (never None)") finally: await memory.delete_bank(bank_id, request_context=request_context) @@ -851,8 +853,8 @@ async def test_metadata_storage_and_retrieval(memory, request_context): assert len(result.results) > 0, "Should recall stored facts" - print(f"✓ Successfully stored and retrieved facts") - print(f" (Note: Metadata support depends on API implementation)") + print("✓ Successfully stored and retrieved facts") + print(" (Note: Metadata support depends on API implementation)") finally: await memory.delete_bank(bank_id, request_context=request_context) @@ -954,7 +956,7 @@ async def test_mixed_content_batch(memory, request_context): short_units = len(unit_ids[0]) long_units = len(unit_ids[1]) - print(f"✓ Mixed batch processed successfully") + print("✓ Mixed batch processed successfully") print(f" Short content: {short_units} units") print(f" Long content: {long_units} units") @@ -1356,7 +1358,7 @@ async def test_chunks_truncation_behavior(memory, request_context): if truncated_chunks: print(f" {len(truncated_chunks)} chunks were truncated due to token limit") else: - print(f" No chunks were truncated (content within limit)") + print(" No chunks were truncated (content within limit)") else: print("✓ No chunks returned (may be under token limit)") @@ -2210,9 +2212,10 @@ async def test_custom_extraction_mode(): custom guidelines while keeping structural parts intact. """ import os + from hindsight_api import LLMConfig + from hindsight_api.config import _get_raw_config, clear_config_cache from hindsight_api.engine.retain.fact_extraction import extract_facts_from_text - from hindsight_api.config import clear_config_cache, _get_raw_config # Save original env vars original_mode = os.getenv("HINDSIGHT_API_RETAIN_EXTRACTION_MODE") @@ -2284,7 +2287,7 @@ async def test_custom_extraction_mode(): if found_english_only: logger.warning(f"⚠ Found English-only keywords in facts: {found_english_only}") logger.warning(f" Facts: {all_facts_text}") - logger.warning(f" This may indicate the LLM is not strictly following language-specific custom guidelines") + logger.warning(" This may indicate the LLM is not strictly following language-specific custom guidelines") # Log but don't fail - LLM behavior can vary else: logger.info("✓ Successfully extracted only Italian facts, ignored English facts") @@ -2315,6 +2318,213 @@ async def test_custom_extraction_mode(): clear_config_cache() +def test_apply_strategy(): + """ + Unit test for apply_strategy: + - Known strategy applies overrides on top of resolved config + - Unknown strategy returns config unchanged with a warning + - Non-hierarchical fields in a strategy are silently ignored + - entity_labels and entities_allow_free_form are overridable + """ + from hindsight_api.config import _get_raw_config, clear_config_cache + from hindsight_api.config_resolver import apply_strategy + + clear_config_cache() + base_config = _get_raw_config() + + strategies = { + "documents": { + "retain_extraction_mode": "chunks", + "retain_chunk_size": 800, + "entities_allow_free_form": False, + }, + "bad_field": { + "database_url": "should-be-ignored", # static field, not hierarchical + "retain_extraction_mode": "verbose", + }, + } + config_with_strategies = base_config.__class__( + **{**base_config.__dict__, "retain_strategies": strategies} + ) + + # Known strategy: overrides applied + result = apply_strategy(config_with_strategies, "documents") + assert result.retain_extraction_mode == "chunks" + assert result.retain_chunk_size == 800 + assert result.entities_allow_free_form is False + + # Non-hierarchical field silently ignored, hierarchical one applied + result2 = apply_strategy(config_with_strategies, "bad_field") + assert result2.retain_extraction_mode == "verbose" + assert result2.database_url == base_config.database_url # unchanged + + # Unknown strategy: config returned unchanged + result3 = apply_strategy(config_with_strategies, "nonexistent") + assert result3.retain_extraction_mode == base_config.retain_extraction_mode + + +def test_collapse_to_verbatim_single_fact_per_chunk(): + """ + Unit test for _collapse_to_verbatim: + - One fact per chunk → text overridden with original chunk text + - Two facts from same chunk → collapsed to one, entities merged + """ + from hindsight_api.engine.retain.fact_extraction import _collapse_to_verbatim + from hindsight_api.engine.retain.types import ChunkMetadata, ExtractedFact + + chunks = [ + ChunkMetadata(chunk_text="Alice went to Paris.", fact_count=1, content_index=0, chunk_index=0), + ChunkMetadata(chunk_text="Bob fixed the bug yesterday.", fact_count=2, content_index=0, chunk_index=1), + ] + + facts = [ + ExtractedFact(fact_text="LLM paraphrase of Alice in Paris", fact_type="world", entities=["Alice", "Paris"], chunk_index=0, content_index=0), + ExtractedFact(fact_text="LLM first fact about Bob", fact_type="world", entities=["Bob"], chunk_index=1, content_index=0), + ExtractedFact(fact_text="LLM second fact about bug", fact_type="world", entities=["bug"], chunk_index=1, content_index=0), + ] + + result = _collapse_to_verbatim(facts, chunks) + + assert len(result) == 2, "Should produce exactly one fact per chunk" + + # Chunk 0: text overridden with original chunk text + assert result[0].fact_text == "Alice went to Paris.", "Text must be the raw chunk text" + assert result[0].entities == ["Alice", "Paris"] + + # Chunk 1: collapsed to one fact, entities merged from both LLM facts + assert result[1].fact_text == "Bob fixed the bug yesterday.", "Text must be the raw chunk text" + assert "Bob" in result[1].entities + assert "bug" in result[1].entities + + +def test_chunks_extraction_mode(): + """ + Unit test for chunks mode: no LLM, chunks stored as-is, zero token usage. + """ + import asyncio + import os + + from hindsight_api.config import _get_raw_config, clear_config_cache + from hindsight_api.engine.retain.fact_extraction import extract_facts_from_contents + from hindsight_api.engine.retain.types import RetainContent + + original_mode = os.getenv("HINDSIGHT_API_RETAIN_EXTRACTION_MODE") + + try: + os.environ["HINDSIGHT_API_RETAIN_EXTRACTION_MODE"] = "chunks" + clear_config_cache() + + contents = [ + RetainContent( + content="Alice joined the infrastructure team on March 5, 2024.", + event_date=datetime(2024, 3, 10, tzinfo=timezone.utc), + entities=[{"text": "Alice"}, {"text": "infrastructure team"}], + ), + RetainContent(content="Bob fixed the critical bug in the payment service."), + ] + + facts, chunks, usage = asyncio.get_event_loop().run_until_complete( + extract_facts_from_contents( + contents=contents, + llm_config=None, # Must not be called + agent_name="TestAgent", + config=_get_raw_config(), + ) + ) + + # One fact per chunk (both contents fit in one chunk each) + assert len(facts) == len(chunks) == 2 + + # Text preserved exactly + assert facts[0].fact_text == contents[0].content + assert facts[1].fact_text == contents[1].content + + # No LLM-extracted entities (user-provided entities handled downstream) + assert facts[0].entities == [] + assert facts[1].entities == [] + + # Zero token usage + assert usage.total_tokens == 0 + + logger.info("✓ chunks mode: no LLM call, chunks stored as-is, zero token usage") + + finally: + if original_mode is not None: + os.environ["HINDSIGHT_API_RETAIN_EXTRACTION_MODE"] = original_mode + else: + os.environ.pop("HINDSIGHT_API_RETAIN_EXTRACTION_MODE", None) + clear_config_cache() + + +@pytest.mark.asyncio +async def test_verbatim_extraction_mode(): + """ + Integration test for verbatim extraction mode. + + Verifies that: + 1. Each chunk produces exactly one fact + 2. The fact text is the original chunk text, not a paraphrase + 3. Entities are still extracted by the LLM + 4. Temporal info (occurred_start) is still extracted + """ + import os + + from hindsight_api import LLMConfig + from hindsight_api.config import _get_raw_config, clear_config_cache + from hindsight_api.engine.retain.fact_extraction import extract_facts_from_contents + from hindsight_api.engine.retain.types import RetainContent + + original_mode = os.getenv("HINDSIGHT_API_RETAIN_EXTRACTION_MODE") + + try: + os.environ["HINDSIGHT_API_RETAIN_EXTRACTION_MODE"] = "verbatim" + clear_config_cache() + + text = ( + "Alice joined the infrastructure team on March 5, 2024. " + "She holds a CKA certification and has 5 years of Kubernetes experience." + ) + + llm_config = LLMConfig.for_memory() + contents = [RetainContent(content=text, event_date=datetime(2024, 3, 10, tzinfo=timezone.utc), context="onboarding notes")] + facts, chunks, _ = await extract_facts_from_contents( + contents=contents, + llm_config=llm_config, + agent_name="TestAgent", + config=_get_raw_config(), + ) + + logger.info(f"Verbatim mode extracted {len(facts)} facts from {len(chunks)} chunks") + for i, f in enumerate(facts): + logger.info(f" fact[{i}]: {f.fact_text!r} entities={f.entities}") + + # One fact per chunk + assert len(facts) == len(chunks), "Verbatim mode must produce exactly one fact per chunk" + + # Text must match the original chunk exactly + for fact, chunk in zip(facts, chunks): + assert fact.fact_text == chunk.chunk_text, ( + f"fact_text must equal original chunk text.\n" + f" expected: {chunk.chunk_text!r}\n" + f" got: {fact.fact_text!r}" + ) + + # Entities should still be extracted + all_entities = [e for f in facts for e in f.entities] + assert any("alice" in e.lower() for e in all_entities), ( + f"Expected entity 'Alice' to be extracted. Entities: {all_entities}" + ) + + logger.info("✓ Verbatim mode preserves chunk text and still extracts entities") + + finally: + if original_mode is not None: + os.environ["HINDSIGHT_API_RETAIN_EXTRACTION_MODE"] = original_mode + else: + os.environ.pop("HINDSIGHT_API_RETAIN_EXTRACTION_MODE", None) + clear_config_cache() + + @pytest.mark.asyncio async def test_retain_batch_with_per_item_tags_on_document(memory, request_context): """ @@ -2349,7 +2559,7 @@ async def test_retain_batch_with_per_item_tags_on_document(memory, request_conte ) assert len(result) > 0, "Should have retained content" - print(f"\n=== Retained content with tags ===") + print("\n=== Retained content with tags ===") # Retrieve the document doc = await memory.get_document( @@ -2380,6 +2590,7 @@ async def test_retain_batch_with_per_item_tags_on_document(memory, request_conte def test_retain_mission_injected_into_prompt(): """Test that retain_mission is injected as a FOCUS section into any extraction mode.""" from unittest.mock import MagicMock + from hindsight_api.engine.retain.fact_extraction import _build_extraction_prompt_and_schema spec = "Focus on technical decisions and architecture choices only." @@ -2405,6 +2616,7 @@ def test_retain_mission_injected_into_prompt(): def test_retain_mission_absent_when_not_set(): """Test that no FOCUS section appears when retain_mission is not set.""" from unittest.mock import MagicMock + from hindsight_api.engine.retain.fact_extraction import _build_extraction_prompt_and_schema config = MagicMock() @@ -2421,6 +2633,7 @@ def test_retain_mission_absent_when_not_set(): def test_retain_mission_config_loaded_from_env(): """Test that retain_mission is loaded from env and is a configurable field.""" import os + from hindsight_api.config import HindsightConfig, _get_raw_config, clear_config_cache original = os.getenv("HINDSIGHT_API_RETAIN_MISSION") @@ -2436,3 +2649,141 @@ def test_retain_mission_config_loaded_from_env(): else: os.environ["HINDSIGHT_API_RETAIN_MISSION"] = original clear_config_cache() + + +def test_strategy_overrides_extraction_mode_for_chunks(): + """ + Unit test: a named strategy with retain_extraction_mode=chunks causes + extract_facts_from_contents to skip the LLM and return verbatim chunks. + """ + import asyncio + + from hindsight_api.config import _get_raw_config, clear_config_cache + from hindsight_api.config_resolver import apply_strategy + from hindsight_api.engine.retain.fact_extraction import extract_facts_from_contents + from hindsight_api.engine.retain.types import RetainContent + + clear_config_cache() + base_config = _get_raw_config() + + # Build a config that has a strategy overriding to chunks + strategies = {"fast": {"retain_extraction_mode": "chunks"}} + config_with_strategies = base_config.__class__( + **{**base_config.__dict__, "retain_strategies": strategies} + ) + strategy_config = apply_strategy(config_with_strategies, "fast") + assert strategy_config.retain_extraction_mode == "chunks" + + contents = [ + RetainContent(content="Alice deployed the new API on Monday."), + RetainContent(content="Bob reviewed the pull request."), + ] + + facts, chunks, usage = asyncio.get_event_loop().run_until_complete( + extract_facts_from_contents( + contents=contents, + llm_config=None, # chunks must not call the LLM + agent_name="TestAgent", + config=strategy_config, + ) + ) + + assert len(facts) == 2 + assert facts[0].fact_text == contents[0].content + assert facts[1].fact_text == contents[1].content + assert usage.total_tokens == 0 + logger.info("✓ strategy with chunks mode: no LLM, verbatim chunks, zero tokens") + + +def test_retain_request_per_item_strategy_field(): + """ + Unit test: MemoryItem accepts a strategy field; items with different strategies + are grouped correctly by per-item strategy. + """ + from hindsight_api.api.http import RetainRequest + + request = RetainRequest.model_validate( + { + "items": [ + {"content": "Alice joined.", "strategy": "fast"}, + {"content": "Bob left.", "strategy": "detailed"}, + {"content": "Carol arrived."}, # no strategy — falls back to bank default + ], + } + ) + + assert request.items[0].strategy == "fast" + assert request.items[1].strategy == "detailed" + assert request.items[2].strategy is None + + # Simulate grouping logic from api_retain handler + strategy_groups: dict = {} + for item in request.items: + strategy_groups.setdefault(item.strategy, []).append(item.content) + + assert set(strategy_groups.keys()) == {"fast", "detailed", None} + assert strategy_groups["fast"] == ["Alice joined."] + assert strategy_groups["detailed"] == ["Bob left."] + assert strategy_groups[None] == ["Carol arrived."] + logger.info("✓ per-item strategy grouping works correctly") + + +@pytest.mark.asyncio +async def test_named_strategy_applied_end_to_end(memory, request_context): + """ + Integration test: a named strategy stored in bank config is actually applied + during retain_batch_async. + + Regression test for the bug where strategy was passed through the HTTP layer + but the extraction mode override was silently ignored, always using the bank + default (e.g. 'concise') instead of the strategy's override (e.g. 'chunks'). + """ + from hindsight_api.config_resolver import ConfigResolver + + bank_id = f"test_strategy_e2e_{datetime.now(timezone.utc).timestamp()}" + + try: + # Seed the bank so the row exists before we write config to it + # (update_bank_config is a plain UPDATE — it silently no-ops on missing rows) + await memory.retain_batch_async( + bank_id=bank_id, + contents=[{"content": "seed"}], + request_context=request_context, + ) + + # Now configure the bank with a named strategy that overrides to chunks + await memory._config_resolver.update_bank_config( + bank_id, + { + "retain_extraction_mode": "concise", # bank default + "retain_strategies": { + "chunks": {"retain_extraction_mode": "chunks"}, + }, + }, + request_context, + ) + + contents = [{"content": "Alice deployed the new API on Monday."}] + + # Retain using the named strategy + unit_ids_by_content, usage = await memory.retain_batch_async( + bank_id=bank_id, + contents=contents, + strategy="chunks", + request_context=request_context, + return_usage=True, + ) + + # chunks produces exactly one fact per chunk (verbatim) and calls no LLM + assert usage.total_tokens == 0, f"chunks should use zero LLM tokens, got {usage.total_tokens}" + assert len(unit_ids_by_content) == 1 + assert len(unit_ids_by_content[0]) == 1, "chunks should produce exactly one fact per content item" + + # Verify the stored fact is the verbatim content + facts = await memory.recall_async(bank_id, "Alice", request_context=request_context) + assert any("Alice" in f.text for f in facts.results), "Verbatim content should be retrievable" + + logger.info("✓ named strategy 'chunks' with chunks applied end-to-end: no LLM, verbatim storage") + + finally: + await memory.delete_bank(bank_id, request_context=request_context) diff --git a/hindsight-cli/src/commands/memory.rs b/hindsight-cli/src/commands/memory.rs index 5bf6e2984..1bf004257 100644 --- a/hindsight-cli/src/commands/memory.rs +++ b/hindsight-cli/src/commands/memory.rs @@ -390,6 +390,7 @@ pub fn retain( entities: None, tags: None, observation_scopes: None, + strategy: None, }; let request = RetainRequest { diff --git a/hindsight-clients/go/api/openapi.yaml b/hindsight-clients/go/api/openapi.yaml index b559bf3f4..85871a740 100644 --- a/hindsight-clients/go/api/openapi.yaml +++ b/hindsight-clients/go/api/openapi.yaml @@ -4014,6 +4014,9 @@ components: type: array observation_scopes: $ref: '#/components/schemas/ObservationScopes' + strategy: + nullable: true + type: string required: - content title: MemoryItem @@ -4795,6 +4798,11 @@ components: operation_id: nullable: true type: string + operation_ids: + items: + type: string + nullable: true + type: array usage: $ref: '#/components/schemas/TokenUsage' required: diff --git a/hindsight-clients/go/model_memory_item.go b/hindsight-clients/go/model_memory_item.go index 327c166f6..22c47585c 100644 --- a/hindsight-clients/go/model_memory_item.go +++ b/hindsight-clients/go/model_memory_item.go @@ -29,6 +29,7 @@ type MemoryItem struct { Entities []EntityInput `json:"entities,omitempty"` Tags []string `json:"tags,omitempty"` ObservationScopes NullableObservationScopes `json:"observation_scopes,omitempty"` + Strategy NullableString `json:"strategy,omitempty"` } type _MemoryItem MemoryItem @@ -342,6 +343,48 @@ func (o *MemoryItem) UnsetObservationScopes() { o.ObservationScopes.Unset() } +// GetStrategy returns the Strategy field value if set, zero value otherwise (both if not set or set to explicit null). +func (o *MemoryItem) GetStrategy() string { + if o == nil || IsNil(o.Strategy.Get()) { + var ret string + return ret + } + return *o.Strategy.Get() +} + +// GetStrategyOk returns a tuple with the Strategy field value if set, nil otherwise +// and a boolean to check if the value has been set. +// NOTE: If the value is an explicit nil, `nil, true` will be returned +func (o *MemoryItem) GetStrategyOk() (*string, bool) { + if o == nil { + return nil, false + } + return o.Strategy.Get(), o.Strategy.IsSet() +} + +// HasStrategy returns a boolean if a field has been set. +func (o *MemoryItem) HasStrategy() bool { + if o != nil && o.Strategy.IsSet() { + return true + } + + return false +} + +// SetStrategy gets a reference to the given NullableString and assigns it to the Strategy field. +func (o *MemoryItem) SetStrategy(v string) { + o.Strategy.Set(&v) +} +// SetStrategyNil sets the value for Strategy to be an explicit nil +func (o *MemoryItem) SetStrategyNil() { + o.Strategy.Set(nil) +} + +// UnsetStrategy ensures that no value is present for Strategy, not even an explicit nil +func (o *MemoryItem) UnsetStrategy() { + o.Strategy.Unset() +} + func (o MemoryItem) MarshalJSON() ([]byte, error) { toSerialize,err := o.ToMap() if err != nil { @@ -374,6 +417,9 @@ func (o MemoryItem) ToMap() (map[string]interface{}, error) { if o.ObservationScopes.IsSet() { toSerialize["observation_scopes"] = o.ObservationScopes.Get() } + if o.Strategy.IsSet() { + toSerialize["strategy"] = o.Strategy.Get() + } return toSerialize, nil } diff --git a/hindsight-clients/go/model_retain_response.go b/hindsight-clients/go/model_retain_response.go index 88ede0dcf..8278f47c7 100644 --- a/hindsight-clients/go/model_retain_response.go +++ b/hindsight-clients/go/model_retain_response.go @@ -27,6 +27,7 @@ type RetainResponse struct { // Whether the operation was processed asynchronously Async bool `json:"async"` OperationId NullableString `json:"operation_id,omitempty"` + OperationIds []string `json:"operation_ids,omitempty"` Usage NullableTokenUsage `json:"usage,omitempty"` } @@ -191,6 +192,39 @@ func (o *RetainResponse) UnsetOperationId() { o.OperationId.Unset() } +// GetOperationIds returns the OperationIds field value if set, zero value otherwise (both if not set or set to explicit null). +func (o *RetainResponse) GetOperationIds() []string { + if o == nil { + var ret []string + return ret + } + return o.OperationIds +} + +// GetOperationIdsOk returns a tuple with the OperationIds field value if set, nil otherwise +// and a boolean to check if the value has been set. +// NOTE: If the value is an explicit nil, `nil, true` will be returned +func (o *RetainResponse) GetOperationIdsOk() ([]string, bool) { + if o == nil || IsNil(o.OperationIds) { + return nil, false + } + return o.OperationIds, true +} + +// HasOperationIds returns a boolean if a field has been set. +func (o *RetainResponse) HasOperationIds() bool { + if o != nil && !IsNil(o.OperationIds) { + return true + } + + return false +} + +// SetOperationIds gets a reference to the given []string and assigns it to the OperationIds field. +func (o *RetainResponse) SetOperationIds(v []string) { + o.OperationIds = v +} + // GetUsage returns the Usage field value if set, zero value otherwise (both if not set or set to explicit null). func (o *RetainResponse) GetUsage() TokenUsage { if o == nil || IsNil(o.Usage.Get()) { @@ -250,6 +284,9 @@ func (o RetainResponse) ToMap() (map[string]interface{}, error) { if o.OperationId.IsSet() { toSerialize["operation_id"] = o.OperationId.Get() } + if o.OperationIds != nil { + toSerialize["operation_ids"] = o.OperationIds + } if o.Usage.IsSet() { toSerialize["usage"] = o.Usage.Get() } diff --git a/hindsight-clients/python/hindsight_client_api/models/memory_item.py b/hindsight-clients/python/hindsight_client_api/models/memory_item.py index 6eacbbfa1..9c88ce8d4 100644 --- a/hindsight-clients/python/hindsight_client_api/models/memory_item.py +++ b/hindsight-clients/python/hindsight_client_api/models/memory_item.py @@ -37,7 +37,8 @@ class MemoryItem(BaseModel): entities: Optional[List[EntityInput]] = None tags: Optional[List[StrictStr]] = None observation_scopes: Optional[ObservationScopes] = None - __properties: ClassVar[List[str]] = ["content", "timestamp", "context", "metadata", "document_id", "entities", "tags", "observation_scopes"] + strategy: Optional[StrictStr] = None + __properties: ClassVar[List[str]] = ["content", "timestamp", "context", "metadata", "document_id", "entities", "tags", "observation_scopes", "strategy"] model_config = ConfigDict( populate_by_name=True, @@ -126,6 +127,11 @@ def to_dict(self) -> Dict[str, Any]: if self.observation_scopes is None and "observation_scopes" in self.model_fields_set: _dict['observation_scopes'] = None + # set to None if strategy (nullable) is None + # and model_fields_set contains the field + if self.strategy is None and "strategy" in self.model_fields_set: + _dict['strategy'] = None + return _dict @classmethod @@ -145,7 +151,8 @@ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: "document_id": obj.get("document_id"), "entities": [EntityInput.from_dict(_item) for _item in obj["entities"]] if obj.get("entities") is not None else None, "tags": obj.get("tags"), - "observation_scopes": ObservationScopes.from_dict(obj["observation_scopes"]) if obj.get("observation_scopes") is not None else None + "observation_scopes": ObservationScopes.from_dict(obj["observation_scopes"]) if obj.get("observation_scopes") is not None else None, + "strategy": obj.get("strategy") }) return _obj diff --git a/hindsight-clients/python/hindsight_client_api/models/retain_response.py b/hindsight-clients/python/hindsight_client_api/models/retain_response.py index 6bd86c14b..07e120063 100644 --- a/hindsight-clients/python/hindsight_client_api/models/retain_response.py +++ b/hindsight-clients/python/hindsight_client_api/models/retain_response.py @@ -32,8 +32,9 @@ class RetainResponse(BaseModel): items_count: StrictInt var_async: StrictBool = Field(description="Whether the operation was processed asynchronously", alias="async") operation_id: Optional[StrictStr] = None + operation_ids: Optional[List[StrictStr]] = None usage: Optional[TokenUsage] = None - __properties: ClassVar[List[str]] = ["success", "bank_id", "items_count", "async", "operation_id", "usage"] + __properties: ClassVar[List[str]] = ["success", "bank_id", "items_count", "async", "operation_id", "operation_ids", "usage"] model_config = ConfigDict( populate_by_name=True, @@ -82,6 +83,11 @@ def to_dict(self) -> Dict[str, Any]: if self.operation_id is None and "operation_id" in self.model_fields_set: _dict['operation_id'] = None + # set to None if operation_ids (nullable) is None + # and model_fields_set contains the field + if self.operation_ids is None and "operation_ids" in self.model_fields_set: + _dict['operation_ids'] = None + # set to None if usage (nullable) is None # and model_fields_set contains the field if self.usage is None and "usage" in self.model_fields_set: @@ -104,6 +110,7 @@ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: "items_count": obj.get("items_count"), "async": obj.get("async"), "operation_id": obj.get("operation_id"), + "operation_ids": obj.get("operation_ids"), "usage": TokenUsage.from_dict(obj["usage"]) if obj.get("usage") is not None else None }) return _obj diff --git a/hindsight-clients/rust/src/lib.rs b/hindsight-clients/rust/src/lib.rs index 7632ce74d..c416afbc3 100644 --- a/hindsight-clients/rust/src/lib.rs +++ b/hindsight-clients/rust/src/lib.rs @@ -72,6 +72,7 @@ mod tests { entities: None, tags: None, observation_scopes: None, + strategy: None, }, types::MemoryItem { content: "Bob works with Alice on the search team".to_string(), @@ -82,6 +83,7 @@ mod tests { entities: None, tags: None, observation_scopes: None, + strategy: None, }, ], document_tags: None, diff --git a/hindsight-clients/typescript/generated/types.gen.ts b/hindsight-clients/typescript/generated/types.gen.ts index c9de29558..ff163810c 100644 --- a/hindsight-clients/typescript/generated/types.gen.ts +++ b/hindsight-clients/typescript/generated/types.gen.ts @@ -1244,6 +1244,12 @@ export type MemoryItem = { | "all_combinations" | Array> | null; + /** + * Strategy + * + * Named retain strategy for this item. Overrides the bank's default strategy for this item only. Strategies are defined in the bank config under 'retain_strategies'. + */ + strategy?: string | null; }; /** @@ -1958,9 +1964,15 @@ export type RetainResponse = { /** * Operation Id * - * Operation ID for tracking async operations. Use GET /v1/default/banks/{bank_id}/operations to list operations. Only present when async=true. + * Operation ID for tracking async operations. Use GET /v1/default/banks/{bank_id}/operations to list operations. Only present when async=true. When items use different per-item strategies, use operation_ids instead. */ operation_id?: string | null; + /** + * Operation Ids + * + * Operation IDs when items were submitted as multiple strategy groups (async=true with mixed per-item strategies). operation_id is set to the first entry for backward compatibility. + */ + operation_ids?: Array | null; /** * Token usage metrics for LLM calls during fact extraction (only present for synchronous operations) */ diff --git a/hindsight-clients/typescript/src/index.ts b/hindsight-clients/typescript/src/index.ts index dcf52f560..21817a31e 100644 --- a/hindsight-clients/typescript/src/index.ts +++ b/hindsight-clients/typescript/src/index.ts @@ -82,6 +82,7 @@ export interface MemoryItemInput { entities?: EntityInput[]; tags?: string[]; observation_scopes?: "per_tag" | "combined" | "all_combinations" | string[][]; + strategy?: string; } export class HindsightClient { @@ -190,6 +191,7 @@ export class HindsightClient { entities: item.entities, tags: item.tags, observation_scopes: item.observation_scopes, + strategy: item.strategy, timestamp: item.timestamp instanceof Date ? item.timestamp.toISOString() diff --git a/hindsight-control-plane/src/components/bank-config-view.tsx b/hindsight-control-plane/src/components/bank-config-view.tsx index ba1c63c7a..081ac6af7 100644 --- a/hindsight-control-plane/src/components/bank-config-view.tsx +++ b/hindsight-control-plane/src/components/bank-config-view.tsx @@ -1,6 +1,6 @@ "use client"; -import { useState, useEffect, useMemo, type ReactNode } from "react"; +import { useState, useEffect, useRef, useMemo, type ReactNode } from "react"; import { useBank } from "@/lib/bank-context"; import { useFeatures } from "@/lib/features-context"; import { client } from "@/lib/api"; @@ -15,6 +15,16 @@ import { SelectValue, } from "@/components/ui/select"; import { Alert, AlertDescription } from "@/components/ui/alert"; +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from "@/components/ui/alert-dialog"; import { Switch } from "@/components/ui/switch"; import { Checkbox } from "@/components/ui/checkbox"; import { Label } from "@/components/ui/label"; @@ -35,6 +45,13 @@ type RetainEdits = { retain_extraction_mode: string | null; retain_mission: string | null; retain_custom_instructions: string | null; + entities_allow_free_form: boolean | null; + entity_labels: LabelGroup[] | null; +}; + +type StrategiesEdits = { + retain_default_strategy: string | null; + retain_strategies: Record> | null; }; type ObservationsEdits = { @@ -55,11 +72,6 @@ type LabelGroup = { values: LabelValue[]; }; -type EntityLabelsEdits = { - entity_labels: LabelGroup[] | null; - entities_allow_free_form: boolean; -}; - type MCPEdits = { mcp_enabled_tools: string[] | null; }; @@ -134,12 +146,28 @@ const ALL_TOOLS: string[] = MCP_TOOL_GROUPS.flatMap((g) => g.tools); // ─── Slice helpers ──────────────────────────────────────────────────────────── +function parseEntityLabels(raw: unknown): LabelGroup[] | null { + if (Array.isArray(raw)) return raw as LabelGroup[]; + if (raw && typeof raw === "object" && Array.isArray((raw as any).attributes)) + return (raw as any).attributes as LabelGroup[]; + return null; +} + function retainSlice(config: Record): RetainEdits { return { retain_chunk_size: config.retain_chunk_size ?? null, retain_extraction_mode: config.retain_extraction_mode ?? null, retain_mission: config.retain_mission ?? null, retain_custom_instructions: config.retain_custom_instructions ?? null, + entities_allow_free_form: config.entities_allow_free_form ?? null, + entity_labels: parseEntityLabels(config.entity_labels), + }; +} + +function strategiesSlice(config: Record): StrategiesEdits { + return { + retain_default_strategy: config.retain_default_strategy ?? null, + retain_strategies: config.retain_strategies ?? null, }; } @@ -154,20 +182,6 @@ function observationsSlice(config: Record): ObservationsEdits { }; } -function entityLabelsSlice(config: Record): EntityLabelsEdits { - const raw = config.entity_labels; - let attrs: LabelGroup[] | null = null; - if (Array.isArray(raw)) { - attrs = raw as LabelGroup[]; - } else if (raw && typeof raw === "object" && Array.isArray(raw.attributes)) { - attrs = raw.attributes as LabelGroup[]; - } - return { - entity_labels: attrs, - entities_allow_free_form: config.entities_allow_free_form ?? true, - }; -} - function mcpSlice(config: Record): MCPEdits { return { mcp_enabled_tools: config.mcp_enabled_tools ?? null, @@ -201,12 +215,10 @@ export function BankConfigView() { // Per-section local edits const [retainEdits, setRetainEdits] = useState(retainSlice({})); + const [strategiesEdits, setStrategiesEdits] = useState(strategiesSlice({})); const [observationsEdits, setObservationsEdits] = useState( observationsSlice({}) ); - const [entityLabelsEdits, setEntityLabelsEdits] = useState( - entityLabelsSlice({}) - ); const [reflectEdits, setReflectEdits] = useState(DEFAULT_PROFILE); const [mcpEdits, setMcpEdits] = useState(mcpSlice({})); const [geminiEdits, setGeminiEdits] = useState(geminiSlice({})); @@ -214,32 +226,26 @@ export function BankConfigView() { // Per-section saving/error state const [retainSaving, setRetainSaving] = useState(false); const [observationsSaving, setObservationsSaving] = useState(false); - const [entityLabelsSaving, setEntityLabelsSaving] = useState(false); const [reflectSaving, setReflectSaving] = useState(false); const [mcpSaving, setMcpSaving] = useState(false); const [geminiSaving, setGeminiSaving] = useState(false); const [retainError, setRetainError] = useState(null); const [observationsError, setObservationsError] = useState(null); - const [entityLabelsError, setEntityLabelsError] = useState(null); const [reflectError, setReflectError] = useState(null); const [mcpError, setMcpError] = useState(null); const [geminiError, setGeminiError] = useState(null); - // Reset dialog - // Dirty tracking const retainDirty = useMemo( - () => JSON.stringify(retainEdits) !== JSON.stringify(retainSlice(baseConfig)), - [retainEdits, baseConfig] + () => + JSON.stringify(retainEdits) !== JSON.stringify(retainSlice(baseConfig)) || + JSON.stringify(strategiesEdits) !== JSON.stringify(strategiesSlice(baseConfig)), + [retainEdits, strategiesEdits, baseConfig] ); const observationsDirty = useMemo( () => JSON.stringify(observationsEdits) !== JSON.stringify(observationsSlice(baseConfig)), [observationsEdits, baseConfig] ); - const entityLabelsDirty = useMemo( - () => JSON.stringify(entityLabelsEdits) !== JSON.stringify(entityLabelsSlice(baseConfig)), - [entityLabelsEdits, baseConfig] - ); const reflectDirty = useMemo( () => JSON.stringify(reflectEdits) !== JSON.stringify(baseProfile), [reflectEdits, baseProfile] @@ -277,8 +283,8 @@ export function BankConfigView() { setBaseConfig(cfg); setBaseProfile(prof); setRetainEdits(retainSlice(cfg)); + setStrategiesEdits(strategiesSlice(cfg)); setObservationsEdits(observationsSlice(cfg)); - setEntityLabelsEdits(entityLabelsSlice(cfg)); setReflectEdits(prof); setMcpEdits(mcpSlice(cfg)); setGeminiEdits(geminiSlice(cfg)); @@ -294,8 +300,9 @@ export function BankConfigView() { setRetainSaving(true); setRetainError(null); try { - await client.updateBankConfig(bankId, retainEdits); - setBaseConfig((prev) => ({ ...prev, ...retainEdits })); + const payload = { ...retainEdits, ...strategiesEdits }; + await client.updateBankConfig(bankId, payload); + setBaseConfig((prev) => ({ ...prev, ...payload })); } catch (err: any) { setRetainError(err.message || "Failed to save retain settings"); } finally { @@ -317,24 +324,6 @@ export function BankConfigView() { } }; - const saveEntityLabels = async () => { - if (!bankId) return; - setEntityLabelsSaving(true); - setEntityLabelsError(null); - try { - const payload = { - entity_labels: entityLabelsEdits.entity_labels, - entities_allow_free_form: entityLabelsEdits.entities_allow_free_form, - }; - await client.updateBankConfig(bankId, payload); - setBaseConfig((prev) => ({ ...prev, ...payload })); - } catch (err: any) { - setEntityLabelsError(err.message || "Failed to save entity labels settings"); - } finally { - setEntityLabelsSaving(false); - } - }; - const saveReflect = async () => { if (!bankId) return; setReflectSaving(true); @@ -416,111 +405,49 @@ export function BankConfigView() { return ( <>
- {/* Retain Section */} + {/* Retain + Strategies Section */} - - setRetainEdits((prev) => ({ + - setRetainEdits((prev) => ({ ...prev, retain_extraction_mode: val })) - } > - {["concise", "verbose", "custom"].map((opt) => ( - - {opt} + + Default + + {Object.keys(strategiesEdits.retain_strategies ?? {}).map((name) => ( + + {name} ))} - {retainEdits.retain_extraction_mode === "custom" && ( - - setRetainEdits((prev) => ({ ...prev, retain_custom_instructions: v || null })) - } - rows={5} - /> - )} - - - {/* Entity Labels Section */} - - -
- - - setEntityLabelsEdits((prev) => ({ ...prev, entities_allow_free_form: v })) - } - /> -
- - - setEntityLabelsEdits((prev) => ({ - ...prev, - entity_labels: attrs.length > 0 ? attrs : null, - })) + setRetainEdits((prev) => ({ ...prev, ...patch }))} + strategies={strategiesEdits.retain_strategies} + onStrategiesChange={(v) => + setStrategiesEdits((prev) => ({ ...prev, retain_strategies: v })) } /> @@ -755,6 +682,340 @@ export function BankConfigView() { ); } +// ─── Retain strategies panel ────────────────────────────────────────────────── + +type RetainFormValues = { + retain_extraction_mode: string | null; + retain_chunk_size: number | null; + retain_mission: string | null; + retain_custom_instructions: string | null; + entities_allow_free_form: boolean | null; + entity_labels: LabelGroup[] | null; +}; + +const EXTRACTION_MODES = ["concise", "verbose", "verbatim", "chunks", "custom"]; +const INHERIT_SENTINEL = "__inherit__"; + +function RetainStrategyForm({ + values, + onChange, + isOverride = false, +}: { + values: RetainFormValues; + onChange: (patch: Partial) => void; + isOverride?: boolean; +}) { + const modeValue = values.retain_extraction_mode ?? (isOverride ? INHERIT_SENTINEL : ""); + const showCustomField = values.retain_extraction_mode === "custom"; + + return ( +
+ + + + + + onChange({ retain_chunk_size: e.target.value ? parseFloat(e.target.value) : null }) + } + placeholder={isOverride ? "Inherited from default" : undefined} + /> + + onChange({ retain_mission: v || null })} + placeholder={ + isOverride + ? "Inherited from default" + : "e.g. Always include technical decisions, API design choices, and architectural trade-offs." + } + rows={3} + /> + {showCustomField && ( + onChange({ retain_custom_instructions: v || null })} + rows={5} + /> + )} + +
+ + onChange({ entities_allow_free_form: v })} + /> +
+ + onChange({ entity_labels: attrs.length > 0 ? attrs : null })} + /> +
+ ); +} + +type LocalStrategy = { id: number; name: string; values: RetainFormValues }; + +function fromStrategiesDict(dict: Record> | null): LocalStrategy[] { + if (!dict) return []; + return Object.entries(dict).map(([name, overrides], i) => ({ + id: i, + name, + values: { + retain_extraction_mode: overrides.retain_extraction_mode ?? null, + retain_chunk_size: overrides.retain_chunk_size ?? null, + retain_mission: overrides.retain_mission ?? null, + retain_custom_instructions: overrides.retain_custom_instructions ?? null, + entities_allow_free_form: overrides.entities_allow_free_form ?? null, + entity_labels: parseEntityLabels(overrides.entity_labels), + }, + })); +} + +function toStrategiesDict(local: LocalStrategy[]): Record> | null { + const dict: Record> = {}; + for (const s of local) { + if (!s.name.trim()) continue; + const overrides: Record = {}; + if (s.values.retain_extraction_mode !== null) + overrides.retain_extraction_mode = s.values.retain_extraction_mode; + if (s.values.retain_chunk_size !== null) + overrides.retain_chunk_size = s.values.retain_chunk_size; + if (s.values.retain_mission) overrides.retain_mission = s.values.retain_mission; + if (s.values.retain_custom_instructions) + overrides.retain_custom_instructions = s.values.retain_custom_instructions; + if (s.values.entities_allow_free_form !== null) + overrides.entities_allow_free_form = s.values.entities_allow_free_form; + if (s.values.entity_labels !== null) overrides.entity_labels = s.values.entity_labels; + dict[s.name.trim()] = overrides; + } + return Object.keys(dict).length > 0 ? dict : null; +} + +function RetainStrategiesPanel({ + defaultValues, + onDefaultChange, + strategies, + onStrategiesChange, +}: { + defaultValues: RetainFormValues; + onDefaultChange: (patch: Partial) => void; + strategies: Record> | null; + onStrategiesChange: (v: Record> | null) => void; +}) { + const [local, setLocal] = useState(() => fromStrategiesDict(strategies)); + const [selectedTab, setSelectedTab] = useState("default"); + const [pendingDelete, setPendingDelete] = useState(null); + const skipSyncRef = useRef(false); + + const strategiesKey = JSON.stringify(strategies); + useEffect(() => { + if (skipSyncRef.current) { + skipSyncRef.current = false; + return; + } + setLocal(fromStrategiesDict(strategies)); + }, [strategiesKey]); + + const updateLocal = (next: LocalStrategy[]) => { + skipSyncRef.current = true; + setLocal(next); + onStrategiesChange(toStrategiesDict(next)); + }; + + const addStrategy = () => { + const id = Date.now(); + const next = [ + ...local, + { + id, + name: "", + values: { + retain_extraction_mode: null, + retain_chunk_size: null, + retain_mission: null, + retain_custom_instructions: null, + entities_allow_free_form: null, + entity_labels: null, + }, + }, + ]; + updateLocal(next); + setSelectedTab(id); + }; + + const removeStrategy = (id: number) => { + const next = local.filter((s) => s.id !== id); + updateLocal(next); + if (selectedTab === id) setSelectedTab("default"); + }; + + const updateStrategy = (id: number, patch: Partial) => { + updateLocal(local.map((s) => (s.id === id ? { ...s, ...patch } : s))); + }; + + const activeStrategy = selectedTab !== "default" ? local.find((s) => s.id === selectedTab) : null; + + return ( +
+ {/* Tab bar */} +
+ {/* Default tab */} + + + {/* Named strategy tabs */} + {local.map((s) => ( +
setSelectedTab(s.id)} + > + + {s.name || unnamed} + + +
+ ))} + + +
+ + {/* Form */} +
+ {selectedTab === "default" ? ( + + ) : activeStrategy ? ( +
+
+ +
+ updateStrategy(activeStrategy.id, { name: e.target.value })} + placeholder="strategy name (e.g. fast)" + className={`h-7 text-xs font-mono max-w-[200px] ${!activeStrategy.name.trim() ? "border-destructive focus-visible:ring-destructive" : ""}`} + /> + {!activeStrategy.name.trim() && ( +

Name is required

+ )} +
+
+ + updateStrategy(activeStrategy.id, { + values: { ...activeStrategy.values, ...patch }, + }) + } + isOverride + /> +
+ ) : null} +
+ + { + if (!open) setPendingDelete(null); + }} + > + + + + Delete strategy “{pendingDelete?.name || "unnamed"}”? + + + This will remove the strategy and all its overrides. This cannot be undone. + + + + Cancel + { + if (pendingDelete) { + removeStrategy(pendingDelete.id); + setPendingDelete(null); + } + }} + > + Delete + + + + +
+ ); +} + // ─── ToolSelector ───────────────────────────────────────────────────────────── function ToolSelector({ diff --git a/hindsight-control-plane/src/components/bank-selector.tsx b/hindsight-control-plane/src/components/bank-selector.tsx index 34c7173a2..a00ee0b5d 100644 --- a/hindsight-control-plane/src/components/bank-selector.tsx +++ b/hindsight-control-plane/src/components/bank-selector.tsx @@ -80,8 +80,22 @@ function BankSelectorInner() { "document" ); const [docAsync, setDocAsync] = React.useState(false); + const [docStrategy, setDocStrategy] = React.useState(""); const [isCreatingDoc, setIsCreatingDoc] = React.useState(false); + // Available strategies for the current bank + const [bankStrategies, setBankStrategies] = React.useState([]); + React.useEffect(() => { + if (!docDialogOpen || !currentBank) return; + client + .getBankConfig(currentBank) + .then((resp) => { + const strategies = resp.config?.retain_strategies; + setBankStrategies(strategies ? Object.keys(strategies) : []); + }) + .catch(() => setBankStrategies([])); + }, [docDialogOpen, currentBank]); + // File upload state const [selectedFiles, setSelectedFiles] = React.useState([]); const [filesMetadata, setFilesMetadata] = React.useState< @@ -91,6 +105,8 @@ function BankSelectorInner() { document_id: string; tags: string; metadata: string; + strategy: string; + advancedTab: "document" | "tags" | "source"; expanded: boolean; }[] >([]); @@ -195,6 +211,8 @@ function BankSelectorInner() { document_id: "", tags: "", metadata: "", + strategy: "", + advancedTab: "document" as "document" | "tags" | "source", expanded: false, }); @@ -214,7 +232,14 @@ function BankSelectorInner() { const updateFileMeta = ( index: number, - field: "context" | "timestamp" | "document_id" | "tags" | "metadata", + field: + | "context" + | "timestamp" + | "document_id" + | "tags" + | "metadata" + | "strategy" + | "advancedTab", value: string ) => { setFilesMetadata((prev) => prev.map((m, i) => (i === index ? { ...m, [field]: value } : m))); @@ -246,6 +271,7 @@ function BankSelectorInner() { .filter(Boolean), }), ...(meta.metadata && { metadata: parseMetadata(meta.metadata) }), + ...(meta.strategy && { strategy: meta.strategy }), })); await client.uploadFiles({ @@ -293,6 +319,7 @@ function BankSelectorInner() { observation_scopes?: "per_tag" | "combined" | "all_combinations" | string[][]; metadata?: Record; entities?: Array<{ text: string }>; + strategy?: string; } = { content: docContent }; if (docContext) item.context = docContext; if (docEventDate) item.timestamp = docEventDate + ":00"; @@ -320,6 +347,7 @@ function BankSelectorInner() { if (parsedMeta) item.metadata = parsedMeta; const parsedEntities = parseEntities(docEntities); if (parsedEntities) item.entities = parsedEntities; + if (docStrategy) item.strategy = docStrategy; await client.retain({ bank_id: currentBank, @@ -340,6 +368,7 @@ function BankSelectorInner() { setDocEntities(""); setDocAdvancedTab("document"); setDocAsync(false); + setDocStrategy(""); // Navigate to documents view to see the new document router.push(`/banks/${currentBank}?view=documents`); @@ -647,74 +676,148 @@ function BankSelectorInner() { {/* Per-file metadata form */} {meta?.expanded && ( -
-
- - - updateFileMeta(index, "context", e.target.value) - } - placeholder="Optional context..." - className="h-8 text-sm" - /> -
-
-
- - - updateFileMeta(index, "timestamp", e.target.value) - } - className="h-8 text-sm text-foreground" - /> +
+ updateFileMeta(index, "advancedTab", v)} + > + + {(["document", "tags", "source"] as const).map((t) => ( + + {t} + + ))} + +
+ +
+
+ + + updateFileMeta(index, "timestamp", e.target.value) + } + className="h-8 text-sm text-foreground" + /> +
+
+ + + updateFileMeta( + index, + "document_id", + e.target.value + ) + } + placeholder="Optional ID..." + className="h-8 text-sm" + /> +
+
+
+ + {bankStrategies.length > 0 ? ( + + ) : ( + + updateFileMeta(index, "strategy", e.target.value) + } + placeholder="Strategy name (optional)..." + className="h-8 text-sm" + /> + )} +
+ + +
+ + + updateFileMeta(index, "tags", e.target.value) + } + placeholder="tag1, tag2..." + className="h-8 text-sm" + /> +

+ Comma-separated — used to filter memories during + recall/reflect +

+
+ + +
+ + + updateFileMeta(index, "context", e.target.value) + } + placeholder="Optional context..." + className="h-8 text-sm" + /> +
+
+ +