From 92debb12e204676e635caaf459ba813014119f14 Mon Sep 17 00:00:00 2001 From: matano Date: Wed, 10 Dec 2025 16:20:53 +0200 Subject: [PATCH 01/51] add control over openrag ingestion params --- src/api/settings.py | 8 ++++++++ src/main.py | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/src/api/settings.py b/src/api/settings.py index 982d9272b..2e8aae292 100644 --- a/src/api/settings.py +++ b/src/api/settings.py @@ -1417,10 +1417,18 @@ async def rollback_onboarding(request, session_manager, task_service): # Only allow rollback if config was marked as edited (onboarding completed) if not current_config.edited: + logger.info("No onboarding configuration to rollback") return JSONResponse( {"error": "No onboarding configuration to rollback"}, status_code=400 ) + from config.settings import INDEX_NAME + if await clients.opensearch.indices.exists(index=INDEX_NAME): + # DELETE / + logger.info(f"Deleting index '{INDEX_NAME}'...") + resp = await clients.opensearch.indices.delete(index=INDEX_NAME) + logger.info(f"Deleted '{INDEX_NAME}': {resp}") + user = request.state.user jwt_token = session_manager.get_effective_jwt_token(user.user_id, request.state.jwt_token) diff --git a/src/main.py b/src/main.py index 1c3d065e6..d912aa970 100644 --- a/src/main.py +++ b/src/main.py @@ -188,6 +188,12 @@ async def init_index(): endpoint=getattr(embedding_provider_config, "endpoint", None) ) + if await clients.opensearch.indices.exists(index=INDEX_NAME): + # DELETE / + logger.info(f"Deleting index '{INDEX_NAME}'...") + resp = await clients.opensearch.indices.delete(index=INDEX_NAME) + logger.info(f"Deleted '{INDEX_NAME}': {resp}") + # Create documents index if not await clients.opensearch.indices.exists(index=INDEX_NAME): await clients.opensearch.indices.create( From 24fa7a9e0cb35af90f52d12976bfb4001e8840b8 Mon Sep 17 00:00:00 2001 From: matano Date: Sun, 28 Dec 2025 16:20:59 +0200 Subject: [PATCH 02/51] support changes to the index name --- src/api/documents.py | 6 ++--- src/api/settings.py | 38 ++++++++++++++++++++++++++------ src/api/v1/documents.py | 4 ++-- src/main.py | 32 ++++++++++++++------------- src/models/processors.py | 21 +++++++++--------- src/services/document_service.py | 5 +++-- src/services/flows_service.py | 11 +++++++++ src/services/search_service.py | 10 +++++---- src/utils/embedding_fields.py | 4 ++-- 9 files changed, 86 insertions(+), 45 deletions(-) diff --git a/src/api/documents.py b/src/api/documents.py index 048f746a3..54437aaf9 100644 --- a/src/api/documents.py +++ b/src/api/documents.py @@ -1,7 +1,7 @@ from starlette.requests import Request from starlette.responses import JSONResponse from utils.logging_config import get_logger -from config.settings import INDEX_NAME +import config.settings as settings logger = get_logger(__name__) @@ -30,7 +30,7 @@ async def check_filename_exists(request: Request, document_service, session_mana logger.debug(f"Checking filename existence: {filename}") response = await opensearch_client.search( - index=INDEX_NAME, + index=settings.INDEX_NAME, body=search_body ) @@ -79,7 +79,7 @@ async def delete_documents_by_filename(request: Request, document_service, sessi logger.debug(f"Deleting documents with filename: {filename}") result = await opensearch_client.delete_by_query( - index=INDEX_NAME, + index=settings.INDEX_NAME, body=delete_query, conflicts="proceed" ) diff --git a/src/api/settings.py b/src/api/settings.py index 2e8aae292..0e6eb29c4 100644 --- a/src/api/settings.py +++ b/src/api/settings.py @@ -202,6 +202,7 @@ async def update_settings(request, session_manager): "llm_model", "llm_provider", "system_prompt", + "index_name", "chunk_size", "chunk_overlap", "table_structure", @@ -513,6 +514,29 @@ async def update_settings(request, session_manager): except Exception as e: logger.error(f"Failed to update docling settings in flow: {str(e)}") + if "index_name" in body: + import config.settings as settings + settings.INDEX_NAME = body["index_name"] + # also update ? + # current_config.knowledge.index_name = + config_updated = True + #await TelemetryClient.send_event( + # Category.SETTINGS_OPERATIONS, + # MessageId.ORB_SETTINGS_UPDATED + #) + + # Also update the ingest flow with the new index name + try: + flows_service = _get_flows_service() + await flows_service.update_ingest_flow_index_name(settings.INDEX_NAME) + logger.info( + f"Successfully updated ingest flow index name to '{settings.INDEX_NAME}'." + ) + except Exception as e: + logger.error(f"Failed to update ingest flow index name: {str(e)}") + # Don't fail the entire settings update if flow update fails + # The config will still be saved + if "chunk_size" in body: current_config.knowledge.chunk_size = body["chunk_size"] config_updated = True @@ -1422,12 +1446,12 @@ async def rollback_onboarding(request, session_manager, task_service): {"error": "No onboarding configuration to rollback"}, status_code=400 ) - from config.settings import INDEX_NAME - if await clients.opensearch.indices.exists(index=INDEX_NAME): + import config.settings as settings + if await clients.opensearch.indices.exists(index=settings.INDEX_NAME): # DELETE / - logger.info(f"Deleting index '{INDEX_NAME}'...") - resp = await clients.opensearch.indices.delete(index=INDEX_NAME) - logger.info(f"Deleted '{INDEX_NAME}': {resp}") + logger.info(f"Deleting index '{settings.INDEX_NAME}'...") + resp = await clients.opensearch.indices.delete(index=settings.INDEX_NAME) + logger.info(f"Deleted '{settings.INDEX_NAME}': {resp}") user = request.state.user jwt_token = session_manager.get_effective_jwt_token(user.user_id, request.state.jwt_token) @@ -1474,12 +1498,12 @@ async def rollback_onboarding(request, session_manager, task_service): # Delete documents by filename from utils.opensearch_queries import build_filename_delete_body - from config.settings import INDEX_NAME + import config.settings as settings delete_query = build_filename_delete_body(filename) result = await opensearch_client.delete_by_query( - index=INDEX_NAME, + index=settings.INDEX_NAME, body=delete_query, conflicts="proceed" ) diff --git a/src/api/v1/documents.py b/src/api/v1/documents.py index f2a30c3cb..470eeebdd 100644 --- a/src/api/v1/documents.py +++ b/src/api/v1/documents.py @@ -115,7 +115,7 @@ async def delete_document_endpoint(request: Request, document_service, session_m user = request.state.user try: - from config.settings import INDEX_NAME + import config.settings as settings from utils.opensearch_queries import build_filename_delete_body # Get OpenSearch client (API key auth uses internal client) @@ -127,7 +127,7 @@ async def delete_document_endpoint(request: Request, document_service, session_m delete_query = build_filename_delete_body(filename) result = await opensearch_client.delete_by_query( - index=INDEX_NAME, + index=settings.INDEX_NAME, body=delete_query, conflicts="proceed" ) diff --git a/src/main.py b/src/main.py index 788fb35e8..69aabdabc 100644 --- a/src/main.py +++ b/src/main.py @@ -66,7 +66,6 @@ API_KEYS_INDEX_NAME, DISABLE_INGEST_WITH_LANGFLOW, INDEX_BODY, - INDEX_NAME, SESSION_SECRET, clients, get_embedding_model, @@ -150,17 +149,18 @@ async def configure_alerting_security(): async def _ensure_opensearch_index(): """Ensure OpenSearch index exists when using traditional connector service.""" + import config.settings as settings try: # Check if index already exists - if await clients.opensearch.indices.exists(index=INDEX_NAME): - logger.debug("OpenSearch index already exists", index_name=INDEX_NAME) + if await clients.opensearch.indices.exists(index=settings.INDEX_NAME): + logger.debug("OpenSearch index already exists", index_name=settings.INDEX_NAME) return # Create the index with hard-coded INDEX_BODY (uses OpenAI embedding dimensions) - await clients.opensearch.indices.create(index=INDEX_NAME, body=INDEX_BODY) + await clients.opensearch.indices.create(index=settings.INDEX_NAME, body=INDEX_BODY) logger.info( "Created OpenSearch index for traditional connector service", - index_name=INDEX_NAME, + index_name=settings.INDEX_NAME, vector_dimensions=INDEX_BODY["mappings"]["properties"]["chunk_embedding"][ "dimension" ], @@ -171,7 +171,7 @@ async def _ensure_opensearch_index(): logger.error( "Failed to initialize OpenSearch index for traditional connector service", error=str(e), - index_name=INDEX_NAME, + index_name=settings.INDEX_NAME, ) await TelemetryClient.send_event(Category.OPENSEARCH_INDEX, MessageId.ORB_OS_INDEX_CREATE_FAIL) # Don't raise the exception to avoid breaking the initialization @@ -180,6 +180,7 @@ async def _ensure_opensearch_index(): async def init_index(): """Initialize OpenSearch index and security roles""" + import config.settings as settings await wait_for_opensearch() # Get the configured embedding model from user configuration @@ -196,27 +197,27 @@ async def init_index(): endpoint=getattr(embedding_provider_config, "endpoint", None) ) - if await clients.opensearch.indices.exists(index=INDEX_NAME): + if await clients.opensearch.indices.exists(index=settings.INDEX_NAME): # DELETE / - logger.info(f"Deleting index '{INDEX_NAME}'...") - resp = await clients.opensearch.indices.delete(index=INDEX_NAME) - logger.info(f"Deleted '{INDEX_NAME}': {resp}") + logger.info(f"Deleting index '{settings.INDEX_NAME}'...") + resp = await clients.opensearch.indices.delete(index=settings.INDEX_NAME) + logger.info(f"Deleted '{settings.INDEX_NAME}': {resp}") # Create documents index - if not await clients.opensearch.indices.exists(index=INDEX_NAME): + if not await clients.opensearch.indices.exists(index=settings.INDEX_NAME): await clients.opensearch.indices.create( - index=INDEX_NAME, body=dynamic_index_body + index=settings.INDEX_NAME, body=dynamic_index_body ) logger.info( "Created OpenSearch index", - index_name=INDEX_NAME, + index_name=settings.INDEX_NAME, embedding_model=embedding_model, ) await TelemetryClient.send_event(Category.OPENSEARCH_INDEX, MessageId.ORB_OS_INDEX_CREATED) else: logger.info( "Index already exists, skipping creation", - index_name=INDEX_NAME, + index_name=settings.INDEX_NAME, embedding_model=embedding_model, ) await TelemetryClient.send_event(Category.OPENSEARCH_INDEX, MessageId.ORB_OS_INDEX_EXISTS) @@ -586,6 +587,7 @@ async def startup_tasks(services): async def initialize_services(): + import config.settings as settings """Initialize all services and their dependencies""" await TelemetryClient.send_event(Category.SERVICE_INITIALIZATION, MessageId.ORB_SVC_INIT_START) # Generate JWT keys if they don't exist @@ -626,7 +628,7 @@ async def initialize_services(): patched_async_client=clients, # Pass the clients object itself process_pool=process_pool, embed_model=get_embedding_model(), - index_name=INDEX_NAME, + index_name=settings.INDEX_NAME, task_service=task_service, session_manager=session_manager, ) diff --git a/src/models/processors.py b/src/models/processors.py index d8de30c56..4742bd517 100644 --- a/src/models/processors.py +++ b/src/models/processors.py @@ -20,7 +20,7 @@ async def check_document_exists( Check if a document with the given hash already exists in OpenSearch. Consolidated hash checking for all processors. """ - from config.settings import INDEX_NAME + import config.settings as settings import asyncio max_retries = 3 @@ -28,7 +28,7 @@ async def check_document_exists( for attempt in range(max_retries): try: - exists = await opensearch_client.exists(index=INDEX_NAME, id=file_hash) + exists = await opensearch_client.exists(index=settings.INDEX_NAME, id=file_hash) return exists except (asyncio.TimeoutError, Exception) as e: if attempt == max_retries - 1: @@ -64,7 +64,7 @@ async def check_filename_exists( Check if a document with the given filename already exists in OpenSearch. Returns True if any chunks with this filename exist. """ - from config.settings import INDEX_NAME + import config.settings as settings from utils.opensearch_queries import build_filename_search_body import asyncio @@ -77,7 +77,7 @@ async def check_filename_exists( search_body = build_filename_search_body(filename, size=1, source=False) response = await opensearch_client.search( - index=INDEX_NAME, + index=settings.INDEX_NAME, body=search_body ) @@ -118,7 +118,7 @@ async def delete_document_by_filename( """ Delete all chunks of a document with the given filename from OpenSearch. """ - from config.settings import INDEX_NAME + import config.settings as settings from utils.opensearch_queries import build_filename_delete_body try: @@ -126,7 +126,7 @@ async def delete_document_by_filename( delete_body = build_filename_delete_body(filename) response = await opensearch_client.delete_by_query( - index=INDEX_NAME, + index=settings.INDEX_NAME, body=delete_body ) @@ -168,7 +168,8 @@ async def process_document_standard( embedding model from settings) """ import datetime - from config.settings import INDEX_NAME, clients, get_embedding_model + from config.settings import clients, get_embedding_model + import config.settings as settings from services.document_service import chunk_texts_for_embeddings from utils.document_processing import extract_relevant from utils.embedding_fields import get_embedding_field_name, ensure_embedding_field_exists @@ -187,7 +188,7 @@ async def process_document_standard( # Ensure the embedding field exists for this model embedding_field_name = await ensure_embedding_field_exists( - opensearch_client, embedding_model, INDEX_NAME + opensearch_client, embedding_model, settings.INDEX_NAME ) logger.info( @@ -265,7 +266,7 @@ async def process_document_standard( chunk_id = f"{file_hash}_{i}" try: await opensearch_client.index( - index=INDEX_NAME, id=chunk_id, body=chunk_doc + index=settings.INDEX_NAME, id=chunk_id, body=chunk_doc ) except Exception as e: logger.error( @@ -601,7 +602,7 @@ async def process_item( import time import asyncio import datetime - from config.settings import INDEX_NAME, clients, get_embedding_model + from config.settings import clients, get_embedding_model from services.document_service import chunk_texts_for_embeddings from utils.document_processing import process_document_sync diff --git a/src/services/document_service.py b/src/services/document_service.py index f40c3d823..f00197948 100644 --- a/src/services/document_service.py +++ b/src/services/document_service.py @@ -12,7 +12,7 @@ logger = get_logger(__name__) -from config.settings import clients, INDEX_NAME, get_embedding_model +from config.settings import clients, get_embedding_model from utils.document_processing import extract_relevant, process_document_sync from utils.telemetry import TelemetryClient, Category, MessageId @@ -153,7 +153,8 @@ async def process_upload_file( ) try: - exists = await opensearch_client.exists(index=INDEX_NAME, id=file_hash) + import config.settings as settings + exists = await opensearch_client.exists(index=settings.INDEX_NAME, id=file_hash) except Exception as e: logger.error( "OpenSearch exists check failed", file_hash=file_hash, error=str(e) diff --git a/src/services/flows_service.py b/src/services/flows_service.py index e97ac2d3a..d69cd769b 100644 --- a/src/services/flows_service.py +++ b/src/services/flows_service.py @@ -931,6 +931,17 @@ async def update_flow_docling_preset(self, preset: str, preset_config: dict): await self._update_flow_field(LANGFLOW_INGEST_FLOW_ID, "docling_serve_opts", preset_config, node_display_name=DOCLING_COMPONENT_DISPLAY_NAME) + async def update_ingest_flow_index_name(self, index_name: str): + """Helper function to update index name in the ingest flow""" + if not LANGFLOW_INGEST_FLOW_ID: + raise ValueError("LANGFLOW_INGEST_FLOW_ID is not configured") + await self._update_flow_field( + LANGFLOW_INGEST_FLOW_ID, + "index_name", + index_name, + node_display_name="OpenSearch (Multi-Model Multi-Embedding)", + ) + async def update_ingest_flow_chunk_size(self, chunk_size: int): """Helper function to update chunk size in the ingest flow""" if not LANGFLOW_INGEST_FLOW_ID: diff --git a/src/services/search_service.py b/src/services/search_service.py index b0927d0f8..b1a79f4a8 100644 --- a/src/services/search_service.py +++ b/src/services/search_service.py @@ -1,7 +1,8 @@ import copy from typing import Any, Dict from agentd.tool_decorator import tool -from config.settings import EMBED_MODEL, clients, INDEX_NAME, get_embedding_model, WATSONX_EMBEDDING_DIMENSIONS +import config.settings as settings +from config.settings import EMBED_MODEL, clients, get_embedding_model, WATSONX_EMBEDDING_DIMENSIONS from auth_context import get_auth_context from utils.logging_config import get_logger @@ -120,7 +121,7 @@ async def search_tool(self, query: str, embedding_model: str = None) -> Dict[str } agg_result = await opensearch_client.search( - index=INDEX_NAME, body=agg_query, params={"terminate_after": 0} + index=settings.INDEX_NAME, body=agg_query, params={"terminate_after": 0} ) buckets = agg_result.get("aggregations", {}).get("embedding_models", {}).get("buckets", []) available_models = [b["key"] for b in buckets if b["key"]] @@ -395,8 +396,9 @@ async def embed_with_model(model_name): search_params = {"terminate_after": 0} try: + logger.info(f"Sending query to index '{settings.INDEX_NAME}'..") results = await opensearch_client.search( - index=INDEX_NAME, body=search_body, params=search_params + index=settings.INDEX_NAME, body=search_body, params=search_params ) except RequestError as e: error_message = str(e) @@ -409,7 +411,7 @@ async def embed_with_model(model_name): ) try: results = await opensearch_client.search( - index=INDEX_NAME, + index=settings.INDEX_NAME, body=fallback_search_body, params=search_params, ) diff --git a/src/utils/embedding_fields.py b/src/utils/embedding_fields.py index 990cb116d..34a0d765c 100644 --- a/src/utils/embedding_fields.py +++ b/src/utils/embedding_fields.py @@ -86,11 +86,11 @@ async def ensure_embedding_field_exists( Raises: Exception: If unable to add the field mapping """ - from config.settings import INDEX_NAME + import config.settings as settings from utils.embeddings import get_embedding_dimensions if index_name is None: - index_name = INDEX_NAME + index_name = settings.INDEX_NAME field_name = get_embedding_field_name(model_name) dimensions = await get_embedding_dimensions(model_name) From 7a372d17e0934daa6e721e201d4328567a31d97b Mon Sep 17 00:00:00 2001 From: matano Date: Mon, 29 Dec 2025 11:47:31 +0200 Subject: [PATCH 03/51] a script for reading and writing to the text splitter component in the ingestion pipeline --- scripts/update_split_test_component.py | 50 ++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 scripts/update_split_test_component.py diff --git a/scripts/update_split_test_component.py b/scripts/update_split_test_component.py new file mode 100644 index 000000000..4544e3eaf --- /dev/null +++ b/scripts/update_split_test_component.py @@ -0,0 +1,50 @@ +# !/usr/bin/env python3 +import subprocess +import sys + +flows_dir = "../flows" +flow_file = flows_dir + "/ingestion_flow.json" +code_file = flows_dir + "/components/split_text.py" +display_name = "Split Text" + + +def main(): + read_component() + # write_component() + +def read_component(): + metadata_module = None # "mypkg.flow_meta" # OPTIONAL + match_index = None # OPTIONAL + output = code_file # OPTIONAL + + # Build the command + cmd = [sys.executable, "extract_flow_component.py", "--flow-file", flow_file] + + if display_name: + cmd += ["--display-name", display_name] + if metadata_module: + cmd += ["--metadata-module", metadata_module] + if match_index is not None: + cmd += ["--match-index", str(match_index)] + if output: + cmd += ["--output", output] + + # Run the command + print("Running:", " ".join(cmd)) + subprocess.run(cmd) + +def write_component(): + # Build the command + cmd = [sys.executable, "update_flow_components.py", + "--code-file", code_file, + "--display-name", display_name, + "--flows-dir", flows_dir + ] + + # Run the command + print("Running:", " ".join(cmd)) + subprocess.run(cmd) + + +if __name__ == "__main__": + main() \ No newline at end of file From 9181522c1fda933333bc88bb55ab85e16f81caea Mon Sep 17 00:00:00 2001 From: matano Date: Wed, 31 Dec 2025 14:57:06 +0200 Subject: [PATCH 04/51] update split text component with the option to choose a text splitter type --- flows/components/split_text.py | 421 ++++++++++++++++++ ...nent.py => update_split_text_component.py} | 10 +- 2 files changed, 427 insertions(+), 4 deletions(-) create mode 100644 flows/components/split_text.py rename scripts/{update_split_test_component.py => update_split_text_component.py} (90%) diff --git a/flows/components/split_text.py b/flows/components/split_text.py new file mode 100644 index 000000000..decfa0b0f --- /dev/null +++ b/flows/components/split_text.py @@ -0,0 +1,421 @@ +import copy +import re +from typing import Iterable + +from langchain_text_splitters import CharacterTextSplitter + +from lfx.custom.custom_component.component import Component +from lfx.io import DropdownInput, HandleInput, IntInput, MessageTextInput, Output +from lfx.schema.data import Data +from lfx.schema.dataframe import DataFrame +from lfx.schema.message import Message +from lfx.utils.util import unescape_string +from langchain_core.documents import Document + + +class SplitTextComponent(Component): + display_name: str = "Split Text" + description: str = "Split text into chunks based on specified criteria." + documentation: str = "https://docs.langflow.org/components-processing#split-text" + icon = "scissors-line-dashed" + name = "SplitText" + + inputs = [ + HandleInput( + name="data_inputs", + display_name="Input", + info="The data with texts to split in chunks.", + input_types=["Data", "DataFrame", "Message"], + required=True, + ), + IntInput( + name="chunk_overlap", + display_name="Chunk Overlap", + info="Number of characters to overlap between chunks.", + value=200, + ), + IntInput( + name="chunk_size", + display_name="Chunk Size", + info=( + "The maximum length of each chunk. Text is first split by separator, " + "then chunks are merged up to this size. " + "Individual splits larger than this won't be further divided." + ), + value=1000, + ), + MessageTextInput( + name="separator", + display_name="Separator", + info=( + "The character to split on. Use \\n for newline. " + "Examples: \\n\\n for paragraphs, \\n for lines, . for sentences" + ), + value="\n", + ), + MessageTextInput( + name="text_key", + display_name="Text Key", + info="The key to use for the text column.", + value="text", + advanced=True, + ), + DropdownInput( + name="keep_separator", + display_name="Keep Separator", + info="Whether to keep the separator in the output chunks and where to place it.", + options=["False", "True", "Start", "End"], + value="False", + advanced=True, + ), + DropdownInput( + name="splitter_type", + display_name="Splitter Type", + info="Which text splitter to use to chunk the documents.", + options=["CharacterTextSplitter", "TableAwareTextSplitter", "LineBasedTextSplitter"], + value="CharacterTextSplitter", + advanced=True, + ), + MessageTextInput( + name="model_id", + display_name="Model ID", + info="The name of the model that will be used for computing embeddings.", + value="ibm-granite/granite-embedding-30m-english", + advanced=True, + ), + ] + + outputs = [ + Output(display_name="Chunks", name="dataframe", method="split_text"), + ] + + def _docs_to_data(self, docs) -> list[Data]: + return [Data(text=doc.page_content, data=doc.metadata) for doc in docs] + + def _fix_separator(self, separator: str) -> str: + """Fix common separator issues and convert to proper format.""" + if separator == "/n": + return "\n" + if separator == "/t": + return "\t" + return separator + + def split_text_base(self): + separator = self._fix_separator(self.separator) + separator = unescape_string(separator) + + if isinstance(self.data_inputs, DataFrame): + if not len(self.data_inputs): + msg = "DataFrame is empty" + raise TypeError(msg) + + self.data_inputs.text_key = self.text_key + try: + documents = self.data_inputs.to_lc_documents() + except Exception as e: + msg = f"Error converting DataFrame to documents: {e}" + raise TypeError(msg) from e + elif isinstance(self.data_inputs, Message): + self.data_inputs = [self.data_inputs.to_data()] + return self.split_text_base() + else: + if not self.data_inputs: + msg = "No data inputs provided" + raise TypeError(msg) + + documents = [] + if isinstance(self.data_inputs, Data): + self.data_inputs.text_key = self.text_key + documents = [self.data_inputs.to_lc_document()] + else: + try: + documents = [input_.to_lc_document() for input_ in self.data_inputs if isinstance(input_, Data)] + if not documents: + msg = f"No valid Data inputs found in {type(self.data_inputs)}" + raise TypeError(msg) + except AttributeError as e: + msg = f"Invalid input type in collection: {e}" + raise TypeError(msg) from e + try: + if self.splitter_type == "CharacterTextSplitter": + # Convert string 'False'/'True' to boolean + keep_sep = self.keep_separator + if isinstance(keep_sep, str): + if keep_sep.lower() == "false": + keep_sep = False + elif keep_sep.lower() == "true": + keep_sep = True + # 'start' and 'end' are kept as strings + + print(f"Creating a CharacterTextSplitter..") + splitter = CharacterTextSplitter( + chunk_overlap=self.chunk_overlap, + chunk_size=self.chunk_size, + separator=separator, + keep_separator=keep_sep, + ) + elif self.splitter_type == "LineBasedTextSplitter": + print(f"Creating a LineBasedTextSplitter with chunk_size={self.chunk_size} and model_id '{self.model_id}'.") + splitter = LineBasedTextSplitter( + chunk_size=self.chunk_size, + model_id=self.model_id + ) + elif self.splitter_type == "TableAwareTextSplitter": + print(f"Creating a TableAwareTextSplitter with chunk_size={self.chunk_size} and model_id '{self.model_id}'.") + splitter = TableAwareTextSplitter( + chunk_size=self.chunk_size, + model_id=self.model_id + ) + else: + raise RuntimeError(f"Unknown splitter type value '{self.splitter_type}'.") + return splitter.split_documents(documents) + except Exception as e: + msg = f"Error splitting text: {e}" + raise TypeError(msg) from e + + def split_text(self) -> DataFrame: + return DataFrame(self._docs_to_data(self.split_text_base())) + +class LineBasedTextSplitter: + def __init__( + self, + chunk_size: int, + model_id: str, + prefix: str = "", + ): + self._chunk_size = chunk_size + + from transformers import AutoTokenizer + self._tokenizer = AutoTokenizer.from_pretrained( + pretrained_model_name_or_path=model_id, + ) + + prefix_len = len(self._tokenizer.encode(prefix, add_special_tokens=False)) + if prefix_len >= self._chunk_size: + raise RuntimeError( + f"Chunks prefix: {prefix} is too long for chunk size {self._chunk_size}" + ) + else: + self._prefix = prefix + self._prefixLen = prefix_len + + def split_documents(self, documents: Iterable[Document]) -> list[Document]: + """Given Documents, chunk the text to smaller pieces and return them as list of Documents""" + + chunks = [] + for document in documents: + chunks.extend(self._chunk_document(document)) + return chunks + + def _chunk_document(self, document: Document): + document_text = document.page_content + document_metadata = document.metadata + chunks = [] + chunk_seq_num = 0 + current = self._prefix + current_len = self._prefixLen + first_character_index = document_metadata.get("start_index", 0) + + new_line_token_count = len(self._tokenizer.encode("\n", add_special_tokens=False)) + lines = document_text.split("\n") + for line in lines: + line_tokens = self._tokenizer.encode(line, add_special_tokens=False) + + while ( + len(line_tokens) > self._chunk_size - current_len + ): # line cannot fit into current + num_available_tokens_in_chunk = ( + self._chunk_size - current_len + if len(line_tokens) + self._prefixLen > self._chunk_size + else 0 + ) # if whole line can fit into a new chunk, do not add anything to current chunk, + # otherwise, split the line between current and next chunks. + + if num_available_tokens_in_chunk > 0: + # split line + if current: + current += "\n" + current_len += new_line_token_count + current += self._tokenizer.decode( + line_tokens[:num_available_tokens_in_chunk] + ) + current_len += num_available_tokens_in_chunk + + # add current chunk + chunks.append( + self._new_chunk( + current, chunk_seq_num, first_character_index, document_metadata + ) + ) + + # new current chunk + first_character_index += len(current) + chunk_seq_num += 1 + current = self._prefix + current_len = self._prefixLen + line_tokens = line_tokens[num_available_tokens_in_chunk:] + + # rest of line fits into current + if len(line_tokens) > 0: + if current: + current += "\n" + current_len += new_line_token_count + current += self._tokenizer.decode(line_tokens) + current_len += len(line_tokens) + + # final chunk + chunks.append( + self._new_chunk( + current, chunk_seq_num, first_character_index, document_metadata + ) + ) + + return chunks + + @staticmethod + def _new_chunk( + text: str, seq_no: int, start_index: int, doc_metadata: dict + ) -> Document: + chunk_metadata = copy.deepcopy(doc_metadata) + chunk_metadata["sequence_number"] = seq_no + chunk_metadata["start_index"] = start_index + return Document(page_content=text, metadata=chunk_metadata) + + +class TableAwareTextSplitter: + + def __init__(self, chunk_size: int, model_id: str): + self.chunk_size = chunk_size + self.model_id = model_id + + def split_documents(self, documents: Iterable[Document]) -> list[Document]: + """Given Documents, chunk the text to smaller pieces and return them as list of Documents""" + + chunks = [] + for document in documents: + chunks.extend(self._chunk_document(document)) + return chunks + + def _chunk_document(self, document: Document) -> list[Document]: + segments = self._get_segments(document) + + chunks = [] + for segment in segments: + line_splitter = LineBasedTextSplitter( + chunk_size=self.chunk_size, + model_id=self.model_id, + prefix=self.get_prefix(segment) + ) + chunks.extend(line_splitter.split_documents([segment])) + + return chunks + + # fix me: does not indicate sub headers + def _get_segments(self, doc): + segments = [] + doc_metadata = doc.metadata + segments_count = 0 + start_index = doc.metadata.get("start_index", 0) + current_segment = Document( + page_content="", + metadata={"type": "text", "seq_no": segments_count, "start_index": start_index} + | doc_metadata, + ) + separator_found = False + lines = doc.page_content.split("\n") + for line in lines: + + if self._is_table_line(line): + if current_segment.metadata["type"] != "table": # first table line + segments.append(current_segment) + segments_count += 1 + start_index += len(current_segment.page_content) + current_segment = Document( + page_content="", + metadata={ + "type": "table", + "caption": self.get_caption(current_segment), + "header": self.condense_table_row(line), + "seq_no": segments_count, + "start_index": start_index, + } + | doc_metadata, + ) + separator_found = False + elif self._is_table_seperator(line): + + separator_found = True + current_segment.metadata[ + "header" + ] += "\n" + self.condense_separator(line) + elif not separator_found: + + current_segment.metadata[ + "header" + ] += "\n" + self.condense_table_row(line) + else: + current_segment.page_content += "\n" + line + + else: # text line + if current_segment.metadata["type"] == "table": + segments.append(current_segment) + segments_count += 1 + start_index += len(current_segment.page_content) + current_segment = Document( + page_content="", + metadata={ + "type": "text", + "seq_no": segments_count, + "start_index": start_index, + } + | doc_metadata, + ) + current_segment.page_content += "\n" + line + + # last segment + segments.append(current_segment) + return [c for c in segments if len(c.page_content.strip()) > 0] + + @staticmethod + def get_prefix(segment: Document) -> str: + if segment.metadata["type"] == "text": + return "" + elif segment.metadata["type"] == "table": + result = segment.metadata["caption"] + if result: + result += "\n" + result += segment.metadata["header"] + return result + else: + raise RuntimeError(f"Internal error: unknown segment type '{segment['type']}' for segment {segment}.") + + # returns last sentence before table + @staticmethod + def get_caption(prev_segment) -> str: + last_sentence = prev_segment.page_content.strip().split("\n")[-1].split(".")[-1] + return last_sentence + + # each line starting with | is included in table + @staticmethod + def _is_table_line(line: str): + return line.startswith("|") + + @staticmethod + def _is_table_seperator(line: str): + cells = [c.strip() for c in line.strip().split("|")] + return all( + re.match(r"[-]+", cell.strip()) for cell in cells if len(cell.strip()) > 0 + ) + + @staticmethod + def condense_separator(line: str): + numCells = len(line.strip().split("|")) - 2 + return "| --- " * numCells + "|" + + @staticmethod + def condense_table_row(line: str) -> str: + if sum([t.isalnum() for t in line]) == 0: + return "" + cells = [c.strip() for c in line.strip().split("|")] + + return " | ".join(cells).strip() diff --git a/scripts/update_split_test_component.py b/scripts/update_split_text_component.py similarity index 90% rename from scripts/update_split_test_component.py rename to scripts/update_split_text_component.py index 4544e3eaf..ec3c3b541 100644 --- a/scripts/update_split_test_component.py +++ b/scripts/update_split_text_component.py @@ -8,9 +8,11 @@ display_name = "Split Text" -def main(): - read_component() - # write_component() +def main(read: bool): + if read: + read_component() + else: + write_component() def read_component(): metadata_module = None # "mypkg.flow_meta" # OPTIONAL @@ -47,4 +49,4 @@ def write_component(): if __name__ == "__main__": - main() \ No newline at end of file + main(read=False) From 5436f162e1900ebf8fd630dd4b4b92fbc439181c Mon Sep 17 00:00:00 2001 From: matano Date: Wed, 31 Dec 2025 16:14:50 +0200 Subject: [PATCH 05/51] add the option to control the splitter_type with the settings endpoint --- src/api/settings.py | 100 ++++++++++++++++++++++++---------- src/config/config_manager.py | 3 + src/services/flows_service.py | 11 ++++ 3 files changed, 85 insertions(+), 29 deletions(-) diff --git a/src/api/settings.py b/src/api/settings.py index 0e6eb29c4..9cef11d2f 100644 --- a/src/api/settings.py +++ b/src/api/settings.py @@ -2,6 +2,9 @@ import platform import time from starlette.responses import JSONResponse + +from langchain_text_splitters import CharacterTextSplitter +from openrag.flows.components.split_text import TableAwareTextSplitter, LineBasedTextSplitter from utils.container_utils import transform_localhost_url from utils.logging_config import get_logger from utils.telemetry import TelemetryClient, Category, MessageId @@ -178,6 +181,46 @@ async def get_settings(request, session_manager): {"error": f"Failed to retrieve settings: {str(e)}"}, status_code=500 ) +async def validate_enum_str( + body: dict, + field_name: str, + allowed_values: list[str], +): + """ + Validate that body[field_name], if present, is: + - a string, + - non-empty after stripping whitespace, + - and one of the allowed_values. + + """ + # If not provided, no validation needed + if field_name not in body: + return None + + value = body[field_name] + + # Must be a string + if not isinstance(value, str): + return JSONResponse( + {"error": f"{field_name} must be a string"}, + status_code=400, + ) + + # Must be non-empty after trimming + trimmed = value.strip() + if not trimmed: + return JSONResponse( + {"error": f"{field_name} must be a non-empty string"}, + status_code=400, + ) + + # Must be among allowed values + if trimmed not in allowed_values: + allowed_str = ", ".join(allowed_values) + return JSONResponse( + {"error": f"{field_name} must be one of: {allowed_str}"}, + status_code=400, + ) async def update_settings(request, session_manager): """Update application settings""" @@ -205,6 +248,7 @@ async def update_settings(request, session_manager): "index_name", "chunk_size", "chunk_overlap", + "splitter_type", "table_structure", "ocr", "picture_descriptions", @@ -269,36 +313,13 @@ async def update_settings(request, session_manager): status_code=400, ) - if "llm_provider" in body: - if ( - not isinstance(body["llm_provider"], str) - or not body["llm_provider"].strip() - ): - return JSONResponse( - {"error": "llm_provider must be a non-empty string"}, - status_code=400, - ) - if body["llm_provider"] not in ["openai", "anthropic", "watsonx", "ollama"]: - return JSONResponse( - {"error": "llm_provider must be one of: openai, anthropic, watsonx, ollama"}, - status_code=400, - ) + validate_enum_str(body, field_name="splitter_type", allowed_values=[ + CharacterTextSplitter.__name__, LineBasedTextSplitter.__name__, TableAwareTextSplitter.__name__ + ]) - if "embedding_provider" in body: - if ( - not isinstance(body["embedding_provider"], str) - or not body["embedding_provider"].strip() - ): - return JSONResponse( - {"error": "embedding_provider must be a non-empty string"}, - status_code=400, - ) - # Anthropic doesn't have embeddings - if body["embedding_provider"] not in ["openai", "watsonx", "ollama"]: - return JSONResponse( - {"error": "embedding_provider must be one of: openai, watsonx, ollama"}, - status_code=400, - ) + validate_enum_str(body, field_name="llm_provider", allowed_values=["openai", "anthropic", "watsonx", "ollama"]) + + validate_enum_str(body, field_name="embedding_provider", allowed_values=["openai", "watsonx", "ollama"]) # Validate provider-specific fields for key in ["openai_api_key", "anthropic_api_key", "watsonx_api_key"]: @@ -537,6 +558,27 @@ async def update_settings(request, session_manager): # Don't fail the entire settings update if flow update fails # The config will still be saved + if "splitter_type" in body: + new_splitter_type = body["splitter_type"] + current_config.knowledge.splitter_type = body["splitter_type"] + config_updated = True + await TelemetryClient.send_event( + Category.SETTINGS_OPERATIONS, + MessageId.ORB_SETTINGS_CHUNK_UPDATED + ) + + # Also update the ingest flow with the new splitter type + try: + flows_service = _get_flows_service() + await flows_service.update_ingest_flow_splitter_type(new_splitter_type) + logger.info( + f"Successfully updated ingest flow splitter type to {new_splitter_type}" + ) + except Exception as e: + logger.error(f"Failed to update ingest flow splitter type: {str(e)}") + # Don't fail the entire settings update if flow update fails + # The config will still be saved + if "chunk_size" in body: current_config.knowledge.chunk_size = body["chunk_size"] config_updated = True diff --git a/src/config/config_manager.py b/src/config/config_manager.py index e52318890..0e0293b6b 100644 --- a/src/config/config_manager.py +++ b/src/config/config_manager.py @@ -5,6 +5,8 @@ from pathlib import Path from typing import Dict, Any, Optional from dataclasses import dataclass, asdict + +from langchain_text_splitters import CharacterTextSplitter from utils.logging_config import get_logger logger = get_logger(__name__) @@ -71,6 +73,7 @@ class KnowledgeConfig: embedding_provider: str = "openai" # Which provider to use for embeddings chunk_size: int = 1000 chunk_overlap: int = 200 + splitter_type: str = CharacterTextSplitter.__name__ table_structure: bool = True ocr: bool = False picture_descriptions: bool = False diff --git a/src/services/flows_service.py b/src/services/flows_service.py index d69cd769b..ad07e34a0 100644 --- a/src/services/flows_service.py +++ b/src/services/flows_service.py @@ -964,6 +964,17 @@ async def update_ingest_flow_chunk_overlap(self, chunk_overlap: int): node_display_name="Split Text", ) + async def update_ingest_flow_splitter_type(self, splitter_type: str): + """Helper function to update splitter type in the ingest flow""" + if not LANGFLOW_INGEST_FLOW_ID: + raise ValueError("LANGFLOW_INGEST_FLOW_ID is not configured") + await self._update_flow_field( + LANGFLOW_INGEST_FLOW_ID, + "splitter_type", + splitter_type, + node_display_name="Split Text", + ) + async def update_ingest_flow_embedding_model(self, embedding_model: str, provider: str): """Helper function to update embedding model in the ingest flow""" if not LANGFLOW_INGEST_FLOW_ID: From f4578909d760305cfdb6d54985b169f4b4f7bb7c Mon Sep 17 00:00:00 2001 From: matano Date: Wed, 31 Dec 2025 16:45:50 +0200 Subject: [PATCH 06/51] remove not working imports, replace with string for class names --- src/api/settings.py | 14 +++++++------- src/config/config_manager.py | 3 +-- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/api/settings.py b/src/api/settings.py index 9cef11d2f..5222f69d9 100644 --- a/src/api/settings.py +++ b/src/api/settings.py @@ -3,8 +3,6 @@ import time from starlette.responses import JSONResponse -from langchain_text_splitters import CharacterTextSplitter -from openrag.flows.components.split_text import TableAwareTextSplitter, LineBasedTextSplitter from utils.container_utils import transform_localhost_url from utils.logging_config import get_logger from utils.telemetry import TelemetryClient, Category, MessageId @@ -313,13 +311,15 @@ async def update_settings(request, session_manager): status_code=400, ) - validate_enum_str(body, field_name="splitter_type", allowed_values=[ - CharacterTextSplitter.__name__, LineBasedTextSplitter.__name__, TableAwareTextSplitter.__name__ - ]) + await validate_enum_str(body, field_name="splitter_type", + allowed_values=[ + "CharacterTextSplitter", "LineBasedTextSplitter", "TableAwareTextSplitter" + ] + ) - validate_enum_str(body, field_name="llm_provider", allowed_values=["openai", "anthropic", "watsonx", "ollama"]) + await validate_enum_str(body, field_name="llm_provider", allowed_values=["openai", "anthropic", "watsonx", "ollama"]) - validate_enum_str(body, field_name="embedding_provider", allowed_values=["openai", "watsonx", "ollama"]) + await validate_enum_str(body, field_name="embedding_provider", allowed_values=["openai", "watsonx", "ollama"]) # Validate provider-specific fields for key in ["openai_api_key", "anthropic_api_key", "watsonx_api_key"]: diff --git a/src/config/config_manager.py b/src/config/config_manager.py index 0e0293b6b..c364ee132 100644 --- a/src/config/config_manager.py +++ b/src/config/config_manager.py @@ -6,7 +6,6 @@ from typing import Dict, Any, Optional from dataclasses import dataclass, asdict -from langchain_text_splitters import CharacterTextSplitter from utils.logging_config import get_logger logger = get_logger(__name__) @@ -73,7 +72,7 @@ class KnowledgeConfig: embedding_provider: str = "openai" # Which provider to use for embeddings chunk_size: int = 1000 chunk_overlap: int = 200 - splitter_type: str = CharacterTextSplitter.__name__ + splitter_type: str = "CharacterTextSplitter" table_structure: bool = True ocr: bool = False picture_descriptions: bool = False From 560ec04c17e063edef30bd176c8af6560717cd6e Mon Sep 17 00:00:00 2001 From: matano Date: Wed, 31 Dec 2025 16:46:09 +0200 Subject: [PATCH 07/51] support tokenization with tiktoken for OpenAI models --- flows/components/split_text.py | 38 +++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/flows/components/split_text.py b/flows/components/split_text.py index decfa0b0f..6f34f7259 100644 --- a/flows/components/split_text.py +++ b/flows/components/split_text.py @@ -184,13 +184,22 @@ def __init__( prefix: str = "", ): self._chunk_size = chunk_size + self.use_tiktoken = False + if model_id in ["text-embedding-3-small", "text-embedding-3-large"]: + self.use_tiktoken = True + print(f"Initializing tokenizer for model '{model_id}', use_tiktoken = {self.use_tiktoken}.") + + if self.use_tiktoken: + import tiktoken + # The tokenizer for text-embedding-3-small + self._tokenizer = tiktoken.get_encoding("cl100k_base") + else: + from transformers import AutoTokenizer + self._tokenizer = AutoTokenizer.from_pretrained( + pretrained_model_name_or_path=model_id, + ) - from transformers import AutoTokenizer - self._tokenizer = AutoTokenizer.from_pretrained( - pretrained_model_name_or_path=model_id, - ) - - prefix_len = len(self._tokenizer.encode(prefix, add_special_tokens=False)) + prefix_len = len(self.tokenize(prefix)) if prefix_len >= self._chunk_size: raise RuntimeError( f"Chunks prefix: {prefix} is too long for chunk size {self._chunk_size}" @@ -199,6 +208,15 @@ def __init__( self._prefix = prefix self._prefixLen = prefix_len + def tokenize(self, text: str) -> list[int]: + if self.use_tiktoken: + return self._tokenizer.encode(text) + else: + return self._tokenizer.encode(text, add_special_tokens=False) + + def decode_tokens(self, tokens: list[int]): + return self._tokenizer.decode(tokens) + def split_documents(self, documents: Iterable[Document]) -> list[Document]: """Given Documents, chunk the text to smaller pieces and return them as list of Documents""" @@ -216,10 +234,10 @@ def _chunk_document(self, document: Document): current_len = self._prefixLen first_character_index = document_metadata.get("start_index", 0) - new_line_token_count = len(self._tokenizer.encode("\n", add_special_tokens=False)) + new_line_token_count = len(self.tokenize("\n")) lines = document_text.split("\n") for line in lines: - line_tokens = self._tokenizer.encode(line, add_special_tokens=False) + line_tokens = self.tokenize(line) while ( len(line_tokens) > self._chunk_size - current_len @@ -236,7 +254,7 @@ def _chunk_document(self, document: Document): if current: current += "\n" current_len += new_line_token_count - current += self._tokenizer.decode( + current += self.decode_tokens( line_tokens[:num_available_tokens_in_chunk] ) current_len += num_available_tokens_in_chunk @@ -260,7 +278,7 @@ def _chunk_document(self, document: Document): if current: current += "\n" current_len += new_line_token_count - current += self._tokenizer.decode(line_tokens) + current += self.decode_tokens(line_tokens) current_len += len(line_tokens) # final chunk From 199263273fa960a4834d639957858c12567fb125 Mon Sep 17 00:00:00 2001 From: matano Date: Wed, 31 Dec 2025 17:02:28 +0200 Subject: [PATCH 08/51] fix validate_enum_str and its use --- src/api/settings.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/api/settings.py b/src/api/settings.py index 5222f69d9..0da4a4bc9 100644 --- a/src/api/settings.py +++ b/src/api/settings.py @@ -183,7 +183,7 @@ async def validate_enum_str( body: dict, field_name: str, allowed_values: list[str], -): +) -> JSONResponse | None: """ Validate that body[field_name], if present, is: - a string, @@ -220,6 +220,8 @@ async def validate_enum_str( status_code=400, ) + return None + async def update_settings(request, session_manager): """Update application settings""" try: @@ -311,15 +313,21 @@ async def update_settings(request, session_manager): status_code=400, ) - await validate_enum_str(body, field_name="splitter_type", + response = await validate_enum_str(body, field_name="splitter_type", allowed_values=[ "CharacterTextSplitter", "LineBasedTextSplitter", "TableAwareTextSplitter" ] ) + if response: + return response - await validate_enum_str(body, field_name="llm_provider", allowed_values=["openai", "anthropic", "watsonx", "ollama"]) + response = await validate_enum_str(body, field_name="llm_provider", allowed_values=["openai", "anthropic", "watsonx", "ollama"]) + if response: + return response - await validate_enum_str(body, field_name="embedding_provider", allowed_values=["openai", "watsonx", "ollama"]) + response = await validate_enum_str(body, field_name="embedding_provider", allowed_values=["openai", "watsonx", "ollama"]) + if response: + return response # Validate provider-specific fields for key in ["openai_api_key", "anthropic_api_key", "watsonx_api_key"]: @@ -560,7 +568,7 @@ async def update_settings(request, session_manager): if "splitter_type" in body: new_splitter_type = body["splitter_type"] - current_config.knowledge.splitter_type = body["splitter_type"] + current_config.knowledge.splitter_type = new_splitter_type config_updated = True await TelemetryClient.send_event( Category.SETTINGS_OPERATIONS, From 579ba6e56103a9625fa176539bc9a82b444da980 Mon Sep 17 00:00:00 2001 From: matano Date: Thu, 1 Jan 2026 12:15:31 +0200 Subject: [PATCH 09/51] add an update of the model_id in the text split component, when a new embedding model is set --- src/api/settings.py | 13 +++++++++++++ src/services/flows_service.py | 11 +++++++++++ 2 files changed, 24 insertions(+) diff --git a/src/api/settings.py b/src/api/settings.py index 0da4a4bc9..d749b3aba 100644 --- a/src/api/settings.py +++ b/src/api/settings.py @@ -486,6 +486,19 @@ async def update_settings(request, session_manager): Category.SETTINGS_OPERATIONS, MessageId.ORB_SETTINGS_EMBED_MODEL ) + + # Also update the ingest flow with a new model id + try: + flows_service = _get_flows_service() + await flows_service.update_ingest_flow_model_id_in_text_splitter(model_id=new_embedding_model) + logger.info( + f"Successfully updated ingest flow model id in text splitter to {new_embedding_model}" + ) + except Exception as e: + logger.error(f"Failed to update ingest flow model id in text splitter: {str(e)}") + # Don't fail the entire settings update if flow update fails + # The config will still be saved + logger.info(f"Embedding model changed from {old_model} to {new_embedding_model}") if "embedding_provider" in body: diff --git a/src/services/flows_service.py b/src/services/flows_service.py index ad07e34a0..1d6fb6b99 100644 --- a/src/services/flows_service.py +++ b/src/services/flows_service.py @@ -975,6 +975,17 @@ async def update_ingest_flow_splitter_type(self, splitter_type: str): node_display_name="Split Text", ) + async def update_ingest_flow_model_id_in_text_splitter(self, model_id: str): + """Helper function to update splitter type in the ingest flow""" + if not LANGFLOW_INGEST_FLOW_ID: + raise ValueError("LANGFLOW_INGEST_FLOW_ID is not configured") + await self._update_flow_field( + LANGFLOW_INGEST_FLOW_ID, + "model_id", + model_id, + node_display_name="Split Text", + ) + async def update_ingest_flow_embedding_model(self, embedding_model: str, provider: str): """Helper function to update embedding model in the ingest flow""" if not LANGFLOW_INGEST_FLOW_ID: From 8c27821f0b08e9a449f05778727b885cdf2fea06 Mon Sep 17 00:00:00 2001 From: matano Date: Sun, 4 Jan 2026 21:12:17 +0200 Subject: [PATCH 10/51] recover for too long prefixes in _chunk_document --- flows/components/split_text.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/flows/components/split_text.py b/flows/components/split_text.py index 6f34f7259..20681c943 100644 --- a/flows/components/split_text.py +++ b/flows/components/split_text.py @@ -319,11 +319,21 @@ def _chunk_document(self, document: Document) -> list[Document]: chunks = [] for segment in segments: - line_splitter = LineBasedTextSplitter( - chunk_size=self.chunk_size, - model_id=self.model_id, - prefix=self.get_prefix(segment) - ) + prefix = self.get_prefix(segment) + try: + line_splitter = LineBasedTextSplitter( + chunk_size=self.chunk_size, + model_id=self.model_id, + prefix=prefix + ) + except RuntimeError as e: + print(f"Cannot create a line splitter with prefix '{prefix}' for segment:\n{segment}\n***\nError: {e}") + print(f"Skipping the prefix ..") + line_splitter = LineBasedTextSplitter( + chunk_size=self.chunk_size, + model_id=self.model_id, + ) + chunks.extend(line_splitter.split_documents([segment])) return chunks From 7fdbb80fff1be9405c7e62d14014b5691936b25e Mon Sep 17 00:00:00 2001 From: matano Date: Tue, 6 Jan 2026 20:14:05 +0200 Subject: [PATCH 11/51] remove recovery of LineBasedTextSplitter init --- flows/components/split_text.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/flows/components/split_text.py b/flows/components/split_text.py index 20681c943..9dfc5b62d 100644 --- a/flows/components/split_text.py +++ b/flows/components/split_text.py @@ -320,19 +320,11 @@ def _chunk_document(self, document: Document) -> list[Document]: chunks = [] for segment in segments: prefix = self.get_prefix(segment) - try: - line_splitter = LineBasedTextSplitter( - chunk_size=self.chunk_size, - model_id=self.model_id, - prefix=prefix - ) - except RuntimeError as e: - print(f"Cannot create a line splitter with prefix '{prefix}' for segment:\n{segment}\n***\nError: {e}") - print(f"Skipping the prefix ..") - line_splitter = LineBasedTextSplitter( - chunk_size=self.chunk_size, - model_id=self.model_id, - ) + line_splitter = LineBasedTextSplitter( + chunk_size=self.chunk_size, + model_id=self.model_id, + prefix=prefix + ) chunks.extend(line_splitter.split_documents([segment])) From b2286293eb5b81ee74b7aacba74019765c253d39 Mon Sep 17 00:00:00 2001 From: matano Date: Wed, 7 Jan 2026 11:04:02 +0200 Subject: [PATCH 12/51] Optionally delete the index when doing onboarding. --- src/api/settings.py | 11 ++++++++--- src/main.py | 13 ++++++++++--- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/api/settings.py b/src/api/settings.py index d749b3aba..b8086e252 100644 --- a/src/api/settings.py +++ b/src/api/settings.py @@ -773,6 +773,7 @@ async def onboarding(request, flows_service, session_manager=None): "llm_model", "embedding_provider", "embedding_model", + "delete_existing_index", "sample_data", # Provider-specific fields "openai_api_key", @@ -786,9 +787,11 @@ async def onboarding(request, flows_service, session_manager=None): # Check for invalid fields invalid_fields = set(body.keys()) - allowed_fields if invalid_fields: + error_message = f"Invalid fields: {', '.join(invalid_fields)}. Allowed fields: {', '.join(allowed_fields)}" + logger.error(error_message) return JSONResponse( { - "error": f"Invalid fields: {', '.join(invalid_fields)}. Allowed fields: {', '.join(allowed_fields)}" + "error": error_message }, status_code=400, ) @@ -1062,10 +1065,12 @@ async def onboarding(request, flows_service, session_manager=None): # Import here to avoid circular imports from main import init_index + delete_existing_index = body.get("delete_existing_index", False) + delete_existing_index = bool(delete_existing_index) logger.info( - "Initializing OpenSearch index after onboarding configuration" + f"Initializing OpenSearch index after onboarding configuration (delete_existing_index={delete_existing_index})", ) - await init_index() + await init_index(delete_existing=delete_existing_index) logger.info("OpenSearch index initialization completed successfully") except Exception as e: if isinstance(e, ValueError): diff --git a/src/main.py b/src/main.py index 69aabdabc..b92859023 100644 --- a/src/main.py +++ b/src/main.py @@ -178,7 +178,7 @@ async def _ensure_opensearch_index(): # The service can still function, document operations might fail later -async def init_index(): +async def init_index(delete_existing: bool = False): """Initialize OpenSearch index and security roles""" import config.settings as settings await wait_for_opensearch() @@ -197,14 +197,21 @@ async def init_index(): endpoint=getattr(embedding_provider_config, "endpoint", None) ) - if await clients.opensearch.indices.exists(index=settings.INDEX_NAME): + index_exists = clients.opensearch.indices.exists(index=settings.INDEX_NAME) + logger.info( + "Initializing OpenSearch index ..", + index_name=settings.INDEX_NAME, + embedding_model=embedding_model, + delete_existing=delete_existing, + ) + if index_exists and delete_existing: # DELETE / logger.info(f"Deleting index '{settings.INDEX_NAME}'...") resp = await clients.opensearch.indices.delete(index=settings.INDEX_NAME) logger.info(f"Deleted '{settings.INDEX_NAME}': {resp}") # Create documents index - if not await clients.opensearch.indices.exists(index=settings.INDEX_NAME): + if not index_exists: await clients.opensearch.indices.create( index=settings.INDEX_NAME, body=dynamic_index_body ) From d3d19ae95c06d7f3e6679775cbcec67089c7065a Mon Sep 17 00:00:00 2001 From: matano Date: Wed, 7 Jan 2026 11:17:45 +0200 Subject: [PATCH 13/51] fix index exists check --- src/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main.py b/src/main.py index b92859023..12fc9601b 100644 --- a/src/main.py +++ b/src/main.py @@ -197,12 +197,13 @@ async def init_index(delete_existing: bool = False): endpoint=getattr(embedding_provider_config, "endpoint", None) ) - index_exists = clients.opensearch.indices.exists(index=settings.INDEX_NAME) + index_exists = await clients.opensearch.indices.exists(index=settings.INDEX_NAME) logger.info( "Initializing OpenSearch index ..", index_name=settings.INDEX_NAME, embedding_model=embedding_model, delete_existing=delete_existing, + index_exists=index_exists, ) if index_exists and delete_existing: # DELETE / From ac01a4ae7a78ec8814d40faa7acc3096fd62caff Mon Sep 17 00:00:00 2001 From: matano Date: Wed, 7 Jan 2026 12:04:59 +0200 Subject: [PATCH 14/51] remember the index is deleted after deletion --- src/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main.py b/src/main.py index 12fc9601b..494fdd88b 100644 --- a/src/main.py +++ b/src/main.py @@ -210,6 +210,7 @@ async def init_index(delete_existing: bool = False): logger.info(f"Deleting index '{settings.INDEX_NAME}'...") resp = await clients.opensearch.indices.delete(index=settings.INDEX_NAME) logger.info(f"Deleted '{settings.INDEX_NAME}': {resp}") + index_exists = False # Create documents index if not index_exists: From 737249efc52a70e9ebbf69c0fe030162ff3332b5 Mon Sep 17 00:00:00 2001 From: matano Date: Wed, 7 Jan 2026 15:27:03 +0200 Subject: [PATCH 15/51] control over use_document_title flag --- src/api/settings.py | 22 ++++++++++++++++++++++ src/services/flows_service.py | 11 +++++++++++ 2 files changed, 33 insertions(+) diff --git a/src/api/settings.py b/src/api/settings.py index b8086e252..2c593c1c6 100644 --- a/src/api/settings.py +++ b/src/api/settings.py @@ -249,6 +249,7 @@ async def update_settings(request, session_manager): "chunk_size", "chunk_overlap", "splitter_type", + "use_document_title", "table_structure", "ocr", "picture_descriptions", @@ -600,6 +601,27 @@ async def update_settings(request, session_manager): # Don't fail the entire settings update if flow update fails # The config will still be saved + if "use_document_title" in body: + new_use_document_title = body["use_document_title"] + current_config.knowledge.use_document_title = new_use_document_title + config_updated = True + await TelemetryClient.send_event( + Category.SETTINGS_OPERATIONS, + MessageId.ORB_SETTINGS_CHUNK_UPDATED + ) + + # Also update the ingest flow with the new splitter type + try: + flows_service = _get_flows_service() + await flows_service.update_ingest_flow_use_document_title(new_use_document_title) + logger.info( + f"Successfully updated ingest flow use_document_title to {new_use_document_title}" + ) + except Exception as e: + logger.error(f"Failed to update ingest flow use_document_title do: {str(e)}") + # Don't fail the entire settings update if flow update fails + # The config will still be saved + if "chunk_size" in body: current_config.knowledge.chunk_size = body["chunk_size"] config_updated = True diff --git a/src/services/flows_service.py b/src/services/flows_service.py index 1d6fb6b99..1d7832274 100644 --- a/src/services/flows_service.py +++ b/src/services/flows_service.py @@ -975,6 +975,17 @@ async def update_ingest_flow_splitter_type(self, splitter_type: str): node_display_name="Split Text", ) + async def update_ingest_flow_use_document_title(self, use_document_title: bool): + """Helper function to update splitter type in the ingest flow""" + if not LANGFLOW_INGEST_FLOW_ID: + raise ValueError("LANGFLOW_INGEST_FLOW_ID is not configured") + await self._update_flow_field( + LANGFLOW_INGEST_FLOW_ID, + "use_document_title", + str(use_document_title), + node_display_name="Split Text", + ) + async def update_ingest_flow_model_id_in_text_splitter(self, model_id: str): """Helper function to update splitter type in the ingest flow""" if not LANGFLOW_INGEST_FLOW_ID: From d123b0ee27510c5674581d56dff1c77bc2bd6eb4 Mon Sep 17 00:00:00 2001 From: matano Date: Wed, 7 Jan 2026 15:30:09 +0200 Subject: [PATCH 16/51] add handling of use_document_title and fixed list of titles --- flows/components/split_text.py | 89 ++++++++++++++++++++++++++++++---- 1 file changed, 79 insertions(+), 10 deletions(-) diff --git a/flows/components/split_text.py b/flows/components/split_text.py index 9dfc5b62d..70eb4e75b 100644 --- a/flows/components/split_text.py +++ b/flows/components/split_text.py @@ -10,6 +10,8 @@ from lfx.schema.dataframe import DataFrame from lfx.schema.message import Message from lfx.utils.util import unescape_string +from lfx.log import logger + from langchain_core.documents import Document @@ -83,6 +85,14 @@ class SplitTextComponent(Component): value="ibm-granite/granite-embedding-30m-english", advanced=True, ), + DropdownInput( + name="use_document_title", + display_name="Use Document Title", + info="Whether to use the document title as a prefix in each chunk.", + options=["False", "True"], + value="False", + advanced=True, + ), ] outputs = [ @@ -147,7 +157,7 @@ def split_text_base(self): keep_sep = True # 'start' and 'end' are kept as strings - print(f"Creating a CharacterTextSplitter..") + logger.debug("SPLIT: Creating a CharacterTextSplitter..") splitter = CharacterTextSplitter( chunk_overlap=self.chunk_overlap, chunk_size=self.chunk_size, @@ -155,13 +165,14 @@ def split_text_base(self): keep_separator=keep_sep, ) elif self.splitter_type == "LineBasedTextSplitter": - print(f"Creating a LineBasedTextSplitter with chunk_size={self.chunk_size} and model_id '{self.model_id}'.") + logger.debug(f"SPLIT: Creating a LineBasedTextSplitter with chunk_size={self.chunk_size} and model_id '{self.model_id}'.") splitter = LineBasedTextSplitter( chunk_size=self.chunk_size, - model_id=self.model_id + model_id=self.model_id, + use_document_title=self.use_document_title, ) elif self.splitter_type == "TableAwareTextSplitter": - print(f"Creating a TableAwareTextSplitter with chunk_size={self.chunk_size} and model_id '{self.model_id}'.") + logger.debug(f"SPLIT: Creating a TableAwareTextSplitter with chunk_size={self.chunk_size} and model_id '{self.model_id}'.") splitter = TableAwareTextSplitter( chunk_size=self.chunk_size, model_id=self.model_id @@ -182,23 +193,30 @@ def __init__( chunk_size: int, model_id: str, prefix: str = "", + use_document_title: bool = False, ): self._chunk_size = chunk_size self.use_tiktoken = False if model_id in ["text-embedding-3-small", "text-embedding-3-large"]: self.use_tiktoken = True - print(f"Initializing tokenizer for model '{model_id}', use_tiktoken = {self.use_tiktoken}.") + logger.debug(f"SPLIT: Initializing LineBasedTextSplitter, for model '{model_id}', use_tiktoken = {self.use_tiktoken}, use_document_title={use_document_title}.") if self.use_tiktoken: import tiktoken - # The tokenizer for text-embedding-3-small + # The tokenizer for text-embedding-3-small, text-embedding-3-large self._tokenizer = tiktoken.get_encoding("cl100k_base") else: from transformers import AutoTokenizer self._tokenizer = AutoTokenizer.from_pretrained( pretrained_model_name_or_path=model_id, ) + self._prefix = "" + self._prefix_len = 0 + self.set_prefix(prefix) + self.use_document_title = use_document_title + def set_prefix(self, prefix): + logger.debug(f"SPLIT: setting prefix to '{prefix}'..") prefix_len = len(self.tokenize(prefix)) if prefix_len >= self._chunk_size: raise RuntimeError( @@ -206,7 +224,7 @@ def __init__( ) else: self._prefix = prefix - self._prefixLen = prefix_len + self._prefix_len = prefix_len def tokenize(self, text: str) -> list[int]: if self.use_tiktoken: @@ -231,8 +249,14 @@ def _chunk_document(self, document: Document): chunks = [] chunk_seq_num = 0 current = self._prefix - current_len = self._prefixLen + current_len = self._prefix_len first_character_index = document_metadata.get("start_index", 0) + if self.use_document_title: + file_name = document_metadata.get("filename", "unknown-file-name") + logger.debug(f"SPLIT: Chunking document with file name '{file_name}'..") + document_title = get_title(file_name) + logger.debug(f"SPLIT: Found title '{document_title}'..") + self.set_prefix(document_title) new_line_token_count = len(self.tokenize("\n")) lines = document_text.split("\n") @@ -244,7 +268,7 @@ def _chunk_document(self, document: Document): ): # line cannot fit into current num_available_tokens_in_chunk = ( self._chunk_size - current_len - if len(line_tokens) + self._prefixLen > self._chunk_size + if len(line_tokens) + self._prefix_len > self._chunk_size else 0 ) # if whole line can fit into a new chunk, do not add anything to current chunk, # otherwise, split the line between current and next chunks. @@ -270,7 +294,7 @@ def _chunk_document(self, document: Document): first_character_index += len(current) chunk_seq_num += 1 current = self._prefix - current_len = self._prefixLen + current_len = self._prefix_len line_tokens = line_tokens[num_available_tokens_in_chunk:] # rest of line fits into current @@ -439,3 +463,48 @@ def condense_table_row(line: str) -> str: cells = [c.strip() for c in line.strip().split("|")] return " | ".join(cells).strip() + +def get_title(file_name: str) -> str: + file_name_to_title = { + "docling.pdf": "Docling Technical Report" + } + file_name_to_title.update(filename_to_output) + return file_name_to_title.get(file_name, "") + + +filename_to_output = { + "Alaska-2017.pdf": """This document is the 2017 annual report (Form 10-K) of Alaska Air Group, Inc., filed with the United States Securities and Exchange Commission (SEC). The report covers the fiscal year ended December 31, 2017. Important entities mentioned include: + +* Alaska Air Group, Inc. (the company) +* United States Securities and Exchange Commission (SEC) +* New York Stock Exchange (where the company's common stock is registered) + +Important dates mentioned include: + +* December 31, 2017 (end of the fiscal year) +* January 31, 2018 (date of share outstanding total) +* June 30, 2017 (date used to calculate aggregate market value of shares held by nonaffiliates)""", + "Alaska-2018.pdf": """This document is the 2018 annual report (Form 10-K) of Alaska Air Group, Inc., filed with the United States Securities and Exchange Commission (SEC). The report covers the fiscal year ended December 31, 2018. Important entities mentioned include: + +* Alaska Air Group, Inc. (the company) +* United States Securities and Exchange Commission (SEC) +* New York Stock Exchange (where the company's common stock is listed) + +Important dates mentioned include: + +* December 31, 2018 (end of the fiscal year) +* January 31, 2019 (date of share outstanding total) +* June 30, 2018 (date used to calculate aggregate market value of shares held by nonaffiliates)""", + "AmericanAirlines-2017.pdf": "This document is the 2017 annual report (Form 10-K) of American Airlines Group Inc., filed with the United States Securities and Exchange Commission (SEC).", + "AmericanAirlines-2018.pdf": "The document \"AmericanAirlines-2018.pdf\" is the 2018 Annual Report on Form 10-K for American Airlines Group Inc.", + "AmericanAirlines-2019.pdf": "This document is the 2020 Annual Report on Form 10-K for American Airlines Group Inc., filed for the year ending 2019.", + "Delta-2017.pdf": "This document is the 2017 annual report (Form 10-K) of Delta Air Lines, Inc. for the fiscal year ended December 31, 2017.", + "Delta-2018.pdf": "This document is the 2018 annual report (Form 10-K) of Delta Air Lines, Inc. for the fiscal year ended December 31, 2018.", + "Delta-2019.pdf": "This document is the 2019 annual report (Form 10-K) of Delta Air Lines, Inc. for the fiscal year ended December 31, 2019.", + "Southwest-2017.pdf": "This document is the 2017 Annual Report to Shareholders of Southwest Airlines Co.", + "Southwest-2018.pdf": "This document is the 2018 Annual Report to Shareholders of Southwest Airlines Co.", + "Southwest-2019.pdf": "This document is the 2019 Annual Report to Shareholders of Southwest Airlines Co.", + "United-2017.pdf": "This document is the 2017 annual report (Form 10-K) of United Continental Holdings, Inc. and United Airlines, Inc.", + "United-2018.pdf": "This document is the 2018 annual report (Form 10-K) of United Continental Holdings, Inc. and United Airlines, Inc.", + "United-2019.pdf": "This document is the 2019 annual report (Form 10-K) of United Airlines Holdings, Inc. and United Airlines, Inc." +} From 078b0a30b54b720127cae164ed670ab4dfa1c9e7 Mon Sep 17 00:00:00 2001 From: matano Date: Wed, 7 Jan 2026 15:30:28 +0200 Subject: [PATCH 17/51] print index name in debu message --- src/api/documents.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/documents.py b/src/api/documents.py index 54437aaf9..acfe20308 100644 --- a/src/api/documents.py +++ b/src/api/documents.py @@ -27,7 +27,7 @@ async def check_filename_exists(request: Request, document_service, session_mana search_body = build_filename_search_body(filename, size=1, source=["filename"]) - logger.debug(f"Checking filename existence: {filename}") + logger.debug(f"Checking filename existence", filename=filename, index_name=settings.INDEX_NAME) response = await opensearch_client.search( index=settings.INDEX_NAME, From 556597dcc433814a3b3f869bda76c19747dc4ce6 Mon Sep 17 00:00:00 2001 From: matano Date: Wed, 7 Jan 2026 16:18:15 +0200 Subject: [PATCH 18/51] fix use_document_title use fix first chunk bug --- flows/components/split_text.py | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/flows/components/split_text.py b/flows/components/split_text.py index 70eb4e75b..ef1100885 100644 --- a/flows/components/split_text.py +++ b/flows/components/split_text.py @@ -110,6 +110,17 @@ def _fix_separator(self, separator: str) -> str: return "\t" return separator + @staticmethod + def to_bool(val): + if isinstance(val, str): + if val.lower() == "false": + return False + elif val.lower() == "true": + return True + elif isinstance(val, bool): + return val + raise RuntimeError(f"Cannot convert value {val} to a boolean value. Expected 'True' or 'False'.") + def split_text_base(self): separator = self._fix_separator(self.separator) separator = unescape_string(separator) @@ -149,14 +160,7 @@ def split_text_base(self): try: if self.splitter_type == "CharacterTextSplitter": # Convert string 'False'/'True' to boolean - keep_sep = self.keep_separator - if isinstance(keep_sep, str): - if keep_sep.lower() == "false": - keep_sep = False - elif keep_sep.lower() == "true": - keep_sep = True - # 'start' and 'end' are kept as strings - + keep_sep = self.to_bool(self.keep_separator) logger.debug("SPLIT: Creating a CharacterTextSplitter..") splitter = CharacterTextSplitter( chunk_overlap=self.chunk_overlap, @@ -165,11 +169,11 @@ def split_text_base(self): keep_separator=keep_sep, ) elif self.splitter_type == "LineBasedTextSplitter": - logger.debug(f"SPLIT: Creating a LineBasedTextSplitter with chunk_size={self.chunk_size} and model_id '{self.model_id}'.") + use_document_title = self.to_bool(self.use_document_title) splitter = LineBasedTextSplitter( chunk_size=self.chunk_size, model_id=self.model_id, - use_document_title=self.use_document_title, + use_document_title=use_document_title, ) elif self.splitter_type == "TableAwareTextSplitter": logger.debug(f"SPLIT: Creating a TableAwareTextSplitter with chunk_size={self.chunk_size} and model_id '{self.model_id}'.") @@ -199,8 +203,7 @@ def __init__( self.use_tiktoken = False if model_id in ["text-embedding-3-small", "text-embedding-3-large"]: self.use_tiktoken = True - logger.debug(f"SPLIT: Initializing LineBasedTextSplitter, for model '{model_id}', use_tiktoken = {self.use_tiktoken}, use_document_title={use_document_title}.") - + logger.debug(f"SPLIT: Initializing LineBasedTextSplitter, chunk_size = {chunk_size}, model_id = '{model_id}', use_tiktoken = {self.use_tiktoken}, use_document_title={use_document_title}.") if self.use_tiktoken: import tiktoken # The tokenizer for text-embedding-3-small, text-embedding-3-large @@ -248,8 +251,7 @@ def _chunk_document(self, document: Document): document_metadata = document.metadata chunks = [] chunk_seq_num = 0 - current = self._prefix - current_len = self._prefix_len + first_character_index = document_metadata.get("start_index", 0) if self.use_document_title: file_name = document_metadata.get("filename", "unknown-file-name") @@ -258,6 +260,9 @@ def _chunk_document(self, document: Document): logger.debug(f"SPLIT: Found title '{document_title}'..") self.set_prefix(document_title) + current = self._prefix + current_len = self._prefix_len + new_line_token_count = len(self.tokenize("\n")) lines = document_text.split("\n") for line in lines: From dd4662f757bf2c714a645f771284f8fdae104ded Mon Sep 17 00:00:00 2001 From: matano Date: Thu, 8 Jan 2026 00:19:56 +0200 Subject: [PATCH 19/51] when updating the index_name also update it in the agent retrieval tool --- src/api/settings.py | 6 +++--- src/services/flows_service.py | 8 +++++++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/api/settings.py b/src/api/settings.py index 2c593c1c6..37f1b704e 100644 --- a/src/api/settings.py +++ b/src/api/settings.py @@ -568,12 +568,12 @@ async def update_settings(request, session_manager): # MessageId.ORB_SETTINGS_UPDATED #) - # Also update the ingest flow with the new index name + # Also update the flows with the new index name try: flows_service = _get_flows_service() - await flows_service.update_ingest_flow_index_name(settings.INDEX_NAME) + await flows_service.update_flows_index_name(settings.INDEX_NAME) logger.info( - f"Successfully updated ingest flow index name to '{settings.INDEX_NAME}'." + f"Successfully updated flows index name to '{settings.INDEX_NAME}'." ) except Exception as e: logger.error(f"Failed to update ingest flow index name: {str(e)}") diff --git a/src/services/flows_service.py b/src/services/flows_service.py index 1d7832274..8ef36391f 100644 --- a/src/services/flows_service.py +++ b/src/services/flows_service.py @@ -931,7 +931,7 @@ async def update_flow_docling_preset(self, preset: str, preset_config: dict): await self._update_flow_field(LANGFLOW_INGEST_FLOW_ID, "docling_serve_opts", preset_config, node_display_name=DOCLING_COMPONENT_DISPLAY_NAME) - async def update_ingest_flow_index_name(self, index_name: str): + async def update_flows_index_name(self, index_name: str): """Helper function to update index name in the ingest flow""" if not LANGFLOW_INGEST_FLOW_ID: raise ValueError("LANGFLOW_INGEST_FLOW_ID is not configured") @@ -941,6 +941,12 @@ async def update_ingest_flow_index_name(self, index_name: str): index_name, node_display_name="OpenSearch (Multi-Model Multi-Embedding)", ) + await self._update_flow_field( + LANGFLOW_CHAT_FLOW_ID, + "index_name", + index_name, + node_display_name="OpenSearch (Multi-Model Multi-Embedding)", + ) async def update_ingest_flow_chunk_size(self, chunk_size: int): """Helper function to update chunk size in the ingest flow""" From e566a1a58ca6e63ca9c9c14073e531326e447bdf Mon Sep 17 00:00:00 2001 From: matano Date: Mon, 19 Jan 2026 11:49:40 +0200 Subject: [PATCH 20/51] add the option of deleting an existing index during onboarding --- src/api/settings.py | 21 +++++++++++++++++++-- src/main.py | 12 ++++++++++-- src/utils/telemetry/message_id.py | 2 ++ 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/api/settings.py b/src/api/settings.py index 982d9272b..e8db90965 100644 --- a/src/api/settings.py +++ b/src/api/settings.py @@ -686,6 +686,7 @@ async def onboarding(request, flows_service, session_manager=None): "llm_model", "embedding_provider", "embedding_model", + "delete_existing_index", "sample_data", # Provider-specific fields "openai_api_key", @@ -975,10 +976,26 @@ async def onboarding(request, flows_service, session_manager=None): # Import here to avoid circular imports from main import init_index + # Handle delete_existing_index + delete_existing_index = False + if "delete_existing_index" in body: + delete_existing_index = body["delete_existing_index"] + if not isinstance(delete_existing_index, bool): + return JSONResponse( + {"error": "delete_existing_index must be a boolean value"}, status_code=400 + ) + if delete_existing_index: + await TelemetryClient.send_event( + Category.ONBOARDING, + MessageId.ORB_ONBOARD_DELETE_EXISTING_INDEX + ) + logger.info("Delete existing index requested during onboarding") + logger.info( - "Initializing OpenSearch index after onboarding configuration" + f"Initializing OpenSearch index after onboarding configuration" ) - await init_index() + await init_index(delete_existing=delete_existing_index) + logger.info("OpenSearch index initialization completed successfully") except Exception as e: if isinstance(e, ValueError): diff --git a/src/main.py b/src/main.py index fd2bf2a7f..303973905 100644 --- a/src/main.py +++ b/src/main.py @@ -179,7 +179,7 @@ async def _ensure_opensearch_index(): # The service can still function, document operations might fail later -async def init_index(): +async def init_index(delete_existing: bool = False): """Initialize OpenSearch index and security roles""" await wait_for_opensearch() @@ -197,8 +197,16 @@ async def init_index(): endpoint=getattr(embedding_provider_config, "endpoint", None) ) + index_exists = await clients.opensearch.indices.exists(index=INDEX_NAME) + if index_exists and delete_existing: + # Asked to delete the existing index .. + logger.info(f"Deleting index '{INDEX_NAME}'...") + resp = await clients.opensearch.indices.delete(index=INDEX_NAME) + logger.info(f"Deleted index '{INDEX_NAME}', response: {resp}") + index_exists = False + # Create documents index - if not await clients.opensearch.indices.exists(index=INDEX_NAME): + if not index_exists: await clients.opensearch.indices.create( index=INDEX_NAME, body=dynamic_index_body ) diff --git a/src/utils/telemetry/message_id.py b/src/utils/telemetry/message_id.py index c00e5eb35..a429982c8 100644 --- a/src/utils/telemetry/message_id.py +++ b/src/utils/telemetry/message_id.py @@ -197,6 +197,8 @@ class MessageId: ORB_ONBOARD_EMBED_MODEL = "ORB_ONBOARD_EMBED_MODEL" # Message: Sample data ingestion requested ORB_ONBOARD_SAMPLE_DATA = "ORB_ONBOARD_SAMPLE_DATA" + # Message: Delete existing index requested + ORB_ONBOARD_DELETE_EXISTING_INDEX = "ORB_ONBOARD_DELETE_EXISTING_INDEX" # Message: Configuration marked as edited ORB_ONBOARD_CONFIG_EDITED = "ORB_ONBOARD_CONFIG_EDITED" # Message: Onboarding rolled back due to all files failing From a4693d06ec2891e62d5f1a40786f05e3eecbb282 Mon Sep 17 00:00:00 2001 From: matano Date: Mon, 19 Jan 2026 11:52:12 +0200 Subject: [PATCH 21/51] remove redundant f-string --- src/api/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/settings.py b/src/api/settings.py index e8db90965..cee83beb8 100644 --- a/src/api/settings.py +++ b/src/api/settings.py @@ -992,7 +992,7 @@ async def onboarding(request, flows_service, session_manager=None): logger.info("Delete existing index requested during onboarding") logger.info( - f"Initializing OpenSearch index after onboarding configuration" + "Initializing OpenSearch index after onboarding configuration" ) await init_index(delete_existing=delete_existing_index) From 8fa0cc5699590152149ffd464e16cbfe47006363 Mon Sep 17 00:00:00 2001 From: matano Date: Tue, 20 Jan 2026 15:40:13 +0200 Subject: [PATCH 22/51] support a custom openai api base --- docker-compose.yml | 4 +++- src/api/provider_validation.py | 17 +++++++++++++---- src/services/models_service.py | 6 +++++- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 305745061..c88b59bac 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -67,6 +67,7 @@ services: - OPENSEARCH_USERNAME=admin - OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD} - OPENAI_API_KEY=${OPENAI_API_KEY} + - OPENAI_API_BASE=${OPENAI_API_BASE:-None} - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY} - WATSONX_API_KEY=${WATSONX_API_KEY} - WATSONX_ENDPOINT=${WATSONX_ENDPOINT} @@ -114,6 +115,7 @@ services: - LANGFUSE_PUBLIC_KEY=${LANGFUSE_PUBLIC_KEY:-} - LANGFUSE_HOST=${LANGFUSE_HOST:-} - OPENAI_API_KEY=${OPENAI_API_KEY:-None} + - OPENAI_API_BASE=${OPENAI_API_BASE:-None} - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-None} - WATSONX_API_KEY=${WATSONX_API_KEY:-None} - WATSONX_ENDPOINT=${WATSONX_ENDPOINT:-None} @@ -133,7 +135,7 @@ services: - MIMETYPE=None - FILESIZE=0 - SELECTED_EMBEDDING_MODEL=${SELECTED_EMBEDDING_MODEL:-} - - LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD,OWNER,OWNER_NAME,OWNER_EMAIL,CONNECTOR_TYPE,FILENAME,MIMETYPE,FILESIZE,SELECTED_EMBEDDING_MODEL,OPENAI_API_KEY,ANTHROPIC_API_KEY,WATSONX_API_KEY,WATSONX_ENDPOINT,WATSONX_PROJECT_ID,OLLAMA_BASE_URL + - LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD,OWNER,OWNER_NAME,OWNER_EMAIL,CONNECTOR_TYPE,FILENAME,MIMETYPE,FILESIZE,SELECTED_EMBEDDING_MODEL,OPENAI_API_KEY,OPENAI_API_BASE,ANTHROPIC_API_KEY,WATSONX_API_KEY,WATSONX_ENDPOINT,WATSONX_PROJECT_ID,OLLAMA_BASE_URL - LANGFLOW_LOG_LEVEL=DEBUG - LANGFLOW_AUTO_LOGIN=${LANGFLOW_AUTO_LOGIN} - LANGFLOW_SUPERUSER=${LANGFLOW_SUPERUSER} diff --git a/src/api/provider_validation.py b/src/api/provider_validation.py index 813826a18..2f587fb49 100644 --- a/src/api/provider_validation.py +++ b/src/api/provider_validation.py @@ -1,6 +1,8 @@ """Provider validation utilities for testing API keys and models during onboarding.""" import json +import os + import httpx from utils.container_utils import transform_localhost_url from utils.logging_config import get_logger @@ -107,6 +109,9 @@ def _extract_error_details(response: httpx.Response) -> str: return parsed return response_text +def get_openai_url(endpoint: str) -> str: + api_base = os.environ.get("OPENAI_API_BASE", "https://api.openai.com") + return f"{api_base}{endpoint}" async def validate_provider_setup( provider: str, @@ -248,10 +253,11 @@ async def _test_openai_lightweight_health(api_key: str) -> None: "Content-Type": "application/json", } + url = get_openai_url(endpoint="/v1/models") async with httpx.AsyncClient() as client: # Use /v1/models endpoint which validates the key without consuming credits response = await client.get( - "https://api.openai.com/v1/models", + url=url, headers=headers, timeout=10.0, # Short timeout for lightweight check ) @@ -309,8 +315,9 @@ async def _test_openai_completion_with_tools(api_key: str, llm_model: str) -> No async with httpx.AsyncClient() as client: # Try with max_tokens first payload = {**base_payload, "max_tokens": 50} + url = get_openai_url(endpoint="/v1/chat/completions") response = await client.post( - "https://api.openai.com/v1/chat/completions", + url=url, headers=headers, json=payload, timeout=30.0, @@ -320,8 +327,9 @@ async def _test_openai_completion_with_tools(api_key: str, llm_model: str) -> No if response.status_code != 200: logger.info("max_tokens parameter failed, trying max_completion_tokens instead") payload = {**base_payload, "max_completion_tokens": 50} + url = get_openai_url(endpoint="/v1/chat/completions") response = await client.post( - "https://api.openai.com/v1/chat/completions", + url=url, headers=headers, json=payload, timeout=30.0, @@ -356,8 +364,9 @@ async def _test_openai_embedding(api_key: str, embedding_model: str) -> None: } async with httpx.AsyncClient() as client: + url = get_openai_url(endpoint="/v1/embeddings") response = await client.post( - "https://api.openai.com/v1/embeddings", + url=url, headers=headers, json=payload, timeout=30.0, diff --git a/src/services/models_service.py b/src/services/models_service.py index cd08b7085..318dfecea 100644 --- a/src/services/models_service.py +++ b/src/services/models_service.py @@ -1,3 +1,5 @@ +import os + import httpx from typing import Dict, List from utils.container_utils import transform_localhost_url @@ -54,8 +56,10 @@ async def get_openai_models(self, api_key: str) -> Dict[str, List[Dict[str, str] async with httpx.AsyncClient() as client: # Lightweight validation: just check if API key is valid # This doesn't consume credits, only validates the key + from api.provider_validation import get_openai_url + url = get_openai_url(endpoint="/v1/models") response = await client.get( - "https://api.openai.com/v1/models", headers=headers, timeout=10.0 + url, headers=headers, timeout=10.0 ) if response.status_code == 200: From cc2187b3eccac473837f10f182b1790f145c2f7a Mon Sep 17 00:00:00 2001 From: matano Date: Thu, 29 Jan 2026 13:17:30 +0200 Subject: [PATCH 23/51] fix merge --- docker-compose.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8a5e8d41e..f28e682a6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -139,11 +139,7 @@ services: - MIMETYPE=None - FILESIZE=0 - SELECTED_EMBEDDING_MODEL=${SELECTED_EMBEDDING_MODEL:-} -<<<<<<< HEAD - - LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD,OWNER,OWNER_NAME,OWNER_EMAIL,CONNECTOR_TYPE,FILENAME,MIMETYPE,FILESIZE,SELECTED_EMBEDDING_MODEL,OPENAI_API_KEY,OPENAI_API_BASE,ANTHROPIC_API_KEY,WATSONX_API_KEY,WATSONX_ENDPOINT,WATSONX_PROJECT_ID,OLLAMA_BASE_URL -======= - - LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD,OPENSEARCH_URL,DOCLING_SERVE_URL,OWNER,OWNER_NAME,OWNER_EMAIL,CONNECTOR_TYPE,FILENAME,MIMETYPE,FILESIZE,SELECTED_EMBEDDING_MODEL,OPENAI_API_KEY,ANTHROPIC_API_KEY,WATSONX_API_KEY,WATSONX_ENDPOINT,WATSONX_PROJECT_ID,OLLAMA_BASE_URL ->>>>>>> remotes/upstream/main + - LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD,OPENSEARCH_URL,DOCLING_SERVE_URL,OWNER,OWNER_NAME,OWNER_EMAIL,CONNECTOR_TYPE,FILENAME,MIMETYPE,FILESIZE,SELECTED_EMBEDDING_MODEL,OPENAI_API_KEY,OPENAI_API_BASE,ANTHROPIC_API_KEY,WATSONX_API_KEY,WATSONX_ENDPOINT,WATSONX_PROJECT_ID,OLLAMA_BASE_URL - LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD,OPENSEARCH_URL,DOCLING_SERVE_URL,OWNER,OWNER_NAME,OWNER_EMAIL,CONNECTOR_TYPE,FILENAME,MIMETYPE,FILESIZE,SELECTED_EMBEDDING_MODEL,OPENAI_API_KEY,ANTHROPIC_API_KEY,WATSONX_API_KEY,WATSONX_ENDPOINT,WATSONX_PROJECT_ID,OLLAMA_BASE_URL>>>>>>> remotes/upstream/main - LANGFLOW_LOG_LEVEL=DEBUG - LANGFLOW_AUTO_LOGIN=${LANGFLOW_AUTO_LOGIN} - LANGFLOW_SUPERUSER=${LANGFLOW_SUPERUSER} From 1794b0df13deb8c6002c17f9f39e084a9a1260c4 Mon Sep 17 00:00:00 2001 From: matano Date: Thu, 29 Jan 2026 14:43:03 +0200 Subject: [PATCH 24/51] further changes to support custom endpoints --- src/api/models.py | 18 ++++++++++++- src/api/provider_health.py | 2 +- src/api/provider_validation.py | 49 +++++++++++++++++----------------- src/services/models_service.py | 6 ++--- 4 files changed, 46 insertions(+), 29 deletions(-) diff --git a/src/api/models.py b/src/api/models.py index 118126029..961e1d8c9 100644 --- a/src/api/models.py +++ b/src/api/models.py @@ -1,3 +1,5 @@ +import os + from starlette.responses import JSONResponse from utils.logging_config import get_logger from config.settings import get_openrag_config @@ -10,9 +12,11 @@ async def get_openai_models(request, models_service, session_manager): try: # Get API key from request body api_key = None + api_base = None try: body = await request.json() api_key = body.get("api_key") if body else None + api_base = body.get("api_base") if body else None except Exception: # Body might be empty or invalid JSON, continue to fallback pass @@ -36,7 +40,19 @@ async def get_openai_models(request, models_service, session_manager): status_code=400, ) - models = await models_service.get_openai_models(api_key=api_key) + if not api_base: + try: + config = get_openrag_config() + api_base = config.providers.openai.endpoint + logger.info( + f"Retrieved OpenAI API base from config: {'yes' if api_base else 'no'}" + ) + except Exception as e: + logger.error(f"Failed to get config: {e}") + if not api_base: + api_base = os.environ.get("OPENAI_BASE_API", "https://api.openai.com") + + models = await models_service.get_openai_models(api_key=api_key, api_base=api_base) return JSONResponse(models) except Exception as e: logger.error(f"Failed to get OpenAI models: {str(e)}") diff --git a/src/api/provider_health.py b/src/api/provider_health.py index 3802d8dcb..36238e279 100644 --- a/src/api/provider_health.py +++ b/src/api/provider_health.py @@ -42,7 +42,7 @@ async def check_provider_health(request): provider = current_config.agent.llm_provider # Validate provider name - valid_providers = ["openai", "ollama", "watsonx", "anthropic"] + valid_providers = ["openai_ete", "openai", "ollama", "watsonx", "anthropic"] if provider not in valid_providers: return JSONResponse( { diff --git a/src/api/provider_validation.py b/src/api/provider_validation.py index 2f587fb49..358264998 100644 --- a/src/api/provider_validation.py +++ b/src/api/provider_validation.py @@ -109,10 +109,6 @@ def _extract_error_details(response: httpx.Response) -> str: return parsed return response_text -def get_openai_url(endpoint: str) -> str: - api_base = os.environ.get("OPENAI_API_BASE", "https://api.openai.com") - return f"{api_base}{endpoint}" - async def validate_provider_setup( provider: str, api_key: str = None, @@ -130,7 +126,7 @@ async def validate_provider_setup( api_key: API key for the provider (optional for ollama) embedding_model: Embedding model to test llm_model: LLM model to test - endpoint: Provider endpoint (required for ollama and watsonx) + endpoint: Provider endpoint (required for ollama and watsonx, optional for openai) project_id: Project ID (required for watsonx) test_completion: If True, performs full validation with completion/embedding tests (consumes credits). If False, performs lightweight validation (no credits consumed). Default: False. @@ -143,11 +139,14 @@ async def validate_provider_setup( try: logger.info(f"Starting validation for provider: {provider_lower} (test_completion={test_completion})") + if provider == "openai" and not endpoint: + endpoint = os.environ.get("OPENAI_BASE_API", "https://api.openai.com") + if test_completion: # Full validation with completion/embedding tests (consumes credits) if embedding_model: # Test embedding - await test_embedding( + await _test_embedding( provider=provider_lower, api_key=api_key, embedding_model=embedding_model, @@ -156,7 +155,7 @@ async def validate_provider_setup( ) elif llm_model: # Test completion with tool calling - await test_completion_with_tools( + await _test_completion_with_tools( provider=provider_lower, api_key=api_key, llm_model=llm_model, @@ -165,7 +164,7 @@ async def validate_provider_setup( ) else: # Lightweight validation (no credits consumed) - await test_lightweight_health( + await _test_lightweight_health( provider=provider_lower, api_key=api_key, endpoint=endpoint, @@ -180,7 +179,7 @@ async def validate_provider_setup( raise -async def test_lightweight_health( +async def _test_lightweight_health( provider: str, api_key: str = None, endpoint: str = None, @@ -188,8 +187,8 @@ async def test_lightweight_health( ) -> None: """Test provider health with lightweight check (no credits consumed).""" - if provider == "openai": - await _test_openai_lightweight_health(api_key) + if provider.startswith("openai"): + await _test_openai_lightweight_health(api_key, endpoint) elif provider == "watsonx": await _test_watsonx_lightweight_health(api_key, endpoint, project_id) elif provider == "ollama": @@ -200,7 +199,7 @@ async def test_lightweight_health( raise ValueError(f"Unknown provider: {provider}") -async def test_completion_with_tools( +async def _test_completion_with_tools( provider: str, api_key: str = None, llm_model: str = None, @@ -209,8 +208,8 @@ async def test_completion_with_tools( ) -> None: """Test completion with tool calling for the provider.""" - if provider == "openai": - await _test_openai_completion_with_tools(api_key, llm_model) + if provider.startswith("openai"): + await _test_openai_completion_with_tools(api_key, llm_model, endpoint) elif provider == "watsonx": await _test_watsonx_completion_with_tools(api_key, llm_model, endpoint, project_id) elif provider == "ollama": @@ -221,7 +220,7 @@ async def test_completion_with_tools( raise ValueError(f"Unknown provider: {provider}") -async def test_embedding( +async def _test_embedding( provider: str, api_key: str = None, embedding_model: str = None, @@ -230,8 +229,8 @@ async def test_embedding( ) -> None: """Test embedding generation for the provider.""" - if provider == "openai": - await _test_openai_embedding(api_key, embedding_model) + if provider.startswith("openai"): + await _test_openai_embedding(api_key, embedding_model, endpoint) elif provider == "watsonx": await _test_watsonx_embedding(api_key, embedding_model, endpoint, project_id) elif provider == "ollama": @@ -241,7 +240,7 @@ async def test_embedding( # OpenAI validation functions -async def _test_openai_lightweight_health(api_key: str) -> None: +async def _test_openai_lightweight_health(api_key: str, endpoint: str) -> None: """Test OpenAI API key validity with lightweight check. Only checks if the API key is valid without consuming credits. @@ -253,7 +252,8 @@ async def _test_openai_lightweight_health(api_key: str) -> None: "Content-Type": "application/json", } - url = get_openai_url(endpoint="/v1/models") + url = f"{endpoint}/v1/models" + logger.debug("Testing openai lightweight health", url=url) async with httpx.AsyncClient() as client: # Use /v1/models endpoint which validates the key without consuming credits response = await client.get( @@ -277,7 +277,7 @@ async def _test_openai_lightweight_health(api_key: str) -> None: raise -async def _test_openai_completion_with_tools(api_key: str, llm_model: str) -> None: +async def _test_openai_completion_with_tools(api_key: str, llm_model: str, endpoint: str) -> None: """Test OpenAI completion with tool calling.""" try: headers = { @@ -315,7 +315,8 @@ async def _test_openai_completion_with_tools(api_key: str, llm_model: str) -> No async with httpx.AsyncClient() as client: # Try with max_tokens first payload = {**base_payload, "max_tokens": 50} - url = get_openai_url(endpoint="/v1/chat/completions") + url = f"{endpoint}/v1/chat/completions" + logger.debug("Test openai completion tools", url=url) response = await client.post( url=url, headers=headers, @@ -327,7 +328,7 @@ async def _test_openai_completion_with_tools(api_key: str, llm_model: str) -> No if response.status_code != 200: logger.info("max_tokens parameter failed, trying max_completion_tokens instead") payload = {**base_payload, "max_completion_tokens": 50} - url = get_openai_url(endpoint="/v1/chat/completions") + logger.debug("Test openai completion tools", url=url) response = await client.post( url=url, headers=headers, @@ -350,7 +351,7 @@ async def _test_openai_completion_with_tools(api_key: str, llm_model: str) -> No raise -async def _test_openai_embedding(api_key: str, embedding_model: str) -> None: +async def _test_openai_embedding(api_key: str, embedding_model: str, endpoint: str) -> None: """Test OpenAI embedding generation.""" try: headers = { @@ -364,7 +365,7 @@ async def _test_openai_embedding(api_key: str, embedding_model: str) -> None: } async with httpx.AsyncClient() as client: - url = get_openai_url(endpoint="/v1/embeddings") + url = f"{endpoint}/v1/embeddings" response = await client.post( url=url, headers=headers, diff --git a/src/services/models_service.py b/src/services/models_service.py index 318dfecea..61fc574f1 100644 --- a/src/services/models_service.py +++ b/src/services/models_service.py @@ -45,7 +45,7 @@ class ModelsService: def __init__(self): self.session_manager = None - async def get_openai_models(self, api_key: str) -> Dict[str, List[Dict[str, str]]]: + async def get_openai_models(self, api_key: str, api_base: str) -> Dict[str, List[Dict[str, str]]]: """Fetch available models from OpenAI API with lightweight validation""" try: headers = { @@ -56,8 +56,8 @@ async def get_openai_models(self, api_key: str) -> Dict[str, List[Dict[str, str] async with httpx.AsyncClient() as client: # Lightweight validation: just check if API key is valid # This doesn't consume credits, only validates the key - from api.provider_validation import get_openai_url - url = get_openai_url(endpoint="/v1/models") + url = f"{api_base}/v1/models" + logger.debug("Getting openai models.", url=url) response = await client.get( url, headers=headers, timeout=10.0 ) From 16a1ad4a4c17cd10d99453afaf76f007fd111974 Mon Sep 17 00:00:00 2001 From: matano Date: Thu, 29 Jan 2026 15:14:28 +0200 Subject: [PATCH 25/51] add missing field --- src/config/config_manager.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/config/config_manager.py b/src/config/config_manager.py index 6af07db4d..46caad0c2 100644 --- a/src/config/config_manager.py +++ b/src/config/config_manager.py @@ -14,6 +14,7 @@ class OpenAIConfig: """OpenAI provider configuration.""" api_key: str = "" + endpoint: str = "" configured: bool = False @@ -223,6 +224,8 @@ def _load_env_overrides( # OpenAI provider settings if os.getenv("OPENAI_API_KEY"): config_data["providers"]["openai"]["api_key"] = os.getenv("OPENAI_API_KEY") + if os.getenv("OPENAI_API_BASE"): + config_data["providers"]["openai"]["endpoint"] = os.getenv("OPENAI_API_BASE") # Anthropic provider settings if os.getenv("ANTHROPIC_API_KEY"): From c72358e8facd4f2d20cf0dbfada842b63d19ce47 Mon Sep 17 00:00:00 2001 From: matano Date: Thu, 29 Jan 2026 15:14:41 +0200 Subject: [PATCH 26/51] fix env var name --- src/api/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/models.py b/src/api/models.py index 961e1d8c9..bc43d1bb2 100644 --- a/src/api/models.py +++ b/src/api/models.py @@ -50,7 +50,7 @@ async def get_openai_models(request, models_service, session_manager): except Exception as e: logger.error(f"Failed to get config: {e}") if not api_base: - api_base = os.environ.get("OPENAI_BASE_API", "https://api.openai.com") + api_base = os.environ.get("OPENAI_API_BASE", "https://api.openai.com") models = await models_service.get_openai_models(api_key=api_key, api_base=api_base) return JSONResponse(models) From e200f6b38e60e26d0f0c83ebb71160b83134c80a Mon Sep 17 00:00:00 2001 From: matano Date: Sun, 1 Feb 2026 10:05:12 +0200 Subject: [PATCH 27/51] openai providers startwith 'openai' --- src/api/provider_validation.py | 10 +++++----- src/config/config_manager.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/api/provider_validation.py b/src/api/provider_validation.py index 358264998..ed051ad88 100644 --- a/src/api/provider_validation.py +++ b/src/api/provider_validation.py @@ -139,8 +139,8 @@ async def validate_provider_setup( try: logger.info(f"Starting validation for provider: {provider_lower} (test_completion={test_completion})") - if provider == "openai" and not endpoint: - endpoint = os.environ.get("OPENAI_BASE_API", "https://api.openai.com") + if provider.startswith("openai") and not endpoint: + endpoint = os.environ.get("OPENAI_API_BASE", "https://api.openai.com") if test_completion: # Full validation with completion/embedding tests (consumes credits) @@ -253,7 +253,7 @@ async def _test_openai_lightweight_health(api_key: str, endpoint: str) -> None: } url = f"{endpoint}/v1/models" - logger.debug("Testing openai lightweight health", url=url) + logger.info("Testing openai lightweight health", url=url) async with httpx.AsyncClient() as client: # Use /v1/models endpoint which validates the key without consuming credits response = await client.get( @@ -316,7 +316,7 @@ async def _test_openai_completion_with_tools(api_key: str, llm_model: str, endpo # Try with max_tokens first payload = {**base_payload, "max_tokens": 50} url = f"{endpoint}/v1/chat/completions" - logger.debug("Test openai completion tools", url=url) + logger.info("Test openai completion tools", url=url) response = await client.post( url=url, headers=headers, @@ -328,7 +328,7 @@ async def _test_openai_completion_with_tools(api_key: str, llm_model: str, endpo if response.status_code != 200: logger.info("max_tokens parameter failed, trying max_completion_tokens instead") payload = {**base_payload, "max_completion_tokens": 50} - logger.debug("Test openai completion tools", url=url) + logger.info("Test openai completion tools", url=url) response = await client.post( url=url, headers=headers, diff --git a/src/config/config_manager.py b/src/config/config_manager.py index 46caad0c2..902552f07 100644 --- a/src/config/config_manager.py +++ b/src/config/config_manager.py @@ -52,7 +52,7 @@ class ProvidersConfig: def get_provider_config(self, provider: str): """Get configuration for a specific provider.""" provider_lower = provider.lower() - if provider_lower == "openai": + if provider_lower.startswith("openai"): return self.openai elif provider_lower == "anthropic": return self.anthropic From 830271f195bec462fbdffe57d6a052f2be555abd Mon Sep 17 00:00:00 2001 From: matano Date: Sun, 1 Feb 2026 10:35:04 +0200 Subject: [PATCH 28/51] revert to a single openai provider --- src/api/provider_validation.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/api/provider_validation.py b/src/api/provider_validation.py index ed051ad88..458190193 100644 --- a/src/api/provider_validation.py +++ b/src/api/provider_validation.py @@ -139,7 +139,7 @@ async def validate_provider_setup( try: logger.info(f"Starting validation for provider: {provider_lower} (test_completion={test_completion})") - if provider.startswith("openai") and not endpoint: + if provider == "openai" and not endpoint: endpoint = os.environ.get("OPENAI_API_BASE", "https://api.openai.com") if test_completion: @@ -187,7 +187,7 @@ async def _test_lightweight_health( ) -> None: """Test provider health with lightweight check (no credits consumed).""" - if provider.startswith("openai"): + if provider == "openai": await _test_openai_lightweight_health(api_key, endpoint) elif provider == "watsonx": await _test_watsonx_lightweight_health(api_key, endpoint, project_id) @@ -208,7 +208,7 @@ async def _test_completion_with_tools( ) -> None: """Test completion with tool calling for the provider.""" - if provider.startswith("openai"): + if provider == "openai": await _test_openai_completion_with_tools(api_key, llm_model, endpoint) elif provider == "watsonx": await _test_watsonx_completion_with_tools(api_key, llm_model, endpoint, project_id) @@ -229,7 +229,7 @@ async def _test_embedding( ) -> None: """Test embedding generation for the provider.""" - if provider.startswith("openai"): + if provider == "openai": await _test_openai_embedding(api_key, embedding_model, endpoint) elif provider == "watsonx": await _test_watsonx_embedding(api_key, embedding_model, endpoint, project_id) From fe5562220062753279bdc4f2054332fe10f7a709 Mon Sep 17 00:00:00 2001 From: matano Date: Sun, 1 Feb 2026 10:36:00 +0200 Subject: [PATCH 29/51] revert to a single openai provider --- src/api/provider_health.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/provider_health.py b/src/api/provider_health.py index 36238e279..3802d8dcb 100644 --- a/src/api/provider_health.py +++ b/src/api/provider_health.py @@ -42,7 +42,7 @@ async def check_provider_health(request): provider = current_config.agent.llm_provider # Validate provider name - valid_providers = ["openai_ete", "openai", "ollama", "watsonx", "anthropic"] + valid_providers = ["openai", "ollama", "watsonx", "anthropic"] if provider not in valid_providers: return JSONResponse( { From 7f6ebe6992a7027b0bddbb967afd40d1a73c1874 Mon Sep 17 00:00:00 2001 From: matano Date: Sun, 1 Feb 2026 10:39:17 +0200 Subject: [PATCH 30/51] revert to a single openai provider --- src/config/config_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/config_manager.py b/src/config/config_manager.py index 902552f07..46caad0c2 100644 --- a/src/config/config_manager.py +++ b/src/config/config_manager.py @@ -52,7 +52,7 @@ class ProvidersConfig: def get_provider_config(self, provider: str): """Get configuration for a specific provider.""" provider_lower = provider.lower() - if provider_lower.startswith("openai"): + if provider_lower == "openai": return self.openai elif provider_lower == "anthropic": return self.anthropic From 787e5da6c15eadb66c1d8fb0364c59bd5277c1ed Mon Sep 17 00:00:00 2001 From: matano Date: Sun, 1 Feb 2026 10:42:44 +0200 Subject: [PATCH 31/51] fix LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index f28e682a6..d5466c8ff 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -139,7 +139,7 @@ services: - MIMETYPE=None - FILESIZE=0 - SELECTED_EMBEDDING_MODEL=${SELECTED_EMBEDDING_MODEL:-} - - LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD,OPENSEARCH_URL,DOCLING_SERVE_URL,OWNER,OWNER_NAME,OWNER_EMAIL,CONNECTOR_TYPE,FILENAME,MIMETYPE,FILESIZE,SELECTED_EMBEDDING_MODEL,OPENAI_API_KEY,OPENAI_API_BASE,ANTHROPIC_API_KEY,WATSONX_API_KEY,WATSONX_ENDPOINT,WATSONX_PROJECT_ID,OLLAMA_BASE_URL - LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD,OPENSEARCH_URL,DOCLING_SERVE_URL,OWNER,OWNER_NAME,OWNER_EMAIL,CONNECTOR_TYPE,FILENAME,MIMETYPE,FILESIZE,SELECTED_EMBEDDING_MODEL,OPENAI_API_KEY,ANTHROPIC_API_KEY,WATSONX_API_KEY,WATSONX_ENDPOINT,WATSONX_PROJECT_ID,OLLAMA_BASE_URL>>>>>>> remotes/upstream/main + - LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD,OPENSEARCH_URL,DOCLING_SERVE_URL,OWNER,OWNER_NAME,OWNER_EMAIL,CONNECTOR_TYPE,FILENAME,MIMETYPE,FILESIZE,SELECTED_EMBEDDING_MODEL,OPENAI_API_KEY,OPENAI_API_BASE,ANTHROPIC_API_KEY,WATSONX_API_KEY,WATSONX_ENDPOINT,WATSONX_PROJECT_ID,OLLAMA_BASE_URL - LANGFLOW_LOG_LEVEL=DEBUG - LANGFLOW_AUTO_LOGIN=${LANGFLOW_AUTO_LOGIN} - LANGFLOW_SUPERUSER=${LANGFLOW_SUPERUSER} From 371029f88e14ef161902b5d9f100a1e33111e596 Mon Sep 17 00:00:00 2001 From: matano Date: Sun, 1 Feb 2026 10:47:55 +0200 Subject: [PATCH 32/51] throw if api_base is not in request or in configuration (as done for the api_key) --- src/api/models.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/api/models.py b/src/api/models.py index bc43d1bb2..8d4ceb281 100644 --- a/src/api/models.py +++ b/src/api/models.py @@ -50,7 +50,12 @@ async def get_openai_models(request, models_service, session_manager): except Exception as e: logger.error(f"Failed to get config: {e}") if not api_base: - api_base = os.environ.get("OPENAI_API_BASE", "https://api.openai.com") + return JSONResponse( + { + "error": "OpenAI API base is required either in request body or in configuration" + }, + status_code=400, + ) models = await models_service.get_openai_models(api_key=api_key, api_base=api_base) return JSONResponse(models) From a9a1c7c22d6dbadd7df740030f5fc4ffbf32c57d Mon Sep 17 00:00:00 2001 From: matano Date: Fri, 6 Feb 2026 00:11:25 +0200 Subject: [PATCH 33/51] make the index name configurable --- .../mutations/useUpdateSettingsMutation.ts | 1 + .../app/api/queries/useGetSettingsQuery.ts | 1 + frontend/app/settings/page.tsx | 30 +++++++++++++++ src/api/documents.py | 6 +-- src/api/settings.py | 38 ++++++++++++++++++- src/api/v1/documents.py | 4 +- src/config/config_manager.py | 1 + src/config/settings.py | 6 ++- src/main.py | 24 ++++++------ src/models/processors.py | 18 ++++----- src/services/document_service.py | 4 +- src/services/flows_service.py | 11 ++++++ src/services/search_service.py | 8 ++-- src/utils/embedding_fields.py | 3 +- src/utils/telemetry/message_id.py | 2 + 15 files changed, 122 insertions(+), 35 deletions(-) diff --git a/frontend/app/api/mutations/useUpdateSettingsMutation.ts b/frontend/app/api/mutations/useUpdateSettingsMutation.ts index f0e6ce586..ae8aab854 100644 --- a/frontend/app/api/mutations/useUpdateSettingsMutation.ts +++ b/frontend/app/api/mutations/useUpdateSettingsMutation.ts @@ -20,6 +20,7 @@ export interface UpdateSettingsRequest { picture_descriptions?: boolean; embedding_model?: string; embedding_provider?: string; + index_name?: string; // Provider-specific settings (for dialogs) model_provider?: string; // Deprecated, kept for backward compatibility diff --git a/frontend/app/api/queries/useGetSettingsQuery.ts b/frontend/app/api/queries/useGetSettingsQuery.ts index 5ee5e74c8..1acc3a172 100644 --- a/frontend/app/api/queries/useGetSettingsQuery.ts +++ b/frontend/app/api/queries/useGetSettingsQuery.ts @@ -15,6 +15,7 @@ export interface KnowledgeSettings { embedding_provider?: string; chunk_size?: number; chunk_overlap?: number; + index_name?: string; table_structure?: boolean; ocr?: boolean; picture_descriptions?: boolean; diff --git a/frontend/app/settings/page.tsx b/frontend/app/settings/page.tsx index d5e56ca96..11d57b64c 100644 --- a/frontend/app/settings/page.tsx +++ b/frontend/app/settings/page.tsx @@ -128,6 +128,7 @@ function KnowledgeSourcesPage() { const [systemPrompt, setSystemPrompt] = useState(""); const [chunkSize, setChunkSize] = useState(1024); const [chunkOverlap, setChunkOverlap] = useState(50); + const [indexName, setIndexName] = useState("documents"); const [tableStructure, setTableStructure] = useState(true); const [ocr, setOcr] = useState(false); const [pictureDescriptions, setPictureDescriptions] = @@ -324,6 +325,12 @@ function KnowledgeSourcesPage() { } }, [settings.knowledge?.chunk_size]); + useEffect(() => { + if (settings.knowledge?.index_name) { + setIndexName(settings.knowledge.index_name); + } + }, [settings.knowledge?.index_name]); + useEffect(() => { if (settings.knowledge?.chunk_overlap) { setChunkOverlap(settings.knowledge.chunk_overlap); @@ -416,6 +423,12 @@ function KnowledgeSourcesPage() { debouncedUpdate({ chunk_overlap: numValue }); }; + // Update index name setting with debounce + const handleIndexNameChange = (value: string) => { + setIndexName(value); + debouncedUpdate({ index_name: value }); + }; + // Update docling settings const handleTableStructureChange = (checked: boolean) => { setTableStructure(checked); @@ -1238,6 +1251,23 @@ function KnowledgeSourcesPage() { /> +
+ + handleIndexNameChange(e.target.value)} + className="w-full" + placeholder="documents" + /> + +
diff --git a/src/api/documents.py b/src/api/documents.py index 048f746a3..a31183f0f 100644 --- a/src/api/documents.py +++ b/src/api/documents.py @@ -1,7 +1,7 @@ from starlette.requests import Request from starlette.responses import JSONResponse from utils.logging_config import get_logger -from config.settings import INDEX_NAME +from config.settings import get_index_name logger = get_logger(__name__) @@ -30,7 +30,7 @@ async def check_filename_exists(request: Request, document_service, session_mana logger.debug(f"Checking filename existence: {filename}") response = await opensearch_client.search( - index=INDEX_NAME, + index=get_index_name(), body=search_body ) @@ -79,7 +79,7 @@ async def delete_documents_by_filename(request: Request, document_service, sessi logger.debug(f"Deleting documents with filename: {filename}") result = await opensearch_client.delete_by_query( - index=INDEX_NAME, + index=get_index_name(), body=delete_query, conflicts="proceed" ) diff --git a/src/api/settings.py b/src/api/settings.py index 2b88f0bbe..6ca747cff 100644 --- a/src/api/settings.py +++ b/src/api/settings.py @@ -105,6 +105,7 @@ async def get_settings(request, session_manager): "table_structure": knowledge_config.table_structure, "ocr": knowledge_config.ocr, "picture_descriptions": knowledge_config.picture_descriptions, + "index_name": knowledge_config.index_name, }, "agent": { "llm_model": agent_config.llm_model, @@ -219,6 +220,7 @@ async def update_settings(request, session_manager): "picture_descriptions", "embedding_model", "embedding_provider", + "index_name", # Provider-specific fields (structured as provider_name.field_name) "openai_api_key", "anthropic_api_key", @@ -278,6 +280,16 @@ async def update_settings(request, session_manager): status_code=400, ) + if "index_name" in body: + if ( + not isinstance(body["index_name"], str) + or not body["index_name"].strip() + ): + return JSONResponse( + {"error": "index_name must be a non-empty string"}, + status_code=400, + ) + if "llm_provider" in body: if ( not isinstance(body["llm_provider"], str) @@ -563,6 +575,28 @@ async def update_settings(request, session_manager): except Exception as e: logger.error(f"Failed to update ingest flow chunk overlap: {str(e)}") # Don't fail the entire settings update if flow update fails + if "index_name" in body: + old_index_name = current_config.knowledge.index_name + new_index_name = body["index_name"].strip() + current_config.knowledge.index_name = new_index_name + config_updated = True + await TelemetryClient.send_event( + Category.SETTINGS_OPERATIONS, + MessageId.ORB_SETTINGS_INDEX_NAME_UPDATED + ) + logger.info(f"Index name changed from {old_index_name} to {new_index_name}") + + # Also update the ingest flow with the new index name + try: + flows_service = _get_flows_service() + await flows_service.update_ingest_flow_index_name(new_index_name) + logger.info( + f"Successfully updated ingest flow index name to {new_index_name}" + ) + except Exception as e: + logger.error(f"Failed to update ingest flow index name: {str(e)}") + # Don't fail the entire settings update if flow update fails + # The config will still be saved # Update provider-specific settings @@ -1539,12 +1573,12 @@ async def rollback_onboarding(request, session_manager, task_service): # Delete documents by filename from utils.opensearch_queries import build_filename_delete_body - from config.settings import INDEX_NAME + from config.settings import get_index_name delete_query = build_filename_delete_body(filename) result = await opensearch_client.delete_by_query( - index=INDEX_NAME, + index=get_index_name(), body=delete_query, conflicts="proceed" ) diff --git a/src/api/v1/documents.py b/src/api/v1/documents.py index f2a30c3cb..66b91ea97 100644 --- a/src/api/v1/documents.py +++ b/src/api/v1/documents.py @@ -115,7 +115,7 @@ async def delete_document_endpoint(request: Request, document_service, session_m user = request.state.user try: - from config.settings import INDEX_NAME + from config.settings import get_index_name from utils.opensearch_queries import build_filename_delete_body # Get OpenSearch client (API key auth uses internal client) @@ -127,7 +127,7 @@ async def delete_document_endpoint(request: Request, document_service, session_m delete_query = build_filename_delete_body(filename) result = await opensearch_client.delete_by_query( - index=INDEX_NAME, + index=get_index_name(), body=delete_query, conflicts="proceed" ) diff --git a/src/config/config_manager.py b/src/config/config_manager.py index 6af07db4d..40ca3c80b 100644 --- a/src/config/config_manager.py +++ b/src/config/config_manager.py @@ -74,6 +74,7 @@ class KnowledgeConfig: table_structure: bool = True ocr: bool = False picture_descriptions: bool = False + index_name: str = "documents" # OpenSearch index name @dataclass diff --git a/src/config/settings.py b/src/config/settings.py index 78cf03a65..ffc6ffedb 100644 --- a/src/config/settings.py +++ b/src/config/settings.py @@ -78,7 +78,6 @@ def is_no_auth_mode(): ) # No default - must be explicitly configured # OpenSearch configuration -INDEX_NAME = "documents" VECTOR_DIM = 1536 EMBED_MODEL = "text-embedding-3-small" @@ -817,3 +816,8 @@ def get_agent_config(): def get_embedding_model() -> str: """Return the currently configured embedding model.""" return get_openrag_config().knowledge.embedding_model or EMBED_MODEL if DISABLE_INGEST_WITH_LANGFLOW else "" + + +def get_index_name() -> str: + """Return the currently configured index name.""" + return get_openrag_config().knowledge.index_name diff --git a/src/main.py b/src/main.py index f943cdbfd..44a07fe86 100644 --- a/src/main.py +++ b/src/main.py @@ -67,10 +67,10 @@ API_KEYS_INDEX_NAME, DISABLE_INGEST_WITH_LANGFLOW, INDEX_BODY, - INDEX_NAME, SESSION_SECRET, clients, get_embedding_model, + get_index_name, is_no_auth_mode, get_openrag_config, ) @@ -152,16 +152,17 @@ async def configure_alerting_security(): async def _ensure_opensearch_index(): """Ensure OpenSearch index exists when using traditional connector service.""" try: + index_name = get_index_name() # Check if index already exists - if await clients.opensearch.indices.exists(index=INDEX_NAME): - logger.debug("OpenSearch index already exists", index_name=INDEX_NAME) + if await clients.opensearch.indices.exists(index=index_name): + logger.debug("OpenSearch index already exists", index_name=index_name) return # Create the index with hard-coded INDEX_BODY (uses OpenAI embedding dimensions) - await clients.opensearch.indices.create(index=INDEX_NAME, body=INDEX_BODY) + await clients.opensearch.indices.create(index=index_name, body=INDEX_BODY) logger.info( "Created OpenSearch index for traditional connector service", - index_name=INDEX_NAME, + index_name=index_name, vector_dimensions=INDEX_BODY["mappings"]["properties"]["chunk_embedding"][ "dimension" ], @@ -172,7 +173,7 @@ async def _ensure_opensearch_index(): logger.error( "Failed to initialize OpenSearch index for traditional connector service", error=str(e), - index_name=INDEX_NAME, + index_name=get_index_name(), ) await TelemetryClient.send_event(Category.OPENSEARCH_INDEX, MessageId.ORB_OS_INDEX_CREATE_FAIL) # Don't raise the exception to avoid breaking the initialization @@ -198,20 +199,21 @@ async def init_index(): ) # Create documents index - if not await clients.opensearch.indices.exists(index=INDEX_NAME): + index_name = get_index_name() + if not await clients.opensearch.indices.exists(index=index_name): await clients.opensearch.indices.create( - index=INDEX_NAME, body=dynamic_index_body + index=index_name, body=dynamic_index_body ) logger.info( "Created OpenSearch index", - index_name=INDEX_NAME, + index_name=index_name, embedding_model=embedding_model, ) await TelemetryClient.send_event(Category.OPENSEARCH_INDEX, MessageId.ORB_OS_INDEX_CREATED) else: logger.info( "Index already exists, skipping creation", - index_name=INDEX_NAME, + index_name=index_name, embedding_model=embedding_model, ) await TelemetryClient.send_event(Category.OPENSEARCH_INDEX, MessageId.ORB_OS_INDEX_EXISTS) @@ -644,7 +646,7 @@ async def initialize_services(): patched_async_client=clients, # Pass the clients object itself process_pool=process_pool, embed_model=get_embedding_model(), - index_name=INDEX_NAME, + index_name=get_index_name(), task_service=task_service, session_manager=session_manager, ) diff --git a/src/models/processors.py b/src/models/processors.py index d8de30c56..ce12d7f57 100644 --- a/src/models/processors.py +++ b/src/models/processors.py @@ -20,7 +20,7 @@ async def check_document_exists( Check if a document with the given hash already exists in OpenSearch. Consolidated hash checking for all processors. """ - from config.settings import INDEX_NAME + from config.settings import get_index_name import asyncio max_retries = 3 @@ -28,7 +28,7 @@ async def check_document_exists( for attempt in range(max_retries): try: - exists = await opensearch_client.exists(index=INDEX_NAME, id=file_hash) + exists = await opensearch_client.exists(index=get_index_name(), id=file_hash) return exists except (asyncio.TimeoutError, Exception) as e: if attempt == max_retries - 1: @@ -64,7 +64,7 @@ async def check_filename_exists( Check if a document with the given filename already exists in OpenSearch. Returns True if any chunks with this filename exist. """ - from config.settings import INDEX_NAME + from config.settings import get_index_name from utils.opensearch_queries import build_filename_search_body import asyncio @@ -77,7 +77,7 @@ async def check_filename_exists( search_body = build_filename_search_body(filename, size=1, source=False) response = await opensearch_client.search( - index=INDEX_NAME, + index=get_index_name(), body=search_body ) @@ -118,7 +118,7 @@ async def delete_document_by_filename( """ Delete all chunks of a document with the given filename from OpenSearch. """ - from config.settings import INDEX_NAME + from config.settings import get_index_name from utils.opensearch_queries import build_filename_delete_body try: @@ -126,7 +126,7 @@ async def delete_document_by_filename( delete_body = build_filename_delete_body(filename) response = await opensearch_client.delete_by_query( - index=INDEX_NAME, + index=get_index_name(), body=delete_body ) @@ -168,7 +168,7 @@ async def process_document_standard( embedding model from settings) """ import datetime - from config.settings import INDEX_NAME, clients, get_embedding_model + from config.settings import clients, get_embedding_model, get_index_name from services.document_service import chunk_texts_for_embeddings from utils.document_processing import extract_relevant from utils.embedding_fields import get_embedding_field_name, ensure_embedding_field_exists @@ -265,7 +265,7 @@ async def process_document_standard( chunk_id = f"{file_hash}_{i}" try: await opensearch_client.index( - index=INDEX_NAME, id=chunk_id, body=chunk_doc + index=get_index_name(), id=chunk_id, body=chunk_doc ) except Exception as e: logger.error( @@ -601,7 +601,7 @@ async def process_item( import time import asyncio import datetime - from config.settings import INDEX_NAME, clients, get_embedding_model + from config.settings import clients, get_embedding_model, get_index_name from services.document_service import chunk_texts_for_embeddings from utils.document_processing import process_document_sync diff --git a/src/services/document_service.py b/src/services/document_service.py index f40c3d823..f010b6c5a 100644 --- a/src/services/document_service.py +++ b/src/services/document_service.py @@ -12,7 +12,7 @@ logger = get_logger(__name__) -from config.settings import clients, INDEX_NAME, get_embedding_model +from config.settings import clients, get_embedding_model, get_index_name from utils.document_processing import extract_relevant, process_document_sync from utils.telemetry import TelemetryClient, Category, MessageId @@ -153,7 +153,7 @@ async def process_upload_file( ) try: - exists = await opensearch_client.exists(index=INDEX_NAME, id=file_hash) + exists = await opensearch_client.exists(index=get_index_name(), id=file_hash) except Exception as e: logger.error( "OpenSearch exists check failed", file_hash=file_hash, error=str(e) diff --git a/src/services/flows_service.py b/src/services/flows_service.py index e97ac2d3a..b60706963 100644 --- a/src/services/flows_service.py +++ b/src/services/flows_service.py @@ -952,6 +952,17 @@ async def update_ingest_flow_chunk_overlap(self, chunk_overlap: int): chunk_overlap, node_display_name="Split Text", ) + async def update_ingest_flow_index_name(self, index_name: str): + """Helper function to update index name in the ingest flow""" + if not LANGFLOW_INGEST_FLOW_ID: + raise ValueError("LANGFLOW_INGEST_FLOW_ID is not configured") + await self._update_flow_field( + LANGFLOW_INGEST_FLOW_ID, + "index_name", + index_name, + node_display_name="OpenSearch (Multi-Model Multi-Embedding)", + ) + async def update_ingest_flow_embedding_model(self, embedding_model: str, provider: str): """Helper function to update embedding model in the ingest flow""" diff --git a/src/services/search_service.py b/src/services/search_service.py index b0927d0f8..da3b53cc7 100644 --- a/src/services/search_service.py +++ b/src/services/search_service.py @@ -1,7 +1,7 @@ import copy from typing import Any, Dict from agentd.tool_decorator import tool -from config.settings import EMBED_MODEL, clients, INDEX_NAME, get_embedding_model, WATSONX_EMBEDDING_DIMENSIONS +from config.settings import EMBED_MODEL, clients, get_embedding_model, get_index_name, WATSONX_EMBEDDING_DIMENSIONS from auth_context import get_auth_context from utils.logging_config import get_logger @@ -120,7 +120,7 @@ async def search_tool(self, query: str, embedding_model: str = None) -> Dict[str } agg_result = await opensearch_client.search( - index=INDEX_NAME, body=agg_query, params={"terminate_after": 0} + index=get_index_name(), body=agg_query, params={"terminate_after": 0} ) buckets = agg_result.get("aggregations", {}).get("embedding_models", {}).get("buckets", []) available_models = [b["key"] for b in buckets if b["key"]] @@ -396,7 +396,7 @@ async def embed_with_model(model_name): try: results = await opensearch_client.search( - index=INDEX_NAME, body=search_body, params=search_params + index=get_index_name(), body=search_body, params=search_params ) except RequestError as e: error_message = str(e) @@ -409,7 +409,7 @@ async def embed_with_model(model_name): ) try: results = await opensearch_client.search( - index=INDEX_NAME, + index=get_index_name(), body=fallback_search_body, params=search_params, ) diff --git a/src/utils/embedding_fields.py b/src/utils/embedding_fields.py index 990cb116d..5240c4d06 100644 --- a/src/utils/embedding_fields.py +++ b/src/utils/embedding_fields.py @@ -86,8 +86,9 @@ async def ensure_embedding_field_exists( Raises: Exception: If unable to add the field mapping """ - from config.settings import INDEX_NAME + from config.settings import get_index_name from utils.embeddings import get_embedding_dimensions + from config.settings import get_index_name if index_name is None: index_name = INDEX_NAME diff --git a/src/utils/telemetry/message_id.py b/src/utils/telemetry/message_id.py index c00e5eb35..a5b17656e 100644 --- a/src/utils/telemetry/message_id.py +++ b/src/utils/telemetry/message_id.py @@ -176,6 +176,8 @@ class MessageId: ORB_SETTINGS_CHUNK_UPDATED = "ORB_SETTINGS_CHUNK_UPDATED" # Message: Docling settings updated ORB_SETTINGS_DOCLING_UPDATED = "ORB_SETTINGS_DOCLING_UPDATED" + # Message: Index name updated + ORB_SETTINGS_INDEX_NAME_UPDATED = "ORB_SETTINGS_INDEX_NAME_UPDATED" # Message: Provider credentials updated ORB_SETTINGS_PROVIDER_CREDS = "ORB_SETTINGS_PROVIDER_CREDS" From 1d8b690896435b3f6c10856530a91d350e2e462d Mon Sep 17 00:00:00 2001 From: matano Date: Fri, 6 Feb 2026 00:17:02 +0200 Subject: [PATCH 34/51] additional replacements of INDEX_NAME with get_index_name() --- scripts/migrate_embedding_model_field.py | 6 +++--- src/api/connector_router.py | 1 - src/models/processors.py | 2 +- src/utils/embedding_fields.py | 5 ++--- tests/integration/test_api_endpoints.py | 16 ++++++++-------- tests/integration/test_startup_ingest.py | 4 ++-- 6 files changed, 16 insertions(+), 18 deletions(-) diff --git a/scripts/migrate_embedding_model_field.py b/scripts/migrate_embedding_model_field.py index d90f270ea..ec7f093cf 100644 --- a/scripts/migrate_embedding_model_field.py +++ b/scripts/migrate_embedding_model_field.py @@ -59,7 +59,7 @@ OPENSEARCH_PORT, OPENSEARCH_USERNAME, OPENSEARCH_PASSWORD, - INDEX_NAME, + get_index_name, ) from utils.logging_config import get_logger from utils.embedding_fields import get_embedding_field_name @@ -231,7 +231,7 @@ async def migrate_legacy_embeddings( ) -> bool: """Main migration function.""" if index_name is None: - index_name = INDEX_NAME + index_name = get_index_name() new_field_name = get_embedding_field_name(model_name) @@ -359,7 +359,7 @@ def main(): parser.add_argument( '--index', default=None, - help=f'Index name (default: {INDEX_NAME})' + help=f'Index name (default: {get_index_name()})' ) args = parser.parse_args() diff --git a/src/api/connector_router.py b/src/api/connector_router.py index dd98e4749..53f808237 100644 --- a/src/api/connector_router.py +++ b/src/api/connector_router.py @@ -5,7 +5,6 @@ from config.settings import ( DISABLE_INGEST_WITH_LANGFLOW, clients, - INDEX_NAME, INDEX_BODY, ) from utils.logging_config import get_logger diff --git a/src/models/processors.py b/src/models/processors.py index ce12d7f57..bc2d38e32 100644 --- a/src/models/processors.py +++ b/src/models/processors.py @@ -187,7 +187,7 @@ async def process_document_standard( # Ensure the embedding field exists for this model embedding_field_name = await ensure_embedding_field_exists( - opensearch_client, embedding_model, INDEX_NAME + opensearch_client, embedding_model, get_index_name() ) logger.info( diff --git a/src/utils/embedding_fields.py b/src/utils/embedding_fields.py index 5240c4d06..02342407b 100644 --- a/src/utils/embedding_fields.py +++ b/src/utils/embedding_fields.py @@ -78,7 +78,7 @@ async def ensure_embedding_field_exists( Args: opensearch_client: OpenSearch client instance model_name: The embedding model name - index_name: OpenSearch index name (defaults to INDEX_NAME from settings) + index_name: OpenSearch index name (defaults to get_index_name() from settings) Returns: The field name that was ensured to exist @@ -88,10 +88,9 @@ async def ensure_embedding_field_exists( """ from config.settings import get_index_name from utils.embeddings import get_embedding_dimensions - from config.settings import get_index_name if index_name is None: - index_name = INDEX_NAME + index_name = get_index_name() field_name = get_embedding_field_name(model_name) dimensions = await get_embedding_dimensions(model_name) diff --git a/tests/integration/test_api_endpoints.py b/tests/integration/test_api_endpoints.py index ddad3535a..bbc633aa7 100644 --- a/tests/integration/test_api_endpoints.py +++ b/tests/integration/test_api_endpoints.py @@ -143,12 +143,12 @@ async def test_upload_and_search_endpoint(tmp_path: Path, disable_langflow_inges sys.modules.pop(mod, None) from src.main import create_app, startup_tasks import src.api.router as upload_router - from src.config.settings import clients, INDEX_NAME, DISABLE_INGEST_WITH_LANGFLOW + from src.config.settings import clients, get_index_name, DISABLE_INGEST_WITH_LANGFLOW # Ensure a clean index before startup await clients.initialize() try: - await clients.opensearch.indices.delete(index=INDEX_NAME) + await clients.opensearch.indices.delete(index=get_index_name()) # Wait for deletion to complete await asyncio.sleep(1) except Exception: @@ -164,7 +164,7 @@ async def test_upload_and_search_endpoint(tmp_path: Path, disable_langflow_inges # Verify index is truly empty after startup try: - count_response = await clients.opensearch.count(index=INDEX_NAME) + count_response = await clients.opensearch.count(index=get_index_name()) doc_count = count_response.get('count', 0) assert doc_count == 0, f"Index should be empty after startup but contains {doc_count} documents" except Exception as e: @@ -480,11 +480,11 @@ async def test_search_multi_embedding_models( sys.modules.pop(mod, None) from src.main import create_app, startup_tasks - from src.config.settings import clients, INDEX_NAME + from src.config.settings import clients, get_index_name await clients.initialize() try: - await clients.opensearch.indices.delete(index=INDEX_NAME) + await clients.opensearch.indices.delete(index=get_index_name()) await asyncio.sleep(1) except Exception: pass @@ -626,12 +626,12 @@ async def test_router_upload_ingest_traditional(tmp_path: Path, disable_langflow sys.modules.pop(mod, None) from src.main import create_app, startup_tasks import src.api.router as upload_router - from src.config.settings import clients, INDEX_NAME, DISABLE_INGEST_WITH_LANGFLOW + from src.config.settings import clients, get_index_name, DISABLE_INGEST_WITH_LANGFLOW # Ensure a clean index before startup await clients.initialize() try: - await clients.opensearch.indices.delete(index=INDEX_NAME) + await clients.opensearch.indices.delete(index=get_index_name()) # Wait for deletion to complete await asyncio.sleep(1) except Exception: @@ -646,7 +646,7 @@ async def test_router_upload_ingest_traditional(tmp_path: Path, disable_langflow # Verify index is truly empty after startup try: - count_response = await clients.opensearch.count(index=INDEX_NAME) + count_response = await clients.opensearch.count(index=get_index_name()) doc_count = count_response.get('count', 0) assert doc_count == 0, f"Index should be empty after startup but contains {doc_count} documents" except Exception as e: diff --git a/tests/integration/test_startup_ingest.py b/tests/integration/test_startup_ingest.py index 78402392a..4c43cd546 100644 --- a/tests/integration/test_startup_ingest.py +++ b/tests/integration/test_startup_ingest.py @@ -60,12 +60,12 @@ async def test_startup_ingest_creates_task(disable_langflow_ingest: bool): sys.modules.pop(mod, None) from src.main import create_app, startup_tasks - from src.config.settings import clients, INDEX_NAME + from src.config.settings import clients, get_index_name # Ensure a clean index before startup await clients.initialize() try: - await clients.opensearch.indices.delete(index=INDEX_NAME) + await clients.opensearch.indices.delete(index=get_index_name()) except Exception: pass From 8466dc46bb2a156e10e2f8b9c887f408408617cc Mon Sep 17 00:00:00 2001 From: matano Date: Fri, 6 Feb 2026 00:47:10 +0200 Subject: [PATCH 35/51] also update the index nae in the chat flow --- src/api/settings.py | 6 +++--- src/services/flows_service.py | 16 ++++++++++++++-- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/api/settings.py b/src/api/settings.py index 6ca747cff..785efda35 100644 --- a/src/api/settings.py +++ b/src/api/settings.py @@ -586,12 +586,12 @@ async def update_settings(request, session_manager): ) logger.info(f"Index name changed from {old_index_name} to {new_index_name}") - # Also update the ingest flow with the new index name + # Also update both ingest and chat flows with the new index name try: flows_service = _get_flows_service() - await flows_service.update_ingest_flow_index_name(new_index_name) + await flows_service.update_flows_index_name(new_index_name) logger.info( - f"Successfully updated ingest flow index name to {new_index_name}" + f"Successfully updated ingest and chat flow index names to {new_index_name}" ) except Exception as e: logger.error(f"Failed to update ingest flow index name: {str(e)}") diff --git a/src/services/flows_service.py b/src/services/flows_service.py index b60706963..30f31258a 100644 --- a/src/services/flows_service.py +++ b/src/services/flows_service.py @@ -952,16 +952,28 @@ async def update_ingest_flow_chunk_overlap(self, chunk_overlap: int): chunk_overlap, node_display_name="Split Text", ) - async def update_ingest_flow_index_name(self, index_name: str): - """Helper function to update index name in the ingest flow""" + async def update_flows_index_name(self, index_name: str): + """Helper function to update index name in both ingest and chat flows""" if not LANGFLOW_INGEST_FLOW_ID: raise ValueError("LANGFLOW_INGEST_FLOW_ID is not configured") + if not LANGFLOW_CHAT_FLOW_ID: + raise ValueError("LANGFLOW_CHAT_FLOW_ID is not configured") + + # Update ingest flow await self._update_flow_field( LANGFLOW_INGEST_FLOW_ID, "index_name", index_name, node_display_name="OpenSearch (Multi-Model Multi-Embedding)", ) + + # Update chat flow + await self._update_flow_field( + LANGFLOW_CHAT_FLOW_ID, + "index_name", + index_name, + node_display_name="OpenSearch (Multi-Model Multi-Embedding)", + ) async def update_ingest_flow_embedding_model(self, embedding_model: str, provider: str): From 4241f31f2c9d449e815e41be0e15134a4dbbfe8d Mon Sep 17 00:00:00 2001 From: matano Date: Sun, 8 Feb 2026 17:35:52 +0200 Subject: [PATCH 36/51] add OPENAI_API_BASE to add_provider_credentials_to_headers() and build_mcp_global_vars_from_config() --- src/utils/langflow_headers.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/utils/langflow_headers.py b/src/utils/langflow_headers.py index e3447e611..bfc5f1271 100644 --- a/src/utils/langflow_headers.py +++ b/src/utils/langflow_headers.py @@ -14,7 +14,10 @@ def add_provider_credentials_to_headers(headers: Dict[str, str], config) -> None # Add OpenAI credentials if config.providers.openai.api_key: headers["X-LANGFLOW-GLOBAL-VAR-OPENAI_API_KEY"] = str(config.providers.openai.api_key) - + + if config.providers.openai.endpoint: + headers["X-LANGFLOW-GLOBAL-VAR-OPENAI_API_BASE"] = str(config.providers.openai.endpoint) + # Add Anthropic credentials if config.providers.anthropic.api_key: headers["X-LANGFLOW-GLOBAL-VAR-ANTHROPIC_API_KEY"] = str(config.providers.anthropic.api_key) @@ -47,6 +50,9 @@ def build_mcp_global_vars_from_config(config) -> Dict[str, str]: if config.providers.openai.api_key: global_vars["OPENAI_API_KEY"] = config.providers.openai.api_key + if config.providers.openai.endpoint: + global_vars["OPENAI_API_BASE"] = config.providers.openai.endpoint + # Add Anthropic credentials if config.providers.anthropic.api_key: global_vars["ANTHROPIC_API_KEY"] = config.providers.anthropic.api_key From 6d2e809f2d17cc0d4f03336a1e0a326e1648a61c Mon Sep 17 00:00:00 2001 From: matano Date: Mon, 9 Feb 2026 12:51:22 +0200 Subject: [PATCH 37/51] add OPENAI_API_BASE as default for the api_base field of the OPENAI embedder --- flows/ingestion_flow.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flows/ingestion_flow.json b/flows/ingestion_flow.json index 6f65c04e0..78a4ddad9 100644 --- a/flows/ingestion_flow.json +++ b/flows/ingestion_flow.json @@ -5317,7 +5317,7 @@ "_type": "Component", "api_base": { "_input_type": "MessageTextInput", - "advanced": true, + "advanced": false, "display_name": "OpenAI API Base URL", "dynamic": false, "info": "Base URL for the API. Leave empty for default.", @@ -5326,7 +5326,7 @@ ], "list": false, "list_add_label": "Add More", - "load_from_db": false, + "load_from_db": true, "name": "api_base", "override_skip": false, "placeholder": "", @@ -5338,7 +5338,7 @@ "trace_as_metadata": true, "track_in_telemetry": false, "type": "str", - "value": "" + "value": "OPENAI_API_BASE" }, "api_key": { "_input_type": "SecretStrInput", From 73ab1b3a735c78eb375024fbfa4fdce9b1bb29ff Mon Sep 17 00:00:00 2001 From: matano Date: Mon, 9 Feb 2026 14:07:50 +0200 Subject: [PATCH 38/51] add OPENAI_API_BASE to embedder and agent --- flows/openrag_agent.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/flows/openrag_agent.json b/flows/openrag_agent.json index f04405ebe..d8541c598 100644 --- a/flows/openrag_agent.json +++ b/flows/openrag_agent.json @@ -1808,14 +1808,14 @@ }, "openai_api_base": { "_input_type": "StrInput", - "advanced": true, + "advanced": false, "display_name": "OpenAI API Base", "dynamic": false, "info": "The base URL of the OpenAI API. Defaults to https://api.openai.com/v1. You can change this to use other APIs like JinaChat, LocalAI and Prem.", "input_types": [], "list": false, "list_add_label": "Add More", - "load_from_db": false, + "load_from_db": true, "name": "openai_api_base", "override_skip": false, "placeholder": "", @@ -1826,7 +1826,7 @@ "trace_as_metadata": true, "track_in_telemetry": false, "type": "str", - "value": "" + "value": "OPENAI_API_BASE" }, "output_schema": { "_input_type": "TableInput", @@ -2344,7 +2344,7 @@ ], "list": false, "list_add_label": "Add More", - "load_from_db": false, + "load_from_db": true, "name": "api_base", "override_skip": false, "placeholder": "", @@ -2356,7 +2356,7 @@ "trace_as_metadata": true, "track_in_telemetry": false, "type": "str", - "value": "" + "value": "OPENAI_API_BASE" }, "api_key": { "_input_type": "SecretStrInput", From 6624c4c2440854523ad05c2576535d11801d4467 Mon Sep 17 00:00:00 2001 From: matano Date: Mon, 9 Feb 2026 14:09:18 +0200 Subject: [PATCH 39/51] when updating an openai api_base, use the OPENAI_API_BASE --- src/services/flows_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/services/flows_service.py b/src/services/flows_service.py index e97ac2d3a..2d147353d 100644 --- a/src/services/flows_service.py +++ b/src/services/flows_service.py @@ -1393,8 +1393,8 @@ async def _update_component_fields( template["api_key"]["advanced"] = False updated = True if provider == "openai" and "api_base" in template: - template["api_base"]["value"] = "" - template["api_base"]["load_from_db"] = False + template["api_base"]["value"] = "OPENAI_API_BASE" + template["api_base"]["load_from_db"] = True template["api_base"]["show"] = True template["api_base"]["advanced"] = False updated = True From 9fbb7cc206e1b811f9c8055fd8da43578e9a72ae Mon Sep 17 00:00:00 2001 From: matano Date: Mon, 9 Feb 2026 14:15:17 +0200 Subject: [PATCH 40/51] add claude-opus to OpenAI allowed LLMs --- src/services/models_service.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/services/models_service.py b/src/services/models_service.py index 61fc574f1..d99e91e33 100644 --- a/src/services/models_service.py +++ b/src/services/models_service.py @@ -30,6 +30,7 @@ class ModelsService: "o3-pro", "o4-mini", "o4-mini-high", + "claude-opus-4-5-20251101", ] ANTHROPIC_MODELS = [ From e062b692a6dec4785868fee99877c39db1723c56 Mon Sep 17 00:00:00 2001 From: matano Date: Tue, 10 Feb 2026 10:31:39 +0200 Subject: [PATCH 41/51] update openai_api_base when updating the template --- src/services/flows_service.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/services/flows_service.py b/src/services/flows_service.py index c0859c1b6..369ac6d51 100644 --- a/src/services/flows_service.py +++ b/src/services/flows_service.py @@ -1448,6 +1448,12 @@ async def _update_component_fields( template["api_base"]["show"] = True template["api_base"]["advanced"] = False updated = True + if provider == "openai" and "openai_api_base" in template: + template["openai_api_base"]["value"] = "OPENAI_API_BASE" + template["openai_api_base"]["load_from_db"] = True + template["openai_api_base"]["show"] = True + template["openai_api_base"]["advanced"] = False + updated = True if provider == "anthropic" and "api_key" in template: template["api_key"]["value"] = "ANTHROPIC_API_KEY" From bd9db6f0bca32d6e046856c91c78b23c35b075c6 Mon Sep 17 00:00:00 2001 From: matano Date: Wed, 11 Feb 2026 10:16:34 +0200 Subject: [PATCH 42/51] revert frontend changes pertaining to index_name --- .../mutations/useUpdateSettingsMutation.ts | 1 - .../app/api/queries/useGetSettingsQuery.ts | 1 - frontend/app/settings/page.tsx | 30 ------------------- 3 files changed, 32 deletions(-) diff --git a/frontend/app/api/mutations/useUpdateSettingsMutation.ts b/frontend/app/api/mutations/useUpdateSettingsMutation.ts index ae8aab854..f0e6ce586 100644 --- a/frontend/app/api/mutations/useUpdateSettingsMutation.ts +++ b/frontend/app/api/mutations/useUpdateSettingsMutation.ts @@ -20,7 +20,6 @@ export interface UpdateSettingsRequest { picture_descriptions?: boolean; embedding_model?: string; embedding_provider?: string; - index_name?: string; // Provider-specific settings (for dialogs) model_provider?: string; // Deprecated, kept for backward compatibility diff --git a/frontend/app/api/queries/useGetSettingsQuery.ts b/frontend/app/api/queries/useGetSettingsQuery.ts index 1acc3a172..5ee5e74c8 100644 --- a/frontend/app/api/queries/useGetSettingsQuery.ts +++ b/frontend/app/api/queries/useGetSettingsQuery.ts @@ -15,7 +15,6 @@ export interface KnowledgeSettings { embedding_provider?: string; chunk_size?: number; chunk_overlap?: number; - index_name?: string; table_structure?: boolean; ocr?: boolean; picture_descriptions?: boolean; diff --git a/frontend/app/settings/page.tsx b/frontend/app/settings/page.tsx index cda996c7b..f89bb919c 100644 --- a/frontend/app/settings/page.tsx +++ b/frontend/app/settings/page.tsx @@ -129,7 +129,6 @@ function KnowledgeSourcesPage() { const [systemPrompt, setSystemPrompt] = useState(""); const [chunkSize, setChunkSize] = useState(1024); const [chunkOverlap, setChunkOverlap] = useState(50); - const [indexName, setIndexName] = useState("documents"); const [tableStructure, setTableStructure] = useState(true); const [ocr, setOcr] = useState(false); const [pictureDescriptions, setPictureDescriptions] = @@ -326,12 +325,6 @@ function KnowledgeSourcesPage() { } }, [settings.knowledge?.chunk_size]); - useEffect(() => { - if (settings.knowledge?.index_name) { - setIndexName(settings.knowledge.index_name); - } - }, [settings.knowledge?.index_name]); - useEffect(() => { if (settings.knowledge?.chunk_overlap) { setChunkOverlap(settings.knowledge.chunk_overlap); @@ -424,12 +417,6 @@ function KnowledgeSourcesPage() { debouncedUpdate({ chunk_overlap: numValue }); }; - // Update index name setting with debounce - const handleIndexNameChange = (value: string) => { - setIndexName(value); - debouncedUpdate({ index_name: value }); - }; - // Update docling settings const handleTableStructureChange = (checked: boolean) => { setTableStructure(checked); @@ -1324,23 +1311,6 @@ function KnowledgeSourcesPage() { />
-
- - handleIndexNameChange(e.target.value)} - className="w-full" - placeholder="documents" - /> - -
From f28b6ab2e79c4b151cfde3053918917862ea423e Mon Sep 17 00:00:00 2001 From: Lucas Oliveira Date: Wed, 11 Feb 2026 15:46:33 -0300 Subject: [PATCH 43/51] updated flows service to update global variable instead of flows --- src/api/settings.py | 9 ++++----- src/services/flows_service.py | 22 ---------------------- 2 files changed, 4 insertions(+), 27 deletions(-) diff --git a/src/api/settings.py b/src/api/settings.py index 785efda35..24c66f5ed 100644 --- a/src/api/settings.py +++ b/src/api/settings.py @@ -586,15 +586,14 @@ async def update_settings(request, session_manager): ) logger.info(f"Index name changed from {old_index_name} to {new_index_name}") - # Also update both ingest and chat flows with the new index name + # Also update global variable with new index name try: - flows_service = _get_flows_service() - await flows_service.update_flows_index_name(new_index_name) + await clients._create_langflow_global_variable("OPENSEARCH_INDEX_NAME", new_index_name) logger.info( - f"Successfully updated ingest and chat flow index names to {new_index_name}" + f"Successfully updated global variable with new index name {new_index_name}" ) except Exception as e: - logger.error(f"Failed to update ingest flow index name: {str(e)}") + logger.error(f"Failed to update global variable with new index name: {str(e)}") # Don't fail the entire settings update if flow update fails # The config will still be saved diff --git a/src/services/flows_service.py b/src/services/flows_service.py index 30f31258a..6989edf19 100644 --- a/src/services/flows_service.py +++ b/src/services/flows_service.py @@ -952,28 +952,6 @@ async def update_ingest_flow_chunk_overlap(self, chunk_overlap: int): chunk_overlap, node_display_name="Split Text", ) - async def update_flows_index_name(self, index_name: str): - """Helper function to update index name in both ingest and chat flows""" - if not LANGFLOW_INGEST_FLOW_ID: - raise ValueError("LANGFLOW_INGEST_FLOW_ID is not configured") - if not LANGFLOW_CHAT_FLOW_ID: - raise ValueError("LANGFLOW_CHAT_FLOW_ID is not configured") - - # Update ingest flow - await self._update_flow_field( - LANGFLOW_INGEST_FLOW_ID, - "index_name", - index_name, - node_display_name="OpenSearch (Multi-Model Multi-Embedding)", - ) - - # Update chat flow - await self._update_flow_field( - LANGFLOW_CHAT_FLOW_ID, - "index_name", - index_name, - node_display_name="OpenSearch (Multi-Model Multi-Embedding)", - ) async def update_ingest_flow_embedding_model(self, embedding_model: str, provider: str): From 7f45721922871f05ed0b02e8117edd0558332d67 Mon Sep 17 00:00:00 2001 From: matano Date: Thu, 12 Feb 2026 10:21:25 +0200 Subject: [PATCH 44/51] updates following merge --- src/api/settings.py | 24 ------------------------ src/main.py | 13 +++++++------ src/services/flows_service.py | 17 ----------------- 3 files changed, 7 insertions(+), 47 deletions(-) diff --git a/src/api/settings.py b/src/api/settings.py index ca330ec81..c2f6f53ca 100644 --- a/src/api/settings.py +++ b/src/api/settings.py @@ -266,7 +266,6 @@ async def update_settings(request, session_manager): "picture_descriptions", "embedding_model", "embedding_provider", - "index_name", # Provider-specific fields (structured as provider_name.field_name) "openai_api_key", "anthropic_api_key", @@ -584,29 +583,6 @@ async def update_settings(request, session_manager): except Exception as e: logger.error(f"Failed to update docling settings in flow: {str(e)}") - if "index_name" in body: - import config.settings as settings - settings.INDEX_NAME = body["index_name"] - # also update ? - # current_config.knowledge.index_name = - config_updated = True - #await TelemetryClient.send_event( - # Category.SETTINGS_OPERATIONS, - # MessageId.ORB_SETTINGS_UPDATED - #) - - # Also update the flows with the new index name - try: - flows_service = _get_flows_service() - await flows_service.update_flows_index_name(settings.INDEX_NAME) - logger.info( - f"Successfully updated flows index name to '{settings.INDEX_NAME}'." - ) - except Exception as e: - logger.error(f"Failed to update ingest flow index name: {str(e)}") - # Don't fail the entire settings update if flow update fails - # The config will still be saved - if "splitter_type" in body: new_splitter_type = body["splitter_type"] current_config.knowledge.splitter_type = new_splitter_type diff --git a/src/main.py b/src/main.py index c63a724f3..1b778f069 100644 --- a/src/main.py +++ b/src/main.py @@ -202,23 +202,24 @@ async def init_index(delete_existing: bool = False): endpoint=getattr(embedding_provider_config, "endpoint", None) ) - index_exists = await clients.opensearch.indices.exists(index=settings.INDEX_NAME) + index_name = get_index_name() + index_exists = await clients.opensearch.indices.exists(index=index_name) logger.info( "Initializing OpenSearch index ..", - index_name=settings.INDEX_NAME, + index_name=index_name, embedding_model=embedding_model, delete_existing=delete_existing, index_exists=index_exists, ) if index_exists and delete_existing: # DELETE / - logger.info(f"Deleting index '{settings.INDEX_NAME}'...") - resp = await clients.opensearch.indices.delete(index=settings.INDEX_NAME) - logger.info(f"Deleted '{settings.INDEX_NAME}': {resp}") + logger.info(f"Deleting index '{index_name}'...") + resp = await clients.opensearch.indices.delete(index=index_name) + logger.info(f"Deleted '{index_name}': {resp}") index_exists = False # Create documents index - index_name = get_index_name() + if not index_exists: await clients.opensearch.indices.create( index=index_name, body=dynamic_index_body diff --git a/src/services/flows_service.py b/src/services/flows_service.py index 369ac6d51..1fb4ff806 100644 --- a/src/services/flows_service.py +++ b/src/services/flows_service.py @@ -931,23 +931,6 @@ async def update_flow_docling_preset(self, preset: str, preset_config: dict): await self._update_flow_field(LANGFLOW_INGEST_FLOW_ID, "docling_serve_opts", preset_config, node_display_name=DOCLING_COMPONENT_DISPLAY_NAME) - async def update_flows_index_name(self, index_name: str): - """Helper function to update index name in the ingest flow""" - if not LANGFLOW_INGEST_FLOW_ID: - raise ValueError("LANGFLOW_INGEST_FLOW_ID is not configured") - await self._update_flow_field( - LANGFLOW_INGEST_FLOW_ID, - "index_name", - index_name, - node_display_name="OpenSearch (Multi-Model Multi-Embedding)", - ) - await self._update_flow_field( - LANGFLOW_CHAT_FLOW_ID, - "index_name", - index_name, - node_display_name="OpenSearch (Multi-Model Multi-Embedding)", - ) - async def update_ingest_flow_chunk_size(self, chunk_size: int): """Helper function to update chunk size in the ingest flow""" if not LANGFLOW_INGEST_FLOW_ID: From cd1bf20febbdb1ab68b1aea7a43c0337499b89af Mon Sep 17 00:00:00 2001 From: matano Date: Thu, 12 Feb 2026 13:06:30 +0200 Subject: [PATCH 45/51] set modify=true when updating langflow index name remove uses of INDEX_NAME constant --- src/api/connectors.py | 4 ++-- src/api/documents.py | 2 +- src/api/settings.py | 16 +++++++++------- src/connectors/langflow_connector_service.py | 4 ++-- src/services/search_service.py | 5 +++-- 5 files changed, 17 insertions(+), 14 deletions(-) diff --git a/src/api/connectors.py b/src/api/connectors.py index 579fd4211..da76541d2 100644 --- a/src/api/connectors.py +++ b/src/api/connectors.py @@ -1,9 +1,9 @@ from starlette.requests import Request from starlette.responses import JSONResponse, PlainTextResponse from connectors.sharepoint.utils import is_valid_sharepoint_url +from config.settings import get_index_name from utils.logging_config import get_logger from utils.telemetry import TelemetryClient, Category, MessageId -from config.settings import INDEX_NAME logger = get_logger(__name__) @@ -49,7 +49,7 @@ async def get_synced_file_ids_for_connector( } result = await opensearch_client.search( - index=INDEX_NAME, + index=get_index_name(), body=query_body ) diff --git a/src/api/documents.py b/src/api/documents.py index c4ca45871..8d7d386ec 100644 --- a/src/api/documents.py +++ b/src/api/documents.py @@ -27,7 +27,7 @@ async def check_filename_exists(request: Request, document_service, session_mana search_body = build_filename_search_body(filename, size=1, source=["filename"]) - logger.debug(f"Checking filename existence", filename=filename, index_name=settings.INDEX_NAME) + logger.debug(f"Checking filename existence", filename=filename, index_name=get_index_name()) response = await opensearch_client.search( index=get_index_name(), diff --git a/src/api/settings.py b/src/api/settings.py index c2f6f53ca..e1f92a675 100644 --- a/src/api/settings.py +++ b/src/api/settings.py @@ -14,6 +14,7 @@ LANGFLOW_PUBLIC_URL, LOCALHOST_URL, clients, + get_index_name, get_openrag_config, config_manager, is_no_auth_mode, @@ -678,7 +679,7 @@ async def update_settings(request, session_manager): # Also update global variable with new index name try: - await clients._create_langflow_global_variable("OPENSEARCH_INDEX_NAME", new_index_name) + await clients._create_langflow_global_variable("OPENSEARCH_INDEX_NAME", new_index_name, modify=True) logger.info( f"Successfully updated global variable with new index name {new_index_name}" ) @@ -1624,11 +1625,12 @@ async def rollback_onboarding(request, session_manager, task_service): ) import config.settings as settings - if await clients.opensearch.indices.exists(index=settings.INDEX_NAME): + index_name = get_index_name() + if await clients.opensearch.indices.exists(index=index_name): # DELETE / - logger.info(f"Deleting index '{settings.INDEX_NAME}'...") - resp = await clients.opensearch.indices.delete(index=settings.INDEX_NAME) - logger.info(f"Deleted '{settings.INDEX_NAME}': {resp}") + logger.info(f"Deleting index '{index_name}'...") + resp = await clients.opensearch.indices.delete(index=index_name) + logger.info(f"Deleted '{index_name}': {resp}") user = request.state.user jwt_token = session_manager.get_effective_jwt_token(user.user_id, request.state.jwt_token) @@ -1675,12 +1677,12 @@ async def rollback_onboarding(request, session_manager, task_service): # Delete documents by filename from utils.opensearch_queries import build_filename_delete_body - from config.settings import INDEX_NAME + from config.settings import get_index_name delete_query = build_filename_delete_body(filename) result = await opensearch_client.delete_by_query( - index=INDEX_NAME, + index=get_index_name(), body=delete_query, conflicts="proceed" ) diff --git a/src/connectors/langflow_connector_service.py b/src/connectors/langflow_connector_service.py index 9f0e39f67..770649374 100644 --- a/src/connectors/langflow_connector_service.py +++ b/src/connectors/langflow_connector_service.py @@ -75,10 +75,10 @@ async def process_connector_document( # This prevents duplicate chunks when syncing files if self.session_manager: try: - from config.settings import INDEX_NAME + from config.settings import get_index_name opensearch_client = self.session_manager.get_user_opensearch_client(owner_user_id, jwt_token) delete_body = {"query": {"term": {"filename": processed_filename}}} - delete_result = await opensearch_client.delete_by_query(index=INDEX_NAME, body=delete_body) + delete_result = await opensearch_client.delete_by_query(index=get_index_name(), body=delete_body) deleted_count = delete_result.get("deleted", 0) logger.info("Deleted existing chunks before re-ingestion", filename=processed_filename, deleted_count=deleted_count) except Exception as delete_err: diff --git a/src/services/search_service.py b/src/services/search_service.py index 01b4d8a49..ef88bb9d4 100644 --- a/src/services/search_service.py +++ b/src/services/search_service.py @@ -396,9 +396,10 @@ async def embed_with_model(model_name): search_params = {"terminate_after": 0} try: - logger.info(f"Sending query to index '{settings.INDEX_NAME}'..") + index_name = get_index_name() + logger.info(f"Sending query to index '{index_name}'..") results = await opensearch_client.search( - index=get_index_name(), body=search_body, params=search_params + index=index_name, body=search_body, params=search_params ) except RequestError as e: error_message = str(e) From a1199ac97a7aab7336783c7ea1488530e3f1c96d Mon Sep 17 00:00:00 2001 From: matano Date: Thu, 12 Feb 2026 13:06:30 +0200 Subject: [PATCH 46/51] set modify=true when updating langflow index name remove uses of INDEX_NAME constant --- src/api/connectors.py | 4 ++-- src/api/documents.py | 2 +- src/api/settings.py | 3 ++- src/connectors/langflow_connector_service.py | 4 ++-- src/services/search_service.py | 4 +++- 5 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/api/connectors.py b/src/api/connectors.py index 579fd4211..da76541d2 100644 --- a/src/api/connectors.py +++ b/src/api/connectors.py @@ -1,9 +1,9 @@ from starlette.requests import Request from starlette.responses import JSONResponse, PlainTextResponse from connectors.sharepoint.utils import is_valid_sharepoint_url +from config.settings import get_index_name from utils.logging_config import get_logger from utils.telemetry import TelemetryClient, Category, MessageId -from config.settings import INDEX_NAME logger = get_logger(__name__) @@ -49,7 +49,7 @@ async def get_synced_file_ids_for_connector( } result = await opensearch_client.search( - index=INDEX_NAME, + index=get_index_name(), body=query_body ) diff --git a/src/api/documents.py b/src/api/documents.py index a31183f0f..8d7d386ec 100644 --- a/src/api/documents.py +++ b/src/api/documents.py @@ -27,7 +27,7 @@ async def check_filename_exists(request: Request, document_service, session_mana search_body = build_filename_search_body(filename, size=1, source=["filename"]) - logger.debug(f"Checking filename existence: {filename}") + logger.debug(f"Checking filename existence", filename=filename, index_name=get_index_name()) response = await opensearch_client.search( index=get_index_name(), diff --git a/src/api/settings.py b/src/api/settings.py index 24c66f5ed..36b558fe7 100644 --- a/src/api/settings.py +++ b/src/api/settings.py @@ -13,6 +13,7 @@ LANGFLOW_PUBLIC_URL, LOCALHOST_URL, clients, + get_index_name, get_openrag_config, config_manager, is_no_auth_mode, @@ -588,7 +589,7 @@ async def update_settings(request, session_manager): # Also update global variable with new index name try: - await clients._create_langflow_global_variable("OPENSEARCH_INDEX_NAME", new_index_name) + await clients._create_langflow_global_variable("OPENSEARCH_INDEX_NAME", new_index_name, modify=True) logger.info( f"Successfully updated global variable with new index name {new_index_name}" ) diff --git a/src/connectors/langflow_connector_service.py b/src/connectors/langflow_connector_service.py index 9f0e39f67..770649374 100644 --- a/src/connectors/langflow_connector_service.py +++ b/src/connectors/langflow_connector_service.py @@ -75,10 +75,10 @@ async def process_connector_document( # This prevents duplicate chunks when syncing files if self.session_manager: try: - from config.settings import INDEX_NAME + from config.settings import get_index_name opensearch_client = self.session_manager.get_user_opensearch_client(owner_user_id, jwt_token) delete_body = {"query": {"term": {"filename": processed_filename}}} - delete_result = await opensearch_client.delete_by_query(index=INDEX_NAME, body=delete_body) + delete_result = await opensearch_client.delete_by_query(index=get_index_name(), body=delete_body) deleted_count = delete_result.get("deleted", 0) logger.info("Deleted existing chunks before re-ingestion", filename=processed_filename, deleted_count=deleted_count) except Exception as delete_err: diff --git a/src/services/search_service.py b/src/services/search_service.py index f66f5a907..ef88bb9d4 100644 --- a/src/services/search_service.py +++ b/src/services/search_service.py @@ -396,8 +396,10 @@ async def embed_with_model(model_name): search_params = {"terminate_after": 0} try: + index_name = get_index_name() + logger.info(f"Sending query to index '{index_name}'..") results = await opensearch_client.search( - index=get_index_name(), body=search_body, params=search_params + index=index_name, body=search_body, params=search_params ) except RequestError as e: error_message = str(e) From 49f423c904867c472cbfb4046f0ad9c29a207223 Mon Sep 17 00:00:00 2001 From: matano Date: Thu, 12 Feb 2026 15:21:42 +0200 Subject: [PATCH 47/51] remove added newline --- src/services/flows_service.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/services/flows_service.py b/src/services/flows_service.py index 6989edf19..e97ac2d3a 100644 --- a/src/services/flows_service.py +++ b/src/services/flows_service.py @@ -953,7 +953,6 @@ async def update_ingest_flow_chunk_overlap(self, chunk_overlap: int): node_display_name="Split Text", ) - async def update_ingest_flow_embedding_model(self, embedding_model: str, provider: str): """Helper function to update embedding model in the ingest flow""" if not LANGFLOW_INGEST_FLOW_ID: From edd3df23dd5160d6c06d7fe69b93bef718428742 Mon Sep 17 00:00:00 2001 From: matano Date: Sun, 15 Feb 2026 10:41:51 +0200 Subject: [PATCH 48/51] changes to docker compose to support benchmarking --- docker-compose.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 295e306f5..18e95b29d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,7 +29,8 @@ services: - "9200:9200" - "9600:9600" volumes: - - ${OPENSEARCH_DATA_PATH:-./opensearch-data}:/usr/share/opensearch/data:U,z + # If OPENSEARCH_DATA_PATH is set, use host path; otherwise use named volume + - ${OPENSEARCH_DATA_PATH:-opensearch-data}:/usr/share/opensearch/data dashboards: image: opensearchproject/opensearch-dashboards:3.0.0 @@ -81,12 +82,15 @@ services: - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} - OPENSEARCH_INDEX_NAME=${OPENSEARCH_INDEX_NAME:-documents} + - LOG_LEVEL=${LOG_LEVEL} volumes: - ${OPENRAG_DOCUMENTS_PATH:-./openrag-documents}:/app/openrag-documents:Z - ${OPENRAG_KEYS_PATH:-./keys}:/app/keys:U,z - ${OPENRAG_FLOWS_PATH:-./flows}:/app/flows:U,z - ${OPENRAG_CONFIG_PATH:-./config}:/app/config:Z - ${OPENRAG_DATA_PATH:-./data}:/app/data:Z + ports: + - "8000:8000" openrag-frontend: image: langflowai/openrag-frontend:${OPENRAG_VERSION:-latest} @@ -147,7 +151,7 @@ services: - FILESIZE=0 - SELECTED_EMBEDDING_MODEL=${SELECTED_EMBEDDING_MODEL:-} - LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD,OPENSEARCH_URL,DOCLING_SERVE_URL,OWNER,OWNER_NAME,OWNER_EMAIL,CONNECTOR_TYPE,DOCUMENT_ID,SOURCE_URL,ALLOWED_USERS,ALLOWED_GROUPS,FILENAME,MIMETYPE,FILESIZE,SELECTED_EMBEDDING_MODEL,OPENAI_API_KEY,OPENAI_API_BASE,ANTHROPIC_API_KEY,WATSONX_API_KEY,WATSONX_ENDPOINT,WATSONX_PROJECT_ID,OLLAMA_BASE_URL,OPENSEARCH_INDEX_NAME - - LANGFLOW_LOG_LEVEL=DEBUG + - LANGFLOW_LOG_LEVEL=${LOG_LEVEL} - LANGFLOW_AUTO_LOGIN=${LANGFLOW_AUTO_LOGIN} - LANGFLOW_SUPERUSER=${LANGFLOW_SUPERUSER} - LANGFLOW_SUPERUSER_PASSWORD=${LANGFLOW_SUPERUSER_PASSWORD} @@ -155,3 +159,6 @@ services: - LANGFLOW_ENABLE_SUPERUSER_CLI=${LANGFLOW_ENABLE_SUPERUSER_CLI} # - DEFAULT_FOLDER_NAME=OpenRAG - HIDE_GETTING_STARTED_PROGRESS=true + +volumes: + opensearch-data: \ No newline at end of file From ff59505f7962c8827961add07ac7548cc1f22dac Mon Sep 17 00:00:00 2001 From: matano Date: Sun, 15 Feb 2026 10:47:34 +0200 Subject: [PATCH 49/51] update to use get_index_name() instead of INDEX_NAME --- src/main.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main.py b/src/main.py index 40a5f4fdb..8ef755b4d 100644 --- a/src/main.py +++ b/src/main.py @@ -201,12 +201,13 @@ async def init_index(delete_existing: bool = False): endpoint=getattr(embedding_provider_config, "endpoint", None) ) - index_exists = await clients.opensearch.indices.exists(index=INDEX_NAME) + index_name = get_index_name() + index_exists = await clients.opensearch.indices.exists(index=index_name) if index_exists and delete_existing: # Asked to delete the existing index .. - logger.info(f"Deleting index '{INDEX_NAME}'...") - resp = await clients.opensearch.indices.delete(index=INDEX_NAME) - logger.info(f"Deleted index '{INDEX_NAME}', response: {resp}") + logger.info(f"Deleting index '{index_name}'...") + resp = await clients.opensearch.indices.delete(index=index_name) + logger.info(f"Deleted index '{index_name}', response: {resp}") index_exists = False # Create documents index From 7889d5201d344adfed5b56fb1e3cfaf3951b50b0 Mon Sep 17 00:00:00 2001 From: matano Date: Sun, 15 Feb 2026 11:00:57 +0200 Subject: [PATCH 50/51] remove uneeded changes following merge --- src/api/settings.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/api/settings.py b/src/api/settings.py index 8f531a682..f24a60ca5 100644 --- a/src/api/settings.py +++ b/src/api/settings.py @@ -268,7 +268,6 @@ async def update_settings(request, session_manager): "picture_descriptions", "embedding_model", "embedding_provider", - "index_name", # Provider-specific fields (structured as provider_name.field_name) "openai_api_key", "anthropic_api_key", @@ -833,7 +832,6 @@ async def onboarding(request, flows_service, session_manager=None): "embedding_provider", "embedding_model", "delete_existing_index", - "sample_data", # Provider-specific fields "openai_api_key", "anthropic_api_key", @@ -1632,19 +1630,10 @@ async def rollback_onboarding(request, session_manager, task_service): # Only allow rollback if config was marked as edited (onboarding completed) if not current_config.edited: - logger.info("No onboarding configuration to rollback") return JSONResponse( {"error": "No onboarding configuration to rollback"}, status_code=400 ) - import config.settings as settings - index_name = get_index_name() - if await clients.opensearch.indices.exists(index=index_name): - # DELETE / - logger.info(f"Deleting index '{index_name}'...") - resp = await clients.opensearch.indices.delete(index=index_name) - logger.info(f"Deleted '{index_name}': {resp}") - user = request.state.user jwt_token = session_manager.get_effective_jwt_token(user.user_id, request.state.jwt_token) From 94201c7862375d9951ceccb08ade9ead04d487d1 Mon Sep 17 00:00:00 2001 From: matano Date: Sun, 15 Feb 2026 11:02:14 +0200 Subject: [PATCH 51/51] move index_name to correct location --- src/api/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/settings.py b/src/api/settings.py index f24a60ca5..21e35b35c 100644 --- a/src/api/settings.py +++ b/src/api/settings.py @@ -258,7 +258,6 @@ async def update_settings(request, session_manager): "llm_model", "llm_provider", "system_prompt", - "index_name", "chunk_size", "chunk_overlap", "splitter_type", @@ -268,6 +267,7 @@ async def update_settings(request, session_manager): "picture_descriptions", "embedding_model", "embedding_provider", + "index_name", # Provider-specific fields (structured as provider_name.field_name) "openai_api_key", "anthropic_api_key",