Add index discovery mode to support online index creation

src/bin/cmd-mcp.ts

@@ -7,6 +7,7 @@ import { FilesystemStore } from "../stores/filesystem.js";
 import { runMCPServer } from "../clients/mcp-server.js";
 import { parseIndexSpecs } from "../stores/index-spec.js";
 import { CompositeStoreReader } from "../stores/composite.js";
+import { ReadOnlyLayeredStore } from "../stores/read-only-layered-store.js";
 
 // stdio subcommand (stdio-based MCP server for local clients like Claude Desktop)
 const stdioCommand = new Command("stdio")
@@ -15,36 +16,44 @@ const stdioCommand = new Command("stdio")
     "-i, --index <specs...>",
     "Index spec(s): name, path:/path, or s3://bucket/key"
   )
+  .option("--discovery", "Enable discovery mode (read-only, manage indexes via CLI)")
   .option("--search-only", "Disable list_files/read_file tools (search only)")
   .action(async (options) => {
     try {
       const indexSpecs: string[] | undefined = options.index;
+      const discoveryFlag = options.discovery;
 
       let store;
-      let indexNames: string[];
+      let indexNames: string[] | undefined;
+      let discovery: boolean;
 
-      if (indexSpecs && indexSpecs.length > 0) {
-        // Parse index specs and create composite store
+      if (discoveryFlag && indexSpecs && indexSpecs.length > 0) {
+        // Discovery mode WITH remote indexes: merge local + remote
+        const specs = parseIndexSpecs(indexSpecs);
+        const remoteStore = await CompositeStoreReader.fromSpecs(specs);
+        const localStore = new FilesystemStore();
+        store = new ReadOnlyLayeredStore(localStore, remoteStore);
+        indexNames = undefined; // Discovery mode: no fixed list
+        discovery = true;
+      } else if (indexSpecs && indexSpecs.length > 0) {
+        // Fixed mode: use read-only CompositeStoreReader
         const specs = parseIndexSpecs(indexSpecs);
-        store = await CompositeStoreReader.fromSpecs(specs);
         indexNames = specs.map((s) => s.displayName);
+        store = await CompositeStoreReader.fromSpecs(specs);
+        discovery = false;
       } else {
-        // No --index: use default store, list all indexes
+        // Discovery mode only: use FilesystemStore
         store = new FilesystemStore();
-        indexNames = await store.list();
-        if (indexNames.length === 0) {
-          console.error("Error: No indexes found.");
-          console.error("The MCP server requires at least one index to operate.");
-          console.error("Run 'ctxc index --help' to see how to create an index.");
-          process.exit(1);
-        }
+        indexNames = undefined;
+        discovery = true;
       }
 
       // Start MCP server (writes to stdout, reads from stdin)
       await runMCPServer({
         store,
         indexNames,
         searchOnly: options.searchOnly,
+        discovery,
       });
     } catch (error) {
       // Write errors to stderr (stdout is for MCP protocol)
@@ -60,6 +69,7 @@ const httpCommand = new Command("http")
     "-i, --index <specs...>",
     "Index spec(s): name, path:/path, or s3://bucket/key"
   )
+  .option("--discovery", "Enable discovery mode (read-only, manage indexes via CLI)")
   .option("--port <number>", "Port to listen on", "3000")
   .option("--host <host>", "Host to bind to", "localhost")
   .option("--cors <origins>", "CORS origins (comma-separated, or '*' for any)")
@@ -72,26 +82,31 @@ const httpCommand = new Command("http")
   .action(async (options) => {
     try {
       const indexSpecs: string[] | undefined = options.index;
+      const discoveryFlag = options.discovery;
 
       let store;
       let indexNames: string[] | undefined;
+      let discovery: boolean;
 
-      if (indexSpecs && indexSpecs.length > 0) {
-        // Parse index specs and create composite store
+      if (discoveryFlag && indexSpecs && indexSpecs.length > 0) {
+        // Discovery mode WITH remote indexes: merge local + remote
+        const specs = parseIndexSpecs(indexSpecs);
+        const remoteStore = await CompositeStoreReader.fromSpecs(specs);
+        const localStore = new FilesystemStore();
+        store = new ReadOnlyLayeredStore(localStore, remoteStore);
+        indexNames = undefined; // Discovery mode: no fixed list
+        discovery = true;
+      } else if (indexSpecs && indexSpecs.length > 0) {
+        // Fixed mode: use read-only CompositeStoreReader
         const specs = parseIndexSpecs(indexSpecs);
-        store = await CompositeStoreReader.fromSpecs(specs);
         indexNames = specs.map((s) => s.displayName);
+        store = await CompositeStoreReader.fromSpecs(specs);
+        discovery = false;
       } else {
-        // No --index: use default store, serve all
+        // Discovery mode only: use FilesystemStore
         store = new FilesystemStore();
-        const availableIndexes = await store.list();
-        if (availableIndexes.length === 0) {
-          console.error("Error: No indexes found.");
-          console.error("The MCP server requires at least one index to operate.");
-          console.error("Run 'ctxc index --help' to see how to create an index.");
-          process.exit(1);
-        }
         indexNames = undefined;
+        discovery = true;
       }
 
       // Parse CORS option
@@ -112,6 +127,7 @@ const httpCommand = new Command("http")
         store,
         indexNames,
         searchOnly: options.searchOnly,
+        discovery,
         port: parseInt(options.port, 10),
         host: options.host,
         cors,

src/clients/cli-agent.ts

@@ -192,7 +192,6 @@ async function loadModel(
       return new AugmentLanguageModel(modelName, {
         apiKey: credentials.apiKey,
         apiUrl: credentials.apiUrl,
-        clientUserAgent,
       }) as unknown as LanguageModel;
     }
     default:

src/clients/mcp-server.ts

@@ -37,21 +37,23 @@ import {
   CallToolRequestSchema,
   ListToolsRequestSchema,
 } from "@modelcontextprotocol/sdk/types.js";
-import type { IndexStoreReader } from "../stores/types.js";
+import type { IndexStoreReader, IndexStore } from "../stores/types.js";
+import type { Source } from "../sources/types.js";
 import { MultiIndexRunner } from "./multi-index-runner.js";
 import { buildClientUserAgent, type MCPClientInfo } from "../core/utils.js";
 import {
   SEARCH_DESCRIPTION,
   LIST_FILES_DESCRIPTION,
   READ_FILE_DESCRIPTION,
+  withListIndexesReference,
   withIndexList,
 } from "./tool-descriptions.js";
 /**
  * Configuration for the MCP server.
  */
 export interface MCPServerConfig {
-  /** Store to load indexes from */
-  store: IndexStoreReader;
+  /** Store to load indexes from (accepts both reader-only and full store) */
+  store: IndexStoreReader | IndexStore;
   /**
    * Index names to expose. If undefined, all indexes in the store are exposed.
    */
@@ -71,6 +73,13 @@ export interface MCPServerConfig {
    * @default "0.1.0"
    */
   version?: string;
+  /**
+   * Discovery mode flag.
+   * When true: use withListIndexesReference (no enum in schemas), no write tools
+   * When false/undefined: use withIndexList (include enum in schemas), write tools available
+   * @default false
+   */
+  discovery?: boolean;
 }
 /**
  * Create an MCP server instance.
@@ -104,6 +113,7 @@ export async function createMCPServer(
     searchOnly: config.searchOnly,
     clientUserAgent,
   });
+
   const { indexNames, indexes } = runner;
   const searchOnly = !runner.hasFileOperations();
   // Format index list for tool descriptions
@@ -146,13 +156,35 @@ export async function createMCPServer(
       required?: string[];
     };
   };
-  // Tool descriptions with available indexes (from shared module)
-  const searchDescription = withIndexList(SEARCH_DESCRIPTION, indexListStr);
-  const listFilesDescription = withIndexList(LIST_FILES_DESCRIPTION, indexListStr);
-  const readFileDescription = withIndexList(READ_FILE_DESCRIPTION, indexListStr);
+
+  // Tool descriptions: use enum in fixed mode, reference in discovery mode
+  let searchDescription: string;
+  let listFilesDescription: string;
+  let readFileDescription: string;
+
+  if (config.discovery) {
+    // Discovery mode: use reference to list_indexes (no enum)
+    searchDescription = withListIndexesReference(SEARCH_DESCRIPTION);
+    listFilesDescription = withListIndexesReference(LIST_FILES_DESCRIPTION);
+    readFileDescription = withListIndexesReference(READ_FILE_DESCRIPTION);
+  } else {
+    // Fixed mode: include enum with index list
+    searchDescription = withIndexList(SEARCH_DESCRIPTION, indexListStr);
+    listFilesDescription = withIndexList(LIST_FILES_DESCRIPTION, indexListStr);
+    readFileDescription = withIndexList(READ_FILE_DESCRIPTION, indexListStr);
+  }
   // List available tools
   server.setRequestHandler(ListToolsRequestSchema, async () => {
     const tools: Tool[] = [
+      {
+        name: "list_indexes",
+        description: "List all available indexes with their metadata. Call this to discover what indexes are available before using search, list_files, or read_file tools.",
+        inputSchema: {
+          type: "object",
+          properties: {},
+          required: [],
+        },
+      },
       {
         name: "search",
         description: searchDescription,
@@ -162,7 +194,7 @@ export async function createMCPServer(
             index_name: {
               type: "string",
               description: "Name of the index to search.",
-              enum: indexNames,
+              ...(config.discovery ? {} : { enum: runner.indexes.map(i => i.name) }),
             },
             query: {
               type: "string",
@@ -189,7 +221,7 @@ export async function createMCPServer(
               index_name: {
                 type: "string",
                 description: "Name of the index.",
-                enum: indexNames,
+                ...(config.discovery ? {} : { enum: runner.indexes.map(i => i.name) }),
               },
               directory: {
                 type: "string",
@@ -220,7 +252,7 @@ export async function createMCPServer(
               index_name: {
                 type: "string",
                 description: "Name of the index.",
-                enum: indexNames,
+                ...(config.discovery ? {} : { enum: runner.indexes.map(i => i.name) }),
               },
               path: {
                 type: "string",
@@ -261,6 +293,24 @@ export async function createMCPServer(
   // Handle tool calls
   server.setRequestHandler(CallToolRequestSchema, async (request) => {
     const { name, arguments: args } = request.params;
+
+    // Handle list_indexes separately (no index_name required)
+    if (name === "list_indexes") {
+      await runner.refreshIndexList();
+      const { indexes } = runner;
+      if (indexes.length === 0) {
+        return {
+          content: [{ type: "text", text: "No indexes available. Use index_repo to create one." }],
+        };
+      }
+      const lines = indexes.map((i) =>
+        `- ${i.name} (${i.type}://${i.identifier}) - synced ${i.syncedAt}`
+      );
+      return {
+        content: [{ type: "text", text: `Available indexes:\n${lines.join("\n")}` }],
+      };
+    }
+
     try {
       const indexName = args?.index_name as string;
       const client = await runner.getClient(indexName);

src/clients/multi-index-runner.ts

@@ -7,7 +7,7 @@
  * @module clients/multi-index-runner
  */
 
-import type { IndexStoreReader } from "../stores/types.js";
+import type { IndexStoreReader, IndexStore } from "../stores/types.js";
 import type { Source } from "../sources/types.js";
 import type { IndexStateSearchOnly } from "../core/types.js";
 import { getSourceIdentifier, getResolvedRef } from "../core/types.js";
@@ -26,7 +26,7 @@ export interface IndexInfo {
 /** Configuration for MultiIndexRunner */
 export interface MultiIndexRunnerConfig {
   /** Store to load indexes from */
-  store: IndexStoreReader;
+  store: IndexStoreReader | IndexStore;
   /**
    * Index names to expose. If undefined, all indexes in the store are exposed.
    */
@@ -88,19 +88,19 @@ export async function createSourceFromState(state: IndexStateSearchOnly): Promis
  * Lazily initializes SearchClient instances as needed and caches them.
  */
 export class MultiIndexRunner {
-  private readonly store: IndexStoreReader;
+  private readonly store: IndexStoreReader | IndexStore;
   private readonly searchOnly: boolean;
   private clientUserAgent?: string;
   private readonly clientCache = new Map<string, SearchClient>();
 
   /** Available index names */
-  readonly indexNames: string[];
+  indexNames: string[];
 
   /** Metadata about available indexes */
-  readonly indexes: IndexInfo[];
+  indexes: IndexInfo[];
 
   private constructor(
-    store: IndexStoreReader,
+    store: IndexStoreReader | IndexStore,
     indexNames: string[],
     indexes: IndexInfo[],
     searchOnly: boolean,
@@ -130,10 +130,6 @@ export class MultiIndexRunner {
       throw new Error(`Indexes not found: ${missingIndexes.join(", ")}`);
     }
 
-    if (indexNames.length === 0) {
-      throw new Error("No indexes available in store");
-    }
-
     // Load metadata for available indexes, filtering out any that fail to load
     const indexes: IndexInfo[] = [];
     const validIndexNames: string[] = [];
@@ -163,10 +159,9 @@ export class MultiIndexRunner {
     return new MultiIndexRunner(store, validIndexNames, indexes, searchOnly, config.clientUserAgent);
   }
 
-
   /**
    * Update the User-Agent string.
-   * 
+   *
    * Call this after receiving MCP client info to include the client name/version.
    * Note: Only affects future client creations, not existing cached clients.
    */
@@ -205,6 +200,45 @@ export class MultiIndexRunner {
     return client;
   }
 
+  /**
+   * Refresh the list of available indexes from the store.
+   * Call after adding or removing indexes.
+   */
+  async refreshIndexList(): Promise<void> {
+    const allIndexNames = await this.store.list();
+    const newIndexes: IndexInfo[] = [];
+    const newIndexNames: string[] = [];
+
+    for (const name of allIndexNames) {
+      try {
+        const state = await this.store.loadSearch(name);
+        if (state) {
+          newIndexNames.push(name);
+          newIndexes.push({
+            name,
+            type: state.source.type,
+            identifier: getSourceIdentifier(state.source),
+            ref: getResolvedRef(state.source),
+            syncedAt: state.source.syncedAt,
+          });
+        }
+      } catch {
+        // Skip indexes that fail to load
+      }
+    }
+
+    this.indexNames = newIndexNames;
+    this.indexes = newIndexes;
+  }
+
+  /**
+   * Invalidate cached SearchClient for an index.
+   * Call after updating an index to ensure fresh data on next access.
+   */
+  invalidateClient(indexName: string): void {
+    this.clientCache.delete(indexName);
+  }
+
   /** Check if file operations are enabled */
   hasFileOperations(): boolean {
     return !this.searchOnly;

src/clients/search-client.ts

@@ -169,7 +169,6 @@ export class SearchClient {
     this.context = await DirectContext.import(this.state.contextState, {
       apiKey: this.apiKey,
       apiUrl: this.apiUrl,
-      clientUserAgent: this.clientUserAgent,
     });
   }