Merge pull request #932 from multiplex55/codex/refactor-indexer-for-streaming-api

Streamed batched indexer with dedupe and max-index limit

Author: multiplex55

src/gui/mod.rs

@@ -483,6 +483,7 @@ pub struct LauncherApp {
     bookmark_aliases_lc: HashMap<String, Option<String>>,
     plugin_dirs: Option<Vec<String>>,
     index_paths: Option<Vec<String>>,
+    max_indexed_items: Option<usize>,
     enabled_plugins: Option<HashSet<String>>,
     enabled_capabilities: Option<std::collections::HashMap<String, Vec<String>>>,
     visible_flag: Arc<AtomicBool>,
@@ -785,20 +786,30 @@ impl LauncherApp {
                 WatchEvent::Actions => {
                     if let Ok(mut acts) = load_actions(&self.actions_path) {
                         let custom_len = acts.len();
+                        self.custom_len = custom_len;
                         if let Some(paths) = &self.index_paths {
-                            match indexer::index_paths(paths) {
-                                Ok(idx) => acts.extend(idx),
-                                Err(e) => {
-                                    tracing::error!(error = %e, "failed to index paths");
-                                    self.report_error_message(
-                                        "launcher",
-                                        format!("Failed to index paths: {e}"),
-                                    );
+                            let options =
+                                indexer::IndexOptions::with_max_items(self.max_indexed_items);
+                            for batch in indexer::index_paths_batched(paths, options) {
+                                match batch {
+                                    Ok(idx) => {
+                                        acts.extend(idx);
+                                        self.actions = Arc::new(acts.clone());
+                                        self.update_action_cache();
+                                        self.search();
+                                    }
+                                    Err(e) => {
+                                        tracing::error!(error = %e, "failed to index paths");
+                                        self.report_error_message(
+                                            "launcher",
+                                            format!("Failed to index paths: {e}"),
+                                        );
+                                        break;
+                                    }
                                 }
                             }
                         }
                         self.actions = Arc::new(acts);
-                        self.custom_len = custom_len;
                         self.update_action_cache();
                         self.search();
                         crate::actions::bump_actions_version();
@@ -1538,6 +1549,7 @@ impl LauncherApp {
             bookmark_aliases_lc,
             plugin_dirs,
             index_paths,
+            max_indexed_items: settings.max_indexed_items,
             enabled_plugins,
             enabled_capabilities,
             visible_flag: visible_flag.clone(),

src/indexer.rs

@@ -1,32 +1,157 @@
 use crate::actions::Action;
-use walkdir::WalkDir;
+use std::collections::HashSet;
+use std::fs;
+use std::path::PathBuf;
+use walkdir::{IntoIter as WalkDirIter, WalkDir};
 
-/// Index the provided filesystem paths and return a list of [`Action`]s.
+const DEFAULT_BATCH_SIZE: usize = 512;
+const DEFAULT_MAX_ITEMS: usize = 100_000;
+
+#[derive(Debug, Clone, Copy)]
+pub struct IndexOptions {
+    pub batch_size: usize,
+    pub max_items: usize,
+}
+
+impl Default for IndexOptions {
+    fn default() -> Self {
+        Self {
+            batch_size: DEFAULT_BATCH_SIZE,
+            max_items: DEFAULT_MAX_ITEMS,
+        }
+    }
+}
+
+impl IndexOptions {
+    pub fn with_max_items(max_items: Option<usize>) -> Self {
+        Self {
+            max_items: max_items.unwrap_or(DEFAULT_MAX_ITEMS),
+            ..Self::default()
+        }
+    }
+}
+
+/// Lazily indexes files from one or more roots and yields actions in batches.
 ///
-/// Any errors encountered while traversing the directory tree are logged and
-/// returned to the caller.
-pub fn index_paths(paths: &[String]) -> anyhow::Result<Vec<Action>> {
-    let mut results = Vec::new();
-    for p in paths {
-        for entry in WalkDir::new(p).into_iter() {
-            let entry = match entry {
-                Ok(e) => e,
-                Err(e) => {
-                    tracing::error!(path = %p, error = %e, "failed to read directory entry");
-                    return Err(e.into());
+/// Duplicate files are skipped by canonical path. Traversal errors stop
+/// iteration and are returned to the caller.
+pub struct IndexBatchIter {
+    roots: Vec<String>,
+    root_idx: usize,
+    current: Option<WalkDirIter>,
+    seen: HashSet<PathBuf>,
+    options: IndexOptions,
+    produced: usize,
+}
+
+impl IndexBatchIter {
+    fn new(paths: &[String], options: IndexOptions) -> Self {
+        let options = IndexOptions {
+            batch_size: options.batch_size.max(1),
+            max_items: options.max_items.max(1),
+        };
+        Self {
+            roots: paths.to_vec(),
+            root_idx: 0,
+            current: None,
+            seen: HashSet::new(),
+            options,
+            produced: 0,
+        }
+    }
+
+    fn next_root(&mut self) -> Option<String> {
+        let root = self.roots.get(self.root_idx).cloned();
+        if root.is_some() {
+            self.root_idx += 1;
+        }
+        root
+    }
+}
+
+impl Iterator for IndexBatchIter {
+    type Item = anyhow::Result<Vec<Action>>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.produced >= self.options.max_items {
+            return None;
+        }
+
+        let mut batch = Vec::with_capacity(self.options.batch_size);
+        while self.produced < self.options.max_items && batch.len() < self.options.batch_size {
+            if self.current.is_none() {
+                if let Some(root) = self.next_root() {
+                    self.current = Some(WalkDir::new(root).into_iter());
+                } else {
+                    break;
                 }
+            }
+
+            let Some(iter) = self.current.as_mut() else {
+                continue;
             };
-            if entry.file_type().is_file() {
-                if let Some(name) = entry.path().file_name().and_then(|n| n.to_str()) {
-                    results.push(Action {
+
+            match iter.next() {
+                Some(Ok(entry)) => {
+                    if !entry.file_type().is_file() {
+                        continue;
+                    }
+                    let canonical = match fs::canonicalize(entry.path()) {
+                        Ok(path) => path,
+                        Err(err) => {
+                            tracing::error!(
+                                path = %entry.path().display(),
+                                error = %err,
+                                "failed to canonicalize indexed path"
+                            );
+                            return Some(Err(err.into()));
+                        }
+                    };
+                    if !self.seen.insert(canonical.clone()) {
+                        continue;
+                    }
+                    let Some(name) = canonical.file_name().and_then(|n| n.to_str()) else {
+                        continue;
+                    };
+                    let display = canonical.display().to_string();
+                    batch.push(Action {
                         label: name.to_string(),
-                        desc: entry.path().display().to_string(),
-                        action: entry.path().display().to_string(),
+                        desc: display.clone(),
+                        action: display,
                         args: None,
                     });
+                    self.produced += 1;
+                }
+                Some(Err(err)) => {
+                    tracing::error!(error = %err, "failed to read directory entry");
+                    return Some(Err(err.into()));
+                }
+                None => {
+                    self.current = None;
                 }
             }
         }
+
+        if batch.is_empty() {
+            None
+        } else {
+            Some(Ok(batch))
+        }
+    }
+}
+
+pub fn index_paths_batched(paths: &[String], options: IndexOptions) -> IndexBatchIter {
+    IndexBatchIter::new(paths, options)
+}
+
+/// Index the provided filesystem paths and return a list of [`Action`]s.
+///
+/// This compatibility helper exhausts the batched iterator into a single
+/// vector; prefer [`index_paths_batched`] when possible.
+pub fn index_paths(paths: &[String]) -> anyhow::Result<Vec<Action>> {
+    let mut results = Vec::new();
+    for batch in index_paths_batched(paths, IndexOptions::default()) {
+        results.extend(batch?);
     }
     Ok(results)
 }

src/main.rs

@@ -199,7 +199,10 @@ fn main() -> anyhow::Result<()> {
     }
 
     if let Some(paths) = &settings.index_paths {
-        actions_vec.extend(indexer::index_paths(paths)?);
+        let options = indexer::IndexOptions::with_max_items(settings.max_indexed_items);
+        for batch in indexer::index_paths_batched(paths, options) {
+            actions_vec.extend(batch?);
+        }
     }
     let actions = Arc::new(actions_vec);
 

src/settings.rs

@@ -328,6 +328,10 @@ pub struct Settings {
     /// Hotkey to show the quick help overlay. If `None`, the overlay is disabled.
     pub help_hotkey: Option<String>,
     pub index_paths: Option<Vec<String>>,
+    /// Maximum number of filesystem entries to index from `index_paths`.
+    ///
+    /// When missing, a conservative default is applied to protect memory.
+    pub max_indexed_items: Option<usize>,
     pub plugin_dirs: Option<Vec<String>>,
     /// Set of plugin names which should be enabled. If `None`, all loaded
     /// plugins are enabled.
@@ -606,6 +610,7 @@ impl Default for Settings {
             quit_hotkey: None,
             help_hotkey: Some("F1".into()),
             index_paths: None,
+            max_indexed_items: None,
             plugin_dirs: None,
             enabled_plugins: None,
             enabled_capabilities: None,

src/settings_editor.rs

@@ -335,6 +335,7 @@ impl SettingsEditor {
                 Some(self.help_hotkey.clone())
             },
             index_paths: current.index_paths.clone(),
+            max_indexed_items: current.max_indexed_items,
             plugin_dirs: current.plugin_dirs.clone(),
             enabled_plugins: current.enabled_plugins.clone(),
             enabled_capabilities: current.enabled_capabilities.clone(),

tests/indexer.rs

@@ -21,15 +21,46 @@ fn indexer_indexes_files_recursively() {
 
     let expected = [file1, file2, file3];
     for path in expected.iter() {
-        let label = path.file_name().unwrap().to_str().unwrap();
-        let display = path.display().to_string();
+        let canonical = fs::canonicalize(path).expect("canonical path");
+        let label = canonical.file_name().unwrap().to_str().unwrap();
+        let display = canonical.display().to_string();
         assert!(actions.iter().any(|a| a.label == label
             && a.action == display
             && a.desc == display
             && a.args.is_none()));
     }
 }
 
+#[test]
+fn indexer_batches_dedupes_and_honors_max_items() {
+    let dir = tempdir().expect("failed to create temp dir");
+    let one = dir.path().join("one.txt");
+    let two = dir.path().join("two.txt");
+    let three = dir.path().join("three.txt");
+    fs::write(&one, b"1").expect("write one");
+    fs::write(&two, b"2").expect("write two");
+    fs::write(&three, b"3").expect("write three");
+
+    let same_root = dir.path().to_string_lossy().to_string();
+    let paths = vec![same_root.clone(), same_root];
+    let mut iter = multi_launcher::indexer::index_paths_batched(
+        &paths,
+        multi_launcher::indexer::IndexOptions {
+            batch_size: 2,
+            max_items: 2,
+        },
+    );
+
+    let first = iter.next().expect("first batch").expect("first ok");
+    assert_eq!(first.len(), 2);
+    assert!(iter.next().is_none(), "max_items should stop iteration");
+
+    let mut seen = std::collections::HashSet::new();
+    for action in first {
+        assert!(seen.insert(action.action), "deduped paths only");
+    }
+}
+
 // Ensure indexing a missing path returns an error
 #[test]
 fn indexer_errors_on_missing_path() {