Add --path argument that can run tests in a directory

docs/content/docs/configuration.md

@@ -42,7 +42,8 @@ All configuration options that can be set via command-line or environment variab
 | `porcelain` | Boolean | Reduce logging to be stable (machine-readable output) |
 | `interactive` | Boolean | Run toolproof in interactive mode |
 | `all` | Boolean | Run all tests when in interactive mode |
-| `run_name` | String | Exact name of a test to run (case-sensitive) |
+| `name` | String | Exact name of a test to run (case-sensitive) |
+| `path` | String | Path to a test file or directory to run |
 | `browser` | String | Specify which browser to use (`chrome` or `pagebrowse`) |
 | `concurrency` | Number | How many tests should be run concurrently |
 | `timeout` | Number | How long in seconds until a step times out |
@@ -73,6 +74,12 @@ npx toolproof --root ./tests
 # Run a specific test by name
 npx toolproof --name "My Test Name"
 
+# Run a specific test file
+npx toolproof --path tests/my-test.toolproof.yml
+
+# Run all tests in a directory
+npx toolproof --path tests/integration
+
 # Provide placeholders
 npx toolproof --placeholders project_dir="$(pwd)" api_key=$API_KEY
 
@@ -96,6 +103,7 @@ npx toolproof -c 20
 | `--timeout <NUM>` | How long in seconds until a step times out |
 | `--browser-timeout <NUM>` | How long in seconds until actions in a browser time out |
 | `-n, --name <NAME>` | Exact name of a test to run |
+| `-p, --path <PATH>` | Path to a test file or directory to run |
 | `--browser <IMPL>` | Specify which browser to use for tests (chrome or pagebrowse, default: chrome) |
 | `--retry-count <COUNT>` | Number of times to retry failed tests before marking them as failed |
 | `--failure-screenshot-location <DIR>` | If set, Toolproof will screenshot the browser to this location when a test fails |
@@ -110,7 +118,8 @@ Most options can also be set using environment variables:
 | `TOOLPROOF_ROOT` | The location from which to look for toolproof test files |
 | `TOOLPROOF_VERBOSE` | Print verbose logging while running tests |
 | `TOOLPROOF_PORCELAIN` | Reduce logging to be stable |
-| `TOOLPROOF_RUN_NAME` | Run a specific test |
+| `TOOLPROOF_RUN_NAME` | Run a specific test by name |
+| `TOOLPROOF_RUN_PATH` | Path to a test file or directory to run |
 | `TOOLPROOF_BROWSER` | Specify which browser to use (chrome or pagebrowse) |
 | `TOOLPROOF_CONCURRENCY` | How many tests should be run concurrently |
 | `TOOLPROOF_TIMEOUT` | How long in seconds until a step times out |

docs/content/docs/debugger.md

@@ -12,10 +12,14 @@ Toolproof's debugger mode allows you to run tests step-by-step, making it easier
 Run Toolproof with the `--debugger` flag along with a specific test:
 
 ```bash
+# Debug a test by name
 npx toolproof --debugger --name "My Test Name"
+
+# Debug a specific test file
+npx toolproof --debugger --path tests/my-test.toolproof.yml
 ```
 
-Debugger mode requires running a single test. If you don't specify a test name, Toolproof will show an error.
+Debugger mode requires running a single test. If you don't specify a test, Toolproof will show an error.
 
 When running in debugger mode:
 

docs/content/docs/installation.md

@@ -52,9 +52,12 @@ npx toolproof
 # Run in interactive mode (for updating snapshots)
 npx toolproof -i
 
-# Run a specific test
+# Run a test by name
 npx toolproof --name "My Test Name"
 
+# Run a test file or directory
+npx toolproof --path tests/integration
+
 # See all options
 npx toolproof --help
 ```

toolproof/src/interactive.rs

@@ -13,6 +13,7 @@ use crate::{
 pub enum RunMode {
     All,
     One(String),
+    Path(String),
 }
 
 impl From<dialoguer::Error> for ToolproofInternalError {

toolproof/src/main.rs

@@ -389,6 +389,24 @@ async fn main_inner() -> Result<(), ()> {
         };
 
         RunMode::One(path.clone())
+    } else if let Some(run_path) = universe.ctx.params.run_path.as_ref() {
+        // Convert the provided path to an absolute path
+        let absolute_path = if run_path.is_absolute() {
+            run_path.clone()
+        } else {
+            universe.ctx.working_directory.join(run_path)
+        };
+
+        // Normalize the path for comparison
+        let normalized_path = absolute_path.normalize();
+
+        // Check if the path exists and is a file or directory
+        if !absolute_path.exists() {
+            eprintln!("Path does not exist: {}", run_path.display());
+            return Err(());
+        }
+
+        RunMode::Path(normalized_path.to_string_lossy().into_owned())
     } else if universe.ctx.params.interactive && !universe.ctx.params.all {
         match get_run_mode(&universe) {
             Ok(mode) => mode,
@@ -409,6 +427,37 @@ async fn main_inner() -> Result<(), ()> {
         return Err(());
     }
 
+    // Validate that path-based filtering found at least one test
+    if let RunMode::Path(ref filter_path) = run_mode {
+        let test_root = universe.ctx.params.root.as_ref()
+            .cloned()
+            .unwrap_or_else(|| universe.ctx.working_directory.clone());
+
+        let matching_tests = universe
+            .tests
+            .iter()
+            .filter(|(test_path, v)| {
+                if v.r#type != ToolproofFileType::Test {
+                    return false;
+                }
+
+                // Convert relative test path to absolute for comparison
+                let absolute_test_path = test_root.join(test_path).normalize();
+                let absolute_test_path_str = absolute_test_path.to_string_lossy();
+
+                absolute_test_path_str.as_ref() == filter_path || absolute_test_path_str.starts_with(filter_path.as_str())
+            })
+            .count();
+
+        if matching_tests == 0 {
+            eprintln!(
+                "No tests found matching path: {}",
+                universe.ctx.params.run_path.as_ref().unwrap().display()
+            );
+            return Err(());
+        }
+    }
+
     enum HoldingError {
         TestFailure,
         SnapFailure { out: String },
@@ -729,6 +778,40 @@ async fn main_inner() -> Result<(), ()> {
                 holding_err.map_err(|e| (test, e))
             }));
         }
+        RunMode::Path(ref filter_path) => {
+            let test_root = universe.ctx.params.root.as_ref()
+                .cloned()
+                .unwrap_or_else(|| universe.ctx.working_directory.clone());
+
+            for mut test in universe
+                .tests
+                .iter()
+                .filter(|(test_path, v)| {
+                    if v.r#type != ToolproofFileType::Test {
+                        return false;
+                    }
+
+                    // Convert relative test path to absolute for comparison
+                    let absolute_test_path = test_root.join(test_path).normalize();
+                    let absolute_test_path_str = absolute_test_path.to_string_lossy();
+
+                    absolute_test_path_str.as_ref() == filter_path || absolute_test_path_str.starts_with(filter_path.as_str())
+                })
+                .map(|(_, v)| v.clone())
+            {
+                let permit = semaphore.clone().acquire_owned().await.unwrap();
+                let uni = Arc::clone(&universe);
+                hands.push(tokio::spawn(async move {
+                    let start = Instant::now();
+                    let res = run_toolproof_experiment(&mut test, Arc::clone(&uni)).await;
+                    let holding_err = handle_res(uni, (&test, res), start);
+
+                    drop(permit);
+
+                    holding_err.map_err(|e| (test, e))
+                }));
+            }
+        }
     }
 
     let mut results = join_all(hands)

toolproof/src/options.rs

@@ -131,6 +131,13 @@ fn get_cli_matches() -> ArgMatches {
                 .long_help("case-sensitive")
                 .required(false)
         )
+        .arg(
+            arg!(
+                -p --path <PATH> "Path to a specific test file or directory to run")
+                .long_help("Run tests from a specific file or all tests in a directory")
+                .required(false)
+                .value_parser(value_parser!(PathBuf)),
+        )
         .arg(
             arg!(
                 --browser <IMPL> ... "Specify which browser to use when running browser automation tests"
@@ -200,6 +207,10 @@ pub struct ToolproofParams {
     #[setting(env = "TOOLPROOF_RUN_NAME")]
     pub run_name: Option<String>,
 
+    /// Run tests from a specific file or directory path
+    #[setting(env = "TOOLPROOF_RUN_PATH")]
+    pub run_path: Option<PathBuf>,
+
     /// Specify which browser to use when running browser automation tests
     #[setting(env = "TOOLPROOF_BROWSER")]
     pub browser: ToolproofBrowserImpl,
@@ -302,6 +313,10 @@ impl ToolproofParams {
             self.run_name = Some(name.clone());
         }
 
+        if let Some(path) = cli_matches.get_one::<PathBuf>("path") {
+            self.run_path = Some(path.clone());
+        }
+
         if let Some(root) = cli_matches.get_one::<PathBuf>("root") {
             self.root = Some(root.clone());
         }