From 7c086c2df876e7c48afd601b2d8ad5c37cbe423f Mon Sep 17 00:00:00 2001 From: Liam Bigelow <40188355+bglw@users.noreply.github.com> Date: Sun, 5 Oct 2025 16:38:58 +1100 Subject: [PATCH] Add --path argument that can run tests in a directory --- docs/content/docs/configuration.md | 13 ++++- docs/content/docs/debugger.md | 6 ++- docs/content/docs/installation.md | 5 +- toolproof/src/interactive.rs | 1 + toolproof/src/main.rs | 83 ++++++++++++++++++++++++++++++ toolproof/src/options.rs | 15 ++++++ 6 files changed, 119 insertions(+), 4 deletions(-) diff --git a/docs/content/docs/configuration.md b/docs/content/docs/configuration.md index 6d7f239..d4681c7 100644 --- a/docs/content/docs/configuration.md +++ b/docs/content/docs/configuration.md @@ -42,7 +42,8 @@ All configuration options that can be set via command-line or environment variab | `porcelain` | Boolean | Reduce logging to be stable (machine-readable output) | | `interactive` | Boolean | Run toolproof in interactive mode | | `all` | Boolean | Run all tests when in interactive mode | -| `run_name` | String | Exact name of a test to run (case-sensitive) | +| `name` | String | Exact name of a test to run (case-sensitive) | +| `path` | String | Path to a test file or directory to run | | `browser` | String | Specify which browser to use (`chrome` or `pagebrowse`) | | `concurrency` | Number | How many tests should be run concurrently | | `timeout` | Number | How long in seconds until a step times out | @@ -73,6 +74,12 @@ npx toolproof --root ./tests # Run a specific test by name npx toolproof --name "My Test Name" +# Run a specific test file +npx toolproof --path tests/my-test.toolproof.yml + +# Run all tests in a directory +npx toolproof --path tests/integration + # Provide placeholders npx toolproof --placeholders project_dir="$(pwd)" api_key=$API_KEY @@ -96,6 +103,7 @@ npx toolproof -c 20 | `--timeout ` | How long in seconds until a step times out | | `--browser-timeout ` | How long in seconds until actions in a browser time out | | `-n, --name ` | Exact name of a test to run | +| `-p, --path ` | Path to a test file or directory to run | | `--browser ` | Specify which browser to use for tests (chrome or pagebrowse, default: chrome) | | `--retry-count ` | Number of times to retry failed tests before marking them as failed | | `--failure-screenshot-location ` | If set, Toolproof will screenshot the browser to this location when a test fails | @@ -110,7 +118,8 @@ Most options can also be set using environment variables: | `TOOLPROOF_ROOT` | The location from which to look for toolproof test files | | `TOOLPROOF_VERBOSE` | Print verbose logging while running tests | | `TOOLPROOF_PORCELAIN` | Reduce logging to be stable | -| `TOOLPROOF_RUN_NAME` | Run a specific test | +| `TOOLPROOF_RUN_NAME` | Run a specific test by name | +| `TOOLPROOF_RUN_PATH` | Path to a test file or directory to run | | `TOOLPROOF_BROWSER` | Specify which browser to use (chrome or pagebrowse) | | `TOOLPROOF_CONCURRENCY` | How many tests should be run concurrently | | `TOOLPROOF_TIMEOUT` | How long in seconds until a step times out | diff --git a/docs/content/docs/debugger.md b/docs/content/docs/debugger.md index f434a37..8c27288 100644 --- a/docs/content/docs/debugger.md +++ b/docs/content/docs/debugger.md @@ -12,10 +12,14 @@ Toolproof's debugger mode allows you to run tests step-by-step, making it easier Run Toolproof with the `--debugger` flag along with a specific test: ```bash +# Debug a test by name npx toolproof --debugger --name "My Test Name" + +# Debug a specific test file +npx toolproof --debugger --path tests/my-test.toolproof.yml ``` -Debugger mode requires running a single test. If you don't specify a test name, Toolproof will show an error. +Debugger mode requires running a single test. If you don't specify a test, Toolproof will show an error. When running in debugger mode: diff --git a/docs/content/docs/installation.md b/docs/content/docs/installation.md index 3d2cfd0..7c79207 100644 --- a/docs/content/docs/installation.md +++ b/docs/content/docs/installation.md @@ -52,9 +52,12 @@ npx toolproof # Run in interactive mode (for updating snapshots) npx toolproof -i -# Run a specific test +# Run a test by name npx toolproof --name "My Test Name" +# Run a test file or directory +npx toolproof --path tests/integration + # See all options npx toolproof --help ``` diff --git a/toolproof/src/interactive.rs b/toolproof/src/interactive.rs index 8e24050..cedcacd 100644 --- a/toolproof/src/interactive.rs +++ b/toolproof/src/interactive.rs @@ -13,6 +13,7 @@ use crate::{ pub enum RunMode { All, One(String), + Path(String), } impl From for ToolproofInternalError { diff --git a/toolproof/src/main.rs b/toolproof/src/main.rs index 5ea19ce..edc553d 100644 --- a/toolproof/src/main.rs +++ b/toolproof/src/main.rs @@ -389,6 +389,24 @@ async fn main_inner() -> Result<(), ()> { }; RunMode::One(path.clone()) + } else if let Some(run_path) = universe.ctx.params.run_path.as_ref() { + // Convert the provided path to an absolute path + let absolute_path = if run_path.is_absolute() { + run_path.clone() + } else { + universe.ctx.working_directory.join(run_path) + }; + + // Normalize the path for comparison + let normalized_path = absolute_path.normalize(); + + // Check if the path exists and is a file or directory + if !absolute_path.exists() { + eprintln!("Path does not exist: {}", run_path.display()); + return Err(()); + } + + RunMode::Path(normalized_path.to_string_lossy().into_owned()) } else if universe.ctx.params.interactive && !universe.ctx.params.all { match get_run_mode(&universe) { Ok(mode) => mode, @@ -409,6 +427,37 @@ async fn main_inner() -> Result<(), ()> { return Err(()); } + // Validate that path-based filtering found at least one test + if let RunMode::Path(ref filter_path) = run_mode { + let test_root = universe.ctx.params.root.as_ref() + .cloned() + .unwrap_or_else(|| universe.ctx.working_directory.clone()); + + let matching_tests = universe + .tests + .iter() + .filter(|(test_path, v)| { + if v.r#type != ToolproofFileType::Test { + return false; + } + + // Convert relative test path to absolute for comparison + let absolute_test_path = test_root.join(test_path).normalize(); + let absolute_test_path_str = absolute_test_path.to_string_lossy(); + + absolute_test_path_str.as_ref() == filter_path || absolute_test_path_str.starts_with(filter_path.as_str()) + }) + .count(); + + if matching_tests == 0 { + eprintln!( + "No tests found matching path: {}", + universe.ctx.params.run_path.as_ref().unwrap().display() + ); + return Err(()); + } + } + enum HoldingError { TestFailure, SnapFailure { out: String }, @@ -729,6 +778,40 @@ async fn main_inner() -> Result<(), ()> { holding_err.map_err(|e| (test, e)) })); } + RunMode::Path(ref filter_path) => { + let test_root = universe.ctx.params.root.as_ref() + .cloned() + .unwrap_or_else(|| universe.ctx.working_directory.clone()); + + for mut test in universe + .tests + .iter() + .filter(|(test_path, v)| { + if v.r#type != ToolproofFileType::Test { + return false; + } + + // Convert relative test path to absolute for comparison + let absolute_test_path = test_root.join(test_path).normalize(); + let absolute_test_path_str = absolute_test_path.to_string_lossy(); + + absolute_test_path_str.as_ref() == filter_path || absolute_test_path_str.starts_with(filter_path.as_str()) + }) + .map(|(_, v)| v.clone()) + { + let permit = semaphore.clone().acquire_owned().await.unwrap(); + let uni = Arc::clone(&universe); + hands.push(tokio::spawn(async move { + let start = Instant::now(); + let res = run_toolproof_experiment(&mut test, Arc::clone(&uni)).await; + let holding_err = handle_res(uni, (&test, res), start); + + drop(permit); + + holding_err.map_err(|e| (test, e)) + })); + } + } } let mut results = join_all(hands) diff --git a/toolproof/src/options.rs b/toolproof/src/options.rs index aab2986..f7da561 100644 --- a/toolproof/src/options.rs +++ b/toolproof/src/options.rs @@ -131,6 +131,13 @@ fn get_cli_matches() -> ArgMatches { .long_help("case-sensitive") .required(false) ) + .arg( + arg!( + -p --path "Path to a specific test file or directory to run") + .long_help("Run tests from a specific file or all tests in a directory") + .required(false) + .value_parser(value_parser!(PathBuf)), + ) .arg( arg!( --browser ... "Specify which browser to use when running browser automation tests" @@ -200,6 +207,10 @@ pub struct ToolproofParams { #[setting(env = "TOOLPROOF_RUN_NAME")] pub run_name: Option, + /// Run tests from a specific file or directory path + #[setting(env = "TOOLPROOF_RUN_PATH")] + pub run_path: Option, + /// Specify which browser to use when running browser automation tests #[setting(env = "TOOLPROOF_BROWSER")] pub browser: ToolproofBrowserImpl, @@ -302,6 +313,10 @@ impl ToolproofParams { self.run_name = Some(name.clone()); } + if let Some(path) = cli_matches.get_one::("path") { + self.run_path = Some(path.clone()); + } + if let Some(root) = cli_matches.get_one::("root") { self.root = Some(root.clone()); }