From 386371f6fce7566f0f2f7fcaa3a0b995ec3c4bbb Mon Sep 17 00:00:00 2001 From: Liam Bigelow <40188355+bglw@users.noreply.github.com> Date: Fri, 29 Nov 2024 11:51:53 +1300 Subject: [PATCH 1/3] Add screenshot capabilities --- docs/content/docs/functions.md | 10 +- toolproof/src/civilization.rs | 4 + toolproof/src/definitions/browser/mod.rs | 179 +++++++++++++++++- toolproof/src/main.rs | 67 ++++--- toolproof/src/options.rs | 14 ++ .../browser/screenshot.toolproof.yml | 21 ++ 6 files changed, 258 insertions(+), 37 deletions(-) create mode 100644 toolproof/test_suite/browser/screenshot.toolproof.yml diff --git a/docs/content/docs/functions.md b/docs/content/docs/functions.md index 1769277..f5cd1ac 100644 --- a/docs/content/docs/functions.md +++ b/docs/content/docs/functions.md @@ -7,7 +7,7 @@ weight: 6 Toolproof provides the following Instructions: -## Filesystem +## Filesystem Instructions: - `I have a {filename} file with the content {contents}` @@ -16,7 +16,7 @@ Retrievals: - `The file {filename}` - Returns a string value -## Process +## Process Instructions: - `I have the environment variable {name} set to {value}` @@ -29,16 +29,18 @@ Retrievals: - `stderr` - Returns a string value -## Hosting +## Hosting Instructions: - `I serve the directory {dir}` -## Browser +## Browser Instructions: - `In my browser, I load {url}` - `In my browser, I evaluate {js}` +- `In my browser, I screenshot the viewport to {filepath}` +- `In my browser, I screenshot the element {selector} to {filepath}` Retrievals: - `In my browser, the result of {js}` diff --git a/toolproof/src/civilization.rs b/toolproof/src/civilization.rs index ed884e3..f57ba88 100644 --- a/toolproof/src/civilization.rs +++ b/toolproof/src/civilization.rs @@ -74,6 +74,10 @@ impl<'u> Civilization<'u> { tmp_dir.join(PathBuf::from(filename)) } + pub fn ensure_path(&mut self, file_path: &PathBuf) { + fs::create_dir_all(file_path.parent().unwrap()).unwrap(); + } + pub fn write_file(&mut self, filename: &str, contents: &str) { let file_path = self.tmp_file_path(filename); fs::create_dir_all(file_path.parent().unwrap()).unwrap(); diff --git a/toolproof/src/definitions/browser/mod.rs b/toolproof/src/definitions/browser/mod.rs index 33cc4bd..0c6e788 100644 --- a/toolproof/src/definitions/browser/mod.rs +++ b/toolproof/src/definitions/browser/mod.rs @@ -1,13 +1,21 @@ +use std::path::PathBuf; use std::sync::Arc; use async_trait::async_trait; +use chromiumoxide::cdp::browser_protocol::page::{ + CaptureScreenshotFormat, CaptureScreenshotParams, +}; use chromiumoxide::cdp::browser_protocol::target::CreateTargetParams; use chromiumoxide::error::CdpError; +use chromiumoxide::handler::viewport::Viewport; +use chromiumoxide::page::ScreenshotParams; use futures::StreamExt; use tokio::task::JoinHandle; use crate::civilization::Civilization; -use crate::errors::{ToolproofInputError, ToolproofInternalError, ToolproofStepError}; +use crate::errors::{ + ToolproofInputError, ToolproofInternalError, ToolproofStepError, ToolproofTestFailure, +}; use crate::options::ToolproofParams; use super::{SegmentArgs, ToolproofInstruction, ToolproofRetriever}; @@ -34,7 +42,21 @@ async fn try_launch_browser(mut max: usize) -> (Browser, chromiumoxide::Handler) let mut launch = Err(CdpError::NotFound); while launch.is_err() && max > 0 { max -= 1; - launch = Browser::launch(BrowserConfig::builder().build().unwrap()).await; + launch = Browser::launch( + BrowserConfig::builder() + .headless_mode(chromiumoxide::browser::HeadlessMode::New) + .viewport(Some(Viewport { + width: 1600, + height: 900, + device_scale_factor: Some(2.0), + emulating_mobile: false, + is_landscape: true, + has_touch: false, + })) + .build() + .unwrap(), + ) + .await; } match launch { Ok(res) => res, @@ -44,6 +66,29 @@ async fn try_launch_browser(mut max: usize) -> (Browser, chromiumoxide::Handler) } } +fn chrome_image_format(filepath: &PathBuf) -> Result { + match filepath.extension() { + Some(ext) => { + let ext = ext.to_string_lossy().to_lowercase(); + match ext.as_str() { + "png" => Ok(CaptureScreenshotFormat::Png), + "webp" => Ok(CaptureScreenshotFormat::Webp), + "jpg" | "jpeg" => Ok(CaptureScreenshotFormat::Jpeg), + _ => Err(ToolproofStepError::External( + ToolproofInputError::StepRequirementsNotMet { + reason: "Image file extension must be png, webp, jpeg, or jpg".to_string(), + }, + )), + } + } + None => Err(ToolproofStepError::External( + ToolproofInputError::StepRequirementsNotMet { + reason: "Image file path must have an extension".to_string(), + }, + )), + } +} + impl BrowserTester { async fn initialize(params: &ToolproofParams) -> Self { match params.browser { @@ -145,6 +190,63 @@ impl BrowserWindow { .map_err(|inner| ToolproofStepError::Internal(inner.into())), } } + + async fn screenshot_page(&self, filepath: PathBuf) -> Result<(), ToolproofStepError> { + match self { + BrowserWindow::Chrome(page) => { + let image_format = chrome_image_format(&filepath)?; + + page.save_screenshot( + ScreenshotParams { + cdp_params: CaptureScreenshotParams { + format: Some(image_format), + ..CaptureScreenshotParams::default() + }, + full_page: Some(false), + omit_background: Some(false), + }, + filepath, + ) + .await + .map(|_| ()) + .map_err(|e| ToolproofStepError::Internal(e.into())) + } + BrowserWindow::Pagebrowse(_) => Err(ToolproofStepError::Internal( + ToolproofInternalError::Custom { + msg: "Screenshots not yet implemented for Pagebrowse".to_string(), + }, + )), + } + } + + async fn screenshot_element( + &self, + selector: &str, + filepath: PathBuf, + ) -> Result<(), ToolproofStepError> { + match self { + BrowserWindow::Chrome(page) => { + let image_format = chrome_image_format(&filepath)?; + + let element = page.find_element(selector).await.map_err(|e| { + ToolproofStepError::Assertion(ToolproofTestFailure::Custom { + msg: format!("Element {selector} could not be screenshot: {e}"), + }) + })?; + + element + .save_screenshot(image_format, filepath) + .await + .map(|_| ()) + .map_err(|e| ToolproofStepError::Internal(e.into())) + } + BrowserWindow::Pagebrowse(_) => Err(ToolproofStepError::Internal( + ToolproofInternalError::Custom { + msg: "Screenshots not yet implemented for Pagebrowse".to_string(), + }, + )), + } + } } mod load_page { @@ -318,3 +420,76 @@ mod eval_js { } } } + +mod screenshots { + use crate::errors::{ToolproofInternalError, ToolproofTestFailure}; + + use super::*; + + pub struct ScreenshotViewport; + + inventory::submit! { + &ScreenshotViewport as &dyn ToolproofInstruction + } + + #[async_trait] + impl ToolproofInstruction for ScreenshotViewport { + fn segments(&self) -> &'static str { + "In my browser, I screenshot the viewport to {filepath}" + } + + async fn run( + &self, + args: &SegmentArgs<'_>, + civ: &mut Civilization, + ) -> Result<(), ToolproofStepError> { + let filepath = args.get_string("filepath")?; + let resolved_path = civ.tmp_file_path(&filepath); + civ.ensure_path(&resolved_path); + + let Some(window) = civ.window.as_ref() else { + return Err(ToolproofStepError::External( + ToolproofInputError::StepRequirementsNotMet { + reason: "no page has been loaded into the browser for this test".into(), + }, + )); + }; + + window.screenshot_page(resolved_path).await + } + } + + pub struct ScreenshotElement; + + inventory::submit! { + &ScreenshotElement as &dyn ToolproofInstruction + } + + #[async_trait] + impl ToolproofInstruction for ScreenshotElement { + fn segments(&self) -> &'static str { + "In my browser, I screenshot the element {selector} to {filepath}" + } + + async fn run( + &self, + args: &SegmentArgs<'_>, + civ: &mut Civilization, + ) -> Result<(), ToolproofStepError> { + let selector = args.get_string("selector")?; + let filepath = args.get_string("filepath")?; + let resolved_path = civ.tmp_file_path(&filepath); + civ.ensure_path(&resolved_path); + + let Some(window) = civ.window.as_ref() else { + return Err(ToolproofStepError::External( + ToolproofInputError::StepRequirementsNotMet { + reason: "no page has been loaded into the browser for this test".into(), + }, + )); + }; + + window.screenshot_element(&selector, resolved_path).await + } + } +} diff --git a/toolproof/src/main.rs b/toolproof/src/main.rs index 4eae2ea..2f833c1 100644 --- a/toolproof/src/main.rs +++ b/toolproof/src/main.rs @@ -192,40 +192,45 @@ fn closest_strings<'o>(target: &String, options: &'o Vec) -> Vec<(&'o St async fn main_inner() -> Result<(), ()> { let ctx = configure(); - for before in &ctx.params.before_all { - let before_cmd = &before.command; - let mut command = Command::new("sh"); - command - .arg("-c") - .current_dir(&ctx.working_directory) - .arg(before_cmd); - - command.stdout(Stdio::piped()); - command.stderr(Stdio::piped()); + if ctx.params.skip_hooks { + println!("{}", "Skipping before_all commands".yellow().bold()); + } else { + for before in &ctx.params.before_all { + let before_cmd = &before.command; + let mut command = Command::new("sh"); + command + .arg("-c") + .current_dir(&ctx.working_directory) + .arg(before_cmd); - println!( - "{}{}", - "Running before_all command: ".blue().bold(), - before_cmd.cyan().bold(), - ); + command.stdout(Stdio::piped()); + command.stderr(Stdio::piped()); - let running = command - .spawn() - .map_err(|_| eprintln!("Failed to run command: {before_cmd}"))?; + println!( + "{}{}", + "Running before_all command: ".blue().bold(), + before_cmd.cyan().bold(), + ); - let Ok(_) = - (match tokio::time::timeout(Duration::from_secs(300), running.wait_with_output()).await - { - Ok(out) => out, - Err(_) => { - eprintln!("Failed to run command due to timeout: {before_cmd}"); - return Err(()); - } - }) - else { - eprintln!("Failed to run command: {before_cmd}"); - return Err(()); - }; + let running = command + .spawn() + .map_err(|_| eprintln!("Failed to run command: {before_cmd}"))?; + + let Ok(_) = + (match tokio::time::timeout(Duration::from_secs(300), running.wait_with_output()) + .await + { + Ok(out) => out, + Err(_) => { + eprintln!("Failed to run command due to timeout: {before_cmd}"); + return Err(()); + } + }) + else { + eprintln!("Failed to run command: {before_cmd}"); + return Err(()); + }; + } } let start = Instant::now(); diff --git a/toolproof/src/options.rs b/toolproof/src/options.rs index bfec3b7..b797b28 100644 --- a/toolproof/src/options.rs +++ b/toolproof/src/options.rs @@ -106,6 +106,12 @@ fn get_cli_matches() -> ArgMatches { ) .action(clap::ArgAction::SetTrue), ) + .arg( + arg!( + -s --skiphooks ... "Skip running any hooks (e.g. before_all)" + ) + .action(clap::ArgAction::SetTrue), + ) .arg( arg!( -n --name "Exact name of a test to run") @@ -180,6 +186,10 @@ pub struct ToolproofParams { /// Commands to run in the working directory before starting to run Toolproof tests pub before_all: Vec, + + /// Skip running any of the before_all hooks + #[setting(env = "TOOLPROOF_SKIPHOOKS")] + pub skip_hooks: bool, } // The configuration object used internally @@ -224,6 +234,10 @@ impl ToolproofParams { self.all = true; } + if cli_matches.get_flag("skiphooks") { + self.skip_hooks = true; + } + if let Some(name) = cli_matches.get_one::("name") { self.run_name = Some(name.clone()); } diff --git a/toolproof/test_suite/browser/screenshot.toolproof.yml b/toolproof/test_suite/browser/screenshot.toolproof.yml new file mode 100644 index 0000000..df2a9d5 --- /dev/null +++ b/toolproof/test_suite/browser/screenshot.toolproof.yml @@ -0,0 +1,21 @@ +name: Browser can screenshot + +steps: + - step: I have a "my_test.toolproof.yml" file with the content {yaml} + yaml: |- + name: Inner passing test + + steps: + - I have a "public/index.html" file with the content "

Hello World

" + - I serve the directory "public" + - In my browser, I load "/" + - step: In my browser, I evaluate {js} + js: |- + await toolproof.querySelector("p"); + - In my browser, I screenshot the element "p" to "shot/p.webp" + - I run "ls shot" + - stdout should contain "p.webp" + - I run "%toolproof_path%" + - step: "stdout should contain 'Passing tests: 1'" + - step: "stdout should contain 'Failing tests: 0'" + - step: "stdout should contain 'All tests passed'" From 28699cf97b944928d98cb2a0eece4c449dbb85e6 Mon Sep 17 00:00:00 2001 From: Liam Bigelow <40188355+bglw@users.noreply.github.com> Date: Fri, 29 Nov 2024 14:24:59 +1300 Subject: [PATCH 2/3] Add extracts --- docs/content/docs/_index.md | 14 ++++++ toolproof/src/main.rs | 20 +++++++++ toolproof/src/parser.rs | 20 +++++++++ toolproof/src/runner.rs | 45 +++++++++++++++++++ toolproof/src/segments.rs | 10 +++++ .../test_suite/base/extract.toolproof.yml | 29 ++++++++++++ 6 files changed, 138 insertions(+) create mode 100644 toolproof/test_suite/base/extract.toolproof.yml diff --git a/docs/content/docs/_index.md b/docs/content/docs/_index.md index 95d8672..b34a979 100644 --- a/docs/content/docs/_index.md +++ b/docs/content/docs/_index.md @@ -135,6 +135,20 @@ steps: In future runs, Toolproof will ensure the retrieved value matches the `snapshot_content` key. Running Toolproof in interactive mode (`-i`) will also allow you to accept the changes and update the file automatically. +### Snapshots + +Any Retrieval can also drive an extract. To do so, place the step inside an object under a `extract` key, alongside an `extract_location` key: +```yaml +steps: + - extract: stdout + extract_location: "%toolproof_process_directory%/extracted_stdout.txt" +``` + +After running Toolproof, the value will be written to that file. + +Toolproof never reads this file, so this step doesn't have any bearing on the success of the test. +Instead, this is intended to pull information from tests to use in other tooling. + ## Test environment Toolproof automatically runs tests in a temporary directory that is discarded at the end of a run. diff --git a/toolproof/src/main.rs b/toolproof/src/main.rs index 2f833c1..44ce420 100644 --- a/toolproof/src/main.rs +++ b/toolproof/src/main.rs @@ -118,6 +118,14 @@ pub enum ToolproofTestStep { state: ToolproofTestStepState, platforms: Option>, }, + Extract { + extract: ToolproofSegments, + extract_location: String, + args: HashMap, + orig: String, + state: ToolproofTestStepState, + platforms: Option>, + }, } impl Display for ToolproofTestStep { @@ -137,6 +145,9 @@ impl Display for ToolproofTestStep { Snapshot { orig, .. } => { write!(f, "snapshot: {}", orig) } + Extract { orig, .. } => { + write!(f, "extract: {}", orig) + } } } } @@ -170,6 +181,7 @@ impl ToolproofTestStep { | Macro { state, .. } | Instruction { state, .. } | Assertion { state, .. } + | Extract { state, .. } | Snapshot { state, .. } => state.clone(), } } @@ -623,6 +635,14 @@ async fn main_inner() -> Result<(), ()> { } } } + ToolproofTestStep::Extract { + extract, + extract_location, + args, + orig, + state, + platforms, + } => todo!(), ToolproofTestStep::Snapshot { snapshot, snapshot_content, diff --git a/toolproof/src/parser.rs b/toolproof/src/parser.rs index e68be61..c8922e8 100644 --- a/toolproof/src/parser.rs +++ b/toolproof/src/parser.rs @@ -79,6 +79,13 @@ enum RawToolproofTestStep { #[serde(flatten)] other: Map, }, + Extract { + extract: String, + extract_location: String, + platforms: Option>, + #[serde(flatten)] + other: Map, + }, } impl TryFrom for ToolproofTestFile { @@ -169,6 +176,19 @@ impl TryFrom for ToolproofTestStep { state: ToolproofTestStepState::Dormant, platforms, }), + RawToolproofTestStep::Extract { + extract, + extract_location, + platforms, + other, + } => Ok(ToolproofTestStep::Extract { + extract: parse_segments(&extract)?, + extract_location, + args: HashMap::from_iter(other.into_iter()), + orig: extract, + state: ToolproofTestStepState::Dormant, + platforms, + }), } } } diff --git a/toolproof/src/runner.rs b/toolproof/src/runner.rs index cd4e6c0..260b983 100644 --- a/toolproof/src/runner.rs +++ b/toolproof/src/runner.rs @@ -317,6 +317,51 @@ async fn run_toolproof_steps( *state = ToolproofTestStepState::Skipped; } } + crate::ToolproofTestStep::Extract { + extract, + extract_location, + args, + orig: _, + state, + platforms, + } => { + let Some((reference_ret, retrieval_step)) = + civ.universe.retrievers.get_key_value(extract) + else { + return Err(mark_and_return_step_error( + ToolproofStepError::External(ToolproofInputError::NonexistentStep), + state, + )); + }; + + let retrieval_args = SegmentArgs::build( + reference_ret, + extract, + args, + Some(&civ), + transient_placeholders.as_ref(), + ) + .map_err(|e| mark_and_return_step_error(e.into(), state))?; + + if platform_matches(platforms) { + let value = retrieval_step + .run(&retrieval_args, civ) + .await + .map_err(|e| mark_and_return_step_error(e.into(), state))?; + + let value_content = match &value { + serde_json::Value::String(s) => s.clone(), + _ => serde_yaml::to_string(&value).expect("extract value is serializable"), + }; + + let location = retrieval_args.process_external_string(&extract_location); + civ.write_file(&location, &value_content); + + *state = ToolproofTestStepState::Passed; + } else { + *state = ToolproofTestStepState::Skipped; + } + } } } diff --git a/toolproof/src/segments.rs b/toolproof/src/segments.rs index 073e9d4..08149ff 100644 --- a/toolproof/src/segments.rs +++ b/toolproof/src/segments.rs @@ -212,6 +212,16 @@ impl<'a> SegmentArgs<'a> { expected: "string".to_string(), }); } + + /// Process an arbitrary string as if it were one of the contained arguments + pub fn process_external_string(&self, raw_value: impl AsRef) -> String { + let mut value = Value::String(raw_value.as_ref().to_string()); + replace_inside_value(&mut value, &self.placeholder_delim, &self.placeholders); + match value { + Value::String(st) => st, + _ => unreachable!(), + } + } } fn replace_inside_value(value: &mut Value, delim: &str, placeholders: &HashMap) { diff --git a/toolproof/test_suite/base/extract.toolproof.yml b/toolproof/test_suite/base/extract.toolproof.yml new file mode 100644 index 0000000..b41b2e8 --- /dev/null +++ b/toolproof/test_suite/base/extract.toolproof.yml @@ -0,0 +1,29 @@ +name: Extract to file + +steps: + - step: I have a "my_test.toolproof.yml" file with the content {yaml} + yaml: |- + name: Inner passing snapshot test + + steps: + - I run 'echo "Aenean eu leo quam"' + - extract: stdout + extract_location: "%toolproof_test_directory%/file.txt" + - snapshot: The file "%toolproof_test_directory%/file.txt" + snapshot_content: |- + ╎Aenean eu leo quam + - I run "%toolproof_path% --porcelain" + - snapshot: stdout + snapshot_content: |- + ╎ + ╎Running tests + ╎ + ╎✓ Inner passing snapshot test + ╎ + ╎Finished running tests + ╎ + ╎Passing tests: 1 + ╎Failing tests: 0 + ╎Skipped tests: 0 + ╎ + ╎All tests passed From aaf4e532ca8dfb4e4d196ecc5038fbe1b31d11fc Mon Sep 17 00:00:00 2001 From: Liam Bigelow <40188355+bglw@users.noreply.github.com> Date: Fri, 29 Nov 2024 14:26:34 +1300 Subject: [PATCH 3/3] Changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf76cb1..01346b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ ## Unreleased +* Add screenshot instructions to Toolproof +* Add `extract` concept to pull retrievals to disk + ## v0.6.1 (November 28, 2024) * Log inner macro steps