From 1cf1ece23cd053a11dbf8f72ee7dbfa69ce3a99a Mon Sep 17 00:00:00 2001 From: Liam Bigelow <40188355+bglw@users.noreply.github.com> Date: Tue, 3 Dec 2024 15:16:13 +1300 Subject: [PATCH] Add timeouts to steps, add element click+hover instructions --- CHANGELOG.md | 3 + docs/content/docs/functions.md | 4 + toolproof/src/definitions/browser/mod.rs | 291 +++++++++++++++++- toolproof/src/options.rs | 16 + toolproof/src/runner.rs | 93 ++++-- .../test_suite/browser/click.toolproof.yml | 22 ++ 6 files changed, 406 insertions(+), 23 deletions(-) create mode 100644 toolproof/test_suite/browser/click.toolproof.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a99044..1fb54c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ ## Unreleased +* Add instructions for clicking and hovering elements on a page +* Added a timeout to all test steps + ## v0.7.0 (November 29, 2024) * Add screenshot instructions to Toolproof diff --git a/docs/content/docs/functions.md b/docs/content/docs/functions.md index f5cd1ac..f7ea1c1 100644 --- a/docs/content/docs/functions.md +++ b/docs/content/docs/functions.md @@ -41,6 +41,10 @@ Instructions: - `In my browser, I evaluate {js}` - `In my browser, I screenshot the viewport to {filepath}` - `In my browser, I screenshot the element {selector} to {filepath}` +- `In my browser, I click {text}` +- `In my browser, I hover {text}` +- `In my browser, I click the selector {selector}` +- `In my browser, I hover the selector {selector}` Retrievals: - `In my browser, the result of {js}` diff --git a/toolproof/src/definitions/browser/mod.rs b/toolproof/src/definitions/browser/mod.rs index 0c6e788..74af811 100644 --- a/toolproof/src/definitions/browser/mod.rs +++ b/toolproof/src/definitions/browser/mod.rs @@ -5,11 +5,14 @@ use async_trait::async_trait; use chromiumoxide::cdp::browser_protocol::page::{ CaptureScreenshotFormat, CaptureScreenshotParams, }; -use chromiumoxide::cdp::browser_protocol::target::CreateTargetParams; +use chromiumoxide::cdp::browser_protocol::target::{ + CreateBrowserContextParams, CreateTargetParams, +}; use chromiumoxide::error::CdpError; use chromiumoxide::handler::viewport::Viewport; use chromiumoxide::page::ScreenshotParams; use futures::StreamExt; +use tempfile::tempdir; use tokio::task::JoinHandle; use crate::civilization::Civilization; @@ -45,6 +48,7 @@ async fn try_launch_browser(mut max: usize) -> (Browser, chromiumoxide::Handler) launch = Browser::launch( BrowserConfig::builder() .headless_mode(chromiumoxide::browser::HeadlessMode::New) + .user_data_dir(tempdir().expect("testing on a system with a temp dir")) .viewport(Some(Viewport { width: 1600, height: 900, @@ -89,6 +93,11 @@ fn chrome_image_format(filepath: &PathBuf) -> Result Self { match params.browser { @@ -127,13 +136,22 @@ impl BrowserTester { BrowserWindow::Pagebrowse(pb.get_window().await.unwrap()) } BrowserTester::Chrome { browser, .. } => { + let context = browser + .create_browser_context(CreateBrowserContextParams { + dispose_on_detach: Some(true), + proxy_server: None, + proxy_bypass_list: None, + origins_with_universal_network_access: None, + }) + .await + .unwrap(); let page = browser .new_page(CreateTargetParams { url: "about:blank".to_string(), for_tab: None, width: None, height: None, - browser_context_id: None, + browser_context_id: Some(context), enable_begin_frame_control: None, new_window: None, background: None, @@ -247,6 +265,143 @@ impl BrowserWindow { )), } } + + async fn interact_text( + &self, + text: &str, + interaction: InteractionType, + ) -> Result<(), ToolproofStepError> { + match self { + BrowserWindow::Chrome(page) => { + let text = text.to_lowercase().replace('\'', "\\'"); + let el_xpath = |el: &str| { + format!("//{el}[contains(translate(., 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz'), '{text}')]") + }; + let xpath = [el_xpath("a"), el_xpath("button"), el_xpath("input")].join(" | "); + let elements = page.find_xpaths(xpath).await.map_err(|e| { + ToolproofStepError::Assertion(ToolproofTestFailure::Custom { + msg: format!("Element with text '{text}' could not be clicked: {e}"), + }) + })?; + + if elements.is_empty() { + return Err(ToolproofStepError::Assertion( + ToolproofTestFailure::Custom { + msg: format!( + "Clickable element containing text '{text}' does not exist. Did you mean to use 'I click the selector'?" + ), + }, + )); + } + + if elements.len() > 1 { + return Err(ToolproofStepError::Assertion( + ToolproofTestFailure::Custom { + msg: format!( + "Found more than one clickable element containing text '{text}'." + ), + }, + )); + } + + elements[0].scroll_into_view().await.map_err(|e| { + ToolproofStepError::Assertion(ToolproofTestFailure::Custom { + msg: format!( + "Element with text '{text}' could not be scrolled into view: {e}" + ), + }) + })?; + + let center = elements[0].clickable_point().await.map_err(|e| { + ToolproofStepError::Assertion(ToolproofTestFailure::Custom { + msg: format!( + "Could not find a clickable point for element with text '{text}': {e}" + ), + }) + })?; + + match interaction { + InteractionType::Click => { + page.click(center).await.map_err(|e| { + ToolproofStepError::Assertion(ToolproofTestFailure::Custom { + msg: format!( + "Element with text '{text}' could not be clicked: {e}" + ), + }) + })?; + } + InteractionType::Hover => { + page.move_mouse(center).await.map_err(|e| { + ToolproofStepError::Assertion(ToolproofTestFailure::Custom { + msg: format!( + "Element with text '{text}' could not be hovered: {e}" + ), + }) + })?; + } + } + + Ok(()) + } + BrowserWindow::Pagebrowse(_) => Err(ToolproofStepError::Internal( + ToolproofInternalError::Custom { + msg: "Clicks not yet implemented for Pagebrowse".to_string(), + }, + )), + } + } + + async fn interact_selector( + &self, + selector: &str, + interaction: InteractionType, + ) -> Result<(), ToolproofStepError> { + match self { + BrowserWindow::Chrome(page) => { + let element = page.find_element(selector).await.map_err(|e| { + ToolproofStepError::Assertion(ToolproofTestFailure::Custom { + msg: format!("Element {selector} could not be clicked: {e}"), + }) + })?; + + element.scroll_into_view().await.map_err(|e| { + ToolproofStepError::Assertion(ToolproofTestFailure::Custom { + msg: format!("Element {selector} could not be scrolled into view: {e}"), + }) + })?; + + let center = element.clickable_point().await.map_err(|e| { + ToolproofStepError::Assertion(ToolproofTestFailure::Custom { + msg: format!("Could not find a clickable point for {selector}: {e}"), + }) + })?; + + match interaction { + InteractionType::Click => { + page.click(center).await.map_err(|e| { + ToolproofStepError::Assertion(ToolproofTestFailure::Custom { + msg: format!("Element {selector} could not be clicked: {e}"), + }) + })?; + } + InteractionType::Hover => { + page.move_mouse(center).await.map_err(|e| { + ToolproofStepError::Assertion(ToolproofTestFailure::Custom { + msg: format!("Element {selector} could not be hovered: {e}"), + }) + })?; + } + } + + Ok(()) + } + BrowserWindow::Pagebrowse(_) => Err(ToolproofStepError::Internal( + ToolproofInternalError::Custom { + msg: "Clicks not yet implemented for Pagebrowse".to_string(), + }, + )), + } + } } mod load_page { @@ -493,3 +648,135 @@ mod screenshots { } } } + +mod interactions { + use super::*; + + pub struct ClickText; + + inventory::submit! { + &ClickText as &dyn ToolproofInstruction + } + + #[async_trait] + impl ToolproofInstruction for ClickText { + fn segments(&self) -> &'static str { + "In my browser, I click {text}" + } + + async fn run( + &self, + args: &SegmentArgs<'_>, + civ: &mut Civilization, + ) -> Result<(), ToolproofStepError> { + let text = args.get_string("text")?; + + let Some(window) = civ.window.as_ref() else { + return Err(ToolproofStepError::External( + ToolproofInputError::StepRequirementsNotMet { + reason: "no page has been loaded into the browser for this test".into(), + }, + )); + }; + + window.interact_text(&text, InteractionType::Click).await + } + } + + pub struct HoverText; + + inventory::submit! { + &HoverText as &dyn ToolproofInstruction + } + + #[async_trait] + impl ToolproofInstruction for HoverText { + fn segments(&self) -> &'static str { + "In my browser, I hover {text}" + } + + async fn run( + &self, + args: &SegmentArgs<'_>, + civ: &mut Civilization, + ) -> Result<(), ToolproofStepError> { + let text = args.get_string("text")?; + + let Some(window) = civ.window.as_ref() else { + return Err(ToolproofStepError::External( + ToolproofInputError::StepRequirementsNotMet { + reason: "no page has been loaded into the browser for this test".into(), + }, + )); + }; + + window.interact_text(&text, InteractionType::Hover).await + } + } + + pub struct ClickSelector; + + inventory::submit! { + &ClickSelector as &dyn ToolproofInstruction + } + + #[async_trait] + impl ToolproofInstruction for ClickSelector { + fn segments(&self) -> &'static str { + "In my browser, I click the selector {selector}" + } + + async fn run( + &self, + args: &SegmentArgs<'_>, + civ: &mut Civilization, + ) -> Result<(), ToolproofStepError> { + let selector = args.get_string("selector")?; + + let Some(window) = civ.window.as_ref() else { + return Err(ToolproofStepError::External( + ToolproofInputError::StepRequirementsNotMet { + reason: "no page has been loaded into the browser for this test".into(), + }, + )); + }; + + window + .interact_selector(&selector, InteractionType::Click) + .await + } + } + + pub struct HoverSelector; + + inventory::submit! { + &HoverSelector as &dyn ToolproofInstruction + } + + #[async_trait] + impl ToolproofInstruction for HoverSelector { + fn segments(&self) -> &'static str { + "In my browser, I hover the selector {selector}" + } + + async fn run( + &self, + args: &SegmentArgs<'_>, + civ: &mut Civilization, + ) -> Result<(), ToolproofStepError> { + let selector = args.get_string("selector")?; + + let Some(window) = civ.window.as_ref() else { + return Err(ToolproofStepError::External( + ToolproofInputError::StepRequirementsNotMet { + reason: "no page has been loaded into the browser for this test".into(), + }, + )); + }; + + window + .interact_selector(&selector, InteractionType::Hover) + .await + } + } +} diff --git a/toolproof/src/options.rs b/toolproof/src/options.rs index b797b28..d457e1e 100644 --- a/toolproof/src/options.rs +++ b/toolproof/src/options.rs @@ -112,6 +112,13 @@ fn get_cli_matches() -> ArgMatches { ) .action(clap::ArgAction::SetTrue), ) + .arg( + arg!( + --timeout "How long in seconds until a step times out" + ) + .required(false) + .value_parser(value_parser!(u64)), + ) .arg( arg!( -n --name "Exact name of a test to run") @@ -176,6 +183,11 @@ pub struct ToolproofParams { #[setting(default = 10)] pub concurrency: usize, + /// How long in seconds until a step times out + #[setting(env = "TOOLPROOF_TIMEOUT")] + #[setting(default = 10)] + pub timeout: u64, + /// What delimiter should be used when replacing placeholders #[setting(env = "TOOLPROOF_PLACEHOLDER_DELIM")] #[setting(default = "%")] @@ -250,6 +262,10 @@ impl ToolproofParams { self.concurrency = *concurrency; } + if let Some(timeout) = cli_matches.get_one::("timeout") { + self.timeout = *timeout; + } + if let Some(placeholder_delimiter) = cli_matches.get_one::("placeholder-delimiter") { self.placeholder_delimiter = placeholder_delimiter.clone(); diff --git a/toolproof/src/runner.rs b/toolproof/src/runner.rs index 260b983..a7a900d 100644 --- a/toolproof/src/runner.rs +++ b/toolproof/src/runner.rs @@ -3,13 +3,14 @@ use futures::FutureExt; use normalize_path::NormalizePath; use similar_string::find_best_similarity; use std::{collections::HashMap, path::PathBuf, sync::Arc}; +use tokio::time::{self, Duration}; use console::style; use crate::{ civilization::Civilization, definitions::ToolproofInstruction, - errors::{ToolproofInputError, ToolproofStepError, ToolproofTestError}, + errors::{ToolproofInputError, ToolproofStepError, ToolproofTestError, ToolproofTestFailure}, platforms::platform_matches, segments::SegmentArgs, universe::Universe, @@ -49,6 +50,8 @@ async fn run_toolproof_steps( civ: &mut Civilization<'_>, transient_placeholders: Option>, ) -> Result { + let timeout_mins = civ.universe.ctx.params.timeout; + let timeout_dur = Duration::from_secs(timeout_mins); for cur_step in steps.iter_mut() { let marked_base_step = cur_step.clone(); let marked_base_args = cur_step.args_pretty(); @@ -62,6 +65,16 @@ async fn run_toolproof_steps( arg_str: marked_base_args.clone(), } }; + let timeout_and_return_step_error = |state: &mut ToolproofTestStepState| { + *state = ToolproofTestStepState::Failed; + ToolproofTestError { + err: ToolproofStepError::Assertion(ToolproofTestFailure::Custom { + msg: format!("Step timed out after {timeout_mins}s"), + }), + step: marked_base_step.clone(), + arg_str: marked_base_args.clone(), + } + }; match cur_step { crate::ToolproofTestStep::Ref { @@ -200,10 +213,16 @@ async fn run_toolproof_steps( .map_err(|e| mark_and_return_step_error(e.into(), state))?; if platform_matches(platforms) { - instruction - .run(&instruction_args, civ) - .await - .map_err(|e| mark_and_return_step_error(e.into(), state))?; + match time::timeout(timeout_dur, instruction.run(&instruction_args, civ)).await + { + Ok(Ok(_)) => {} + Ok(Err(e)) => { + return Err(mark_and_return_step_error(e.into(), state)); + } + Err(_) => { + return Err(timeout_and_return_step_error(state)); + } + } *state = ToolproofTestStepState::Passed; } else { @@ -237,10 +256,16 @@ async fn run_toolproof_steps( .map_err(|e| mark_and_return_step_error(e.into(), state))?; let value = if platform_matches(platforms) { - retrieval_step - .run(&retrieval_args, civ) - .await - .map_err(|e| mark_and_return_step_error(e.into(), state))? + match time::timeout(timeout_dur, retrieval_step.run(&retrieval_args, civ)).await + { + Ok(Ok(val)) => val, + Ok(Err(e)) => { + return Err(mark_and_return_step_error(e.into(), state)); + } + Err(_) => { + return Err(timeout_and_return_step_error(state)); + } + } } else { serde_json::Value::Null }; @@ -264,10 +289,20 @@ async fn run_toolproof_steps( .map_err(|e| mark_and_return_step_error(e.into(), state))?; if platform_matches(platforms) { - assertion_step - .run(value, &assertion_args, civ) - .await - .map_err(|e| mark_and_return_step_error(e.into(), state))?; + match time::timeout( + timeout_dur, + assertion_step.run(value, &assertion_args, civ), + ) + .await + { + Ok(Ok(_)) => {} + Ok(Err(e)) => { + return Err(mark_and_return_step_error(e.into(), state)); + } + Err(_) => { + return Err(timeout_and_return_step_error(state)); + } + } *state = ToolproofTestStepState::Passed; } else { @@ -301,10 +336,18 @@ async fn run_toolproof_steps( .map_err(|e| mark_and_return_step_error(e.into(), state))?; if platform_matches(platforms) { - let value = retrieval_step - .run(&retrieval_args, civ) - .await - .map_err(|e| mark_and_return_step_error(e.into(), state))?; + let value = + match time::timeout(timeout_dur, retrieval_step.run(&retrieval_args, civ)) + .await + { + Ok(Ok(val)) => val, + Ok(Err(e)) => { + return Err(mark_and_return_step_error(e.into(), state)); + } + Err(_) => { + return Err(timeout_and_return_step_error(state)); + } + }; let value_content = match &value { serde_json::Value::String(s) => s.clone(), @@ -344,10 +387,18 @@ async fn run_toolproof_steps( .map_err(|e| mark_and_return_step_error(e.into(), state))?; if platform_matches(platforms) { - let value = retrieval_step - .run(&retrieval_args, civ) - .await - .map_err(|e| mark_and_return_step_error(e.into(), state))?; + let value = + match time::timeout(timeout_dur, retrieval_step.run(&retrieval_args, civ)) + .await + { + Ok(Ok(val)) => val, + Ok(Err(e)) => { + return Err(mark_and_return_step_error(e.into(), state)); + } + Err(_) => { + return Err(timeout_and_return_step_error(state)); + } + }; let value_content = match &value { serde_json::Value::String(s) => s.clone(), diff --git a/toolproof/test_suite/browser/click.toolproof.yml b/toolproof/test_suite/browser/click.toolproof.yml new file mode 100644 index 0000000..6f5e47e --- /dev/null +++ b/toolproof/test_suite/browser/click.toolproof.yml @@ -0,0 +1,22 @@ +name: Browser can click + +steps: + - step: I have a "setup.toolproof.yml" file with the content {yaml} + yaml: |- + name: Inner passing test + + steps: + - step: I have a "public/index.html" file with the content {html} + html: |- +

Hello

+ - I serve the directory "public" + - In my browser, I load "/" + - In my browser, I click "World" + - step: In my browser, I evaluate {js} + js: |- + toolproof.assert_eq(document.querySelector("button").innerText, "Test"); + - I run "%toolproof_path%" + - step: "stdout should contain 'Passing tests: 1'" + - step: "stdout should contain 'Failing tests: 0'" + - step: "stdout should contain 'All tests passed'" + - stderr should be empty