diff --git a/crates/cli/src/cli/problem.rs b/crates/cli/src/cli/problem.rs index 5953f85..f2bdcd7 100644 --- a/crates/cli/src/cli/problem.rs +++ b/crates/cli/src/cli/problem.rs @@ -1,14 +1,12 @@ use std::fs; -use anyhow::{anyhow, Context, Result}; +use anyhow::{anyhow, bail, Context, Result}; use clap::{value_parser, Arg, ArgAction, ArgMatches, Command}; use crate::config::Settings; -use crate::problem::{ - archive, check, compare, create, generate, - run::{RunnableCategory, RunnableFile}, - solve, test, -}; +use crate::problem::fuzz; +use crate::problem::run::{RunnableCategory, RunnableFile}; +use crate::problem::{archive, check, compare, create, generate, solve, test}; use crate::util::{get_problem_from_cwd, get_project_root}; pub fn cli() -> Command { @@ -38,36 +36,12 @@ pub fn cli() -> Command { ) .subcommand( Command::new("compare") - .about("Compare two solutions") + .about("Compare two or more solutions") .args([ - Arg::new("generate") - .long("generate") - .help("Generate new test cases until the solutions produce different results") - .action(ArgAction::SetTrue), - Arg::new("file1") - .long("file1") - .help("Name of the first solution file") - .action(ArgAction::Set), - Arg::new("lang1") - .long("lang1") - .help("Language of the first solution file (e.g. cpp, py)") - .action(ArgAction::Set), - Arg::new("file2") - .long("file2") - .help("Name of the second solution file") - .action(ArgAction::Set), - Arg::new("lang2") - .long("lang2") - .help("Language of the second solution file (e.g. cpp, py)") - .action(ArgAction::Set), - Arg::new("generator-file") - .long("generator-file") - .help("Name of the generator file") - .action(ArgAction::Set), - Arg::new("generator-lang") - .long("generator-lang") - .help("Language of the generator file (e.g. cpp, py)") - .action(ArgAction::Set), + Arg::new("file") + .long("file") + .help("Name of the solution file") + .action(ArgAction::Append), Arg::new("problem") .short('p') .long("problem") @@ -93,6 +67,25 @@ pub fn cli() -> Command { .required(true), ), ) + .subcommand( + Command::new("fuzz") + .about("Find potential edge cases in two or more solutions") + .args([ + Arg::new("file") + .long("file") + .help("Name of the solution file") + .action(ArgAction::Append), + Arg::new("generator-file") + .long("generator-file") + .help("Name of the generator file") + .action(ArgAction::Set), + Arg::new("problem") + .short('p') + .long("problem") + .help("Problem name (this is not the problem title)") + .action(ArgAction::Set), + ]), + ) .subcommand( Command::new("generate") .about("Generate a test case input with a generator file") @@ -101,10 +94,6 @@ pub fn cli() -> Command { .long("file") .help("Name of the generator file") .action(ArgAction::Set), - Arg::new("lang") - .long("lang") - .help("Language of the generator file (e.g. cpp, py)") - .action(ArgAction::Set), Arg::new("problem") .short('p') .long("problem") @@ -187,38 +176,30 @@ pub fn exec(args: &ArgMatches, settings: &Settings) -> Result<()> { Some(name) => name, None => &get_problem_from_cwd(&problems_dir)?, }; - let generate = (cmd.try_get_one::("generate")?).unwrap_or(&false); - let solution_1 = RunnableFile::new( - settings, - RunnableCategory::Solution, - cmd.try_get_one::("file1")?, - cmd.try_get_one::("lang1")?, - ); + let files: Vec<&String> = cmd + .try_get_many::("file")? + .context("At least two solution files are required for comparison")? + .collect(); - let solution_2 = RunnableFile::new( - settings, - RunnableCategory::Solution, - cmd.try_get_one::("file2")?, - cmd.try_get_one::("lang2")?, - ); + if files.len() < 2 { + bail!("At least two solution files are required for comparison"); + } - let generator = RunnableFile::new( - settings, - RunnableCategory::Generator, - cmd.try_get_one::("generator-file")?, - cmd.try_get_one::("generator-lang")?, - ); + let mut solution_files: Vec = Vec::new(); + for f in files { + let solution_file = + RunnableFile::new(settings, RunnableCategory::Solution, Some(f)); + solution_files.push(solution_file?); + } - compare::compare( - settings, - &problems_dir, - problem_name, - generate, - &solution_1, - &solution_2, - &generator, - )?; + let compare_args = compare::CompareArgs { + problems_dir: &problems_dir, + problem_name: problem_name.to_string(), + solution_files, + }; + + compare::compare(settings, &compare_args)?; } Some(("create", cmd)) => { let problem_name = cmd @@ -231,6 +212,43 @@ pub fn exec(args: &ArgMatches, settings: &Settings) -> Result<()> { create::create(&problems_dir, problem_name, *difficulty)?; } + Some(("fuzz", cmd)) => { + let problem_name = match cmd.try_get_one::("problem")? { + Some(name) => name, + None => &get_problem_from_cwd(&problems_dir)?, + }; + + let files: Vec<&String> = cmd + .try_get_many::("file")? + .context("At least two solution files are required for fuzzing")? + .collect(); + + if files.len() < 2 { + bail!("At least two solution files are required for fuzzing"); + } + + let mut solution_files: Vec = Vec::new(); + for f in files { + let solution_file = + RunnableFile::new(settings, RunnableCategory::Solution, Some(f)); + solution_files.push(solution_file?); + } + + let generator = RunnableFile::new( + settings, + RunnableCategory::Generator, + cmd.try_get_one::("generator-file")?, + )?; + + let fuzz_args = fuzz::FuzzArgs { + problems_dir: &problems_dir, + problem_name: problem_name.to_string(), + solution_files, + generator, + }; + + fuzz::fuzz(settings, &fuzz_args)?; + } Some(("generate", cmd)) => { let problem_name = match cmd.try_get_one::("problem")? { Some(name) => name, @@ -241,8 +259,7 @@ pub fn exec(args: &ArgMatches, settings: &Settings) -> Result<()> { settings, RunnableCategory::Generator, cmd.try_get_one::("file")?, - cmd.try_get_one::("lang")?, - ); + )?; let test_name = cmd .try_get_one::("test-name")? diff --git a/crates/cli/src/problem/compare.rs b/crates/cli/src/problem/compare.rs index ee7cce9..cfea6fb 100644 --- a/crates/cli/src/problem/compare.rs +++ b/crates/cli/src/problem/compare.rs @@ -1,137 +1,106 @@ -use std::fs; use std::path::Path; use std::time::Duration; -use anyhow::{Context, Result}; -use uuid::Uuid; +use anyhow::{bail, Context, Result}; -use super::generate; use super::run::{RunCommand, RunnableFile}; use super::sync_mappings::get_problem; -use crate::util::get_project_root; -use crate::{config::Settings, util::get_input_files_in_directory}; +use crate::config::Settings; +use crate::problem::run::RunResult; +use crate::util::{get_input_files_in_directory, get_project_root}; + +/// Arguments for the compare command. +pub struct CompareArgs<'a> { + pub problems_dir: &'a Path, + pub problem_name: String, + pub solution_files: Vec, +} /// Compare two solutions. -pub fn compare( - settings: &Settings, - problems_dir: &Path, - problem_name: &str, - generate: &bool, - solution_1: &RunnableFile, - solution_2: &RunnableFile, - generator: &RunnableFile, -) -> Result<()> { +pub fn compare(settings: &Settings, compare_args: &CompareArgs) -> Result<()> { let project_root = get_project_root()?; + let CompareArgs { + problems_dir, + problem_name, + solution_files, + } = compare_args; let problem_path = project_root.join(get_problem(problems_dir, problem_name)?); - let run_command_1 = RunCommand::new( - settings, - &problem_path, - solution_1, - problem_path.join("solutions/solution_1.out"), - problem_path.join(format!("{solution_1}")), - )?; - let run_command_2 = RunCommand::new( - settings, - &problem_path, - solution_2, - problem_path.join("solutions/solution_2.out"), - problem_path.join(format!("{solution_2}")), - )?; - - if *generate { - let mut total_tests = 0; - let mut total_time_1: f64 = 0f64; - let mut total_time_2: f64 = 0f64; - - loop { - total_tests += 1; - - let test_name = format!("generated_{}", Uuid::new_v4()); - - generate::generate(settings, problems_dir, problem_name, generator, &test_name) - .context("Failed to generate test case")?; - - let input_file_path = problem_path.join(format!("tests/{}.in", test_name)); - - let result_1 = run_command_1 - .get_result(Some(&input_file_path)) - .context("Failed to get output from solution 1")?; - let result_2 = run_command_2 - .get_result(Some(&input_file_path)) - .context("Failed to get output from solution 2")?; + let mut run_commands: Vec = Vec::new(); + for (i, file) in solution_files.iter().enumerate() { + run_commands.push(RunCommand::new( + settings, + &problem_path, + file, + problem_path.join(format!("solutions/solution_{i}.out")), + problem_path.join(format!("{file}")), + )?); + } - if result_1.output.as_bytes() != result_2.output.as_bytes() { - eprintln!( - " ! Test case {total_tests} (tests/generated.in) failed, time taken: {:.5}s and {:.5}s respectively", - result_1.elapsed_time.as_secs_f64(), - result_2.elapsed_time.as_secs_f64() - ); - break; - } else { - eprintln!( - " + Test case {total_tests} passed, time taken: {:.5}s and {:.5}s respectively", - result_1.elapsed_time.as_secs_f64(), - result_2.elapsed_time.as_secs_f64() - ); - eprintln!( - " Total percentage time difference: {:.5}%", - (total_time_2 - total_time_1) * 100f64 / total_time_1 - ); - fs::remove_file(input_file_path).context("Failed to delete generated test case")?; - } + // If there aren't at least two solutions, we can't compare so return an error + if run_commands.len() < 2 { + bail!("At least two solutions are required to compare."); + } - total_time_1 += result_1.elapsed_time.as_secs_f64(); - total_time_2 += result_2.elapsed_time.as_secs_f64(); - } - } else { - let test_files = get_input_files_in_directory(problem_path.join("tests"))?; + let test_files = get_input_files_in_directory(problem_path.join("tests"))?; - let mut tests_passed = 0; - let mut total_tests = 0; - let mut total_time_1: Duration = Duration::new(0, 0); - let mut total_time_2: Duration = Duration::new(0, 0); + let mut tests_passed = 0; + let mut total_tests = 0; + let mut total_times: Vec = vec![Duration::new(0, 0); run_commands.len()]; - eprintln!("Running the solution files for each test case..."); + eprintln!("Running the solution files for each test case..."); - for test_file in test_files { - let input_file_path = problem_path.join(format!("tests/{}", test_file)); + for test_file in test_files { + let input_file_path = problem_path.join(format!("tests/{}", test_file)); - let result_1 = run_command_1 + let mut results: Vec = Vec::new(); + for (i, run_cmd) in run_commands.iter().enumerate() { + let result = run_cmd .get_result(Some(&input_file_path)) - .context("Failed to get output from solution 1")?; - let result_2 = run_command_2 - .get_result(Some(&input_file_path)) - .context("Failed to get output from solution 2")?; + .context(format!("Failed to get output from solution {i}"))?; + results.push(result); + } - if result_1.output.as_bytes() != result_2.output.as_bytes() { - eprintln!( - " ! Test case failed: {test_file}, time taken: {:.5}s and {:.5}s respectively", - result_1.elapsed_time.as_secs_f64(), - result_2.elapsed_time.as_secs_f64() - ); - } else { + let mut passed = true; + let mut avg_duration = Duration::new(0, 0); + // We verify earlier that there is at least two solutions so indexing 0 is no issue + let result_1 = &results[0]; + for (i, result) in results.iter().enumerate().skip(1) { + // TODO: compare 1st, 2nd and nth result for a "best of three" (if applicable)? + if result_1.output.as_bytes() != result.output.as_bytes() { eprintln!( - " + Test case passed: {test_file}, time taken: {:.5}s and {:.5}s respectively", - result_1.elapsed_time.as_secs_f64(), - result_2.elapsed_time.as_secs_f64() - ); - tests_passed += 1; + " ! Test case failed: {test_file}, solution 0 took {:.5}s, solution {i} took {:.5}s", + result_1.elapsed_time.as_secs_f64(), + result.elapsed_time.as_secs_f64() + ); + passed = false; + break; } - total_tests += 1; - total_time_1 += result_1.elapsed_time; - total_time_2 += result_2.elapsed_time; + avg_duration += result.elapsed_time; + } + + if passed { + eprintln!( + " + Test case passed: {test_file}, average time taken: {:.5}s", + avg_duration.as_secs_f64() / (results.len() as f64) + ); + tests_passed += 1; } - eprintln!( - "{tests_passed} out of {total_tests} test cases passed, time taken: {:.5}s and {:.5}s respectively.", - total_time_1.as_secs_f64(), - total_time_2.as_secs_f64() - ); + total_tests += 1; + for (i, result) in results.iter().enumerate() { + total_times[i] += result.elapsed_time; + } } - run_command_1.cleanup()?; - run_command_2.cleanup()?; + eprintln!("{tests_passed} out of {total_tests} test cases passed"); + for (i, time) in total_times.iter().enumerate() { + eprintln!(" Time taken for solution {i}: {:.5}s", time.as_secs_f64()); + } + + for run_command in run_commands { + run_command.cleanup()?; + } Ok(()) } diff --git a/crates/cli/src/problem/fuzz.rs b/crates/cli/src/problem/fuzz.rs new file mode 100644 index 0000000..6c04013 --- /dev/null +++ b/crates/cli/src/problem/fuzz.rs @@ -0,0 +1,124 @@ +use std::fs; +use std::path::Path; +use std::time::Duration; + +use anyhow::{bail, Context, Result}; +use uuid::Uuid; + +use super::generate; +use super::run::{RunCommand, RunResult, RunnableFile}; +use super::sync_mappings::get_problem; +use crate::{config::Settings, util::get_project_root}; + +pub struct FuzzArgs<'a> { + pub problems_dir: &'a Path, + pub problem_name: String, + pub solution_files: Vec, + pub generator: RunnableFile, +} + +/// Generate new test cases until the solutions produce different results. +pub fn fuzz(settings: &Settings, fuzz_args: &FuzzArgs) -> Result<()> { + let project_root = get_project_root()?; + let FuzzArgs { + problems_dir, + problem_name, + solution_files, + generator, + } = fuzz_args; + let problem_path = project_root.join(get_problem(problems_dir, problem_name)?); + + let mut run_commands: Vec = Vec::new(); + for (i, file) in solution_files.iter().enumerate() { + run_commands.push(RunCommand::new( + settings, + &problem_path, + file, + problem_path.join(format!("solutions/solution_{i}.out")), + problem_path.join(format!("{file}")), + )?); + } + + // If there aren't at least two solutions, we can't compare so return an error + if run_commands.len() < 2 { + bail!("At least two solutions are required for fuzzing."); + } + + let mut total_tests = 0; + let mut total_times: Vec = vec![Duration::new(0, 0); run_commands.len()]; + + // TODO: Set an optional limit? + loop { + total_tests += 1; + + let test_name = format!("generated_{}", Uuid::new_v4()); + + generate::generate(settings, problems_dir, problem_name, generator, &test_name) + .context("Failed to generate test case")?; + + let input_file_path = problem_path.join(format!("tests/{}.in", test_name)); + + let mut results: Vec = Vec::new(); + for (i, run_cmd) in run_commands.iter().enumerate() { + let result = run_cmd + .get_result(Some(&input_file_path)) + .context(format!("Failed to get output from solution {i}"))?; + results.push(result); + } + + let mut passed = true; + let mut avg_duration = Duration::new(0, 0); + // We verify earlier that there is at least two solutions so indexing 0 is no issue + let result_1 = &results[0]; + for (i, result) in results.iter().enumerate().skip(1) { + // TODO: compare 1st, 2nd and nth result for a "best of three" (if applicable)? + if result_1.output.as_bytes() != result.output.as_bytes() { + eprintln!( + " ! Test case {total_tests} (tests/generated.in) failed, solution 0 took {:.5}s, solution {i} took {:.5}s", + result_1.elapsed_time.as_secs_f64(), + result.elapsed_time.as_secs_f64() + ); + passed = false; + break; + } + avg_duration += result.elapsed_time; + } + + if passed { + let max_total_time = total_times + .iter() + .max() + .unwrap_or(&Duration::new(0, 0)) + .to_owned(); + let min_total_time = total_times + .iter() + .min() + .unwrap_or(&Duration::new(0, 0)) + .to_owned(); + + eprintln!( + " + Test case {total_tests} passed, average time taken: {:.5}s", + avg_duration.as_secs_f64() / (results.len() as f64) + ); + eprintln!( + " Total percentage time difference (min, max times): {:.5}%", + (max_total_time.abs_diff(min_total_time)).as_secs_f64() * 100f64 + / min_total_time.as_secs_f64() + ); + + fs::remove_file(input_file_path).context("Failed to delete generated test case")?; + } else { + break; + } + + for (i, result) in results.iter().enumerate() { + total_times[i] += result.elapsed_time; + } + } + + for run_command in run_commands { + run_command.cleanup()?; + } + + Ok(()) +} diff --git a/crates/cli/src/problem/mod.rs b/crates/cli/src/problem/mod.rs index 713cc61..07b08e4 100644 --- a/crates/cli/src/problem/mod.rs +++ b/crates/cli/src/problem/mod.rs @@ -4,6 +4,7 @@ pub mod archive; pub mod check; pub mod compare; pub mod create; +pub mod fuzz; pub mod generate; pub mod run; pub mod solve; diff --git a/crates/cli/src/problem/run.rs b/crates/cli/src/problem/run.rs index bc5d344..47b02ee 100644 --- a/crates/cli/src/problem/run.rs +++ b/crates/cli/src/problem/run.rs @@ -1,13 +1,15 @@ -use anyhow::{bail, Context, Result}; -use normpath::PathExt; use std::ffi::OsStr; use std::fmt; use std::fs::{self, File}; use std::path::{Path, PathBuf}; use std::time::{Duration, Instant}; + +use anyhow::{bail, Context, Result}; +use normpath::PathExt; use subprocess::{Exec, Redirection}; use crate::config::Settings; +use crate::util::get_lang_from_extension; /// Represents the category of a runnable file, either a solution or a generator. #[derive(Eq, PartialEq)] @@ -26,42 +28,55 @@ impl fmt::Display for RunnableCategory { } /// Represents a runnable file, which can be either a solution or a generator. +/// The file must not be a binary file, and is expected to be a script or a +/// source code file pub struct RunnableFile { - pub category: RunnableCategory, - pub name: String, - pub lang: String, + category: RunnableCategory, + name: String, + lang: String, } impl RunnableFile { - /// Sets the name and language if given, otherwise defaults to the category name - /// and the default language from the settings. + /// Sets the file name if given, and infers the language from the file + /// extension. Otherwise defaults to the category name and the default + /// language from the settings. pub fn new( settings: &Settings, category: RunnableCategory, name: Option<&String>, - lang: Option<&String>, - ) -> Self { - Self { - name: name.cloned().unwrap_or(format!("{category}")), - lang: lang - .cloned() - .unwrap_or(if category == RunnableCategory::Solution { - settings.problem.default_lang.clone() - } else { - settings.problem.default_generator_lang.clone() - }), - category, + ) -> Result { + let lang: String; + let filename: String; + if name.is_none() { + lang = match category { + RunnableCategory::Solution => settings.problem.default_lang.clone(), + RunnableCategory::Generator => settings.problem.default_generator_lang.clone(), + }; + filename = format!("{category}.{lang}"); + } else { + lang = + get_lang_from_extension(name.context("Failed to get filename of runnable file")?)?; + filename = name.unwrap().to_string(); } + + Ok(Self { + name: filename, + lang, + category, + }) } } impl fmt::Display for RunnableFile { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}s/{}.{}", self.category, self.name, self.lang) + write!(f, "{}s/{}", self.category, self.name) } } /// Represents a command to run a solution or generator file. +// TODO: Technically it wouldn't really be correct to have a "script_file" +// if the file is only compiled, so we should probably make bin_file and +// script_file mutually exclusive pub struct RunCommand { bin_file: PathBuf, script_file: PathBuf, diff --git a/crates/cli/src/util.rs b/crates/cli/src/util.rs index 24d43a3..0fc36f8 100644 --- a/crates/cli/src/util.rs +++ b/crates/cli/src/util.rs @@ -21,7 +21,11 @@ pub fn get_project_root() -> Result { /// Get the name of the problem from the current working directory if the /// directory is a valid problem folder. pub fn get_problem_from_cwd(problems_dir: &Path) -> Result { + let project_root = get_project_root()?; let path = std::env::current_dir()?; + if project_root == path { + bail!("You are in the project root directory. Please navigate to a problem directory"); + } let problem_name = path .file_name() .context("Failed to get problem name")? @@ -67,3 +71,13 @@ pub fn is_file_empty>(path: P) -> Result { let metadata = fs::metadata(path)?; Ok(metadata.len() == 0) } + +pub fn get_lang_from_extension>(path: P) -> Result { + let lang = path + .as_ref() + .extension() + .and_then(|s| s.to_str()) + .context("Failed to get file extension")? + .to_string(); + Ok(lang) +}