From 4b9348d6a46e9c77b09cbd7266faef6893b12aa9 Mon Sep 17 00:00:00 2001 From: Ray Okamoto Date: Thu, 19 Jun 2025 04:01:39 +0930 Subject: [PATCH 1/5] feat: allow two or more solution files to be compared --- crates/cli/src/cli/problem.rs | 71 +++++------ crates/cli/src/problem/compare.rs | 200 ++++++++++++++++++------------ crates/cli/src/problem/run.rs | 3 + crates/cli/src/util.rs | 10 ++ 4 files changed, 169 insertions(+), 115 deletions(-) diff --git a/crates/cli/src/cli/problem.rs b/crates/cli/src/cli/problem.rs index 5953f85..986c560 100644 --- a/crates/cli/src/cli/problem.rs +++ b/crates/cli/src/cli/problem.rs @@ -1,6 +1,6 @@ use std::fs; -use anyhow::{anyhow, Context, Result}; +use anyhow::{anyhow, bail, Context, Result}; use clap::{value_parser, Arg, ArgAction, ArgMatches, Command}; use crate::config::Settings; @@ -9,7 +9,7 @@ use crate::problem::{ run::{RunnableCategory, RunnableFile}, solve, test, }; -use crate::util::{get_problem_from_cwd, get_project_root}; +use crate::util::{get_lang_from_extension, get_problem_from_cwd, get_project_root}; pub fn cli() -> Command { Command::new("problem") @@ -38,27 +38,15 @@ pub fn cli() -> Command { ) .subcommand( Command::new("compare") - .about("Compare two solutions") + .about("Compare two or more solutions") .args([ Arg::new("generate") .long("generate") .help("Generate new test cases until the solutions produce different results") .action(ArgAction::SetTrue), - Arg::new("file1") - .long("file1") - .help("Name of the first solution file") - .action(ArgAction::Set), - Arg::new("lang1") - .long("lang1") - .help("Language of the first solution file (e.g. cpp, py)") - .action(ArgAction::Set), - Arg::new("file2") - .long("file2") - .help("Name of the second solution file") - .action(ArgAction::Set), - Arg::new("lang2") - .long("lang2") - .help("Language of the second solution file (e.g. cpp, py)") + Arg::new("file") + .long("file") + .help("Name of the solution file") .action(ArgAction::Set), Arg::new("generator-file") .long("generator-file") @@ -189,19 +177,25 @@ pub fn exec(args: &ArgMatches, settings: &Settings) -> Result<()> { }; let generate = (cmd.try_get_one::("generate")?).unwrap_or(&false); - let solution_1 = RunnableFile::new( - settings, - RunnableCategory::Solution, - cmd.try_get_one::("file1")?, - cmd.try_get_one::("lang1")?, - ); + let files: Vec<&String> = cmd + .get_many::("file") + .context("At least two solution files are required for comparison")? + .collect(); - let solution_2 = RunnableFile::new( - settings, - RunnableCategory::Solution, - cmd.try_get_one::("file2")?, - cmd.try_get_one::("lang2")?, - ); + if files.len() < 2 { + bail!("At least two solution files are required for comparison"); + } + + let mut solution_files: Vec = Vec::new(); + for f in files { + let solution_file = RunnableFile::new( + settings, + RunnableCategory::Solution, + Some(f), + Some(&get_lang_from_extension(f)?), + ); + solution_files.push(solution_file); + } let generator = RunnableFile::new( settings, @@ -210,15 +204,14 @@ pub fn exec(args: &ArgMatches, settings: &Settings) -> Result<()> { cmd.try_get_one::("generator-lang")?, ); - compare::compare( - settings, - &problems_dir, - problem_name, - generate, - &solution_1, - &solution_2, - &generator, - )?; + let compare_args = compare::CompareArgs { + problems_dir: &problems_dir, + problem_name: problem_name.to_string(), + solution_files, + generator, + }; + + compare::compare(settings, generate, &compare_args)?; } Some(("create", cmd)) => { let problem_name = cmd diff --git a/crates/cli/src/problem/compare.rs b/crates/cli/src/problem/compare.rs index ee7cce9..7e6b9cc 100644 --- a/crates/cli/src/problem/compare.rs +++ b/crates/cli/src/problem/compare.rs @@ -2,48 +2,56 @@ use std::fs; use std::path::Path; use std::time::Duration; -use anyhow::{Context, Result}; +use anyhow::{bail, Context, Result}; use uuid::Uuid; use super::generate; use super::run::{RunCommand, RunnableFile}; use super::sync_mappings::get_problem; -use crate::util::get_project_root; -use crate::{config::Settings, util::get_input_files_in_directory}; +use crate::config::Settings; +use crate::problem::run::RunResult; +use crate::util::{get_input_files_in_directory, get_project_root}; + +/// Arguments for the compare command. +pub struct CompareArgs<'a> { + pub problems_dir: &'a Path, + pub problem_name: String, + pub solution_files: Vec, + pub generator: RunnableFile, +} /// Compare two solutions. -pub fn compare( - settings: &Settings, - problems_dir: &Path, - problem_name: &str, - generate: &bool, - solution_1: &RunnableFile, - solution_2: &RunnableFile, - generator: &RunnableFile, -) -> Result<()> { +pub fn compare(settings: &Settings, generate: &bool, compare_args: &CompareArgs) -> Result<()> { let project_root = get_project_root()?; + let CompareArgs { + problems_dir, + problem_name, + generator, + solution_files, + } = compare_args; let problem_path = project_root.join(get_problem(problems_dir, problem_name)?); - let run_command_1 = RunCommand::new( - settings, - &problem_path, - solution_1, - problem_path.join("solutions/solution_1.out"), - problem_path.join(format!("{solution_1}")), - )?; - let run_command_2 = RunCommand::new( - settings, - &problem_path, - solution_2, - problem_path.join("solutions/solution_2.out"), - problem_path.join(format!("{solution_2}")), - )?; + let mut run_commands: Vec = Vec::new(); + for (i, file) in solution_files.iter().enumerate() { + run_commands.push(RunCommand::new( + settings, + &problem_path, + file, + problem_path.join(format!("solutions/solution_{i}.out")), + problem_path.join(format!("{file}")), + )?); + } + + // If there aren't at least two solutions, we can't compare so return an error + if run_commands.len() < 2 { + bail!("At least two solutions are required to compare."); + } if *generate { let mut total_tests = 0; - let mut total_time_1: f64 = 0f64; - let mut total_time_2: f64 = 0f64; + let mut total_times: Vec = vec![Duration::new(0, 0); run_commands.len()]; + // TODO: Set an optional limit? loop { total_tests += 1; @@ -54,84 +62,124 @@ pub fn compare( let input_file_path = problem_path.join(format!("tests/{}.in", test_name)); - let result_1 = run_command_1 - .get_result(Some(&input_file_path)) - .context("Failed to get output from solution 1")?; - let result_2 = run_command_2 - .get_result(Some(&input_file_path)) - .context("Failed to get output from solution 2")?; + let mut results: Vec = Vec::new(); + for (i, run_cmd) in run_commands.iter().enumerate() { + let result = run_cmd + .get_result(Some(&input_file_path)) + .context(format!("Failed to get output from solution {i}"))?; + results.push(result); + } + + let mut passed = true; + let mut avg_duration = Duration::new(0, 0); + // We verify earlier that there is at least two solutions so indexing 0 is no issue + let result_1 = &results[0]; + for (i, result) in results.iter().enumerate().skip(1) { + // TODO: compare 1st, 2nd and nth result for a "best of three" (if applicable)? + if result_1.output.as_bytes() != result.output.as_bytes() { + eprintln!( + " ! Test case {total_tests} (tests/generated.in) failed, solution 1 took {:.5}s, solution {i} took {:.5}s", + result_1.elapsed_time.as_secs_f64(), + result.elapsed_time.as_secs_f64() + ); + passed = false; + break; + } + avg_duration += result.elapsed_time; + } + + if passed { + let max_total_time = total_times + .iter() + .max() + .unwrap_or(&Duration::new(0, 0)) + .to_owned(); + let min_total_time = total_times + .iter() + .min() + .unwrap_or(&Duration::new(0, 0)) + .to_owned(); - if result_1.output.as_bytes() != result_2.output.as_bytes() { - eprintln!( - " ! Test case {total_tests} (tests/generated.in) failed, time taken: {:.5}s and {:.5}s respectively", - result_1.elapsed_time.as_secs_f64(), - result_2.elapsed_time.as_secs_f64() - ); - break; - } else { eprintln!( - " + Test case {total_tests} passed, time taken: {:.5}s and {:.5}s respectively", - result_1.elapsed_time.as_secs_f64(), - result_2.elapsed_time.as_secs_f64() + " + Test case {total_tests} passed, average time taken: {:.5}s", + avg_duration.as_secs_f64() / (results.len() as f64) ); eprintln!( - " Total percentage time difference: {:.5}%", - (total_time_2 - total_time_1) * 100f64 / total_time_1 + " Total percentage time difference (min, max times): {:.5}%", + (max_total_time.abs_diff(min_total_time)).as_secs_f64() * 100f64 + / min_total_time.as_secs_f64() ); + fs::remove_file(input_file_path).context("Failed to delete generated test case")?; + } else { + break; } - total_time_1 += result_1.elapsed_time.as_secs_f64(); - total_time_2 += result_2.elapsed_time.as_secs_f64(); + for (i, result) in results.iter().enumerate() { + total_times[i] += result.elapsed_time; + } } } else { let test_files = get_input_files_in_directory(problem_path.join("tests"))?; let mut tests_passed = 0; let mut total_tests = 0; - let mut total_time_1: Duration = Duration::new(0, 0); - let mut total_time_2: Duration = Duration::new(0, 0); + let mut total_times: Vec = vec![Duration::new(0, 0); run_commands.len()]; eprintln!("Running the solution files for each test case..."); for test_file in test_files { let input_file_path = problem_path.join(format!("tests/{}", test_file)); - let result_1 = run_command_1 - .get_result(Some(&input_file_path)) - .context("Failed to get output from solution 1")?; - let result_2 = run_command_2 - .get_result(Some(&input_file_path)) - .context("Failed to get output from solution 2")?; + let mut results: Vec = Vec::new(); + for (i, run_cmd) in run_commands.iter().enumerate() { + let result = run_cmd + .get_result(Some(&input_file_path)) + .context(format!("Failed to get output from solution {i}"))?; + results.push(result); + } - if result_1.output.as_bytes() != result_2.output.as_bytes() { - eprintln!( - " ! Test case failed: {test_file}, time taken: {:.5}s and {:.5}s respectively", - result_1.elapsed_time.as_secs_f64(), - result_2.elapsed_time.as_secs_f64() - ); - } else { + let mut passed = true; + let mut avg_duration = Duration::new(0, 0); + // We verify earlier that there is at least two solutions so indexing 0 is no issue + let result_1 = &results[0]; + for (i, result) in results.iter().enumerate().skip(1) { + // TODO: compare 1st, 2nd and nth result for a "best of three" (if applicable)? + if result_1.output.as_bytes() != result.output.as_bytes() { + eprintln!( + " ! Test case failed: {test_file}, solution 1 took {:.5}s, solution {i} took {:.5}s", + result_1.elapsed_time.as_secs_f64(), + result.elapsed_time.as_secs_f64() + ); + passed = false; + break; + } + avg_duration += result.elapsed_time; + } + + if passed { eprintln!( - " + Test case passed: {test_file}, time taken: {:.5}s and {:.5}s respectively", - result_1.elapsed_time.as_secs_f64(), - result_2.elapsed_time.as_secs_f64() + " + Test case passed: {test_file}, average time taken: {:.5}s", + avg_duration.as_secs_f64() / (results.len() as f64) ); tests_passed += 1; } + total_tests += 1; - total_time_1 += result_1.elapsed_time; - total_time_2 += result_2.elapsed_time; + for (i, result) in results.iter().enumerate() { + total_times[i] += result.elapsed_time; + } } - eprintln!( - "{tests_passed} out of {total_tests} test cases passed, time taken: {:.5}s and {:.5}s respectively.", - total_time_1.as_secs_f64(), - total_time_2.as_secs_f64() - ); + eprintln!("{tests_passed} out of {total_tests} test cases passed"); + for (i, time) in total_times.iter().enumerate() { + eprintln!(" Time taken for solution {i}: {:.5}s", time.as_secs_f64()); + } } - run_command_1.cleanup()?; - run_command_2.cleanup()?; + for run_command in run_commands { + run_command.cleanup()?; + } Ok(()) } diff --git a/crates/cli/src/problem/run.rs b/crates/cli/src/problem/run.rs index bc5d344..cfa002f 100644 --- a/crates/cli/src/problem/run.rs +++ b/crates/cli/src/problem/run.rs @@ -62,6 +62,9 @@ impl fmt::Display for RunnableFile { } /// Represents a command to run a solution or generator file. +// TODO: Technically it wouldn't really be correct to have a "script_file" +// if the file is only compiled, so we should probably make bin_file and +// script_file mutually exclusive pub struct RunCommand { bin_file: PathBuf, script_file: PathBuf, diff --git a/crates/cli/src/util.rs b/crates/cli/src/util.rs index 24d43a3..e13402d 100644 --- a/crates/cli/src/util.rs +++ b/crates/cli/src/util.rs @@ -67,3 +67,13 @@ pub fn is_file_empty>(path: P) -> Result { let metadata = fs::metadata(path)?; Ok(metadata.len() == 0) } + +pub fn get_lang_from_extension>(path: P) -> Result { + let lang = path + .as_ref() + .extension() + .and_then(|s| s.to_str()) + .context("Failed to get file extension")? + .to_string(); + Ok(lang) +} From 314ec0feb126216dd0a5a6e44babbfdcf782be46 Mon Sep 17 00:00:00 2001 From: Ray Okamoto Date: Thu, 19 Jun 2025 16:00:11 +0930 Subject: [PATCH 2/5] feat: improve error reporting for cwd problems --- crates/cli/src/util.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/cli/src/util.rs b/crates/cli/src/util.rs index e13402d..0fc36f8 100644 --- a/crates/cli/src/util.rs +++ b/crates/cli/src/util.rs @@ -21,7 +21,11 @@ pub fn get_project_root() -> Result { /// Get the name of the problem from the current working directory if the /// directory is a valid problem folder. pub fn get_problem_from_cwd(problems_dir: &Path) -> Result { + let project_root = get_project_root()?; let path = std::env::current_dir()?; + if project_root == path { + bail!("You are in the project root directory. Please navigate to a problem directory"); + } let problem_name = path .file_name() .context("Failed to get problem name")? From 374ac12ea754684f7feb616da1a1963011ca09a0 Mon Sep 17 00:00:00 2001 From: Ray Okamoto Date: Thu, 19 Jun 2025 16:11:47 +0930 Subject: [PATCH 3/5] feat!: add problem fuzz command This is a breaking change that moves the `generate` functionality of the problem compare command over to problem fuzz. --- crates/cli/src/cli/problem.rs | 91 ++++++++++++----- crates/cli/src/problem/compare.rs | 159 ++++++++---------------------- crates/cli/src/problem/fuzz.rs | 124 +++++++++++++++++++++++ crates/cli/src/problem/mod.rs | 1 + 4 files changed, 233 insertions(+), 142 deletions(-) create mode 100644 crates/cli/src/problem/fuzz.rs diff --git a/crates/cli/src/cli/problem.rs b/crates/cli/src/cli/problem.rs index 986c560..6ee2eab 100644 --- a/crates/cli/src/cli/problem.rs +++ b/crates/cli/src/cli/problem.rs @@ -4,6 +4,7 @@ use anyhow::{anyhow, bail, Context, Result}; use clap::{value_parser, Arg, ArgAction, ArgMatches, Command}; use crate::config::Settings; +use crate::problem::fuzz; use crate::problem::{ archive, check, compare, create, generate, run::{RunnableCategory, RunnableFile}, @@ -40,22 +41,10 @@ pub fn cli() -> Command { Command::new("compare") .about("Compare two or more solutions") .args([ - Arg::new("generate") - .long("generate") - .help("Generate new test cases until the solutions produce different results") - .action(ArgAction::SetTrue), Arg::new("file") .long("file") .help("Name of the solution file") .action(ArgAction::Set), - Arg::new("generator-file") - .long("generator-file") - .help("Name of the generator file") - .action(ArgAction::Set), - Arg::new("generator-lang") - .long("generator-lang") - .help("Language of the generator file (e.g. cpp, py)") - .action(ArgAction::Set), Arg::new("problem") .short('p') .long("problem") @@ -81,6 +70,29 @@ pub fn cli() -> Command { .required(true), ), ) + .subcommand( + Command::new("fuzz") + .about("Find potential edge cases in two or more solutions") + .args([ + Arg::new("file") + .long("file") + .help("Name of the solution file") + .action(ArgAction::Set), + Arg::new("generator-file") + .long("generator-file") + .help("Name of the generator file") + .action(ArgAction::Set), + Arg::new("generator-lang") + .long("generator-lang") + .help("Language of the generator file (e.g. cpp, py)") + .action(ArgAction::Set), + Arg::new("problem") + .short('p') + .long("problem") + .help("Problem name (this is not the problem title)") + .action(ArgAction::Set), + ]), + ) .subcommand( Command::new("generate") .about("Generate a test case input with a generator file") @@ -175,10 +187,9 @@ pub fn exec(args: &ArgMatches, settings: &Settings) -> Result<()> { Some(name) => name, None => &get_problem_from_cwd(&problems_dir)?, }; - let generate = (cmd.try_get_one::("generate")?).unwrap_or(&false); let files: Vec<&String> = cmd - .get_many::("file") + .try_get_many::("file")? .context("At least two solution files are required for comparison")? .collect(); @@ -197,21 +208,13 @@ pub fn exec(args: &ArgMatches, settings: &Settings) -> Result<()> { solution_files.push(solution_file); } - let generator = RunnableFile::new( - settings, - RunnableCategory::Generator, - cmd.try_get_one::("generator-file")?, - cmd.try_get_one::("generator-lang")?, - ); - let compare_args = compare::CompareArgs { problems_dir: &problems_dir, problem_name: problem_name.to_string(), solution_files, - generator, }; - compare::compare(settings, generate, &compare_args)?; + compare::compare(settings, &compare_args)?; } Some(("create", cmd)) => { let problem_name = cmd @@ -224,6 +227,48 @@ pub fn exec(args: &ArgMatches, settings: &Settings) -> Result<()> { create::create(&problems_dir, problem_name, *difficulty)?; } + Some(("fuzz", cmd)) => { + let problem_name = match cmd.try_get_one::("problem")? { + Some(name) => name, + None => &get_problem_from_cwd(&problems_dir)?, + }; + + let files: Vec<&String> = cmd + .try_get_many::("file")? + .context("At least two solution files are required for fuzzing")? + .collect(); + + if files.len() < 2 { + bail!("At least two solution files are required for fuzzing"); + } + + let mut solution_files: Vec = Vec::new(); + for f in files { + let solution_file = RunnableFile::new( + settings, + RunnableCategory::Solution, + Some(f), + Some(&get_lang_from_extension(f)?), + ); + solution_files.push(solution_file); + } + + let generator = RunnableFile::new( + settings, + RunnableCategory::Generator, + cmd.try_get_one::("generator-file")?, + cmd.try_get_one::("generator-lang")?, + ); + + let fuzz_args = fuzz::FuzzArgs { + problems_dir: &problems_dir, + problem_name: problem_name.to_string(), + solution_files, + generator, + }; + + fuzz::fuzz(settings, &fuzz_args)?; + } Some(("generate", cmd)) => { let problem_name = match cmd.try_get_one::("problem")? { Some(name) => name, diff --git a/crates/cli/src/problem/compare.rs b/crates/cli/src/problem/compare.rs index 7e6b9cc..2365c23 100644 --- a/crates/cli/src/problem/compare.rs +++ b/crates/cli/src/problem/compare.rs @@ -1,11 +1,8 @@ -use std::fs; use std::path::Path; use std::time::Duration; use anyhow::{bail, Context, Result}; -use uuid::Uuid; -use super::generate; use super::run::{RunCommand, RunnableFile}; use super::sync_mappings::get_problem; use crate::config::Settings; @@ -17,16 +14,14 @@ pub struct CompareArgs<'a> { pub problems_dir: &'a Path, pub problem_name: String, pub solution_files: Vec, - pub generator: RunnableFile, } /// Compare two solutions. -pub fn compare(settings: &Settings, generate: &bool, compare_args: &CompareArgs) -> Result<()> { +pub fn compare(settings: &Settings, compare_args: &CompareArgs) -> Result<()> { let project_root = get_project_root()?; let CompareArgs { problems_dir, problem_name, - generator, solution_files, } = compare_args; let problem_path = project_root.join(get_problem(problems_dir, problem_name)?); @@ -47,136 +42,62 @@ pub fn compare(settings: &Settings, generate: &bool, compare_args: &CompareArgs) bail!("At least two solutions are required to compare."); } - if *generate { - let mut total_tests = 0; - let mut total_times: Vec = vec![Duration::new(0, 0); run_commands.len()]; + let test_files = get_input_files_in_directory(problem_path.join("tests"))?; - // TODO: Set an optional limit? - loop { - total_tests += 1; + let mut tests_passed = 0; + let mut total_tests = 0; + let mut total_times: Vec = vec![Duration::new(0, 0); run_commands.len()]; - let test_name = format!("generated_{}", Uuid::new_v4()); + eprintln!("Running the solution files for each test case..."); - generate::generate(settings, problems_dir, problem_name, generator, &test_name) - .context("Failed to generate test case")?; + for test_file in test_files { + let input_file_path = problem_path.join(format!("tests/{}", test_file)); - let input_file_path = problem_path.join(format!("tests/{}.in", test_name)); - - let mut results: Vec = Vec::new(); - for (i, run_cmd) in run_commands.iter().enumerate() { - let result = run_cmd - .get_result(Some(&input_file_path)) - .context(format!("Failed to get output from solution {i}"))?; - results.push(result); - } - - let mut passed = true; - let mut avg_duration = Duration::new(0, 0); - // We verify earlier that there is at least two solutions so indexing 0 is no issue - let result_1 = &results[0]; - for (i, result) in results.iter().enumerate().skip(1) { - // TODO: compare 1st, 2nd and nth result for a "best of three" (if applicable)? - if result_1.output.as_bytes() != result.output.as_bytes() { - eprintln!( - " ! Test case {total_tests} (tests/generated.in) failed, solution 1 took {:.5}s, solution {i} took {:.5}s", - result_1.elapsed_time.as_secs_f64(), - result.elapsed_time.as_secs_f64() - ); - passed = false; - break; - } - avg_duration += result.elapsed_time; - } - - if passed { - let max_total_time = total_times - .iter() - .max() - .unwrap_or(&Duration::new(0, 0)) - .to_owned(); - let min_total_time = total_times - .iter() - .min() - .unwrap_or(&Duration::new(0, 0)) - .to_owned(); - - eprintln!( - " + Test case {total_tests} passed, average time taken: {:.5}s", - avg_duration.as_secs_f64() / (results.len() as f64) - ); - eprintln!( - " Total percentage time difference (min, max times): {:.5}%", - (max_total_time.abs_diff(min_total_time)).as_secs_f64() * 100f64 - / min_total_time.as_secs_f64() - ); - - fs::remove_file(input_file_path).context("Failed to delete generated test case")?; - } else { - break; - } - - for (i, result) in results.iter().enumerate() { - total_times[i] += result.elapsed_time; - } + let mut results: Vec = Vec::new(); + for (i, run_cmd) in run_commands.iter().enumerate() { + let result = run_cmd + .get_result(Some(&input_file_path)) + .context(format!("Failed to get output from solution {i}"))?; + results.push(result); } - } else { - let test_files = get_input_files_in_directory(problem_path.join("tests"))?; - - let mut tests_passed = 0; - let mut total_tests = 0; - let mut total_times: Vec = vec![Duration::new(0, 0); run_commands.len()]; - - eprintln!("Running the solution files for each test case..."); - - for test_file in test_files { - let input_file_path = problem_path.join(format!("tests/{}", test_file)); - let mut results: Vec = Vec::new(); - for (i, run_cmd) in run_commands.iter().enumerate() { - let result = run_cmd - .get_result(Some(&input_file_path)) - .context(format!("Failed to get output from solution {i}"))?; - results.push(result); - } - - let mut passed = true; - let mut avg_duration = Duration::new(0, 0); - // We verify earlier that there is at least two solutions so indexing 0 is no issue - let result_1 = &results[0]; - for (i, result) in results.iter().enumerate().skip(1) { - // TODO: compare 1st, 2nd and nth result for a "best of three" (if applicable)? - if result_1.output.as_bytes() != result.output.as_bytes() { - eprintln!( + let mut passed = true; + let mut avg_duration = Duration::new(0, 0); + // We verify earlier that there is at least two solutions so indexing 0 is no issue + let result_1 = &results[0]; + for (i, result) in results.iter().enumerate().skip(1) { + // TODO: compare 1st, 2nd and nth result for a "best of three" (if applicable)? + if result_1.output.as_bytes() != result.output.as_bytes() { + eprintln!( " ! Test case failed: {test_file}, solution 1 took {:.5}s, solution {i} took {:.5}s", result_1.elapsed_time.as_secs_f64(), result.elapsed_time.as_secs_f64() ); - passed = false; - break; - } - avg_duration += result.elapsed_time; - } - - if passed { - eprintln!( - " + Test case passed: {test_file}, average time taken: {:.5}s", - avg_duration.as_secs_f64() / (results.len() as f64) - ); - tests_passed += 1; + passed = false; + break; } + avg_duration += result.elapsed_time; + } - total_tests += 1; - for (i, result) in results.iter().enumerate() { - total_times[i] += result.elapsed_time; - } + if passed { + eprintln!( + " + Test case passed: {test_file}, average time taken: {:.5}s", + avg_duration.as_secs_f64() / (results.len() as f64) + ); + tests_passed += 1; } - eprintln!("{tests_passed} out of {total_tests} test cases passed"); - for (i, time) in total_times.iter().enumerate() { - eprintln!(" Time taken for solution {i}: {:.5}s", time.as_secs_f64()); + total_tests += 1; + for (i, result) in results.iter().enumerate() { + total_times[i] += result.elapsed_time; } } + eprintln!("{tests_passed} out of {total_tests} test cases passed"); + for (i, time) in total_times.iter().enumerate() { + eprintln!(" Time taken for solution {i}: {:.5}s", time.as_secs_f64()); + } + for run_command in run_commands { run_command.cleanup()?; } diff --git a/crates/cli/src/problem/fuzz.rs b/crates/cli/src/problem/fuzz.rs new file mode 100644 index 0000000..6eaad49 --- /dev/null +++ b/crates/cli/src/problem/fuzz.rs @@ -0,0 +1,124 @@ +use std::fs; +use std::path::Path; +use std::time::Duration; + +use anyhow::{bail, Context, Result}; +use uuid::Uuid; + +use super::generate; +use super::run::{RunCommand, RunResult, RunnableFile}; +use super::sync_mappings::get_problem; +use crate::{config::Settings, util::get_project_root}; + +pub struct FuzzArgs<'a> { + pub problems_dir: &'a Path, + pub problem_name: String, + pub solution_files: Vec, + pub generator: RunnableFile, +} + +/// Generate new test cases until the solutions produce different results. +pub fn fuzz(settings: &Settings, fuzz_args: &FuzzArgs) -> Result<()> { + let project_root = get_project_root()?; + let FuzzArgs { + problems_dir, + problem_name, + solution_files, + generator, + } = fuzz_args; + let problem_path = project_root.join(get_problem(problems_dir, problem_name)?); + + let mut run_commands: Vec = Vec::new(); + for (i, file) in solution_files.iter().enumerate() { + run_commands.push(RunCommand::new( + settings, + &problem_path, + file, + problem_path.join(format!("solutions/solution_{i}.out")), + problem_path.join(format!("{file}")), + )?); + } + + // If there aren't at least two solutions, we can't compare so return an error + if run_commands.len() < 2 { + bail!("At least two solutions are required for fuzzing."); + } + + let mut total_tests = 0; + let mut total_times: Vec = vec![Duration::new(0, 0); run_commands.len()]; + + // TODO: Set an optional limit? + loop { + total_tests += 1; + + let test_name = format!("generated_{}", Uuid::new_v4()); + + generate::generate(settings, problems_dir, problem_name, generator, &test_name) + .context("Failed to generate test case")?; + + let input_file_path = problem_path.join(format!("tests/{}.in", test_name)); + + let mut results: Vec = Vec::new(); + for (i, run_cmd) in run_commands.iter().enumerate() { + let result = run_cmd + .get_result(Some(&input_file_path)) + .context(format!("Failed to get output from solution {i}"))?; + results.push(result); + } + + let mut passed = true; + let mut avg_duration = Duration::new(0, 0); + // We verify earlier that there is at least two solutions so indexing 0 is no issue + let result_1 = &results[0]; + for (i, result) in results.iter().enumerate().skip(1) { + // TODO: compare 1st, 2nd and nth result for a "best of three" (if applicable)? + if result_1.output.as_bytes() != result.output.as_bytes() { + eprintln!( + " ! Test case {total_tests} (tests/generated.in) failed, solution 1 took {:.5}s, solution {i} took {:.5}s", + result_1.elapsed_time.as_secs_f64(), + result.elapsed_time.as_secs_f64() + ); + passed = false; + break; + } + avg_duration += result.elapsed_time; + } + + if passed { + let max_total_time = total_times + .iter() + .max() + .unwrap_or(&Duration::new(0, 0)) + .to_owned(); + let min_total_time = total_times + .iter() + .min() + .unwrap_or(&Duration::new(0, 0)) + .to_owned(); + + eprintln!( + " + Test case {total_tests} passed, average time taken: {:.5}s", + avg_duration.as_secs_f64() / (results.len() as f64) + ); + eprintln!( + " Total percentage time difference (min, max times): {:.5}%", + (max_total_time.abs_diff(min_total_time)).as_secs_f64() * 100f64 + / min_total_time.as_secs_f64() + ); + + fs::remove_file(input_file_path).context("Failed to delete generated test case")?; + } else { + break; + } + + for (i, result) in results.iter().enumerate() { + total_times[i] += result.elapsed_time; + } + } + + for run_command in run_commands { + run_command.cleanup()?; + } + + Ok(()) +} diff --git a/crates/cli/src/problem/mod.rs b/crates/cli/src/problem/mod.rs index 713cc61..07b08e4 100644 --- a/crates/cli/src/problem/mod.rs +++ b/crates/cli/src/problem/mod.rs @@ -4,6 +4,7 @@ pub mod archive; pub mod check; pub mod compare; pub mod create; +pub mod fuzz; pub mod generate; pub mod run; pub mod solve; From d998a4515500d134eca48837172528e0c7955ca5 Mon Sep 17 00:00:00 2001 From: Ray Okamoto Date: Fri, 20 Jun 2025 16:51:30 +0930 Subject: [PATCH 4/5] feat: infer lang from file name in `RunnableFile::new` --- crates/cli/src/cli/problem.rs | 47 +++++++++---------------------- crates/cli/src/problem/run.rs | 52 +++++++++++++++++++++-------------- 2 files changed, 45 insertions(+), 54 deletions(-) diff --git a/crates/cli/src/cli/problem.rs b/crates/cli/src/cli/problem.rs index 6ee2eab..f2bdcd7 100644 --- a/crates/cli/src/cli/problem.rs +++ b/crates/cli/src/cli/problem.rs @@ -5,12 +5,9 @@ use clap::{value_parser, Arg, ArgAction, ArgMatches, Command}; use crate::config::Settings; use crate::problem::fuzz; -use crate::problem::{ - archive, check, compare, create, generate, - run::{RunnableCategory, RunnableFile}, - solve, test, -}; -use crate::util::{get_lang_from_extension, get_problem_from_cwd, get_project_root}; +use crate::problem::run::{RunnableCategory, RunnableFile}; +use crate::problem::{archive, check, compare, create, generate, solve, test}; +use crate::util::{get_problem_from_cwd, get_project_root}; pub fn cli() -> Command { Command::new("problem") @@ -44,7 +41,7 @@ pub fn cli() -> Command { Arg::new("file") .long("file") .help("Name of the solution file") - .action(ArgAction::Set), + .action(ArgAction::Append), Arg::new("problem") .short('p') .long("problem") @@ -77,15 +74,11 @@ pub fn cli() -> Command { Arg::new("file") .long("file") .help("Name of the solution file") - .action(ArgAction::Set), + .action(ArgAction::Append), Arg::new("generator-file") .long("generator-file") .help("Name of the generator file") .action(ArgAction::Set), - Arg::new("generator-lang") - .long("generator-lang") - .help("Language of the generator file (e.g. cpp, py)") - .action(ArgAction::Set), Arg::new("problem") .short('p') .long("problem") @@ -101,10 +94,6 @@ pub fn cli() -> Command { .long("file") .help("Name of the generator file") .action(ArgAction::Set), - Arg::new("lang") - .long("lang") - .help("Language of the generator file (e.g. cpp, py)") - .action(ArgAction::Set), Arg::new("problem") .short('p') .long("problem") @@ -199,13 +188,9 @@ pub fn exec(args: &ArgMatches, settings: &Settings) -> Result<()> { let mut solution_files: Vec = Vec::new(); for f in files { - let solution_file = RunnableFile::new( - settings, - RunnableCategory::Solution, - Some(f), - Some(&get_lang_from_extension(f)?), - ); - solution_files.push(solution_file); + let solution_file = + RunnableFile::new(settings, RunnableCategory::Solution, Some(f)); + solution_files.push(solution_file?); } let compare_args = compare::CompareArgs { @@ -244,21 +229,16 @@ pub fn exec(args: &ArgMatches, settings: &Settings) -> Result<()> { let mut solution_files: Vec = Vec::new(); for f in files { - let solution_file = RunnableFile::new( - settings, - RunnableCategory::Solution, - Some(f), - Some(&get_lang_from_extension(f)?), - ); - solution_files.push(solution_file); + let solution_file = + RunnableFile::new(settings, RunnableCategory::Solution, Some(f)); + solution_files.push(solution_file?); } let generator = RunnableFile::new( settings, RunnableCategory::Generator, cmd.try_get_one::("generator-file")?, - cmd.try_get_one::("generator-lang")?, - ); + )?; let fuzz_args = fuzz::FuzzArgs { problems_dir: &problems_dir, @@ -279,8 +259,7 @@ pub fn exec(args: &ArgMatches, settings: &Settings) -> Result<()> { settings, RunnableCategory::Generator, cmd.try_get_one::("file")?, - cmd.try_get_one::("lang")?, - ); + )?; let test_name = cmd .try_get_one::("test-name")? diff --git a/crates/cli/src/problem/run.rs b/crates/cli/src/problem/run.rs index cfa002f..47b02ee 100644 --- a/crates/cli/src/problem/run.rs +++ b/crates/cli/src/problem/run.rs @@ -1,13 +1,15 @@ -use anyhow::{bail, Context, Result}; -use normpath::PathExt; use std::ffi::OsStr; use std::fmt; use std::fs::{self, File}; use std::path::{Path, PathBuf}; use std::time::{Duration, Instant}; + +use anyhow::{bail, Context, Result}; +use normpath::PathExt; use subprocess::{Exec, Redirection}; use crate::config::Settings; +use crate::util::get_lang_from_extension; /// Represents the category of a runnable file, either a solution or a generator. #[derive(Eq, PartialEq)] @@ -26,38 +28,48 @@ impl fmt::Display for RunnableCategory { } /// Represents a runnable file, which can be either a solution or a generator. +/// The file must not be a binary file, and is expected to be a script or a +/// source code file pub struct RunnableFile { - pub category: RunnableCategory, - pub name: String, - pub lang: String, + category: RunnableCategory, + name: String, + lang: String, } impl RunnableFile { - /// Sets the name and language if given, otherwise defaults to the category name - /// and the default language from the settings. + /// Sets the file name if given, and infers the language from the file + /// extension. Otherwise defaults to the category name and the default + /// language from the settings. pub fn new( settings: &Settings, category: RunnableCategory, name: Option<&String>, - lang: Option<&String>, - ) -> Self { - Self { - name: name.cloned().unwrap_or(format!("{category}")), - lang: lang - .cloned() - .unwrap_or(if category == RunnableCategory::Solution { - settings.problem.default_lang.clone() - } else { - settings.problem.default_generator_lang.clone() - }), - category, + ) -> Result { + let lang: String; + let filename: String; + if name.is_none() { + lang = match category { + RunnableCategory::Solution => settings.problem.default_lang.clone(), + RunnableCategory::Generator => settings.problem.default_generator_lang.clone(), + }; + filename = format!("{category}.{lang}"); + } else { + lang = + get_lang_from_extension(name.context("Failed to get filename of runnable file")?)?; + filename = name.unwrap().to_string(); } + + Ok(Self { + name: filename, + lang, + category, + }) } } impl fmt::Display for RunnableFile { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}s/{}.{}", self.category, self.name, self.lang) + write!(f, "{}s/{}", self.category, self.name) } } From 546cfe993c63080e9d171753ff005ede63cee8b1 Mon Sep 17 00:00:00 2001 From: Ray Okamoto Date: Sat, 21 Jun 2025 01:12:57 +0930 Subject: [PATCH 5/5] fix(problem): use 0-indexing for solution names --- crates/cli/src/problem/compare.rs | 2 +- crates/cli/src/problem/fuzz.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/cli/src/problem/compare.rs b/crates/cli/src/problem/compare.rs index 2365c23..cfea6fb 100644 --- a/crates/cli/src/problem/compare.rs +++ b/crates/cli/src/problem/compare.rs @@ -69,7 +69,7 @@ pub fn compare(settings: &Settings, compare_args: &CompareArgs) -> Result<()> { // TODO: compare 1st, 2nd and nth result for a "best of three" (if applicable)? if result_1.output.as_bytes() != result.output.as_bytes() { eprintln!( - " ! Test case failed: {test_file}, solution 1 took {:.5}s, solution {i} took {:.5}s", + " ! Test case failed: {test_file}, solution 0 took {:.5}s, solution {i} took {:.5}s", result_1.elapsed_time.as_secs_f64(), result.elapsed_time.as_secs_f64() ); diff --git a/crates/cli/src/problem/fuzz.rs b/crates/cli/src/problem/fuzz.rs index 6eaad49..6c04013 100644 --- a/crates/cli/src/problem/fuzz.rs +++ b/crates/cli/src/problem/fuzz.rs @@ -74,7 +74,7 @@ pub fn fuzz(settings: &Settings, fuzz_args: &FuzzArgs) -> Result<()> { // TODO: compare 1st, 2nd and nth result for a "best of three" (if applicable)? if result_1.output.as_bytes() != result.output.as_bytes() { eprintln!( - " ! Test case {total_tests} (tests/generated.in) failed, solution 1 took {:.5}s, solution {i} took {:.5}s", + " ! Test case {total_tests} (tests/generated.in) failed, solution 0 took {:.5}s, solution {i} took {:.5}s", result_1.elapsed_time.as_secs_f64(), result.elapsed_time.as_secs_f64() );