From a4814acfa668f6d5e12c62d62418b9d2c35f6c41 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 18 Jul 2025 05:07:27 +0000 Subject: [PATCH 1/4] Initial plan From 9fec44876ff6a8580045f79be67035aa622fa768 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 18 Jul 2025 05:13:48 +0000 Subject: [PATCH 2/4] Add consolidated results feature - CLI option and core functionality Co-authored-by: deepakmahakale <14993828+deepakmahakale@users.noreply.github.com> --- lib/parallel_tests/cli.rb | 45 +++++++++++++++++++++------ lib/parallel_tests/cucumber/runner.rb | 8 +++-- lib/parallel_tests/gherkin/runner.rb | 10 ++++-- lib/parallel_tests/rspec/runner.rb | 9 ++++-- lib/parallel_tests/test/runner.rb | 10 ++++-- 5 files changed, 65 insertions(+), 17 deletions(-) diff --git a/lib/parallel_tests/cli.rb b/lib/parallel_tests/cli.rb index d53fbef5..96547172 100644 --- a/lib/parallel_tests/cli.rb +++ b/lib/parallel_tests/cli.rb @@ -4,6 +4,7 @@ require 'parallel_tests' require 'shellwords' require 'pathname' +require 'json' module ParallelTests class CLI @@ -70,6 +71,7 @@ def execute_in_parallel(items, num_processes, options) def run_tests_in_parallel(num_processes, options) test_results = nil + consolidated_results = {} run_tests_proc = -> do groups = @runner.tests_in_groups(options[:files], num_processes, options) @@ -80,19 +82,21 @@ def run_tests_in_parallel(num_processes, options) num_processes = 1 end - report_number_of_tests(groups) unless options[:quiet] + report_number_of_tests(groups, consolidated_results) unless options[:quiet] test_results = execute_in_parallel(groups, groups.size, options) do |group, index| run_tests(group, index, num_processes, options) end - report_results(test_results, options) unless options[:quiet] + report_results(test_results, consolidated_results, options) unless options[:quiet] end if options[:quiet] run_tests_proc.call else - report_time_taken(&run_tests_proc) + report_time_taken(consolidated_results, &run_tests_proc) end + write_consolidated_results(consolidated_results, options) + if any_test_failed?(test_results) warn final_fail_message @@ -145,10 +149,12 @@ def lock(lockfile) end end - def report_results(test_results, options) + def report_results(test_results, consolidated_results, options) results = @runner.find_results(test_results.map { |result| result[:stdout] } * "") + @runner.summarize_results(results, consolidated_results) + puts "" - puts @runner.summarize_results(results) + puts consolidated_results[:summary][:message] report_failure_rerun_commmand(test_results, options) end @@ -167,12 +173,19 @@ def report_failure_rerun_commmand(test_results, options) end end - def report_number_of_tests(groups) + def report_number_of_tests(groups, consolidated_results) name = @runner.test_file_name num_processes = groups.size num_tests = groups.map(&:size).sum tests_per_process = (num_processes == 0 ? 0 : num_tests / num_processes) - puts "#{pluralize(num_processes, 'process')} for #{pluralize(num_tests, name)}, ~ #{pluralize(tests_per_process, name)} per process" + message = "#{pluralize(num_processes, 'process')} for #{pluralize(num_tests, name)}, ~ #{pluralize(tests_per_process, name)} per process" + consolidated_results[:tests] = { + message: message, + tests: num_tests, + processes: num_processes, + tests_per_process: tests_per_process + } + puts message end def pluralize(n, singular) @@ -185,6 +198,14 @@ def pluralize(n, singular) end end + def write_consolidated_results(consolidated_results, options) + return if options[:consolidated_results].nil? || options[:consolidated_results].empty? + + File.write(options[:consolidated_results], consolidated_results.to_json) + + puts "Wrote consolidated results to #{options[:consolidated_results]}" + end + # exit with correct status code so rake parallel:test && echo 123 works def any_test_failed?(test_results) test_results.any? { |result| result[:exit_status] != 0 } @@ -334,6 +355,7 @@ def parse_options!(argv) opts.on("--verbose-process-command", "Print the command that will be executed by each process before it begins") { options[:verbose_process_command] = true } opts.on("--verbose-rerun-command", "After a process fails, print the command executed by that process") { options[:verbose_rerun_command] = true } opts.on("--quiet", "Print only tests output") { options[:quiet] = true } + opts.on("--consolidated-results [PATH]", "Location to write consolidated test results") { |path| options[:consolidated_results] = path } opts.on("-v", "--version", "Show Version") do puts ParallelTests::VERSION exit 0 @@ -434,9 +456,14 @@ def execute_command_in_parallel(command, num_processes, options) abort if results.any? { |r| r[:exit_status] != 0 } end - def report_time_taken(&block) + def report_time_taken(consolidated_results, &block) seconds = ParallelTests.delta(&block).to_i - puts "\nTook #{seconds} seconds#{detailed_duration(seconds)}" + message = "Took #{seconds} seconds#{detailed_duration(seconds)}" + consolidated_results[:time_taken] = { + seconds: seconds, + message: message + } + puts "\n#{message}" end def detailed_duration(seconds) diff --git a/lib/parallel_tests/cucumber/runner.rb b/lib/parallel_tests/cucumber/runner.rb index f217131f..991189a1 100644 --- a/lib/parallel_tests/cucumber/runner.rb +++ b/lib/parallel_tests/cucumber/runner.rb @@ -20,7 +20,7 @@ def line_is_result?(line) super || line =~ SCENARIO_REGEX || line =~ SCENARIOS_RESULTS_BOUNDARY_REGEX end - def summarize_results(results) + def summarize_results(results, consolidated_results) output = [] scenario_groups = results.slice_before(SCENARIOS_RESULTS_BOUNDARY_REGEX).group_by(&:first) @@ -31,7 +31,11 @@ def summarize_results(results) output << super - output.join("\n\n") + consolidated_results[:summary] = { + message: output.join("\n\n") + } + + consolidated_results[:summary][:message] end def command_with_seed(cmd, seed) diff --git a/lib/parallel_tests/gherkin/runner.rb b/lib/parallel_tests/gherkin/runner.rb index 5182e8d3..d30373de 100644 --- a/lib/parallel_tests/gherkin/runner.rb +++ b/lib/parallel_tests/gherkin/runner.rb @@ -49,10 +49,10 @@ def build_test_command(file_list, options) # cucumber has 2 result lines per test run, that cannot be added # 1 scenario (1 failed) # 1 step (1 failed) - def summarize_results(results) + def summarize_results(results, consolidated_results) sort_order = ['scenario', 'step', 'failed', 'flaky', 'undefined', 'skipped', 'pending', 'passed'] - ['scenario', 'step'].map do |group| + message = ['scenario', 'step'].map do |group| group_results = results.grep(/^\d+ #{group}/) next if group_results.empty? @@ -64,6 +64,12 @@ def summarize_results(results) end "#{sums[0]} (#{sums[1..].join(", ")})" end.compact.join("\n") + + consolidated_results[:summary] = { + message: message + } + + consolidated_results[:summary][:message] end def cucumber_opts(given) diff --git a/lib/parallel_tests/rspec/runner.rb b/lib/parallel_tests/rspec/runner.rb index 3b351983..c355a769 100644 --- a/lib/parallel_tests/rspec/runner.rb +++ b/lib/parallel_tests/rspec/runner.rb @@ -57,7 +57,7 @@ def command_with_seed(cmd, seed) # Summarize results from threads and colorize results based on failure and pending counts. # - def summarize_results(results) + def summarize_results(results, consolidated_results) text = super return text unless $stdout.tty? sums = sum_up_results(results) @@ -69,7 +69,12 @@ def summarize_results(results) else 32 # green end - "\e[#{color}m#{text}\e[0m" + colorized_text = "\e[#{color}m#{text}\e[0m" + + # Update consolidated results with colorized message + consolidated_results[:summary][:message] = colorized_text + + colorized_text end private diff --git a/lib/parallel_tests/test/runner.rb b/lib/parallel_tests/test/runner.rb index 5e8669e9..48c8a9f1 100644 --- a/lib/parallel_tests/test/runner.rb +++ b/lib/parallel_tests/test/runner.rb @@ -135,9 +135,15 @@ def test_env_number(process_number, options = {}) end end - def summarize_results(results) + def summarize_results(results, consolidated_results) sums = sum_up_results(results) - sums.sort.map { |word, number| "#{number} #{word}#{'s' if number != 1}" }.join(', ') + message = sums.sort.map { |word, number| "#{number} #{word}#{'s' if number != 1}" }.join(', ') + + consolidated_results[:summary] = sums.merge({ + message: message + }) + + consolidated_results[:summary][:message] end # remove old seed and add new seed From b7d6f58f700d7a2519bbc4401ef5d859ed3d4f49 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 18 Jul 2025 05:16:38 +0000 Subject: [PATCH 3/4] Update test specs to pass consolidated_results parameter Co-authored-by: deepakmahakale <14993828+deepakmahakale@users.noreply.github.com> --- spec/parallel_tests/cucumber/runner_spec.rb | 4 ++-- spec/parallel_tests/gherkin/runner_behaviour.rb | 8 ++++---- spec/parallel_tests/rspec/runner_spec.rb | 4 ++-- spec/parallel_tests/test/runner_spec.rb | 10 +++++----- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/spec/parallel_tests/cucumber/runner_spec.rb b/spec/parallel_tests/cucumber/runner_spec.rb index 31cfe8fc..f12fe6ef 100644 --- a/spec/parallel_tests/cucumber/runner_spec.rb +++ b/spec/parallel_tests/cucumber/runner_spec.rb @@ -21,7 +21,7 @@ def call(*args) "Failing Scenarios:", "cucumber features/failure:3", "cucumber features/failure:4", "Failing Scenarios:", "cucumber features/failure:5", "cucumber features/failure:6" ] - output = call(results) + output = call(results, {}) output.gsub!(/.*WARNING.*\n/, "") expect(output).to eq(<<~TXT) Failing Scenarios: @@ -42,7 +42,7 @@ def call(*args) "Failing Scenarios:", "cucumber features/failure:5", "cucumber features/failure:6", "Flaky Scenarios:", "cucumber features/failure:7", "cucumber features/failure:8" ] - expect(call(results)).to eq("Failing Scenarios:\ncucumber features/failure:1\ncucumber features/failure:2\ncucumber features/failure:5\ncucumber features/failure:6\n\nFlaky Scenarios:\ncucumber features/failure:3\ncucumber features/failure:4\ncucumber features/failure:7\ncucumber features/failure:8\n\n") + expect(call(results, {})).to eq("Failing Scenarios:\ncucumber features/failure:1\ncucumber features/failure:2\ncucumber features/failure:5\ncucumber features/failure:6\n\nFlaky Scenarios:\ncucumber features/failure:3\ncucumber features/failure:4\ncucumber features/failure:7\ncucumber features/failure:8\n\n") end end end diff --git a/spec/parallel_tests/gherkin/runner_behaviour.rb b/spec/parallel_tests/gherkin/runner_behaviour.rb index 31e0b98c..eec1ab6b 100644 --- a/spec/parallel_tests/gherkin/runner_behaviour.rb +++ b/spec/parallel_tests/gherkin/runner_behaviour.rb @@ -169,7 +169,7 @@ def call(*args) "4 scenarios (4 passed)", "40 steps (40 passed)", "1 scenario (1 passed)", "1 step (1 passed)" ] - expect(call(results)).to eq("12 scenarios (2 failed, 1 flaky, 9 passed)\n74 steps (3 failed, 2 skipped, 69 passed)") + expect(call(results, {})).to eq("12 scenarios (2 failed, 1 flaky, 9 passed)\n74 steps (3 failed, 2 skipped, 69 passed)") end it "adds same results with plurals" do @@ -177,7 +177,7 @@ def call(*args) "1 scenario (1 passed)", "2 steps (2 passed)", "2 scenarios (2 passed)", "7 steps (7 passed)" ] - expect(call(results)).to eq("3 scenarios (3 passed)\n9 steps (9 passed)") + expect(call(results, {})).to eq("3 scenarios (3 passed)\n9 steps (9 passed)") end it "adds non-similar results" do @@ -185,11 +185,11 @@ def call(*args) "1 scenario (1 passed)", "1 step (1 passed)", "2 scenarios (1 failed, 1 pending)", "2 steps (1 failed, 1 pending)" ] - expect(call(results)).to eq("3 scenarios (1 failed, 1 pending, 1 passed)\n3 steps (1 failed, 1 pending, 1 passed)") + expect(call(results, {})).to eq("3 scenarios (1 failed, 1 pending, 1 passed)\n3 steps (1 failed, 1 pending, 1 passed)") end it "does not pluralize 1" do - expect(call(["1 scenario (1 passed)", "1 step (1 passed)"])).to eq("1 scenario (1 passed)\n1 step (1 passed)") + expect(call(["1 scenario (1 passed)", "1 step (1 passed)"], {})).to eq("1 scenario (1 passed)\n1 step (1 passed)") end end diff --git a/spec/parallel_tests/rspec/runner_spec.rb b/spec/parallel_tests/rspec/runner_spec.rb index 540a4e43..bfe9ce92 100644 --- a/spec/parallel_tests/rspec/runner_spec.rb +++ b/spec/parallel_tests/rspec/runner_spec.rb @@ -137,7 +137,7 @@ def call(*args) before { allow($stdout).to receive(:tty?).and_return false } it 'is not colourized' do - results = ParallelTests::RSpec::Runner.send(:summarize_results, ['1 example, 0 failures, 0 pendings']) + results = ParallelTests::RSpec::Runner.send(:summarize_results, ['1 example, 0 failures, 0 pendings'], {}) expect(results).to eq('1 example, 0 failures, 0 pendings') end @@ -146,7 +146,7 @@ def call(*args) context 'on TTY device' do before { allow($stdout).to receive(:tty?).and_return true } - subject(:colorized_results) { ParallelTests::RSpec::Runner.send(:summarize_results, [result_string]) } + subject(:colorized_results) { ParallelTests::RSpec::Runner.send(:summarize_results, [result_string], {}) } context 'when there are no pending or failed tests' do let(:result_string) { '1 example, 0 failures, 0 pendings' } diff --git a/spec/parallel_tests/test/runner_spec.rb b/spec/parallel_tests/test/runner_spec.rb index 571433dd..65d499d6 100644 --- a/spec/parallel_tests/test/runner_spec.rb +++ b/spec/parallel_tests/test/runner_spec.rb @@ -404,23 +404,23 @@ def call(*args) end it "adds results" do - expect(call(['1 foo 3 bar', '2 foo 5 bar'])).to eq('8 bars, 3 foos') + expect(call(['1 foo 3 bar', '2 foo 5 bar'], {})).to eq('8 bars, 3 foos') end it "adds results with braces" do - expect(call(['1 foo(s) 3 bar(s)', '2 foo 5 bar'])).to eq('8 bars, 3 foos') + expect(call(['1 foo(s) 3 bar(s)', '2 foo 5 bar'], {})).to eq('8 bars, 3 foos') end it "adds same results with plurals" do - expect(call(['1 foo 3 bar', '2 foos 5 bar'])).to eq('8 bars, 3 foos') + expect(call(['1 foo 3 bar', '2 foos 5 bar'], {})).to eq('8 bars, 3 foos') end it "adds non-similar results" do - expect(call(['1 xxx 2 yyy', '1 xxx 2 zzz'])).to eq('2 xxxs, 2 yyys, 2 zzzs') + expect(call(['1 xxx 2 yyy', '1 xxx 2 zzz'], {})).to eq('2 xxxs, 2 yyys, 2 zzzs') end it "does not pluralize 1" do - expect(call(['1 xxx 2 yyy'])).to eq('1 xxx, 2 yyys') + expect(call(['1 xxx 2 yyy'], {})).to eq('1 xxx, 2 yyys') end end From 661434d0494779bdab187b9af32a32ebd62e1869 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 18 Jul 2025 05:21:41 +0000 Subject: [PATCH 4/4] Fix consolidated results to work properly in quiet mode Co-authored-by: deepakmahakale <14993828+deepakmahakale@users.noreply.github.com> --- lib/parallel_tests/cli.rb | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/lib/parallel_tests/cli.rb b/lib/parallel_tests/cli.rb index 96547172..ba67f043 100644 --- a/lib/parallel_tests/cli.rb +++ b/lib/parallel_tests/cli.rb @@ -82,17 +82,24 @@ def run_tests_in_parallel(num_processes, options) num_processes = 1 end - report_number_of_tests(groups, consolidated_results) unless options[:quiet] + # Always populate consolidated results for tests info if consolidated results is requested + report_number_of_tests(groups, consolidated_results, options) if !options[:quiet] || options[:consolidated_results] test_results = execute_in_parallel(groups, groups.size, options) do |group, index| run_tests(group, index, num_processes, options) end - report_results(test_results, consolidated_results, options) unless options[:quiet] + # Always populate consolidated results for summary if consolidated results is requested + report_results(test_results, consolidated_results, options) if !options[:quiet] || options[:consolidated_results] end if options[:quiet] - run_tests_proc.call + # Still capture timing if consolidated results are requested + if options[:consolidated_results] + report_time_taken(consolidated_results, options, &run_tests_proc) + else + run_tests_proc.call + end else - report_time_taken(consolidated_results, &run_tests_proc) + report_time_taken(consolidated_results, options, &run_tests_proc) end write_consolidated_results(consolidated_results, options) @@ -153,8 +160,10 @@ def report_results(test_results, consolidated_results, options) results = @runner.find_results(test_results.map { |result| result[:stdout] } * "") @runner.summarize_results(results, consolidated_results) - puts "" - puts consolidated_results[:summary][:message] + unless options[:quiet] + puts "" + puts consolidated_results[:summary][:message] + end report_failure_rerun_commmand(test_results, options) end @@ -173,7 +182,7 @@ def report_failure_rerun_commmand(test_results, options) end end - def report_number_of_tests(groups, consolidated_results) + def report_number_of_tests(groups, consolidated_results, options = {}) name = @runner.test_file_name num_processes = groups.size num_tests = groups.map(&:size).sum @@ -185,7 +194,7 @@ def report_number_of_tests(groups, consolidated_results) processes: num_processes, tests_per_process: tests_per_process } - puts message + puts message unless options[:quiet] end def pluralize(n, singular) @@ -456,14 +465,14 @@ def execute_command_in_parallel(command, num_processes, options) abort if results.any? { |r| r[:exit_status] != 0 } end - def report_time_taken(consolidated_results, &block) + def report_time_taken(consolidated_results, options = {}, &block) seconds = ParallelTests.delta(&block).to_i message = "Took #{seconds} seconds#{detailed_duration(seconds)}" consolidated_results[:time_taken] = { seconds: seconds, message: message } - puts "\n#{message}" + puts "\n#{message}" unless options[:quiet] end def detailed_duration(seconds)