diff --git a/lib/parallel_tests/cli.rb b/lib/parallel_tests/cli.rb index d53fbef5..ba67f043 100644 --- a/lib/parallel_tests/cli.rb +++ b/lib/parallel_tests/cli.rb @@ -4,6 +4,7 @@ require 'parallel_tests' require 'shellwords' require 'pathname' +require 'json' module ParallelTests class CLI @@ -70,6 +71,7 @@ def execute_in_parallel(items, num_processes, options) def run_tests_in_parallel(num_processes, options) test_results = nil + consolidated_results = {} run_tests_proc = -> do groups = @runner.tests_in_groups(options[:files], num_processes, options) @@ -80,19 +82,28 @@ def run_tests_in_parallel(num_processes, options) num_processes = 1 end - report_number_of_tests(groups) unless options[:quiet] + # Always populate consolidated results for tests info if consolidated results is requested + report_number_of_tests(groups, consolidated_results, options) if !options[:quiet] || options[:consolidated_results] test_results = execute_in_parallel(groups, groups.size, options) do |group, index| run_tests(group, index, num_processes, options) end - report_results(test_results, options) unless options[:quiet] + # Always populate consolidated results for summary if consolidated results is requested + report_results(test_results, consolidated_results, options) if !options[:quiet] || options[:consolidated_results] end if options[:quiet] - run_tests_proc.call + # Still capture timing if consolidated results are requested + if options[:consolidated_results] + report_time_taken(consolidated_results, options, &run_tests_proc) + else + run_tests_proc.call + end else - report_time_taken(&run_tests_proc) + report_time_taken(consolidated_results, options, &run_tests_proc) end + write_consolidated_results(consolidated_results, options) + if any_test_failed?(test_results) warn final_fail_message @@ -145,10 +156,14 @@ def lock(lockfile) end end - def report_results(test_results, options) + def report_results(test_results, consolidated_results, options) results = @runner.find_results(test_results.map { |result| result[:stdout] } * "") - puts "" - puts @runner.summarize_results(results) + @runner.summarize_results(results, consolidated_results) + + unless options[:quiet] + puts "" + puts consolidated_results[:summary][:message] + end report_failure_rerun_commmand(test_results, options) end @@ -167,12 +182,19 @@ def report_failure_rerun_commmand(test_results, options) end end - def report_number_of_tests(groups) + def report_number_of_tests(groups, consolidated_results, options = {}) name = @runner.test_file_name num_processes = groups.size num_tests = groups.map(&:size).sum tests_per_process = (num_processes == 0 ? 0 : num_tests / num_processes) - puts "#{pluralize(num_processes, 'process')} for #{pluralize(num_tests, name)}, ~ #{pluralize(tests_per_process, name)} per process" + message = "#{pluralize(num_processes, 'process')} for #{pluralize(num_tests, name)}, ~ #{pluralize(tests_per_process, name)} per process" + consolidated_results[:tests] = { + message: message, + tests: num_tests, + processes: num_processes, + tests_per_process: tests_per_process + } + puts message unless options[:quiet] end def pluralize(n, singular) @@ -185,6 +207,14 @@ def pluralize(n, singular) end end + def write_consolidated_results(consolidated_results, options) + return if options[:consolidated_results].nil? || options[:consolidated_results].empty? + + File.write(options[:consolidated_results], consolidated_results.to_json) + + puts "Wrote consolidated results to #{options[:consolidated_results]}" + end + # exit with correct status code so rake parallel:test && echo 123 works def any_test_failed?(test_results) test_results.any? { |result| result[:exit_status] != 0 } @@ -334,6 +364,7 @@ def parse_options!(argv) opts.on("--verbose-process-command", "Print the command that will be executed by each process before it begins") { options[:verbose_process_command] = true } opts.on("--verbose-rerun-command", "After a process fails, print the command executed by that process") { options[:verbose_rerun_command] = true } opts.on("--quiet", "Print only tests output") { options[:quiet] = true } + opts.on("--consolidated-results [PATH]", "Location to write consolidated test results") { |path| options[:consolidated_results] = path } opts.on("-v", "--version", "Show Version") do puts ParallelTests::VERSION exit 0 @@ -434,9 +465,14 @@ def execute_command_in_parallel(command, num_processes, options) abort if results.any? { |r| r[:exit_status] != 0 } end - def report_time_taken(&block) + def report_time_taken(consolidated_results, options = {}, &block) seconds = ParallelTests.delta(&block).to_i - puts "\nTook #{seconds} seconds#{detailed_duration(seconds)}" + message = "Took #{seconds} seconds#{detailed_duration(seconds)}" + consolidated_results[:time_taken] = { + seconds: seconds, + message: message + } + puts "\n#{message}" unless options[:quiet] end def detailed_duration(seconds) diff --git a/lib/parallel_tests/cucumber/runner.rb b/lib/parallel_tests/cucumber/runner.rb index f217131f..991189a1 100644 --- a/lib/parallel_tests/cucumber/runner.rb +++ b/lib/parallel_tests/cucumber/runner.rb @@ -20,7 +20,7 @@ def line_is_result?(line) super || line =~ SCENARIO_REGEX || line =~ SCENARIOS_RESULTS_BOUNDARY_REGEX end - def summarize_results(results) + def summarize_results(results, consolidated_results) output = [] scenario_groups = results.slice_before(SCENARIOS_RESULTS_BOUNDARY_REGEX).group_by(&:first) @@ -31,7 +31,11 @@ def summarize_results(results) output << super - output.join("\n\n") + consolidated_results[:summary] = { + message: output.join("\n\n") + } + + consolidated_results[:summary][:message] end def command_with_seed(cmd, seed) diff --git a/lib/parallel_tests/gherkin/runner.rb b/lib/parallel_tests/gherkin/runner.rb index 5182e8d3..d30373de 100644 --- a/lib/parallel_tests/gherkin/runner.rb +++ b/lib/parallel_tests/gherkin/runner.rb @@ -49,10 +49,10 @@ def build_test_command(file_list, options) # cucumber has 2 result lines per test run, that cannot be added # 1 scenario (1 failed) # 1 step (1 failed) - def summarize_results(results) + def summarize_results(results, consolidated_results) sort_order = ['scenario', 'step', 'failed', 'flaky', 'undefined', 'skipped', 'pending', 'passed'] - ['scenario', 'step'].map do |group| + message = ['scenario', 'step'].map do |group| group_results = results.grep(/^\d+ #{group}/) next if group_results.empty? @@ -64,6 +64,12 @@ def summarize_results(results) end "#{sums[0]} (#{sums[1..].join(", ")})" end.compact.join("\n") + + consolidated_results[:summary] = { + message: message + } + + consolidated_results[:summary][:message] end def cucumber_opts(given) diff --git a/lib/parallel_tests/rspec/runner.rb b/lib/parallel_tests/rspec/runner.rb index 3b351983..c355a769 100644 --- a/lib/parallel_tests/rspec/runner.rb +++ b/lib/parallel_tests/rspec/runner.rb @@ -57,7 +57,7 @@ def command_with_seed(cmd, seed) # Summarize results from threads and colorize results based on failure and pending counts. # - def summarize_results(results) + def summarize_results(results, consolidated_results) text = super return text unless $stdout.tty? sums = sum_up_results(results) @@ -69,7 +69,12 @@ def summarize_results(results) else 32 # green end - "\e[#{color}m#{text}\e[0m" + colorized_text = "\e[#{color}m#{text}\e[0m" + + # Update consolidated results with colorized message + consolidated_results[:summary][:message] = colorized_text + + colorized_text end private diff --git a/lib/parallel_tests/test/runner.rb b/lib/parallel_tests/test/runner.rb index 5e8669e9..48c8a9f1 100644 --- a/lib/parallel_tests/test/runner.rb +++ b/lib/parallel_tests/test/runner.rb @@ -135,9 +135,15 @@ def test_env_number(process_number, options = {}) end end - def summarize_results(results) + def summarize_results(results, consolidated_results) sums = sum_up_results(results) - sums.sort.map { |word, number| "#{number} #{word}#{'s' if number != 1}" }.join(', ') + message = sums.sort.map { |word, number| "#{number} #{word}#{'s' if number != 1}" }.join(', ') + + consolidated_results[:summary] = sums.merge({ + message: message + }) + + consolidated_results[:summary][:message] end # remove old seed and add new seed diff --git a/spec/parallel_tests/cucumber/runner_spec.rb b/spec/parallel_tests/cucumber/runner_spec.rb index 31cfe8fc..f12fe6ef 100644 --- a/spec/parallel_tests/cucumber/runner_spec.rb +++ b/spec/parallel_tests/cucumber/runner_spec.rb @@ -21,7 +21,7 @@ def call(*args) "Failing Scenarios:", "cucumber features/failure:3", "cucumber features/failure:4", "Failing Scenarios:", "cucumber features/failure:5", "cucumber features/failure:6" ] - output = call(results) + output = call(results, {}) output.gsub!(/.*WARNING.*\n/, "") expect(output).to eq(<<~TXT) Failing Scenarios: @@ -42,7 +42,7 @@ def call(*args) "Failing Scenarios:", "cucumber features/failure:5", "cucumber features/failure:6", "Flaky Scenarios:", "cucumber features/failure:7", "cucumber features/failure:8" ] - expect(call(results)).to eq("Failing Scenarios:\ncucumber features/failure:1\ncucumber features/failure:2\ncucumber features/failure:5\ncucumber features/failure:6\n\nFlaky Scenarios:\ncucumber features/failure:3\ncucumber features/failure:4\ncucumber features/failure:7\ncucumber features/failure:8\n\n") + expect(call(results, {})).to eq("Failing Scenarios:\ncucumber features/failure:1\ncucumber features/failure:2\ncucumber features/failure:5\ncucumber features/failure:6\n\nFlaky Scenarios:\ncucumber features/failure:3\ncucumber features/failure:4\ncucumber features/failure:7\ncucumber features/failure:8\n\n") end end end diff --git a/spec/parallel_tests/gherkin/runner_behaviour.rb b/spec/parallel_tests/gherkin/runner_behaviour.rb index 31e0b98c..eec1ab6b 100644 --- a/spec/parallel_tests/gherkin/runner_behaviour.rb +++ b/spec/parallel_tests/gherkin/runner_behaviour.rb @@ -169,7 +169,7 @@ def call(*args) "4 scenarios (4 passed)", "40 steps (40 passed)", "1 scenario (1 passed)", "1 step (1 passed)" ] - expect(call(results)).to eq("12 scenarios (2 failed, 1 flaky, 9 passed)\n74 steps (3 failed, 2 skipped, 69 passed)") + expect(call(results, {})).to eq("12 scenarios (2 failed, 1 flaky, 9 passed)\n74 steps (3 failed, 2 skipped, 69 passed)") end it "adds same results with plurals" do @@ -177,7 +177,7 @@ def call(*args) "1 scenario (1 passed)", "2 steps (2 passed)", "2 scenarios (2 passed)", "7 steps (7 passed)" ] - expect(call(results)).to eq("3 scenarios (3 passed)\n9 steps (9 passed)") + expect(call(results, {})).to eq("3 scenarios (3 passed)\n9 steps (9 passed)") end it "adds non-similar results" do @@ -185,11 +185,11 @@ def call(*args) "1 scenario (1 passed)", "1 step (1 passed)", "2 scenarios (1 failed, 1 pending)", "2 steps (1 failed, 1 pending)" ] - expect(call(results)).to eq("3 scenarios (1 failed, 1 pending, 1 passed)\n3 steps (1 failed, 1 pending, 1 passed)") + expect(call(results, {})).to eq("3 scenarios (1 failed, 1 pending, 1 passed)\n3 steps (1 failed, 1 pending, 1 passed)") end it "does not pluralize 1" do - expect(call(["1 scenario (1 passed)", "1 step (1 passed)"])).to eq("1 scenario (1 passed)\n1 step (1 passed)") + expect(call(["1 scenario (1 passed)", "1 step (1 passed)"], {})).to eq("1 scenario (1 passed)\n1 step (1 passed)") end end diff --git a/spec/parallel_tests/rspec/runner_spec.rb b/spec/parallel_tests/rspec/runner_spec.rb index 540a4e43..bfe9ce92 100644 --- a/spec/parallel_tests/rspec/runner_spec.rb +++ b/spec/parallel_tests/rspec/runner_spec.rb @@ -137,7 +137,7 @@ def call(*args) before { allow($stdout).to receive(:tty?).and_return false } it 'is not colourized' do - results = ParallelTests::RSpec::Runner.send(:summarize_results, ['1 example, 0 failures, 0 pendings']) + results = ParallelTests::RSpec::Runner.send(:summarize_results, ['1 example, 0 failures, 0 pendings'], {}) expect(results).to eq('1 example, 0 failures, 0 pendings') end @@ -146,7 +146,7 @@ def call(*args) context 'on TTY device' do before { allow($stdout).to receive(:tty?).and_return true } - subject(:colorized_results) { ParallelTests::RSpec::Runner.send(:summarize_results, [result_string]) } + subject(:colorized_results) { ParallelTests::RSpec::Runner.send(:summarize_results, [result_string], {}) } context 'when there are no pending or failed tests' do let(:result_string) { '1 example, 0 failures, 0 pendings' } diff --git a/spec/parallel_tests/test/runner_spec.rb b/spec/parallel_tests/test/runner_spec.rb index 571433dd..65d499d6 100644 --- a/spec/parallel_tests/test/runner_spec.rb +++ b/spec/parallel_tests/test/runner_spec.rb @@ -404,23 +404,23 @@ def call(*args) end it "adds results" do - expect(call(['1 foo 3 bar', '2 foo 5 bar'])).to eq('8 bars, 3 foos') + expect(call(['1 foo 3 bar', '2 foo 5 bar'], {})).to eq('8 bars, 3 foos') end it "adds results with braces" do - expect(call(['1 foo(s) 3 bar(s)', '2 foo 5 bar'])).to eq('8 bars, 3 foos') + expect(call(['1 foo(s) 3 bar(s)', '2 foo 5 bar'], {})).to eq('8 bars, 3 foos') end it "adds same results with plurals" do - expect(call(['1 foo 3 bar', '2 foos 5 bar'])).to eq('8 bars, 3 foos') + expect(call(['1 foo 3 bar', '2 foos 5 bar'], {})).to eq('8 bars, 3 foos') end it "adds non-similar results" do - expect(call(['1 xxx 2 yyy', '1 xxx 2 zzz'])).to eq('2 xxxs, 2 yyys, 2 zzzs') + expect(call(['1 xxx 2 yyy', '1 xxx 2 zzz'], {})).to eq('2 xxxs, 2 yyys, 2 zzzs') end it "does not pluralize 1" do - expect(call(['1 xxx 2 yyy'])).to eq('1 xxx, 2 yyys') + expect(call(['1 xxx 2 yyy'], {})).to eq('1 xxx, 2 yyys') end end