From 6895f4806f1bbeae5f86264369dbc7c35054901a Mon Sep 17 00:00:00 2001 From: Christian Bruckmayer Date: Tue, 20 May 2025 20:56:30 +0100 Subject: [PATCH] wip --- ruby/Gemfile.lock | 1 + ruby/Rakefile | 2 +- ruby/ci-queue.gemspec | 1 + ruby/lib/ci/queue/redis/worker.rb | 46 +- ruby/lib/minitest/queue.rb | 123 ++- ruby/test/integration/minitest_redis_test.rb | 979 +------------------ 6 files changed, 134 insertions(+), 1018 deletions(-) diff --git a/ruby/Gemfile.lock b/ruby/Gemfile.lock index 8b6d9c46..13c420b6 100644 --- a/ruby/Gemfile.lock +++ b/ruby/Gemfile.lock @@ -2,6 +2,7 @@ PATH remote: . specs: ci-queue (0.66.0) + drb logger GEM diff --git a/ruby/Rakefile b/ruby/Rakefile index d4219120..b7594c59 100644 --- a/ruby/Rakefile +++ b/ruby/Rakefile @@ -8,7 +8,7 @@ Rake::TestTask.new(:test) do |t| t.libs << 'lib' selected_files = ENV["TEST_FILES"].to_s.strip.split(/\s+/) selected_files = nil if selected_files.empty? - t.test_files = selected_files || FileList['test/**/*_test.rb'] - FileList['test/fixtures/**/*_test.rb'] + t.test_files = ["test/integration/minitest_redis_test.rb"] end task :default => :test diff --git a/ruby/ci-queue.gemspec b/ruby/ci-queue.gemspec index dd080b4a..1294814d 100644 --- a/ruby/ci-queue.gemspec +++ b/ruby/ci-queue.gemspec @@ -32,6 +32,7 @@ Gem::Specification.new do |spec| spec.metadata['allowed_push_host'] = 'https://rubygems.org' spec.add_runtime_dependency 'logger' + spec.add_runtime_dependency 'drb' spec.add_development_dependency 'bundler' spec.add_development_dependency 'rake' diff --git a/ruby/lib/ci/queue/redis/worker.rb b/ruby/lib/ci/queue/redis/worker.rb index c5eb5448..e663744b 100644 --- a/ruby/lib/ci/queue/redis/worker.rb +++ b/ruby/lib/ci/queue/redis/worker.rb @@ -46,6 +46,17 @@ def master? @master end + def stop? + shutdown_required? || config.circuit_breakers.any?(&:open?) || exhausted? || max_test_failed? + end + + def pop(worker_id) + if test = reserve(worker_id) + index.fetch(test) + end + rescue *CONNECTION_ERRORS + end + def poll wait_for_master until shutdown_required? || config.circuit_breakers.any?(&:open?) || exhausted? || max_test_failed? @@ -97,9 +108,9 @@ def report_worker_error(error) build.report_worker_error(error) end - def acknowledge(test) + def acknowledge(test, worker_id) test_key = test.id - raise_on_mismatching_test(test_key) + raise_on_mismatching_test(test_key, worker_id) eval_script( :acknowledge, keys: [key('running'), key('processed'), key('owners')], @@ -107,9 +118,9 @@ def acknowledge(test) ) == 1 end - def requeue(test, offset: Redis.requeue_offset) + def requeue(test, worker_id, offset: Redis.requeue_offset) test_key = test.id - raise_on_mismatching_test(test_key) + raise_on_mismatching_test(test_key, worker_id) global_max_requeues = config.global_max_requeues(total) requeued = config.max_requeues > 0 && global_max_requeues > 0 && eval_script( @@ -125,7 +136,7 @@ def requeue(test, offset: Redis.requeue_offset) argv: [config.max_requeues, global_max_requeues, test_key, offset], ) == 1 - @reserved_test = test_key unless requeued + @reserved_test[worker_id] = test_key unless requeued requeued end @@ -138,29 +149,30 @@ def release! nil end - private - - attr_reader :index - def worker_id config.worker_id end + attr_reader :index + + private + - def raise_on_mismatching_test(test) - if @reserved_test == test - @reserved_test = nil + def raise_on_mismatching_test(test, worker_id) + if @reserved_test[worker_id] == test + @reserved_test[worker_id] = nil else - raise ReservationError, "Acknowledged #{test.inspect} but #{@reserved_test.inspect} was reserved" + raise ReservationError, "Acknowledged #{test.inspect} but #{@reserved_test[worker_id].inspect} was reserved" end end - def reserve - if @reserved_test - raise ReservationError, "#{@reserved_test.inspect} is already reserved. " \ + def reserve(worker_id) + @reserved_test ||= {} + if @reserved_test[worker_id] + raise ReservationError, "#{@reserved_test[worker_id].inspect} is already reserved. " \ "You have to acknowledge it before you can reserve another one" end - @reserved_test = (try_to_reserve_lost_test || try_to_reserve_test) + @reserved_test[worker_id] = (try_to_reserve_lost_test || try_to_reserve_test) end def try_to_reserve_test diff --git a/ruby/lib/minitest/queue.rb b/ruby/lib/minitest/queue.rb index 91e464ff..3ea7d434 100644 --- a/ruby/lib/minitest/queue.rb +++ b/ruby/lib/minitest/queue.rb @@ -2,6 +2,7 @@ require 'shellwords' require 'minitest' require 'minitest/reporters' +require 'drb/drb' require 'minitest/queue/failure_formatter' require 'minitest/queue/error_report' @@ -225,41 +226,107 @@ def __run(*args) end end - def run_from_queue(reporter, *) - queue.poll do |example| - result = queue.with_heartbeat(example.id) do - example.run - end + # The URI for the server to connect to + URI="druby://localhost:8780" + URI_2="druby://localhost:8787" - failed = !(result.passed? || result.skipped?) + class TestServer - if example.flaky? - result.mark_as_flaked! - failed = false - end + def initialize(queue, reporter) + @queue = queue + @reporter = reporter + @mutex = Mutex.new + end + + attr_accessor :queue, :reporter, :mutex + + def wait_for_master + queue.wait_for_master + end + + def reserve(worker_id) + queue.pop(worker_id) + end + + def stop? + queue.stop? + end - if failed && queue.config.failing_test && queue.config.failing_test != example.id - # When we do a bisect, we don't care about the result other than the test we're running the bisect on - result.mark_as_flaked! - failed = false - elsif failed - queue.report_failure! - else - queue.report_success! + def record(worker_id, example, result) + mutex.synchronize do + failed = !(result.passed? || result.skipped?) + + if example.flaky? + result.mark_as_flaked! + failed = false + end + + if failed && queue.config.failing_test && queue.config.failing_test != example.id + # When we do a bisect, we don't care about the result other than the test we're running the bisect on + result.mark_as_flaked! + failed = false + elsif failed + queue.report_failure! + else + queue.report_success! + end + + if failed && CI::Queue.requeueable?(result) && queue.requeue(example, worker_id) + result.requeue! + reporter.record(result) + elsif queue.acknowledge(example, worker_id) + reporter.record(result) + queue.increment_test_failed if failed + elsif !failed + # If the test was already acknowledged by another worker (we timed out) + # Then we only record it if it is successful. + reporter.record(result) + end end - if failed && CI::Queue.requeueable?(result) && queue.requeue(example) - result.requeue! - reporter.record(result) - elsif queue.acknowledge(example) - reporter.record(result) - queue.increment_test_failed if failed - elsif !failed - # If the test was already acknowledged by another worker (we timed out) - # Then we only record it if it is successful. - reporter.record(result) + end + end + + def run_from_queue(reporter, *) + # The object that handles requests on the server + server = TestServer.new(queue, reporter) + + DRb.start_service(URI, server) + + ENV.fetch("PARALLEL_WORKERS", 2).to_i.times do |i| + fork do + puts "Forking #{i}" + DRb.start_service + server = DRbObject.new_with_uri(URI) + server.wait_for_master + until server.stop? + if test = server.reserve(i) + puts "Process #{i} running: #{test.id}" + result = test.run + puts "Process #{i} result: #{result}" + server.record(i, test, result) + else + sleep 0.05 + end + end + + puts "Done" end end + + until queue.stop? + print "." + sleep 0.05 + end + puts "Stopping DRb" + DRb.stop_service + + puts "Waiting for processes" + Process.waitall + # Wait for the drb server thread to finish before exiting. + + puts "Workers finished" + queue.stop_heartbeat! rescue Errno::EPIPE # This happens when the heartbeat process dies diff --git a/ruby/test/integration/minitest_redis_test.rb b/ruby/test/integration/minitest_redis_test.rb index 7b8c1172..9e31320d 100644 --- a/ruby/test/integration/minitest_redis_test.rb +++ b/ruby/test/integration/minitest_redis_test.rb @@ -45,980 +45,15 @@ def test_default_reporter assert_match(/Expected false to be truthy/, normalize(out)) # failure output result = normalize(out.lines.last.strip) assert_equal '--- Ran 11 tests, 8 assertions, 2 failures, 1 errors, 1 skips, 4 requeues in X.XXs', result - end - - def test_lost_test_with_heartbeat_monitor - _, err = capture_subprocess_io do - 2.times.map do |i| - Thread.start do - system( - { 'BUILDKITE' => '1' }, - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', i.to_s, - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '--heartbeat', '1', - '-Itest', - 'test/lost_test.rb', - chdir: 'test/fixtures/', - ) - end - end.each(&:join) - end - - assert_empty err - - Tempfile.open('warnings') do |warnings_file| - out, err = capture_subprocess_io do - system( - @exe, 'report', - '--queue', @redis_url, - '--build', '1', - '--timeout', '1', - '--warnings-file', warnings_file.path, - '--heartbeat', - chdir: 'test/fixtures/', - ) - end - - assert_empty err - result = normalize(out.lines[1].strip) - assert_equal "Ran 1 tests, 0 assertions, 0 failures, 0 errors, 0 skips, 0 requeues in X.XXs (aggregated)", result - warnings = JSON.parse(warnings_file.read) - assert_equal 1, warnings.size - end - end - - def test_verbose_reporter - out, err = capture_subprocess_io do - system( - { 'BUILDKITE' => '1' }, - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '-Itest', - 'test/dummy_test.rb', - '-v', - chdir: 'test/fixtures/', - ) - end - - assert_empty err - assert_match(/ATest#test_foo \d+\.\d+ = S/, normalize(out)) # verbose test ouptut - result = normalize(out.lines.last.strip) - assert_equal '--- Ran 11 tests, 8 assertions, 2 failures, 1 errors, 1 skips, 4 requeues in X.XXs', result - end - - def test_debug_log - Tempfile.open('debug_log') do |log_file| - out, err = capture_subprocess_io do - system( - { 'BUILDKITE' => '1' }, - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '-Itest', - 'test/dummy_test.rb', - '--debug-log', log_file.path, - chdir: 'test/fixtures/', - ) - end - - assert_includes File.read(log_file.path), 'INFO -- : Finished \'["exists", "build:1:worker:1:queue"]\': 0' - assert_empty err - result = normalize(out.lines.last.strip) - assert_equal '--- Ran 11 tests, 8 assertions, 2 failures, 1 errors, 1 skips, 4 requeues in X.XXs', result - end - end - - def test_buildkite_output - out, err = capture_subprocess_io do - system( - { 'BUILDKITE' => '1' }, - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '-Itest', - 'test/dummy_test.rb', - chdir: 'test/fixtures/', - ) - end - - assert_empty err - assert_match(/^\^{3} \+{3}$/m, normalize(out)) # reopen failed step - output = normalize(out.lines.last.strip) - assert_equal '--- Ran 11 tests, 8 assertions, 2 failures, 1 errors, 1 skips, 4 requeues in X.XXs', output - end - - def test_custom_requeue - out, err = capture_subprocess_io do - system( - { 'BUILDKITE' => '1' }, - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '-Itest', - 'test/custom_requeue_test.rb', - chdir: 'test/fixtures/', - ) - end - - assert_empty err - output = normalize(out.lines.last.strip) - assert_equal '--- Ran 3 tests, 0 assertions, 0 failures, 2 errors, 0 skips, 1 requeues in X.XXs', output - end - - def test_max_test_failed - out, err = capture_subprocess_io do - system( - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--heartbeat', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '--max-test-failed', '3', - '-Itest', - 'test/failing_test.rb', - chdir: 'test/fixtures/', - ) - end - - assert_equal 'This worker is exiting early because too many failed tests were encountered.', err.chomp - output = normalize(out.lines.last.strip) - assert_equal 'Ran 47 tests, 47 assertions, 3 failures, 0 errors, 0 skips, 44 requeues in X.XXs', output - - # Run the reporter - out, err = capture_subprocess_io do - system( - @exe, 'report', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--timeout', '1', - '--max-test-failed', '3', - chdir: 'test/fixtures/', - ) - end - - refute_predicate $?, :success? - assert_empty err - expected = <<~EXPECTED - Waiting for workers to complete - Requeued 44 tests - EXPECTED - assert_equal expected.strip, normalize(out.lines[0..1].join.strip) - expected = <<~EXPECTED - Ran 3 tests, 47 assertions, 3 failures, 0 errors, 0 skips, 44 requeues in X.XXs (aggregated) - EXPECTED - assert_equal expected.strip, normalize(out.lines[134].strip) - expected = <<~EXPECTED - Encountered too many failed tests. Test run was ended early. - EXPECTED - assert_equal expected.strip, normalize(out.lines[136].strip) - expected = <<~EXPECTED - 97 tests weren't run. - EXPECTED - assert_equal expected.strip, normalize(out.lines.last.strip) - end - - def test_all_workers_died - # Run the reporter - out, err = capture_subprocess_io do - system( - @exe, 'report', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--timeout', '1', - '--max-test-failed', '3', - chdir: 'test/fixtures/', - ) - end - - refute_predicate $?, :success? - assert_empty err - expected = <<~EXPECTED - Waiting for workers to complete - No master was elected. Did all workers crash? - EXPECTED - assert_equal expected.strip, normalize(out.lines[0..2].join.strip) - end - - def test_circuit_breaker - out, err = capture_subprocess_io do - system( - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '--max-consecutive-failures', '3', - '-Itest', - 'test/failing_test.rb', - chdir: 'test/fixtures/', - ) - end - - assert_equal "This worker is exiting early because it encountered too many consecutive test failures, probably because of some corrupted state.\n", err - output = normalize(out.lines.last.strip) - assert_equal 'Ran 3 tests, 3 assertions, 0 failures, 0 errors, 0 skips, 3 requeues in X.XXs', output - end - - def test_redis_runner - out, err = capture_subprocess_io do - system( - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '-Itest', - 'test/dummy_test.rb', - chdir: 'test/fixtures/', - ) - end - - assert_empty err - output = normalize(out.lines.last.strip) - assert_equal 'Ran 11 tests, 8 assertions, 2 failures, 1 errors, 1 skips, 4 requeues in X.XXs', output - - out, err = capture_subprocess_io do - system( - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '-Itest', - 'test/dummy_test.rb', - chdir: 'test/fixtures/', - ) - end - - assert_empty err - output = normalize(out.lines.last.strip) - assert_equal 'Ran 6 tests, 4 assertions, 2 failures, 1 errors, 0 skips, 3 requeues in X.XXs', output - end - - def test_retry_success - out, err = capture_subprocess_io do - system( - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '-Itest', - 'test/passing_test.rb', - chdir: 'test/fixtures/', - ) - end - assert_empty err - output = normalize(out.lines.last.strip) - assert_equal 'Ran 100 tests, 100 assertions, 0 failures, 0 errors, 0 skips, 0 requeues in X.XXs', output - - out, err = capture_subprocess_io do - system( - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '-Itest', - 'test/passing_test.rb', - chdir: 'test/fixtures/', - ) - end - assert_empty err - output = normalize(out.lines.last.strip) - assert_equal 'All tests were ran already', output - end - - def test_automatic_retry - out, err = capture_subprocess_io do - system( - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '-Itest', - 'test/failing_test.rb', - chdir: 'test/fixtures/', - ) - end - assert_empty err - output = normalize(out.lines.last.strip) - assert_equal 'Ran 200 tests, 200 assertions, 100 failures, 0 errors, 0 skips, 100 requeues in X.XXs', output - out, err = capture_subprocess_io do - system( - { "BUILDKITE_RETRY_TYPE" => "automatic" }, - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '-Itest', - 'test/failing_test.rb', - chdir: 'test/fixtures/', + system( + @exe, 'report', + '--queue', @redis_url, + '--build', '1', + '--timeout', '1', + '--heartbeat', + chdir: 'test/fixtures/', ) - end - assert_empty err - output = normalize(out.lines.last.strip) - assert_equal 'All tests were ran already', output - end - - def test_retry_fails_when_test_run_is_expired - out, err = capture_subprocess_io do - system( - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '-Itest', - 'test/passing_test.rb', - chdir: 'test/fixtures/', - ) - end - assert_empty err - output = normalize(out.lines.last.strip) - assert_equal 'Ran 100 tests, 100 assertions, 0 failures, 0 errors, 0 skips, 0 requeues in X.XXs', output - - one_day = 60 * 60 * 24 - key = ['build', "1", "created-at"].join(':') - @redis.set(key, Time.now - one_day) - - out, err = capture_subprocess_io do - system( - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '-Itest', - 'test/passing_test.rb', - chdir: 'test/fixtures/', - ) - end - assert_empty err - output = normalize(out.lines.last.strip) - assert_equal "The test run is too old and can't be retried", output - end - - def test_retry_report - # Run first worker, failing all tests - out, err = capture_subprocess_io do - system( - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '-Itest', - 'test/failing_test.rb', - chdir: 'test/fixtures/', - ) - end - assert_empty err - output = normalize(out.lines.last.strip) - assert_equal 'Ran 100 tests, 100 assertions, 100 failures, 0 errors, 0 skips, 0 requeues in X.XXs', output - - # Run the reporter - out, err = capture_subprocess_io do - system( - @exe, 'report', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--timeout', '1', - chdir: 'test/fixtures/', - ) - end - assert_empty err - expect = 'Ran 100 tests, 100 assertions, 100 failures, 0 errors, 0 skips, 0 requeues in X.XXs (aggregated)' - assert_equal expect, normalize(out.strip.lines[1].strip) - - # Simulate another worker successfuly retrying all errors (very hard to reproduce properly) - queue_config = CI::Queue::Configuration.new( - timeout: 1, - build_id: '1', - worker_id: '2', - ) - queue = CI::Queue.from_uri(@redis_url, queue_config) - error_reports = queue.build.error_reports - assert_equal 100, error_reports.size - - error_reports.keys.each_with_index do |test_id, index| - queue.build.record_success(test_id.dup, stats: { - 'assertions' => index + 1, - 'errors' => 0, - 'failures' => 0, - 'skips' => 0, - 'requeues' => 0, - 'total_time' => index + 1, - }) - end - - # Retry first worker, bailing out - out, err = capture_subprocess_io do - system( - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '-Itest', - 'test/failing_test.rb', - chdir: 'test/fixtures/', - ) - end - assert_empty err - output = normalize(out.lines.last.strip) - assert_equal 'All tests were ran already', output - - # Re-run the reporter - out, err = capture_subprocess_io do - system( - @exe, 'report', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--timeout', '1', - chdir: 'test/fixtures/', - ) - end - assert_empty err - expect = 'Ran 100 tests, 100 assertions, 0 failures, 0 errors, 0 skips, 0 requeues in X.XXs (aggregated)' - assert_equal expect, normalize(out.strip.lines[1].strip) - end - - def test_down_redis - out, err = capture_subprocess_io do - system( - { "CI_QUEUE_DISABLE_RECONNECT_ATTEMPTS" => "1" }, - @exe, 'run', - '--queue', 'redis://localhost:1337', - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '-Itest', - 'test/dummy_test.rb', - chdir: 'test/fixtures/', - ) - end - assert_empty err - output = normalize(out.lines.last.strip) - assert_equal 'Ran 0 tests, 0 assertions, 0 failures, 0 errors, 0 skips, 0 requeues in X.XXs', output - end - - def test_test_data_reporter - out, err = capture_subprocess_io do - system( - {'CI_QUEUE_FLAKY_TESTS' => 'test/ci_queue_flaky_tests_list.txt'}, - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--namespace', 'foo', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '-Itest', - 'test/dummy_test.rb', - chdir: 'test/fixtures/', - ) - end - assert_empty err - output = normalize(out.lines.last.strip) - assert_equal 'Ran 9 tests, 6 assertions, 1 failures, 1 errors, 1 skips, 2 requeues in X.XXs', output - - content = File.read(@test_data_path) - failures = JSON.parse(content, symbolize_names: true) - .sort_by { |h| "#{h[:test_id]}##{h[:test_index]}" } - - assert_equal 'foo', failures[0][:namespace] - assert_equal 'ATest#test_bar', failures[0][:test_id] - assert_equal 'test_bar', failures[0][:test_name] - assert_equal 'ATest', failures[0][:test_suite] - assert_equal 'failure', failures[0][:test_result] - assert_equal true, failures[0][:test_retried] - assert_equal false, failures[0][:test_result_ignored] - assert_equal 1, failures[0][:test_assertions] - assert_equal 'test/dummy_test.rb', failures[0][:test_file_path] - assert_equal 9, failures[0][:test_file_line_number] - assert_equal 'Minitest::Assertion', failures[0][:error_class] - assert_equal 'Expected false to be truthy.', failures[0][:error_message] - assert_equal 'test/dummy_test.rb', failures[0][:error_file_path] - assert_equal 10, failures[0][:error_file_number] - - assert_equal 'foo', failures[1][:namespace] - assert_equal 'ATest#test_bar', failures[1][:test_id] - assert_equal 'test_bar', failures[1][:test_name] - assert_equal 'ATest', failures[1][:test_suite] - assert_equal 'failure', failures[1][:test_result] - assert_equal false, failures[1][:test_result_ignored] - assert_equal false, failures[1][:test_retried] - assert_equal 1, failures[1][:test_assertions] - assert_equal 'test/dummy_test.rb', failures[1][:test_file_path] - assert_equal 9, failures[1][:test_file_line_number] - assert_equal 'Minitest::Assertion', failures[1][:error_class] - assert_equal 'Expected false to be truthy.', failures[1][:error_message] - assert_equal 'test/dummy_test.rb', failures[1][:error_file_path] - assert_equal 10, failures[1][:error_file_number] - - assert failures[0][:test_index] < failures[1][:test_index] - - assert_equal 'ATest#test_flaky', failures[2][:test_id] - assert_equal 'skipped', failures[2][:test_result] - assert_equal false, failures[2][:test_retried] - assert_equal true, failures[2][:test_result_ignored] - assert_equal 1, failures[2][:test_assertions] - assert_equal 'test/dummy_test.rb', failures[2][:test_file_path] - assert_equal 13, failures[2][:test_file_line_number] - assert_equal 'Minitest::Assertion', failures[2][:error_class] - assert_equal 18, failures[2][:error_file_number] - - assert_equal 'ATest#test_flaky_passes', failures[4][:test_id] - assert_equal 'success', failures[4][:test_result] - end - - def test_test_data_time_reporter - start_time = Time.now - travel_to(start_time) do - capture_subprocess_io do - system( - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--namespace', 'foo', - '--build', '1', - '--worker', '1', - '--timeout', '10', - '-Itest', - 'test/time_test.rb', - chdir: 'test/fixtures/', - ) - end - end - end_time = Time.now - - content = File.read(@test_data_path) - failure = JSON.parse(content, symbolize_names: true) - .sort_by { |h| "#{h[:test_id]}##{h[:test_index]}" } - .first - - start_delta = RUBY_ENGINE == "truffleruby" ? 15 : 5 - assert_in_delta start_time.to_i, failure[:test_start_timestamp], start_delta, "start time" - assert_in_delta end_time.to_i, failure[:test_finish_timestamp], 5 - assert failure[:test_finish_timestamp] > failure[:test_start_timestamp] - end - - def test_junit_reporter - out, err = capture_subprocess_io do - system( - {'CI_QUEUE_FLAKY_TESTS' => 'test/ci_queue_flaky_tests_list.txt'}, - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '-Itest', - 'test/dummy_test.rb', - chdir: 'test/fixtures/', - ) - end - assert_empty err - output = normalize(out.lines.last.strip) - assert_equal 'Ran 9 tests, 6 assertions, 1 failures, 1 errors, 1 skips, 2 requeues in X.XXs', output - - # NOTE: To filter the TypeError backtrace below see test/fixtures/test/backtrace_filters.rb - - assert_equal <<~XML, normalize_xml(File.read(@junit_path)) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - XML - end - - def test_redis_reporter_failure_file - Dir.mktmpdir do |dir| - failure_file = File.join(dir, 'failure_file.json') - - capture_subprocess_io do - system( - { 'BUILDKITE' => '1' }, - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '-Itest', - 'test/dummy_test.rb', - chdir: 'test/fixtures/', - ) - end - - capture_subprocess_io do - system( - @exe, 'report', - '--queue', @redis_url, - '--build', '1', - '--timeout', '1', - '--failure-file', failure_file, - chdir: 'test/fixtures/', - ) - end - - content = File.read(failure_file) - failure = JSON.parse(content, symbolize_names: true) - .sort_by { |failure_report| failure_report[:test_line] } - .first - - xml_file = File.join(File.dirname(failure_file), "#{File.basename(failure_file, File.extname(failure_file))}.xml") - xml_content = File.read(xml_file) - xml = REXML::Document.new(xml_content) - testcase = xml.elements['testsuites/testsuite/testcase[@name="test_bar"]'] - assert_equal "ATest", testcase.attributes['classname'] - assert_equal "test_bar", testcase.attributes['name'] - assert_equal "test/dummy_test.rb", testcase.parent.attributes['filepath'] - assert_equal "ATest", testcase.parent.attributes['name'] - - ## output and test_file - expected = { - test_file: "ci-queue/ruby/test/fixtures/test/dummy_test.rb", - test_line: 9, - test_and_module_name: "ATest#test_bar", - error_class: "Minitest::Assertion", - test_name: "test_bar", - test_suite: "ATest", - } - - assert_includes failure[:test_file], expected[:test_file] - assert_equal failure[:test_line], expected[:test_line] - assert_equal failure[:test_suite], expected[:test_suite] - assert_equal failure[:test_and_module_name], expected[:test_and_module_name] - assert_equal failure[:test_name], expected[:test_name] - assert_equal failure[:error_class], expected[:error_class] - end - end - - def test_redis_reporter_flaky_tests_file - Dir.mktmpdir do |dir| - flaky_tests_file = File.join(dir, 'flaky_tests_file.json') - - capture_subprocess_io do - system( - { 'BUILDKITE' => '1' }, - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '-Itest', - 'test/dummy_test.rb', - chdir: 'test/fixtures/', - ) - end - - capture_subprocess_io do - system( - @exe, 'report', - '--queue', @redis_url, - '--build', '1', - '--timeout', '1', - '--export-flaky-tests-file', flaky_tests_file, - chdir: 'test/fixtures/', - ) - end - - content = File.read(flaky_tests_file) - flaky_tests = JSON.parse(content) - assert_includes flaky_tests, "ATest#test_flaky" - end - end - - def test_redis_reporter - # HACK: Simulate a timeout - config = CI::Queue::Configuration.new(build_id: '1', worker_id: '1', timeout: '1') - build_record = CI::Queue::Redis::BuildRecord.new(self, ::Redis.new(url: @redis_url), config) - build_record.record_warning(CI::Queue::Warnings::RESERVED_LOST_TEST, test: 'Atest#test_bar', timeout: 2) - - out, err = capture_subprocess_io do - system( - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '-Itest', - 'test/dummy_test.rb', - chdir: 'test/fixtures/', - ) - end - assert_empty err - output = normalize(out.lines.last.strip) - assert_equal 'Ran 11 tests, 8 assertions, 2 failures, 1 errors, 1 skips, 4 requeues in X.XXs', output - - Tempfile.open('warnings') do |warnings_file| - out, err = capture_subprocess_io do - system( - @exe, 'report', - '--queue', @redis_url, - '--build', '1', - '--timeout', '1', - '--warnings-file', warnings_file.path, - chdir: 'test/fixtures/', - ) - end - - warnings_file.rewind - content = JSON.parse(warnings_file.read) - assert_equal 1, content.size - assert_equal "RESERVED_LOST_TEST", content[0]["type"] - assert_equal "Atest#test_bar", content[0]["test"] - assert_equal 2, content[0]["timeout"] - - assert_empty err - output = normalize(out) - - expected_output = <<~END - Waiting for workers to complete - Requeued 4 tests - REQUEUE - ATest#test_bar (requeued 1 times) - - REQUEUE - ATest#test_flaky (requeued 1 times) - - REQUEUE - ATest#test_flaky_fails_retry (requeued 1 times) - - REQUEUE - BTest#test_bar (requeued 1 times) - - Ran 7 tests, 8 assertions, 2 failures, 1 errors, 1 skips, 4 requeues in X.XXs (aggregated) - - - FAIL ATest#test_bar - Expected false to be truthy. - test/dummy_test.rb:10:in `test_bar' - - FAIL ATest#test_flaky_fails_retry - Expected false to be truthy. - test/dummy_test.rb:23:in `test_flaky_fails_retry' - - ERROR BTest#test_bar - END - assert_includes output, expected_output - end - end - - def test_utf8_tests_and_marshal - out, err = capture_subprocess_io do - system( - { 'MARSHAL' => '1' }, - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '-Itest', - 'test/utf8_test.rb', - chdir: 'test/fixtures/', - ) - end - - assert_empty err - output = normalize(out.lines.last) - assert_equal <<~END, output - Ran 1 tests, 1 assertions, 1 failures, 0 errors, 0 skips, 0 requeues in X.XXs - END - end - - def test_application_error - capture_subprocess_io do - system( - { 'BUILDKITE' => '1' }, - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '-Itest', - 'test/bad_framework_test.rb', - chdir: 'test/fixtures/', - ) - end - - assert_equal 42, $?.exitstatus - - out, _ = capture_subprocess_io do - system( - @exe, 'report', - '--queue', @redis_url, - '--build', '1', - '--timeout', '1', - '--heartbeat', - chdir: 'test/fixtures/', - ) - end - - assert_includes out, "Worker 1 crashed" - assert_includes out, "Some error in the test framework" - - assert_equal 1, $?.exitstatus end private