From 6c61b50576f7675890b55b4b5382a61fd2db58d6 Mon Sep 17 00:00:00 2001 From: Christian Bruckmayer Date: Wed, 18 Jun 2025 10:20:47 +0100 Subject: [PATCH] wip --- ruby/Rakefile | 2 +- ruby/test/fixtures/test/test_helper.rb | 2 + ruby/test/integration/minitest_redis_test.rb | 978 +------------------ 3 files changed, 4 insertions(+), 978 deletions(-) diff --git a/ruby/Rakefile b/ruby/Rakefile index d4219120..b7594c59 100644 --- a/ruby/Rakefile +++ b/ruby/Rakefile @@ -8,7 +8,7 @@ Rake::TestTask.new(:test) do |t| t.libs << 'lib' selected_files = ENV["TEST_FILES"].to_s.strip.split(/\s+/) selected_files = nil if selected_files.empty? - t.test_files = selected_files || FileList['test/**/*_test.rb'] - FileList['test/fixtures/**/*_test.rb'] + t.test_files = ["test/integration/minitest_redis_test.rb"] end task :default => :test diff --git a/ruby/test/fixtures/test/test_helper.rb b/ruby/test/fixtures/test/test_helper.rb index 416324b3..bc62e660 100644 --- a/ruby/test/fixtures/test/test_helper.rb +++ b/ruby/test/fixtures/test/test_helper.rb @@ -8,6 +8,8 @@ require 'minitest/autorun' require_relative './backtrace_filters' +sleep 10 + Minitest.backtrace_filter = BacktraceFilters.new( Minitest.backtrace_filter ) diff --git a/ruby/test/integration/minitest_redis_test.rb b/ruby/test/integration/minitest_redis_test.rb index 0cdd1f00..ecebfeef 100644 --- a/ruby/test/integration/minitest_redis_test.rb +++ b/ruby/test/integration/minitest_redis_test.rb @@ -44,985 +44,9 @@ def test_default_reporter assert_empty err assert_match(/Expected false to be truthy/, normalize(out)) # failure output result = normalize(out.lines.last.strip) + puts out.lines.last.strip assert_equal '--- Ran 11 tests, 8 assertions, 2 failures, 1 errors, 1 skips, 4 requeues in X.XXs', result end - - def test_lost_test_with_heartbeat_monitor - _, err = capture_subprocess_io do - 2.times.map do |i| - Thread.start do - system( - { 'BUILDKITE' => '1' }, - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', i.to_s, - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '--heartbeat', '1', - '-Itest', - 'test/lost_test.rb', - chdir: 'test/fixtures/', - ) - end - end.each(&:join) - end - - assert_empty err - - Tempfile.open('warnings') do |warnings_file| - out, err = capture_subprocess_io do - system( - @exe, 'report', - '--queue', @redis_url, - '--build', '1', - '--timeout', '1', - '--warnings-file', warnings_file.path, - '--heartbeat', - chdir: 'test/fixtures/', - ) - end - - assert_empty err - result = normalize(out.lines[1].strip) - assert_equal "Ran 1 tests, 0 assertions, 0 failures, 0 errors, 0 skips, 0 requeues in X.XXs (aggregated)", result - warnings = JSON.parse(warnings_file.read) - assert_equal 1, warnings.size - end - end - - def test_verbose_reporter - out, err = capture_subprocess_io do - system( - { 'BUILDKITE' => '1' }, - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '-Itest', - 'test/dummy_test.rb', - '-v', - chdir: 'test/fixtures/', - ) - end - - assert_empty err - assert_match(/ATest#test_foo \d+\.\d+ = S/, normalize(out)) # verbose test ouptut - result = normalize(out.lines.last.strip) - assert_equal '--- Ran 11 tests, 8 assertions, 2 failures, 1 errors, 1 skips, 4 requeues in X.XXs', result - end - - def test_debug_log - Tempfile.open('debug_log') do |log_file| - out, err = capture_subprocess_io do - system( - { 'BUILDKITE' => '1' }, - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '-Itest', - 'test/dummy_test.rb', - '--debug-log', log_file.path, - chdir: 'test/fixtures/', - ) - end - - assert_includes File.read(log_file.path), 'INFO -- : Finished \'["exists", "build:1:worker:1:queue"]\': 0' - assert_empty err - result = normalize(out.lines.last.strip) - assert_equal '--- Ran 11 tests, 8 assertions, 2 failures, 1 errors, 1 skips, 4 requeues in X.XXs', result - end - end - - def test_buildkite_output - out, err = capture_subprocess_io do - system( - { 'BUILDKITE' => '1' }, - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '-Itest', - 'test/dummy_test.rb', - chdir: 'test/fixtures/', - ) - end - - assert_empty err - assert_match(/^\^{3} \+{3}$/m, normalize(out)) # reopen failed step - output = normalize(out.lines.last.strip) - assert_equal '--- Ran 11 tests, 8 assertions, 2 failures, 1 errors, 1 skips, 4 requeues in X.XXs', output - end - - def test_custom_requeue - out, err = capture_subprocess_io do - system( - { 'BUILDKITE' => '1' }, - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '-Itest', - 'test/custom_requeue_test.rb', - chdir: 'test/fixtures/', - ) - end - - assert_empty err - output = normalize(out.lines.last.strip) - assert_equal '--- Ran 3 tests, 0 assertions, 0 failures, 2 errors, 0 skips, 1 requeues in X.XXs', output - end - - def test_max_test_failed - out, err = capture_subprocess_io do - system( - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--heartbeat', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '--max-test-failed', '3', - '-Itest', - 'test/failing_test.rb', - chdir: 'test/fixtures/', - ) - end - - assert_equal 'This worker is exiting early because too many failed tests were encountered.', err.chomp - output = normalize(out.lines.last.strip) - assert_equal 'Ran 47 tests, 47 assertions, 3 failures, 0 errors, 0 skips, 44 requeues in X.XXs', output - - # Run the reporter - out, err = capture_subprocess_io do - system( - @exe, 'report', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--timeout', '1', - '--max-test-failed', '3', - chdir: 'test/fixtures/', - ) - end - - refute_predicate $?, :success? - assert_equal 44, $?.exitstatus - assert_empty err - expected = <<~EXPECTED - Waiting for workers to complete - Requeued 44 tests - EXPECTED - assert_equal expected.strip, normalize(out.lines[0..1].join.strip) - expected = <<~EXPECTED - Ran 3 tests, 47 assertions, 3 failures, 0 errors, 0 skips, 44 requeues in X.XXs (aggregated) - EXPECTED - assert_equal expected.strip, normalize(out.lines[134].strip) - expected = <<~EXPECTED - Encountered too many failed tests. Test run was ended early. - EXPECTED - assert_equal expected.strip, normalize(out.lines[136].strip) - expected = <<~EXPECTED - 97 tests weren't run. - EXPECTED - assert_equal expected.strip, normalize(out.lines.last.strip) - end - - def test_all_workers_died - # Run the reporter - out, err = capture_subprocess_io do - system( - @exe, 'report', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--timeout', '1', - '--max-test-failed', '3', - chdir: 'test/fixtures/', - ) - end - - refute_predicate $?, :success? - assert_equal 40, $?.exitstatus - assert_empty err - expected = <<~EXPECTED - Waiting for workers to complete - No leader was elected. This typically means no worker was able to start. Were there any errors during application boot? - EXPECTED - assert_equal expected.strip, normalize(out.lines[0..2].join.strip) - end - - def test_circuit_breaker - out, err = capture_subprocess_io do - system( - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '--max-consecutive-failures', '3', - '-Itest', - 'test/failing_test.rb', - chdir: 'test/fixtures/', - ) - end - - assert_equal "This worker is exiting early because it encountered too many consecutive test failures, probably because of some corrupted state.\n", err - output = normalize(out.lines.last.strip) - assert_equal 'Ran 3 tests, 3 assertions, 0 failures, 0 errors, 0 skips, 3 requeues in X.XXs', output - end - - def test_redis_runner - out, err = capture_subprocess_io do - system( - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '-Itest', - 'test/dummy_test.rb', - chdir: 'test/fixtures/', - ) - end - - assert_empty err - output = normalize(out.lines.last.strip) - assert_equal 'Ran 11 tests, 8 assertions, 2 failures, 1 errors, 1 skips, 4 requeues in X.XXs', output - - out, err = capture_subprocess_io do - system( - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '-Itest', - 'test/dummy_test.rb', - chdir: 'test/fixtures/', - ) - end - - assert_empty err - output = normalize(out.lines.last.strip) - assert_equal 'Ran 6 tests, 4 assertions, 2 failures, 1 errors, 0 skips, 3 requeues in X.XXs', output - end - - def test_retry_success - out, err = capture_subprocess_io do - system( - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '-Itest', - 'test/passing_test.rb', - chdir: 'test/fixtures/', - ) - end - assert_empty err - output = normalize(out.lines.last.strip) - assert_equal 'Ran 100 tests, 100 assertions, 0 failures, 0 errors, 0 skips, 0 requeues in X.XXs', output - - out, err = capture_subprocess_io do - system( - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '-Itest', - 'test/passing_test.rb', - chdir: 'test/fixtures/', - ) - end - assert_empty err - output = normalize(out.lines.last.strip) - assert_equal 'All tests were ran already', output - end - - def test_automatic_retry - out, err = capture_subprocess_io do - system( - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '-Itest', - 'test/failing_test.rb', - chdir: 'test/fixtures/', - ) - end - assert_empty err - output = normalize(out.lines.last.strip) - assert_equal 'Ran 200 tests, 200 assertions, 100 failures, 0 errors, 0 skips, 100 requeues in X.XXs', output - - out, err = capture_subprocess_io do - system( - { "BUILDKITE_RETRY_TYPE" => "automatic" }, - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '-Itest', - 'test/failing_test.rb', - chdir: 'test/fixtures/', - ) - end - assert_empty err - output = normalize(out.lines.last.strip) - assert_equal 'All tests were ran already', output - end - - def test_retry_fails_when_test_run_is_expired - out, err = capture_subprocess_io do - system( - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '-Itest', - 'test/passing_test.rb', - chdir: 'test/fixtures/', - ) - end - assert_empty err - output = normalize(out.lines.last.strip) - assert_equal 'Ran 100 tests, 100 assertions, 0 failures, 0 errors, 0 skips, 0 requeues in X.XXs', output - - one_day = 60 * 60 * 24 - key = ['build', "1", "created-at"].join(':') - @redis.set(key, Time.now - one_day) - - out, err = capture_subprocess_io do - system( - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '-Itest', - 'test/passing_test.rb', - chdir: 'test/fixtures/', - ) - end - assert_empty err - output = normalize(out.lines.last.strip) - assert_equal "The test run is too old and can't be retried", output - end - - def test_retry_report - # Run first worker, failing all tests - out, err = capture_subprocess_io do - system( - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '-Itest', - 'test/failing_test.rb', - chdir: 'test/fixtures/', - ) - end - assert_empty err - output = normalize(out.lines.last.strip) - assert_equal 'Ran 100 tests, 100 assertions, 100 failures, 0 errors, 0 skips, 0 requeues in X.XXs', output - - # Run the reporter - out, err = capture_subprocess_io do - system( - @exe, 'report', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--timeout', '1', - chdir: 'test/fixtures/', - ) - end - assert_empty err - expect = 'Ran 100 tests, 100 assertions, 100 failures, 0 errors, 0 skips, 0 requeues in X.XXs (aggregated)' - assert_equal expect, normalize(out.strip.lines[1].strip) - - # Simulate another worker successfuly retrying all errors (very hard to reproduce properly) - queue_config = CI::Queue::Configuration.new( - timeout: 1, - build_id: '1', - worker_id: '2', - ) - queue = CI::Queue.from_uri(@redis_url, queue_config) - error_reports = queue.build.error_reports - assert_equal 100, error_reports.size - - error_reports.keys.each_with_index do |test_id, index| - queue.build.record_success(test_id.dup, stats: { - 'assertions' => index + 1, - 'errors' => 0, - 'failures' => 0, - 'skips' => 0, - 'requeues' => 0, - 'total_time' => index + 1, - }, acknowledge: false) - end - - # Retry first worker, bailing out - out, err = capture_subprocess_io do - system( - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '-Itest', - 'test/failing_test.rb', - chdir: 'test/fixtures/', - ) - end - assert_empty err - output = normalize(out.lines.last.strip) - assert_equal 'All tests were ran already', output - - # Re-run the reporter - out, err = capture_subprocess_io do - system( - @exe, 'report', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--timeout', '1', - chdir: 'test/fixtures/', - ) - end - assert_empty err - expect = 'Ran 100 tests, 100 assertions, 0 failures, 0 errors, 0 skips, 0 requeues in X.XXs (aggregated)' - assert_equal expect, normalize(out.strip.lines[1].strip) - end - - def test_down_redis - out, err = capture_subprocess_io do - system( - { "CI_QUEUE_DISABLE_RECONNECT_ATTEMPTS" => "1" }, - @exe, 'run', - '--queue', 'redis://localhost:1337', - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '-Itest', - 'test/dummy_test.rb', - chdir: 'test/fixtures/', - ) - end - assert_empty err - output = normalize(out.lines.last.strip) - assert_equal 'Ran 0 tests, 0 assertions, 0 failures, 0 errors, 0 skips, 0 requeues in X.XXs', output - end - - def test_test_data_reporter - out, err = capture_subprocess_io do - system( - {'CI_QUEUE_FLAKY_TESTS' => 'test/ci_queue_flaky_tests_list.txt'}, - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--namespace', 'foo', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '-Itest', - 'test/dummy_test.rb', - chdir: 'test/fixtures/', - ) - end - assert_empty err - output = normalize(out.lines.last.strip) - assert_equal 'Ran 9 tests, 6 assertions, 1 failures, 1 errors, 1 skips, 2 requeues in X.XXs', output - - content = File.read(@test_data_path) - failures = JSON.parse(content, symbolize_names: true) - .sort_by { |h| "#{h[:test_id]}##{h[:test_index]}" } - - assert_equal 'foo', failures[0][:namespace] - assert_equal 'ATest#test_bar', failures[0][:test_id] - assert_equal 'test_bar', failures[0][:test_name] - assert_equal 'ATest', failures[0][:test_suite] - assert_equal 'failure', failures[0][:test_result] - assert_equal true, failures[0][:test_retried] - assert_equal false, failures[0][:test_result_ignored] - assert_equal 1, failures[0][:test_assertions] - assert_equal 'test/dummy_test.rb', failures[0][:test_file_path] - assert_equal 9, failures[0][:test_file_line_number] - assert_equal 'Minitest::Assertion', failures[0][:error_class] - assert_equal 'Expected false to be truthy.', failures[0][:error_message] - assert_equal 'test/dummy_test.rb', failures[0][:error_file_path] - assert_equal 10, failures[0][:error_file_number] - - assert_equal 'foo', failures[1][:namespace] - assert_equal 'ATest#test_bar', failures[1][:test_id] - assert_equal 'test_bar', failures[1][:test_name] - assert_equal 'ATest', failures[1][:test_suite] - assert_equal 'failure', failures[1][:test_result] - assert_equal false, failures[1][:test_result_ignored] - assert_equal false, failures[1][:test_retried] - assert_equal 1, failures[1][:test_assertions] - assert_equal 'test/dummy_test.rb', failures[1][:test_file_path] - assert_equal 9, failures[1][:test_file_line_number] - assert_equal 'Minitest::Assertion', failures[1][:error_class] - assert_equal 'Expected false to be truthy.', failures[1][:error_message] - assert_equal 'test/dummy_test.rb', failures[1][:error_file_path] - assert_equal 10, failures[1][:error_file_number] - - assert failures[0][:test_index] < failures[1][:test_index] - - assert_equal 'ATest#test_flaky', failures[2][:test_id] - assert_equal 'skipped', failures[2][:test_result] - assert_equal false, failures[2][:test_retried] - assert_equal true, failures[2][:test_result_ignored] - assert_equal 1, failures[2][:test_assertions] - assert_equal 'test/dummy_test.rb', failures[2][:test_file_path] - assert_equal 13, failures[2][:test_file_line_number] - assert_equal 'Minitest::Assertion', failures[2][:error_class] - assert_equal 18, failures[2][:error_file_number] - - assert_equal 'ATest#test_flaky_passes', failures[4][:test_id] - assert_equal 'success', failures[4][:test_result] - end - - def test_test_data_time_reporter - start_time = Time.now - travel_to(start_time) do - capture_subprocess_io do - system( - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--namespace', 'foo', - '--build', '1', - '--worker', '1', - '--timeout', '10', - '-Itest', - 'test/time_test.rb', - chdir: 'test/fixtures/', - ) - end - end - end_time = Time.now - - content = File.read(@test_data_path) - failure = JSON.parse(content, symbolize_names: true) - .sort_by { |h| "#{h[:test_id]}##{h[:test_index]}" } - .first - - start_delta = RUBY_ENGINE == "truffleruby" ? 15 : 5 - assert_in_delta start_time.to_i, failure[:test_start_timestamp], start_delta, "start time" - assert_in_delta end_time.to_i, failure[:test_finish_timestamp], 5 - assert failure[:test_finish_timestamp] > failure[:test_start_timestamp] - end - - def test_junit_reporter - out, err = capture_subprocess_io do - system( - {'CI_QUEUE_FLAKY_TESTS' => 'test/ci_queue_flaky_tests_list.txt'}, - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '-Itest', - 'test/dummy_test.rb', - chdir: 'test/fixtures/', - ) - end - assert_empty err - output = normalize(out.lines.last.strip) - assert_equal 'Ran 9 tests, 6 assertions, 1 failures, 1 errors, 1 skips, 2 requeues in X.XXs', output - - # NOTE: To filter the TypeError backtrace below see test/fixtures/test/backtrace_filters.rb - - assert_equal <<~XML, normalize_xml(File.read(@junit_path)) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - XML - end - - def test_redis_reporter_failure_file - Dir.mktmpdir do |dir| - failure_file = File.join(dir, 'failure_file.json') - - capture_subprocess_io do - system( - { 'BUILDKITE' => '1' }, - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '-Itest', - 'test/dummy_test.rb', - chdir: 'test/fixtures/', - ) - end - - capture_subprocess_io do - system( - @exe, 'report', - '--queue', @redis_url, - '--build', '1', - '--timeout', '1', - '--failure-file', failure_file, - chdir: 'test/fixtures/', - ) - end - - content = File.read(failure_file) - failure = JSON.parse(content, symbolize_names: true) - .sort_by { |failure_report| failure_report[:test_line] } - .first - - xml_file = File.join(File.dirname(failure_file), "#{File.basename(failure_file, File.extname(failure_file))}.xml") - xml_content = File.read(xml_file) - xml = REXML::Document.new(xml_content) - testcase = xml.elements['testsuites/testsuite/testcase[@name="test_bar"]'] - assert_equal "ATest", testcase.attributes['classname'] - assert_equal "test_bar", testcase.attributes['name'] - assert_equal "test/dummy_test.rb", testcase.parent.attributes['filepath'] - assert_equal "ATest", testcase.parent.attributes['name'] - - ## output and test_file - expected = { - test_file: "ci-queue/ruby/test/fixtures/test/dummy_test.rb", - test_line: 9, - test_and_module_name: "ATest#test_bar", - error_class: "Minitest::Assertion", - test_name: "test_bar", - test_suite: "ATest", - } - - assert_includes failure[:test_file], expected[:test_file] - assert_equal failure[:test_line], expected[:test_line] - assert_equal failure[:test_suite], expected[:test_suite] - assert_equal failure[:test_and_module_name], expected[:test_and_module_name] - assert_equal failure[:test_name], expected[:test_name] - assert_equal failure[:error_class], expected[:error_class] - end - end - - def test_redis_reporter_flaky_tests_file - Dir.mktmpdir do |dir| - flaky_tests_file = File.join(dir, 'flaky_tests_file.json') - - capture_subprocess_io do - system( - { 'BUILDKITE' => '1' }, - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '-Itest', - 'test/dummy_test.rb', - chdir: 'test/fixtures/', - ) - end - - capture_subprocess_io do - system( - @exe, 'report', - '--queue', @redis_url, - '--build', '1', - '--timeout', '1', - '--export-flaky-tests-file', flaky_tests_file, - chdir: 'test/fixtures/', - ) - end - - content = File.read(flaky_tests_file) - flaky_tests = JSON.parse(content) - assert_includes flaky_tests, "ATest#test_flaky" - end - end - - def test_redis_reporter - # HACK: Simulate a timeout - config = CI::Queue::Configuration.new(build_id: '1', worker_id: '1', timeout: '1') - build_record = CI::Queue::Redis::BuildRecord.new(self, ::Redis.new(url: @redis_url), config) - build_record.record_warning(CI::Queue::Warnings::RESERVED_LOST_TEST, test: 'Atest#test_bar', timeout: 2) - - out, err = capture_subprocess_io do - system( - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '-Itest', - 'test/dummy_test.rb', - chdir: 'test/fixtures/', - ) - end - assert_empty err - output = normalize(out.lines.last.strip) - assert_equal 'Ran 11 tests, 8 assertions, 2 failures, 1 errors, 1 skips, 4 requeues in X.XXs', output - - Tempfile.open('warnings') do |warnings_file| - out, err = capture_subprocess_io do - system( - @exe, 'report', - '--queue', @redis_url, - '--build', '1', - '--timeout', '1', - '--warnings-file', warnings_file.path, - chdir: 'test/fixtures/', - ) - end - - warnings_file.rewind - content = JSON.parse(warnings_file.read) - assert_equal 1, content.size - assert_equal "RESERVED_LOST_TEST", content[0]["type"] - assert_equal "Atest#test_bar", content[0]["test"] - assert_equal 2, content[0]["timeout"] - - assert_empty err - output = normalize(out) - - expected_output = <<~END - Waiting for workers to complete - Requeued 4 tests - REQUEUE - ATest#test_bar (requeued 1 times) - - REQUEUE - ATest#test_flaky (requeued 1 times) - - REQUEUE - ATest#test_flaky_fails_retry (requeued 1 times) - - REQUEUE - BTest#test_bar (requeued 1 times) - - Ran 7 tests, 8 assertions, 2 failures, 1 errors, 1 skips, 4 requeues in X.XXs (aggregated) - - - FAIL ATest#test_bar - Expected false to be truthy. - test/dummy_test.rb:10:in `test_bar' - - FAIL ATest#test_flaky_fails_retry - Expected false to be truthy. - test/dummy_test.rb:23:in `test_flaky_fails_retry' - - ERROR BTest#test_bar - END - assert_includes output, expected_output - end - end - - def test_utf8_tests_and_marshal - out, err = capture_subprocess_io do - system( - { 'MARSHAL' => '1' }, - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '-Itest', - 'test/utf8_test.rb', - chdir: 'test/fixtures/', - ) - end - - assert_empty err - output = normalize(out.lines.last) - assert_equal <<~END, output - Ran 1 tests, 1 assertions, 1 failures, 0 errors, 0 skips, 0 requeues in X.XXs - END - end - - def test_application_error - capture_subprocess_io do - system( - { 'BUILDKITE' => '1' }, - @exe, 'run', - '--queue', @redis_url, - '--seed', 'foobar', - '--build', '1', - '--worker', '1', - '--timeout', '1', - '--max-requeues', '1', - '--requeue-tolerance', '1', - '-Itest', - 'test/bad_framework_test.rb', - chdir: 'test/fixtures/', - ) - end - - assert_equal 42, $?.exitstatus - - out, _ = capture_subprocess_io do - system( - @exe, 'report', - '--queue', @redis_url, - '--build', '1', - '--timeout', '1', - '--heartbeat', - chdir: 'test/fixtures/', - ) - end - - assert_includes out, "Worker 1 crashed" - assert_includes out, "Some error in the test framework" - - assert_equal 42, $?.exitstatus - end - private def normalize_xml(output)