diff --git a/ruby/lib/ci/queue/build_record.rb b/ruby/lib/ci/queue/build_record.rb index 99ebf047..aede320c 100644 --- a/ruby/lib/ci/queue/build_record.rb +++ b/ruby/lib/ci/queue/build_record.rb @@ -23,7 +23,7 @@ def record_error(id, payload, stats: nil) record_stats(stats) end - def record_success(id, stats: nil, skip_flaky_record: false) + def record_success(id, stats: nil, skipped: false, requeued: false) error_reports.delete(id) record_stats(stats) end diff --git a/ruby/lib/ci/queue/configuration.rb b/ruby/lib/ci/queue/configuration.rb index 1941b81e..8477ad84 100644 --- a/ruby/lib/ci/queue/configuration.rb +++ b/ruby/lib/ci/queue/configuration.rb @@ -2,7 +2,7 @@ module CI module Queue class Configuration - attr_accessor :timeout, :worker_id, :max_requeues, :grind_count, :failure_file, :export_flaky_tests_file + attr_accessor :timeout, :worker_id, :max_requeues, :grind_count, :failure_file, :export_flaky_tests_file, :export_skipped_tests_file attr_accessor :requeue_tolerance, :namespace, :failing_test, :statsd_endpoint attr_accessor :max_test_duration, :max_test_duration_percentile, :track_test_duration attr_accessor :max_test_failed, :redis_ttl, :warnings_file, :debug_log, :max_missed_heartbeat_seconds @@ -37,7 +37,8 @@ def initialize( grind_count: nil, max_duration: nil, failure_file: nil, max_test_duration: nil, max_test_duration_percentile: 0.5, track_test_duration: false, max_test_failed: nil, queue_init_timeout: nil, redis_ttl: 8 * 60 * 60, report_timeout: nil, inactive_workers_timeout: nil, - export_flaky_tests_file: nil, warnings_file: nil, debug_log: nil, max_missed_heartbeat_seconds: nil) + export_flaky_tests_file: nil, warnings_file: nil, debug_log: nil, max_missed_heartbeat_seconds: nil, + export_skipped_tests_file: nil) @build_id = build_id @circuit_breakers = [CircuitBreaker::Disabled] @failure_file = failure_file @@ -61,6 +62,7 @@ def initialize( @report_timeout = report_timeout @inactive_workers_timeout = inactive_workers_timeout @export_flaky_tests_file = export_flaky_tests_file + @export_skipped_tests_file = export_skipped_tests_file @warnings_file = warnings_file @debug_log = debug_log @max_missed_heartbeat_seconds = max_missed_heartbeat_seconds diff --git a/ruby/lib/ci/queue/redis/build_record.rb b/ruby/lib/ci/queue/redis/build_record.rb index b1329ff9..5cc46fde 100644 --- a/ruby/lib/ci/queue/redis/build_record.rb +++ b/ruby/lib/ci/queue/redis/build_record.rb @@ -69,13 +69,14 @@ def record_error(id, payload, stats: nil) nil end - def record_success(id, stats: nil, skip_flaky_record: false) - error_reports_deleted_count, requeued_count, _ = redis.pipelined do |pipeline| + def record_success(id, stats: nil, skipped: false, requeued: false) + previous_error_count, previous_requeue_count = redis.pipelined do |pipeline| pipeline.hdel(key('error-reports'), id.dup.force_encoding(Encoding::BINARY)) pipeline.hget(key('requeues-count'), id.b) record_stats(stats, pipeline: pipeline) end - record_flaky(id) if !skip_flaky_record && (error_reports_deleted_count.to_i > 0 || requeued_count.to_i > 0) + record_flaky(id) if !skipped && (previous_error_count.to_i > 0 || previous_requeue_count.to_i > 0) + record_skipped(id) if skipped && !requeued nil end @@ -90,6 +91,17 @@ def record_flaky(id, stats: nil) nil end + def record_skipped(id, stats: nil) + redis.pipelined do |pipeline| + pipeline.sadd?( + key('skipped-reports'), + id.b + ) + pipeline.expire(key('skipped-reports'), config.redis_ttl) + end + nil + end + def max_test_failed? return false if config.max_test_failed.nil? @@ -104,6 +116,10 @@ def flaky_reports redis.smembers(key('flaky-reports')) end + def skipped_reports + redis.smembers(key('skipped-reports')) + end + def fetch_stats(stat_names) counts = redis.pipelined do |pipeline| stat_names.each { |c| pipeline.hvals(key(c)) } diff --git a/ruby/lib/minitest/queue/build_status_recorder.rb b/ruby/lib/minitest/queue/build_status_recorder.rb index abe8911e..59225076 100644 --- a/ruby/lib/minitest/queue/build_status_recorder.rb +++ b/ruby/lib/minitest/queue/build_status_recorder.rb @@ -52,7 +52,7 @@ def record(test) if (test.failure || test.error?) && !test.skipped? build.record_error("#{test.klass}##{test.name}", dump(test), stats: stats) else - build.record_success("#{test.klass}##{test.name}", stats: stats, skip_flaky_record: test.skipped?) + build.record_success("#{test.klass}##{test.name}", stats: stats, skipped: test.skipped?, requeued: test.requeued?) end end diff --git a/ruby/lib/minitest/queue/build_status_reporter.rb b/ruby/lib/minitest/queue/build_status_reporter.rb index b34084cb..9f2ea3c4 100644 --- a/ruby/lib/minitest/queue/build_status_reporter.rb +++ b/ruby/lib/minitest/queue/build_status_reporter.rb @@ -21,6 +21,10 @@ def flaky_reports build.flaky_reports end + def skipped_reports + build.skipped_reports + end + def requeued_tests build.requeued_tests end @@ -90,6 +94,10 @@ def write_flaky_tests_file(file) File.write(file, flaky_reports.to_json) end + def write_skipped_tests_file(file) + File.write(file, skipped_reports.to_json) + end + private attr_reader :build diff --git a/ruby/lib/minitest/queue/runner.rb b/ruby/lib/minitest/queue/runner.rb index 5b437088..dbf8f893 100644 --- a/ruby/lib/minitest/queue/runner.rb +++ b/ruby/lib/minitest/queue/runner.rb @@ -261,6 +261,7 @@ def report_command reporter.report reporter.write_failure_file(queue_config.failure_file) if queue_config.failure_file reporter.write_flaky_tests_file(queue_config.export_flaky_tests_file) if queue_config.export_flaky_tests_file + reporter.write_skipped_tests_file(queue_config.export_skipped_tests_file) if queue_config.export_skipped_tests_file msg = "#{supervisor.size} tests weren't run." @@ -276,6 +277,7 @@ def report_command reporter = BuildStatusReporter.new(build: supervisor.build) reporter.write_failure_file(queue_config.failure_file) if queue_config.failure_file reporter.write_flaky_tests_file(queue_config.export_flaky_tests_file) if queue_config.export_flaky_tests_file + reporter.write_skipped_tests_file(queue_config.export_skipped_tests_file) if queue_config.export_skipped_tests_file reporter.report exit! reporter.success? ? 0 : 1 @@ -551,6 +553,15 @@ def parser queue_config.export_flaky_tests_file = file end + help = <<~EOS + Defines a file where skipped tests during the execution are written to in json format. + Defaults to disabled. + EOS + opts.separator "" + opts.on('--export-skipped-tests-file FILE', help) do |file| + queue_config.export_skipped_tests_file = file + end + help = <<~EOS Defines a file where warnings during the execution are written to. Defaults to disabled. diff --git a/ruby/test/integration/minitest_redis_test.rb b/ruby/test/integration/minitest_redis_test.rb index 05e426ef..a657db69 100644 --- a/ruby/test/integration/minitest_redis_test.rb +++ b/ruby/test/integration/minitest_redis_test.rb @@ -824,6 +824,44 @@ def test_redis_reporter_flaky_tests_file end end + def test_redis_reporter_skipped_tests_file + Dir.mktmpdir do |dir| + skipped_tests_file = File.join(dir, 'skipped_tests_file.json') + + capture_subprocess_io do + system( + { 'BUILDKITE' => '1' }, + @exe, 'run', + '--queue', @redis_url, + '--seed', 'foobar', + '--build', '1', + '--worker', '1', + '--timeout', '1', + '--max-requeues', '1', + '--requeue-tolerance', '1', + '-Itest', + 'test/dummy_test.rb', + chdir: 'test/fixtures/', + ) + end + + capture_subprocess_io do + system( + @exe, 'report', + '--queue', @redis_url, + '--build', '1', + '--timeout', '1', + '--export-skipped-tests-file', skipped_tests_file, + chdir: 'test/fixtures/', + ) + end + + content = File.read(skipped_tests_file) + skipped_tests = JSON.parse(content) + assert_includes skipped_tests, "ATest#test_foo" + end + end + def test_redis_reporter # HACK: Simulate a timeout config = CI::Queue::Configuration.new(build_id: '1', worker_id: '1', timeout: '1') diff --git a/ruby/test/minitest/queue/build_status_recorder_test.rb b/ruby/test/minitest/queue/build_status_recorder_test.rb index a2cf52e0..108149bc 100644 --- a/ruby/test/minitest/queue/build_status_recorder_test.rb +++ b/ruby/test/minitest/queue/build_status_recorder_test.rb @@ -37,6 +37,7 @@ def test_aggregation assert_equal 1, summary.requeues assert_equal 5, summary.error_reports.size assert_equal 0, summary.flaky_reports.size + assert_equal 2, summary.skipped_reports.size end def test_retrying_test