From 6d4e9ccf48dd79ae33f92a7a1e58b88a33bc03c7 Mon Sep 17 00:00:00 2001 From: Christian Bruckmayer Date: Tue, 17 Jun 2025 21:54:36 +0100 Subject: [PATCH] Check we executed all tests --- ruby/lib/ci/queue/redis/build_record.rb | 2 ++ ruby/lib/ci/queue/redis/worker.rb | 14 ++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/ruby/lib/ci/queue/redis/build_record.rb b/ruby/lib/ci/queue/redis/build_record.rb index 1f0fac38..23726922 100644 --- a/ruby/lib/ci/queue/redis/build_record.rb +++ b/ruby/lib/ci/queue/redis/build_record.rb @@ -72,6 +72,8 @@ def record_success(id, stats: nil, skip_flaky_record: false, acknowledge: true) error_reports_deleted_count, requeued_count, _ = redis.pipelined do |pipeline| pipeline.hdel(key('error-reports'), id) pipeline.hget(key('requeues-count'), id) + pipeline.sadd(key('success-reports'), id) + pipeline.expire(key('success-reports'), config.redis_ttl) record_stats(stats, pipeline: pipeline) end record_flaky(id) if !skip_flaky_record && (error_reports_deleted_count.to_i > 0 || requeued_count.to_i > 0) diff --git a/ruby/lib/ci/queue/redis/worker.rb b/ruby/lib/ci/queue/redis/worker.rb index db87e566..d4986eee 100644 --- a/ruby/lib/ci/queue/redis/worker.rb +++ b/ruby/lib/ci/queue/redis/worker.rb @@ -69,6 +69,20 @@ def poll pipeline.expire(key('worker', worker_id, 'queue'), config.redis_ttl) pipeline.expire(key('processed'), config.redis_ttl) end + + # check we executed all tests + # only the master should perform this check because otherwise we will DDoS Redis when all workers + # try to fetch the processed tests at the same time + if master? && exhausted? + executed_tests = (redis.smembers(key('success-reports')) + redis.hkeys(key('error-reports'))) + missing_tests = @index.keys - executed_tests + + if missing_tests.size > 0 + puts "ci-queue did not process all tests!" + puts missing_tests + report_worker_error(StandardError.new("ci-queue did not process all tests!")) + end + end rescue *CONNECTION_ERRORS end