diff --git a/ruby/lib/minitest/queue/build_status_reporter.rb b/ruby/lib/minitest/queue/build_status_reporter.rb
index c81f28a3..4d663648 100644
--- a/ruby/lib/minitest/queue/build_status_reporter.rb
+++ b/ruby/lib/minitest/queue/build_status_reporter.rb
@@ -129,7 +129,7 @@ def report
 
         puts aggregates
 
-        if supervisor.time_left.to_i <= 0
+        if timed_out?
           puts red("Timed out waiting for tests to be executed.")
 
           remaining_tests = supervisor.test_ids
@@ -142,7 +142,7 @@ def report
           end
 
           exit_code = TIMED_OUT_EXIT_CODE
-        elsif supervisor.time_left_with_no_workers.to_i <= 0
+        elsif all_workers_died?
           puts red("All workers died.")
           exit_code = WORKERS_DIED_EXIT_CODE
         elsif supervisor.max_test_failed?
@@ -167,7 +167,11 @@ def report
 
       def success?
         build.error_reports.empty? &&
-          build.worker_errors.empty?
+          build.worker_errors.empty? &&
+          build.queue_exhausted? &&
+          !supervisor.max_test_failed? &&
+          !all_workers_died? &&
+          !timed_out?
       end
 
       def record(*)
@@ -220,6 +224,14 @@ def write_flaky_tests_file(file)
 
       attr_reader :build, :supervisor
 
+      def timed_out?
+        supervisor.time_left.to_i <= 0
+      end
+
+      def all_workers_died?
+        supervisor.time_left_with_no_workers.to_i <= 0
+      end
+
       def aggregates
         success = failures.zero? && errors.zero?
         failures_count = "#{failures} failures, #{errors} errors,"
diff --git a/ruby/test/minitest/queue/build_status_reporter_test.rb b/ruby/test/minitest/queue/build_status_reporter_test.rb
new file mode 100644
index 00000000..b1acab49
--- /dev/null
+++ b/ruby/test/minitest/queue/build_status_reporter_test.rb
@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+require 'test_helper'
+
+module Minitest::Queue
+  class BuildStatusReporterTest < Minitest::Test
+    include ReporterTestHelper
+    include QueueHelper
+
+    def setup
+      @redis_url = ENV.fetch('REDIS_URL', 'redis://localhost:6379/0')
+      @redis = ::Redis.new(url: @redis_url)
+      @redis.flushdb
+      @supervisor = supervisor
+      @recorder = BuildStatusRecorder.new(build: @supervisor.build)
+      @reporter = BuildStatusReporter.new(supervisor: @supervisor)
+    end
+
+    def test_timing_out_waiting_for_tests
+      queue = worker(1)
+      exit_status = nil
+
+      queue.poll do |_test|
+        queue.shutdown!
+      end
+
+      @supervisor.instance_variable_set(:@time_left, 0)
+
+      out, err = capture_subprocess_io do
+        exit_status = @reporter.report
+      end
+
+      assert_equal 43, exit_status
+      assert_includes out, "Timed out waiting for tests to be executed."
+    end
+
+    def test_all_workers_died
+      queue = worker(1)
+      exit_status = nil
+
+      queue.poll do |_test|
+        queue.shutdown!
+      end
+
+      @supervisor.instance_variable_set(:@time_left, 1)
+      @supervisor.instance_variable_set(:@time_left_with_no_workers, 0)
+
+      out, err = capture_subprocess_io do
+        exit_status = @reporter.report
+      end
+
+      assert_equal 45, exit_status
+      assert_includes out, "All workers died."
+    end
+
+    private
+
+    def worker(id)
+      result = nil
+      capture_io do
+        result = CI::Queue::Redis.new(
+          @redis_url,
+          CI::Queue::Configuration.new(
+            build_id: '42',
+            worker_id: id.to_s,
+            timeout: 0.2,
+          ),
+        ).populate([
+          Minitest::Queue::SingleExample.new("Minitest::Test", "a")
+        ])
+      end
+      result
+    end
+
+    def supervisor(timeout: 30, queue_init_timeout: nil)
+      CI::Queue::Redis::Supervisor.new(
+        @redis_url,
+        CI::Queue::Configuration.new(
+          build_id: '42',
+          timeout: timeout,
+          queue_init_timeout: queue_init_timeout
+        ),
+      )
+    end
+  end
+end