From 2567d87e2a14c9dd941076a7b8b2e8e925004ebc Mon Sep 17 00:00:00 2001
From: kokuyouwind <kokuyouwind@gmail.com>
Date: Wed, 17 Jan 2018 17:57:22 +0900
Subject: [PATCH 1/3] :rotating_light: add testcase error occuring outside
 examples

---
 local_test/client_config.rb               | 1 +
 local_test/error_outside_examples_spec.rb | 8 ++++++++
 2 files changed, 9 insertions(+)
 create mode 100644 local_test/error_outside_examples_spec.rb

diff --git a/local_test/client_config.rb b/local_test/client_config.rb
index c6536da..a7949ac 100644
--- a/local_test/client_config.rb
+++ b/local_test/client_config.rb
@@ -22,6 +22,7 @@
       'local_test/success_spec.rb',
       'local_test/fail_spec.rb',
       'local_test/timeout_spec.rb',
+      'local_test/error_outside_examples_spec.rb',
     ]
   end
   conf.setup_command = <<-END
diff --git a/local_test/error_outside_examples_spec.rb b/local_test/error_outside_examples_spec.rb
new file mode 100644
index 0000000..8581d15
--- /dev/null
+++ b/local_test/error_outside_examples_spec.rb
@@ -0,0 +1,8 @@
+describe 'test4' do
+  describe 'outside error' do
+    include_context 'not exist context'
+    it 'cause outside error' do
+      expect(1).to eq(1)
+    end
+  end
+end

From c38c379a516f2151a8891b9460cd18f19377dd80 Mon Sep 17 00:00:00 2001
From: kokuyouwind <kokuyouwind@gmail.com>
Date: Wed, 17 Jan 2018 18:16:35 +0900
Subject: [PATCH 2/3] :bug: fail when error outside examples occurred

---
 rrrspec-client/lib/rrrspec/client/slave_runner.rb | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/rrrspec-client/lib/rrrspec/client/slave_runner.rb b/rrrspec-client/lib/rrrspec/client/slave_runner.rb
index 099bca8..3e11291 100644
--- a/rrrspec-client/lib/rrrspec/client/slave_runner.rb
+++ b/rrrspec-client/lib/rrrspec/client/slave_runner.rb
@@ -12,6 +12,7 @@ class SlaveRunner
       TASKQUEUE_ARBITER_TIMEOUT = 20
       TIMEOUT_EXITCODE = 42
       class SoftTimeoutException < Exception; end
+      class OutsideExamplesError < StandardError; end
 
       def initialize(slave, working_dir, taskset_key)
         @slave = slave
@@ -92,7 +93,9 @@ def work
       end
 
       class RedisReportingFormatter
-        RSpec::Core::Formatters.register(self, :example_passed, :example_pending, :example_failed)
+        RSpec::Core::Formatters.register(
+          self, :example_passed, :example_pending, :example_failed, :dump_summary
+        )
 
         def initialize(_output)
           self.class.reset
@@ -110,6 +113,14 @@ def example_failed(notification)
           self.class.example_failed(notification)
         end
 
+        def dump_summary(notification)
+          # RSpec skips all examples when error outside examples occurred
+          # So we will raise error and make current taskset failure
+          if notification.errors_outside_of_examples_count > 0
+            raise OutsideExamplesError
+          end
+        end
+
         module ClassMethods
           attr_reader :passed, :pending, :failed
 

From 96cc530759031ec5aeabbc8940755ab2056c18fa Mon Sep 17 00:00:00 2001
From: kokuyouwind <kokuyouwind@gmail.com>
Date: Fri, 19 Jan 2018 18:25:08 +0900
Subject: [PATCH 3/3] :see_no_evil: skip error_outside_examples_spec.rb in
 local test

It affect other test cases result.
---
 local_test/client_config.rb | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/local_test/client_config.rb b/local_test/client_config.rb
index a7949ac..5e03c54 100644
--- a/local_test/client_config.rb
+++ b/local_test/client_config.rb
@@ -22,7 +22,8 @@
       'local_test/success_spec.rb',
       'local_test/fail_spec.rb',
       'local_test/timeout_spec.rb',
-      'local_test/error_outside_examples_spec.rb',
+      # this file is commented-out because it cause false positive of other tests.
+      # 'local_test/error_outside_examples_spec.rb',
     ]
   end
   conf.setup_command = <<-END