MAD2-155 - Incorrect Madcow Suite Status Result

madcow-core/src/main/groovy/au/com/ps4impact/madcow/MadcowTestCase.groovy

@@ -46,6 +46,7 @@ class MadcowTestCase implements IJSONSerializable {
     public GrassParser grassParser;
     public MadcowStepRunner stepRunner;
     public boolean ignoreTestCase = false;
+    public boolean testCaseError = false;
 
     public ArrayList<MadcowStep> steps = new ArrayList<MadcowStep>();
     public MadcowStep lastExecutedStep;
@@ -232,6 +233,10 @@ class MadcowTestCase implements IJSONSerializable {
                 status = MadcowStepResult.StatusType.FAIL;
         }
 
+        if(this.testCaseError == true) {
+            status = MadcowStepResult.StatusType.FAIL;
+        }
+
         return [
                 name: this.name != null && this.name.length() > 0 ? this.name : null,
                 suiteName: this.testSuite?.name,

madcow-core/src/main/groovy/au/com/ps4impact/madcow/execution/ParallelTestCaseRunner.groovy

@@ -67,6 +67,7 @@ class ParallelTestCaseRunner {
                             testCase.execute();
                             testCase.logInfo("Test ${testCase.name} Passed");
                         } catch (e) {
+                            testCase.testCaseError = true;
                             testCase.logError("Test ${testCase.name} Failed!\nException: $e");
                             return callback.act(P.p(testCase, some(e)));
                         }

madcow-core/src/main/groovy/au/com/ps4impact/madcow/execution/SingleTestCaseRunner.groovy

@@ -49,6 +49,7 @@ class SingleTestCaseRunner {
                     testCase.execute();
                     testCase.logInfo("Test ${testCase.name} Passed");
                 } catch (e) {
+                    testCase.testCaseError = true;
                     testCase.logError("Test ${testCase.name} Failed!\n\nException: $e");
                 }
             }

madcow-core/src/test/groovy/au/com/ps4impact/madcow/execution/ParallelTestCaseRunnerTest.groovy

@@ -0,0 +1,74 @@
+package au.com.ps4impact.madcow.execution
+
+import au.com.ps4impact.madcow.MadcowTestCase
+import fj.Effect
+import fj.P
+import fj.P2
+import fj.Unit
+import fj.control.parallel.Actor
+import fj.control.parallel.Strategy
+import fj.data.Option
+
+import java.util.concurrent.ConcurrentHashMap
+import java.util.concurrent.ExecutorService
+import java.util.concurrent.Executors
+import java.util.concurrent.atomic.AtomicInteger
+
+class ParallelTestCaseRunnerTest extends GroovyTestCase {
+
+    ArrayList<String> grassScript = new ArrayList<String>()
+
+    ExecutorService pool = Executors.newFixedThreadPool(10)
+    Strategy<Unit> strategy = Strategy.executorStrategy(pool)
+
+    ConcurrentHashMap exceptions = [:]
+    AtomicInteger numberOfTestsRan = new AtomicInteger(0)
+
+    def callback = Actor.queueActor(strategy, { P2<MadcowTestCase, Option<Exception>> result ->
+        numberOfTestsRan.andIncrement;
+        exceptions.put(result._1(), result._2())
+    } as Effect);
+
+    void testTestCaseErrorTrue() {
+        MadcowTestCase madcowTestCase = new MadcowTestCase("SingleTestCaseRunnerTest-testExecutionException", grassScript) {
+            @Override
+            void execute() {
+                throw new Exception()
+            }
+
+            @Override
+            protected void createStepRunner() {
+            }
+        }
+
+        executeTest(madcowTestCase)
+        assertTrue(madcowTestCase.testCaseError == true)
+    }
+
+    void testTestCaseErrorFalse() {
+        MadcowTestCase madcowTestCase = new MadcowTestCase("SingleTestCaseRunnerTest-testExecutionException", grassScript) {
+            @Override
+            void execute() {
+            }
+
+            @Override
+            protected void createStepRunner() {
+            }
+        }
+
+        executeTest(madcowTestCase)
+        assertTrue(madcowTestCase.testCaseError == false)
+    }
+
+
+    private void executeTest(MadcowTestCase madcowTestCase) {
+        def parallelTestCaseRunner = new ParallelTestCaseRunner(strategy, callback)
+        try {
+            parallelTestCaseRunner.act(P.p(madcowTestCase, null))
+        } catch(e) {}
+
+        while (numberOfTestsRan.get() < 1) {
+            Thread.sleep(500);
+        }
+    }
+}

madcow-core/src/test/groovy/au/com/ps4impact/madcow/execution/SingleTestCaseRunnerTest.groovy

@@ -0,0 +1,44 @@
+package au.com.ps4impact.madcow.execution
+
+import au.com.ps4impact.madcow.MadcowTestCase
+import au.com.ps4impact.madcow.report.IMadcowReport
+
+class SingleTestCaseRunnerTest extends GroovyTestCase {
+
+    ArrayList<String> grassScript = new ArrayList<String>()
+
+    void testTestCaseErrorTrue() {
+        MadcowTestCase testCase = new MadcowTestCase("SingleTestCaseRunnerTest-testExecutionException", grassScript) {
+            @Override
+            void execute() {
+                throw new Exception()
+            }
+
+            @Override
+            protected void createStepRunner() {
+            }
+        }
+        try {
+            new SingleTestCaseRunner(testCase, new ArrayList<IMadcowReport>())
+        } catch(e) {}
+
+        assertTrue(testCase.testCaseError == true)
+    }
+
+    void testTestCaseErrorFalse() {
+        MadcowTestCase testCase = new MadcowTestCase("SingleTestCaseRunnerTest-testExecutionException", grassScript) {
+            @Override
+            void execute() {
+            }
+
+            @Override
+            protected void createStepRunner() {
+            }
+        }
+        try {
+            new SingleTestCaseRunner(testCase, new ArrayList<IMadcowReport>())
+        } catch(e) {}
+
+        assertTrue(testCase.testCaseError == false)
+    }
+}