From 642b4126e5e59e8547d527690de893639c504f4b Mon Sep 17 00:00:00 2001 From: Kenneth Lausdahl Date: Thu, 22 Feb 2018 22:56:28 +0100 Subject: [PATCH 01/12] added testing support for vdm sl fixed #668 --- core/interpreter/src/main/java/TestCase.java | 52 +++-- .../interpreter/src/main/java/TestRunner.java | 218 +++++++++++------- 2 files changed, 170 insertions(+), 100 deletions(-) diff --git a/core/interpreter/src/main/java/TestCase.java b/core/interpreter/src/main/java/TestCase.java index 4be9432b2a..ad29b7ee00 100644 --- a/core/interpreter/src/main/java/TestCase.java +++ b/core/interpreter/src/main/java/TestCase.java @@ -1,8 +1,9 @@ +import org.overture.ast.definitions.AExplicitOperationDefinition; +import org.overture.ast.modules.AModuleModules; import org.overture.interpreter.runtime.*; import org.overture.interpreter.values.*; import org.w3c.dom.Document; import org.w3c.dom.Element; -import org.w3c.dom.Node; import org.w3c.dom.NodeList; import org.xml.sax.SAXException; @@ -44,19 +45,17 @@ public static Value reflectionRunTest(Value obj, Value name) success = true; } catch (Exception e) { if (e instanceof ExitException) { - if(((ExitException)e).value.objectValue(null).type.getName().equals("AssertionFailedError")) - { + if (((ExitException) e).value.objectValue(null).type.getName().equals("AssertionFailedError")) { success = false; } throw e; } try { - return ClassInterpreter.getInstance().evaluate("Error`throw(\"" + return ClassInterpreter.getInstance().evaluate("Error`throw(\"" + e.getMessage().replaceAll("\\\\", "\\\\\\\\").replaceAll("\"", "\\\\\"").replaceAll("\'", "\\\'") + "\")", mainContext); - }catch(ExitException e2) - { + } catch (ExitException e2) { error = e2; throw e2; } @@ -81,6 +80,32 @@ public static Value reflectionRunTest(Value obj, Value name) } + public static boolean reflectionRunTest(AModuleModules module, AExplicitOperationDefinition opDef) throws Exception { + String moduleName = module.getName().getName(); + String testName = opDef.getName().getName(); + + long timerStart = System.nanoTime(); + boolean success = false; + ExitException error = null; + try { + ModuleInterpreter.getInstance().evaluate(moduleName + "`" + testName + "()" + , ModuleInterpreter.getInstance().initialContext); + success = true; + } catch (Exception e) { + if (e instanceof ExitException) { + success = false; + } + throw e; + } finally { + long totalExecTime = System.nanoTime() - timerStart; + + if (System.getProperty(vdmUnitReportEnable) != null) { + recordTestResults(moduleName, testName, success, error, totalExecTime); + } + } + return success; + } + private static void recordTestResults(String containerName, String methodName, boolean success, ExitException error, long totalExecTime) throws ParserConfigurationException, IOException, SAXException, XPathExpressionException, TransformerException { File report = new File("TEST-" + containerName + ".xml"); @@ -115,8 +140,7 @@ private static void recordTestResults(String containerName, String methodName, b n = doc.createElement("testcase"); } - while(n.getFirstChild()!=null) - { + while (n.getFirstChild() != null) { n.removeChild(n.getFirstChild()); } @@ -127,20 +151,20 @@ private static void recordTestResults(String containerName, String methodName, b testSuiteNode.setAttribute("tests", String.valueOf(Integer.parseInt(testSuiteNode.getAttribute("tests")) + 1)); - if (error!=null) { + if (error != null) { testSuiteNode.setAttribute("error", String.valueOf(Integer.parseInt(testSuiteNode.getAttribute("errors")) + 1)); Element errorElement = doc.createElement("error"); - errorElement.setAttribute("message",error.number+""); - errorElement.setAttribute("type","ERROR"); + errorElement.setAttribute("message", error.number + ""); + errorElement.setAttribute("type", "ERROR"); StringWriter strOut = new StringWriter(); - error.ctxt.printStackTrace(new PrintWriter(strOut),true); + error.ctxt.printStackTrace(new PrintWriter(strOut), true); errorElement.setTextContent(strOut.toString()); n.appendChild(errorElement); } else if (!success) { testSuiteNode.setAttribute("failures", String.valueOf(Integer.parseInt(testSuiteNode.getAttribute("failures")) + 1)); Element failureElement = doc.createElement("failure"); - failureElement.setAttribute("message",methodName); - failureElement.setAttribute("type","WARNING"); + failureElement.setAttribute("message", methodName); + failureElement.setAttribute("type", "WARNING"); failureElement.setAttribute("time", totalExecTime * 1E-9 + ""); n.appendChild(failureElement); } diff --git a/core/interpreter/src/main/java/TestRunner.java b/core/interpreter/src/main/java/TestRunner.java index 50a1aae924..481ecba4e6 100644 --- a/core/interpreter/src/main/java/TestRunner.java +++ b/core/interpreter/src/main/java/TestRunner.java @@ -1,90 +1,136 @@ +import org.overture.ast.definitions.AExplicitOperationDefinition; +import org.overture.ast.definitions.PDefinition; +import org.overture.ast.definitions.SClassDefinition; +import org.overture.ast.modules.AModuleModules; +import org.overture.interpreter.messages.Console; +import org.overture.interpreter.runtime.*; +import org.overture.interpreter.values.*; + import java.util.List; import java.util.Vector; -import org.overture.ast.definitions.SClassDefinition; -import org.overture.interpreter.runtime.*; -import org.overture.interpreter.values.ObjectValue; -import org.overture.interpreter.values.SetValue; -import org.overture.interpreter.values.Value; -import org.overture.interpreter.values.ValueSet; - -public class TestRunner -{ - public static Value collectTests(Value obj) - { - List tests = new Vector(); - ObjectValue instance = (ObjectValue) obj; - - if (ClassInterpreter.getInstance() instanceof ClassInterpreter) - { - for (SClassDefinition def : ((ClassInterpreter) ClassInterpreter.getInstance()).getClasses()) - { - if (def.getIsAbstract() || !isTestClass(def)) - { - continue; - } - tests.add(def.getName().getName()); - } - } - - Context mainContext = new StateContext(Interpreter.getInstance().getAssistantFactory(), instance.type.getLocation(), "reflection scope"); - - mainContext.putAll(ClassInterpreter.getInstance().initialContext); - mainContext.setThreadState(ClassInterpreter.getInstance().initialContext.threadState.dbgp, ClassInterpreter.getInstance().initialContext.threadState.CPU); - - ValueSet vals = new ValueSet(); - for (String value : tests) - { - try - { - vals.add(ClassInterpreter.getInstance().evaluate("new " + value - + "()", mainContext)); - } catch (Exception e) - { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - - try - { - return new SetValue(vals); - } - catch (ValueException e) - { - return null; // Not reached - } - } - - private static boolean isTestClass(SClassDefinition def) - { - if (def.getIsAbstract() || def.getName().getName().equals("Test") - || def.getName().getName().equals("TestCase") - || def.getName().getName().equals("TestSuite")) - { - return false; - } - - if (checkForSuper(def, "TestSuite")) - { - // the implementation must be upgrade before this work. - // The upgrade should handle the static method for creatint the suire - return false; - } - - return checkForSuper(def, "Test"); - } - - private static boolean checkForSuper(SClassDefinition def, String superName) - { - for (SClassDefinition superDef : def.getSuperDefs()) - { - if (superDef.getName().getName().equals(superName) - || checkForSuper(superDef, superName)) - { - return true; - } - } - return false; - } +public class TestRunner { + public static Value collectTests(Value obj) { + List tests = new Vector(); + ObjectValue instance = (ObjectValue) obj; + + if (ClassInterpreter.getInstance() instanceof ClassInterpreter) { + for (SClassDefinition def : ((ClassInterpreter) ClassInterpreter.getInstance()).getClasses()) { + if (def.getIsAbstract() || !isTestClass(def)) { + continue; + } + tests.add(def.getName().getName()); + } + } + + Context mainContext = new StateContext(Interpreter.getInstance().getAssistantFactory(), instance.type.getLocation(), "reflection scope"); + + mainContext.putAll(ClassInterpreter.getInstance().initialContext); + mainContext.setThreadState(ClassInterpreter.getInstance().initialContext.threadState.dbgp, ClassInterpreter.getInstance().initialContext.threadState.CPU); + + ValueSet vals = new ValueSet(); + for (String value : tests) { + try { + vals.add(ClassInterpreter.getInstance().evaluate("new " + value + + "()", mainContext)); + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + try { + return new SetValue(vals); + } catch (ValueException e) { + return null; // Not reached + } + } + + private static boolean isTestClass(SClassDefinition def) { + if (def.getIsAbstract() || def.getName().getName().equals("Test") + || def.getName().getName().equals("TestCase") + || def.getName().getName().equals("TestSuite")) { + return false; + } + + if (checkForSuper(def, "TestSuite")) { + // the implementation must be upgrade before this work. + // The upgrade should handle the static method for creatint the suire + return false; + } + + return checkForSuper(def, "Test"); + } + + private static boolean checkForSuper(SClassDefinition def, String superName) { + for (SClassDefinition superDef : def.getSuperDefs()) { + if (superDef.getName().getName().equals(superName) + || checkForSuper(superDef, superName)) { + return true; + } + } + return false; + } + + + public static Value run() { + //lets find the modules + List tests = new Vector<>(); + + if (ModuleInterpreter.getInstance() instanceof ModuleInterpreter) { + for (AModuleModules def : ((ModuleInterpreter) ModuleInterpreter.getInstance()).getModules()) { + if (!def.getName().getName().endsWith("Test")) { + continue; + } + tests.add(def); + } + } + + int testCount = 0; + int testFailCount = 0; + int testErrorCount = 0; + + for (AModuleModules module : tests) { + String moduleName = module.getName().getName(); + for (PDefinition def : module.getDefs()) { + + String testName = def.getName().getName(); + if (def instanceof AExplicitOperationDefinition && testName.startsWith("test")) { + try { + testCount++; + Console.out.println("Executing test: " + moduleName + "`" + testName + "()"); + if (TestCase.reflectionRunTest(module, (AExplicitOperationDefinition) def)) { + Console.out.println("\tOK"); + } else { + testFailCount++; + Console.out.println("\tFAIL"); + } + } catch (Exception e) { + testErrorCount++; + Console.out.println("\tERROR"); + } + } + } + } + + String header = "----------------------------------------\n" + + "| TEST RESULTS |\n" + + "|--------------------------------------|"; + Console.out.println(header); + Console.out.println(String.format("| Executed: %1$-" + 27 + "s", testCount) + "|"); + Console.out.println(String.format("| Failures: %1$-" + 27 + "s", testFailCount) + "|"); + Console.out.println(String.format("| Errors : %1$-" + 27 + "s", testErrorCount) + "|"); + Console.out.println("|______________________________________|"); + + + if (testErrorCount == 0 && testFailCount == 0) { + Console.out.println("| SUCCESS |"); + } else { + Console.out.println("| FAILURE |"); + } + Console.out.println("|______________________________________|\n"); + + + return new VoidValue(); + } } From b672ce3f909fb36921ed870b3449a008b3de40a7 Mon Sep 17 00:00:00 2001 From: Kenneth Lausdahl Date: Fri, 2 Mar 2018 11:56:04 +0100 Subject: [PATCH 02/12] do not remove workspace --- Jenkinsfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 3799ee3905..daeeef033b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -52,10 +52,6 @@ node { throw any //rethrow exception to prevent the build from proceeding } finally { - stage ('Clean up workspace'){ - step([$class: 'WsCleanup']) - } - stage('Reporting'){ // Notify on build failure using the Email-ext plugin emailext(body: '${DEFAULT_CONTENT}', mimeType: 'text/html', From a30cba5d26ae995456102de56eb56e30c7777988 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20W=2E=20V=2E=20Tran-J=C3=B8rgensen?= Date: Fri, 2 Mar 2018 13:29:46 +0100 Subject: [PATCH 03/12] Support VDM-SL test failures --- .../interpreter/src/main/java/TestRunner.java | 51 +++++++++++++++++-- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/core/interpreter/src/main/java/TestRunner.java b/core/interpreter/src/main/java/TestRunner.java index 481ecba4e6..2cde651e8d 100644 --- a/core/interpreter/src/main/java/TestRunner.java +++ b/core/interpreter/src/main/java/TestRunner.java @@ -10,6 +10,31 @@ import java.util.Vector; public class TestRunner { + + private static boolean fail = false; + private static String msg = null; + + public static Value markFail() + { + fail = true; + return new VoidValue(); + } + + public static Value setMsg(Value msgVal) + { + if(msgVal instanceof SeqValue) + { + try { + msg = msgVal.stringValue(null); + } catch (ValueException e) { + + Console.out.println("\tReceived unexpected message: " + msgVal); + } + } + + return new VoidValue(); + } + public static Value collectTests(Value obj) { List tests = new Vector(); ObjectValue instance = (ObjectValue) obj; @@ -91,19 +116,37 @@ public static Value run() { int testErrorCount = 0; for (AModuleModules module : tests) { + String moduleName = module.getName().getName(); for (PDefinition def : module.getDefs()) { String testName = def.getName().getName(); if (def instanceof AExplicitOperationDefinition && testName.startsWith("test")) { + + fail = false; + msg = null; + try { testCount++; Console.out.println("Executing test: " + moduleName + "`" + testName + "()"); - if (TestCase.reflectionRunTest(module, (AExplicitOperationDefinition) def)) { - Console.out.println("\tOK"); - } else { + + TestCase.reflectionRunTest(module, (AExplicitOperationDefinition) def); + + if(fail) + { testFailCount++; - Console.out.println("\tFAIL"); + //TODO: is this the right format? + if(msg != null) + { + Console.out.println("\tFAIL: " + msg); + } + else + { + Console.out.println("\tFAIL"); + } + } + else { + Console.out.println("\tOK"); } } catch (Exception e) { testErrorCount++; From 3c43cf5d744d7852c5794f5300a92e30bc2cd240 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20W=2E=20V=2E=20Tran-J=C3=B8rgensen?= Date: Fri, 2 Mar 2018 13:56:11 +0100 Subject: [PATCH 04/12] Support VDM-SL setUp and tearDown --- .../interpreter/src/main/java/TestRunner.java | 49 ++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/core/interpreter/src/main/java/TestRunner.java b/core/interpreter/src/main/java/TestRunner.java index 2cde651e8d..4bf6b65192 100644 --- a/core/interpreter/src/main/java/TestRunner.java +++ b/core/interpreter/src/main/java/TestRunner.java @@ -6,6 +6,7 @@ import org.overture.interpreter.runtime.*; import org.overture.interpreter.values.*; +import java.util.LinkedList; import java.util.List; import java.util.Vector; @@ -116,7 +117,10 @@ public static Value run() { int testErrorCount = 0; for (AModuleModules module : tests) { - + + AExplicitOperationDefinition setup = findLifecycleOp(module.getDefs(), "setUp"); + AExplicitOperationDefinition tearDown = findLifecycleOp(module.getDefs(), "tearDown"); + String moduleName = module.getName().getName(); for (PDefinition def : module.getDefs()) { @@ -126,12 +130,26 @@ public static Value run() { fail = false; msg = null; + boolean tearDownRun = false; + try { testCount++; Console.out.println("Executing test: " + moduleName + "`" + testName + "()"); + if(setup != null) + { + TestCase.reflectionRunTest(module, setup); + } + TestCase.reflectionRunTest(module, (AExplicitOperationDefinition) def); + tearDownRun = true; + + if(tearDown != null) + { + TestCase.reflectionRunTest(module, tearDown); + } + if(fail) { testFailCount++; @@ -152,6 +170,22 @@ public static Value run() { testErrorCount++; Console.out.println("\tERROR"); } + finally + { + if (!tearDownRun) + { + if (tearDown != null) + { + try + { + TestCase.reflectionRunTest(module, tearDown); + } catch (Exception e) + { + e.printStackTrace(); + } + } + } + } } } } @@ -176,4 +210,17 @@ public static Value run() { return new VoidValue(); } + + private static AExplicitOperationDefinition findLifecycleOp(LinkedList defs, String name) { + + for(PDefinition d : defs) + { + if(d instanceof AExplicitOperationDefinition && d.getName().getName().equals(name)) + { + return (AExplicitOperationDefinition) d; + } + } + + return null; + } } From e3a3e8c9f6c3ae88e107d506e65d93a883be9aeb Mon Sep 17 00:00:00 2001 From: Kenneth Lausdahl Date: Fri, 2 Mar 2018 14:47:47 +0100 Subject: [PATCH 05/12] increased log history jenkins --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index daeeef033b..d711f95008 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,7 +2,7 @@ node { try { // Only keep one build - properties([[$class: 'BuildDiscarderProperty', strategy: [$class: 'LogRotator', numToKeepStr: '1']]]) + properties([[$class: 'BuildDiscarderProperty', strategy: [$class: 'LogRotator', numToKeepStr: '25']]]) // Mark the code checkout 'stage'.... stage ('Checkout'){ From f56c00f41b73c5d5c685eb91645caf4e267f084b Mon Sep 17 00:00:00 2001 From: Kenneth Lausdahl Date: Mon, 5 Mar 2018 13:42:44 +0100 Subject: [PATCH 06/12] fixed null pointer, where a name was taken out of a function that uses patterns --- core/interpreter/src/main/java/TestRunner.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/core/interpreter/src/main/java/TestRunner.java b/core/interpreter/src/main/java/TestRunner.java index 4bf6b65192..d108aad833 100644 --- a/core/interpreter/src/main/java/TestRunner.java +++ b/core/interpreter/src/main/java/TestRunner.java @@ -124,9 +124,8 @@ public static Value run() { String moduleName = module.getName().getName(); for (PDefinition def : module.getDefs()) { - String testName = def.getName().getName(); - if (def instanceof AExplicitOperationDefinition && testName.startsWith("test")) { - + if (def instanceof AExplicitOperationDefinition && def.getName().getName().startsWith("test")) { + fail = false; msg = null; @@ -134,7 +133,7 @@ public static Value run() { try { testCount++; - Console.out.println("Executing test: " + moduleName + "`" + testName + "()"); + Console.out.println("Executing test: " + moduleName + "`" + def.getName().getName() + "()"); if(setup != null) { From fdb91cc311e4577906eb69d936622ee31f26d8b4 Mon Sep 17 00:00:00 2001 From: Kenneth Lausdahl Date: Mon, 5 Mar 2018 14:24:20 +0100 Subject: [PATCH 07/12] updated junit report with error messages and fail flag for sl --- core/interpreter/src/main/java/TestCase.java | 9 +++++++-- core/interpreter/src/main/java/TestRunner.java | 12 ++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/core/interpreter/src/main/java/TestCase.java b/core/interpreter/src/main/java/TestCase.java index ad29b7ee00..61240e27fa 100644 --- a/core/interpreter/src/main/java/TestCase.java +++ b/core/interpreter/src/main/java/TestCase.java @@ -90,10 +90,11 @@ public static boolean reflectionRunTest(AModuleModules module, AExplicitOperatio try { ModuleInterpreter.getInstance().evaluate(moduleName + "`" + testName + "()" , ModuleInterpreter.getInstance().initialContext); - success = true; + success = !TestRunner.isFailed(); } catch (Exception e) { + success = false; if (e instanceof ExitException) { - success = false; + error = (ExitException) e; } throw e; } finally { @@ -166,6 +167,10 @@ private static void recordTestResults(String containerName, String methodName, b failureElement.setAttribute("message", methodName); failureElement.setAttribute("type", "WARNING"); failureElement.setAttribute("time", totalExecTime * 1E-9 + ""); + if(TestRunner.getMsg()!=null) + { + failureElement.setTextContent(TestRunner.getMsg()); + } n.appendChild(failureElement); } diff --git a/core/interpreter/src/main/java/TestRunner.java b/core/interpreter/src/main/java/TestRunner.java index d108aad833..65808235c7 100644 --- a/core/interpreter/src/main/java/TestRunner.java +++ b/core/interpreter/src/main/java/TestRunner.java @@ -20,6 +20,11 @@ public static Value markFail() fail = true; return new VoidValue(); } + + public static boolean isFailed() + { + return fail; + } public static Value setMsg(Value msgVal) { @@ -35,6 +40,11 @@ public static Value setMsg(Value msgVal) return new VoidValue(); } + + public static String getMsg() + { + return msg; + } public static Value collectTests(Value obj) { List tests = new Vector(); @@ -184,6 +194,8 @@ public static Value run() { } } } + msg = null; + fail = false; } } } From d5472aef74d62925c3639ef75f18b90e104bf6d7 Mon Sep 17 00:00:00 2001 From: Kenneth Lausdahl Date: Tue, 6 Mar 2018 09:56:32 +0100 Subject: [PATCH 08/12] change junit report message to use error message instead of plain error number --- core/interpreter/src/main/java/TestCase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/interpreter/src/main/java/TestCase.java b/core/interpreter/src/main/java/TestCase.java index 61240e27fa..51d42d8b91 100644 --- a/core/interpreter/src/main/java/TestCase.java +++ b/core/interpreter/src/main/java/TestCase.java @@ -155,7 +155,7 @@ private static void recordTestResults(String containerName, String methodName, b if (error != null) { testSuiteNode.setAttribute("error", String.valueOf(Integer.parseInt(testSuiteNode.getAttribute("errors")) + 1)); Element errorElement = doc.createElement("error"); - errorElement.setAttribute("message", error.number + ""); + errorElement.setAttribute("message", error.getMessage() + ""); errorElement.setAttribute("type", "ERROR"); StringWriter strOut = new StringWriter(); error.ctxt.printStackTrace(new PrintWriter(strOut), true); From 6bf885f8b1162aaf25ce13c5fe5b29ee5420e0b0 Mon Sep 17 00:00:00 2001 From: Kenneth Lausdahl Date: Tue, 6 Mar 2018 13:16:45 +0100 Subject: [PATCH 09/12] fixed setup teardown calls so not test report info is recorded for these + added error message to console and handling of internal interpreter errors in report --- core/interpreter/src/main/java/TestCase.java | 28 +++++++++++++++---- .../interpreter/src/main/java/TestRunner.java | 8 +++--- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/core/interpreter/src/main/java/TestCase.java b/core/interpreter/src/main/java/TestCase.java index 51d42d8b91..113ee2ac8b 100644 --- a/core/interpreter/src/main/java/TestCase.java +++ b/core/interpreter/src/main/java/TestCase.java @@ -86,16 +86,14 @@ public static boolean reflectionRunTest(AModuleModules module, AExplicitOperatio long timerStart = System.nanoTime(); boolean success = false; - ExitException error = null; + Exception error = null; try { ModuleInterpreter.getInstance().evaluate(moduleName + "`" + testName + "()" , ModuleInterpreter.getInstance().initialContext); success = !TestRunner.isFailed(); } catch (Exception e) { success = false; - if (e instanceof ExitException) { - error = (ExitException) e; - } + error = e; throw e; } finally { long totalExecTime = System.nanoTime() - timerStart; @@ -107,7 +105,20 @@ public static boolean reflectionRunTest(AModuleModules module, AExplicitOperatio return success; } - private static void recordTestResults(String containerName, String methodName, boolean success, ExitException error, long totalExecTime) throws ParserConfigurationException, IOException, SAXException, XPathExpressionException, TransformerException { + + public static void reflectionRun(AModuleModules module, AExplicitOperationDefinition opDef) { + String moduleName = module.getName().getName(); + String testName = opDef.getName().getName(); + + try { + ModuleInterpreter.getInstance().evaluate(moduleName + "`" + testName + "()" + , ModuleInterpreter.getInstance().initialContext); + } catch (Exception e) { + // Fine + } + } + + private static void recordTestResults(String containerName, String methodName, boolean success, Exception error, long totalExecTime) throws ParserConfigurationException, IOException, SAXException, XPathExpressionException, TransformerException { File report = new File("TEST-" + containerName + ".xml"); @@ -158,7 +169,12 @@ private static void recordTestResults(String containerName, String methodName, b errorElement.setAttribute("message", error.getMessage() + ""); errorElement.setAttribute("type", "ERROR"); StringWriter strOut = new StringWriter(); - error.ctxt.printStackTrace(new PrintWriter(strOut), true); + if(error instanceof ExitException) { + ((ExitException)error).ctxt.printStackTrace(new PrintWriter(strOut), true); + }else + { + error.printStackTrace(new PrintWriter(strOut)); + } errorElement.setTextContent(strOut.toString()); n.appendChild(errorElement); } else if (!success) { diff --git a/core/interpreter/src/main/java/TestRunner.java b/core/interpreter/src/main/java/TestRunner.java index 65808235c7..e0d9b183dd 100644 --- a/core/interpreter/src/main/java/TestRunner.java +++ b/core/interpreter/src/main/java/TestRunner.java @@ -147,7 +147,7 @@ public static Value run() { if(setup != null) { - TestCase.reflectionRunTest(module, setup); + TestCase.reflectionRun(module, setup); } TestCase.reflectionRunTest(module, (AExplicitOperationDefinition) def); @@ -156,7 +156,7 @@ public static Value run() { if(tearDown != null) { - TestCase.reflectionRunTest(module, tearDown); + TestCase.reflectionRun(module, tearDown); } if(fail) @@ -177,7 +177,7 @@ public static Value run() { } } catch (Exception e) { testErrorCount++; - Console.out.println("\tERROR"); + Console.out.println("\tERROR: "+e.getMessage()); } finally { @@ -187,7 +187,7 @@ public static Value run() { { try { - TestCase.reflectionRunTest(module, tearDown); + TestCase.reflectionRun(module, tearDown); } catch (Exception e) { e.printStackTrace(); From 020ba1a7e08fba6ecacb4b00acbe6f701e713688 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20W=2E=20V=2E=20Tran-J=C3=B8rgensen?= Date: Fri, 23 Mar 2018 13:02:45 +0100 Subject: [PATCH 10/12] Update JUnit4Trans to handle SL Assert and TestRunner --- .../codegen/vdm2java/JUnit4Trans.java | 46 +++++++++++++++++++ .../org/overture/codegen/ir/IRConstants.java | 2 +- 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/core/codegen/javagen/src/main/java/org/overture/codegen/vdm2java/JUnit4Trans.java b/core/codegen/javagen/src/main/java/org/overture/codegen/vdm2java/JUnit4Trans.java index 6002cffa6b..ec05a848e2 100644 --- a/core/codegen/javagen/src/main/java/org/overture/codegen/vdm2java/JUnit4Trans.java +++ b/core/codegen/javagen/src/main/java/org/overture/codegen/vdm2java/JUnit4Trans.java @@ -7,11 +7,14 @@ import org.overture.codegen.assistant.NodeAssistantIR; import org.overture.codegen.ir.IRConstants; import org.overture.codegen.ir.IRGeneratedTag; +import org.overture.codegen.ir.STypeIR; import org.overture.codegen.ir.analysis.AnalysisException; import org.overture.codegen.ir.analysis.DepthFirstAnalysisAdaptor; import org.overture.codegen.ir.declarations.ADefaultClassDeclIR; import org.overture.codegen.ir.declarations.AMethodDeclIR; import org.overture.codegen.ir.declarations.SClassDeclIR; +import org.overture.codegen.ir.statements.APlainCallStmIR; +import org.overture.codegen.ir.types.AClassTypeIR; import org.overture.codegen.trans.assistants.TransAssistantIR; import org.overture.config.Settings; @@ -26,6 +29,12 @@ public class JUnit4Trans extends DepthFirstAnalysisAdaptor private static final String TEST_NAME_PREFIX = "test"; public static final String TEST_ANNOTATION = "@Test"; public static final String JUNI4_IMPORT = "org.junit.*"; + public static final String ASSERT_MODULE = "Assert"; + public static final String ASSERT_TRUE_MSG_METHOD = "assertTrueMsg"; + public static final String ASSERT_FALSE_MSG_METHOD = "assertFalseMsg"; + public static final String JUNIT4_ASSERT_TRUE_METHOD = "assertTrue"; + public static final String JUNIT4_ASSERT_FALSE_METHOD = "assertFalse"; + public TransAssistantIR assist; private JavaCodeGen javaCg; @@ -119,6 +128,43 @@ private void adjustTestClass(ADefaultClassDeclIR copy) { for(AMethodDeclIR m : copy.getMethods()) { m.setStatic(false); + + try + { + m.apply(new DepthFirstAnalysisAdaptor() + { + @Override + public void caseAPlainCallStmIR(APlainCallStmIR node) + throws AnalysisException + { + STypeIR type = node.getClassType(); + + if (type instanceof AClassTypeIR) + { + AClassTypeIR classType = (AClassTypeIR) type; + + if (classType.getName().equals(ASSERT_MODULE)) + { + if (node.getName().equals(ASSERT_FALSE_MSG_METHOD)) + { + node.setName(JUNIT4_ASSERT_FALSE_METHOD); + } else if (node.getName().equals(ASSERT_TRUE_MSG_METHOD)) + { + node.setName(JUNIT4_ASSERT_TRUE_METHOD); + } + } + } + else + { + log.warn("Expected class type but got: " + type); + } + } + }); + } catch (AnalysisException e) + { + log.error("Got unexpected analysis error: " + e.getMessage()); + e.printStackTrace(); + } } for(int i = 0; i < copy.getMethods().size(); i++) diff --git a/core/codegen/platform/src/main/java/org/overture/codegen/ir/IRConstants.java b/core/codegen/platform/src/main/java/org/overture/codegen/ir/IRConstants.java index 5e9e4aa41c..d2c41d2066 100644 --- a/core/codegen/platform/src/main/java/org/overture/codegen/ir/IRConstants.java +++ b/core/codegen/platform/src/main/java/org/overture/codegen/ir/IRConstants.java @@ -38,7 +38,7 @@ public class IRConstants public static final String TEST_MODULE_NAME_POSTFIX = "Test"; - public static final String[] CLASS_NAMES_USED_IN_SL = {"CSV", "IO", "MATH", "VDMUtil"}; + public static final String[] CLASS_NAMES_USED_IN_SL = {"CSV", "IO", "MATH", "VDMUtil", "Assert", "TestRunner"}; public static final String[] CLASS_NAMES_USED_IN_VDM_PP_RT = (String[]) ArrayUtils.addAll(CLASS_NAMES_USED_IN_SL, new String[]{"VDMUnit", "Throwable", "Error", "AssertionFailedError", From 3f58d3f59a39f9219c2f1529e7030b3d3d0ee4bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20W=2E=20V=2E=20Tran-J=C3=B8rgensen?= Date: Fri, 23 Mar 2018 13:20:45 +0100 Subject: [PATCH 11/12] Remove wrong warning --- .../main/java/org/overture/codegen/vdm2java/JUnit4Trans.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/core/codegen/javagen/src/main/java/org/overture/codegen/vdm2java/JUnit4Trans.java b/core/codegen/javagen/src/main/java/org/overture/codegen/vdm2java/JUnit4Trans.java index ff63b62fe2..8c44de8592 100644 --- a/core/codegen/javagen/src/main/java/org/overture/codegen/vdm2java/JUnit4Trans.java +++ b/core/codegen/javagen/src/main/java/org/overture/codegen/vdm2java/JUnit4Trans.java @@ -158,10 +158,6 @@ public void caseAPlainCallStmIR(APlainCallStmIR node) } } } - else - { - log.warn("Expected class type but got: " + type); - } } }); } catch (AnalysisException e) From 8c36cd242f0af169a62798b1879e3727ba768330 Mon Sep 17 00:00:00 2001 From: Kenneth Lausdahl Date: Mon, 9 Jul 2018 22:54:09 +0200 Subject: [PATCH 12/12] fixed issue with test not stopping execution after markfail --- .../ast/messages/InternalException.java | 6 ++++++ core/interpreter/src/main/java/TestCase.java | 17 ++++++++++++++++- core/interpreter/src/main/java/TestRunner.java | 12 +++++++++--- .../org/overture/interpreter/util/Delegate.java | 2 +- 4 files changed, 32 insertions(+), 5 deletions(-) diff --git a/core/ast/src/main/java/org/overture/ast/messages/InternalException.java b/core/ast/src/main/java/org/overture/ast/messages/InternalException.java index 4480278e4e..9cb5575010 100644 --- a/core/ast/src/main/java/org/overture/ast/messages/InternalException.java +++ b/core/ast/src/main/java/org/overture/ast/messages/InternalException.java @@ -36,6 +36,12 @@ public InternalException(int number, String message) this.number = number; } + public InternalException(int number, String message, Throwable cause) + { + super(message,cause); + this.number = number; + } + @Override public String toString() { diff --git a/core/interpreter/src/main/java/TestCase.java b/core/interpreter/src/main/java/TestCase.java index 113ee2ac8b..b766821687 100644 --- a/core/interpreter/src/main/java/TestCase.java +++ b/core/interpreter/src/main/java/TestCase.java @@ -1,4 +1,5 @@ import org.overture.ast.definitions.AExplicitOperationDefinition; +import org.overture.ast.messages.InternalException; import org.overture.ast.modules.AModuleModules; import org.overture.interpreter.runtime.*; import org.overture.interpreter.values.*; @@ -15,6 +16,7 @@ import javax.xml.transform.stream.StreamResult; import javax.xml.xpath.*; import java.io.*; +import java.lang.reflect.InvocationTargetException; public class TestCase { @@ -91,7 +93,20 @@ public static boolean reflectionRunTest(AModuleModules module, AExplicitOperatio ModuleInterpreter.getInstance().evaluate(moduleName + "`" + testName + "()" , ModuleInterpreter.getInstance().initialContext); success = !TestRunner.isFailed(); - } catch (Exception e) { + } catch(TestRunner.TestAsserException e) + { + success =false; + }catch(InternalException e) + { + if(e.getCause() instanceof InvocationTargetException && ((InvocationTargetException)e.getCause()).getTargetException() instanceof TestRunner.TestAsserException) + { + success =false; + }else + { + throw e; + } + } + catch (Exception e) { success = false; error = e; throw e; diff --git a/core/interpreter/src/main/java/TestRunner.java b/core/interpreter/src/main/java/TestRunner.java index e0d9b183dd..d605daaf90 100644 --- a/core/interpreter/src/main/java/TestRunner.java +++ b/core/interpreter/src/main/java/TestRunner.java @@ -11,14 +11,20 @@ import java.util.Vector; public class TestRunner { + + public static class TestAsserException extends RuntimeException + { + public TestAsserException(String message) { + super(message); + } + } private static boolean fail = false; private static String msg = null; - public static Value markFail() - { + public static Value markFail() { fail = true; - return new VoidValue(); + throw new TestAsserException("Assert message: "+getMsg()); } public static boolean isFailed() diff --git a/core/interpreter/src/main/java/org/overture/interpreter/util/Delegate.java b/core/interpreter/src/main/java/org/overture/interpreter/util/Delegate.java index b9510e62b5..16f139bb5d 100644 --- a/core/interpreter/src/main/java/org/overture/interpreter/util/Delegate.java +++ b/core/interpreter/src/main/java/org/overture/interpreter/util/Delegate.java @@ -355,7 +355,7 @@ public Value invokeDelegate(Object delegateObject, Context ctxt) strOut.append(e.getTargetException().getMessage()+"\n"); e.getTargetException().printStackTrace(new PrintWriter(strOut)); throw new InternalException(59, "Failed in native method: " - + strOut); + + strOut,e); } }