Dev prod/junit platform engine

buildSrc/src/main/kotlin/Dependencies.kt

@@ -4,7 +4,6 @@ object Versions {
     val COMMONS_LANG3 = "3.7"
     val JUNIT = "4.12"
     val JSR305 = "3.0.2"
-    val SLF4J = "1.7.16"
     val SPOCK_CORE = "1.2-groovy-2.5"
     val TYPESAFE_CONFIG = "1.2.1"
     val CGLIB = "3.2.7"
@@ -18,7 +17,6 @@ object Libraries {
     val COMMONS_LANG3 = "org.apache.commons:commons-lang3:${Versions.COMMONS_LANG3}"
     val JUNIT = "junit:junit:${Versions.JUNIT}"
     val JSR305 = "com.google.code.findbugs:jsr305:${Versions.JSR305}"
-    val SLF4J = "org.slf4j:slf4j-simple:${Versions.SLF4J}"
     val SPOCK_CORE = "org.spockframework:spock-core:${Versions.SPOCK_CORE}"
     val TYPESAFE_CONFIG = "com.typesafe:config:${Versions.TYPESAFE_CONFIG}"
     val CGLIB = "cglib:cglib-nodep:${Versions.CGLIB}"

docs/build.gradle.kts

@@ -4,6 +4,7 @@ plugins {
 
 dependencies {
     implementation(project(":samples-check"))
+    implementation(Libraries.JUNIT)
 }
 
 tasks.test {

samples-check/build.gradle.kts

@@ -6,7 +6,8 @@ plugins {
 
 dependencies {
     api(project(":samples-discovery"))
-    api(Libraries.JUNIT)
+    implementation(Libraries.JUNIT)
+    implementation("org.junit.platform:junit-platform-engine:1.8.1")
     compileOnly(Libraries.JSR305)
     implementation(Libraries.COMMONS_IO)
     implementation(Libraries.COMMONS_LANG3)

samples-check/src/main/java/org/gradle/exemplar/executor/CliCommandExecutor.java

@@ -15,15 +15,11 @@
  */
 package org.gradle.exemplar.executor;
 
-import java.io.File;
 import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.List;
 
 public class CliCommandExecutor extends CommandExecutor {
-    public CliCommandExecutor(File directory) {
-        super(directory);
-    }
 
     @Override
     public int run(final String executable, final List<String> args, final List<String> flags, final OutputStream output) {

samples-check/src/main/java/org/gradle/exemplar/executor/CommandExecutor.java

@@ -25,16 +25,9 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 
-public abstract class CommandExecutor {
-    private final File directory;
+public abstract class CommandExecutor implements CommandExecutorExtension {
 
-    public CommandExecutor() {
-        this.directory = null;
-    }
-
-    protected CommandExecutor(final File directory) {
-        this.directory = directory;
-    }
+    private File directory;
 
     protected abstract int run(final String executable, final List<String> args, final List<String> flags, final OutputStream output);
 
@@ -159,7 +152,8 @@ private void shutdownExecutor() {
         }
     }
 
-    public CommandExecutionResult execute(final Command command, final ExecutionMetadata executionMetadata) {
+    public CommandExecutionResult execute(final Command command, final ExecutionMetadata executionMetadata, File workingDir) {
+        this.directory = workingDir;
         final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
         final int exitCode = run(command.getExecutable(), command.getArgs(), command.getFlags(), outputStream);
 

samples-check/src/main/java/org/gradle/exemplar/executor/CommandExecutorExtension.java

@@ -0,0 +1,11 @@
+package org.gradle.exemplar.executor;
+
+import org.gradle.exemplar.model.Command;
+
+import java.io.File;
+
+public interface CommandExecutorExtension {
+
+    CommandExecutionResult execute(final Command command, final ExecutionMetadata executionMetadata, File workingDir);
+
+}

samples-check/src/main/java/org/gradle/exemplar/test/engine/SamplesRunnerJUnitEngine.java

@@ -0,0 +1,247 @@
+package org.gradle.exemplar.test.engine;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.gradle.exemplar.executor.*;
+import org.gradle.exemplar.loader.SamplesDiscovery;
+import org.gradle.exemplar.model.Command;
+import org.gradle.exemplar.model.Sample;
+import org.gradle.exemplar.test.normalizer.OutputNormalizer;
+import org.gradle.exemplar.test.runner.SampleModifier;
+import org.gradle.exemplar.test.runner.SamplesRoot;
+import org.gradle.exemplar.test.runner.SamplesRunner;
+import org.gradle.exemplar.test.verifier.AnyOrderLineSegmentedOutputVerifier;
+import org.gradle.exemplar.test.verifier.StrictOrderLineSegmentedOutputVerifier;
+import org.junit.platform.engine.EngineDiscoveryRequest;
+import org.junit.platform.engine.EngineExecutionListener;
+import org.junit.platform.engine.ExecutionRequest;
+import org.junit.platform.engine.TestDescriptor;
+import org.junit.platform.engine.TestEngine;
+import org.junit.platform.engine.TestExecutionResult;
+import org.junit.platform.engine.UniqueId;
+import org.junit.platform.engine.discovery.ClassSelector;
+import org.junit.platform.engine.support.descriptor.AbstractTestDescriptor;
+import org.junit.platform.engine.support.descriptor.ClassSource;
+import org.junit.platform.engine.support.descriptor.EngineDescriptor;
+import org.opentest4j.AssertionFailedError;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static java.util.Collections.emptyList;
+
+public class SamplesRunnerJUnitEngine implements TestEngine {
+
+    private final File tempDir;
+    private final List<? extends OutputNormalizer> normalizers = emptyList();
+    private final List<SampleModifier> sampleModifiers = emptyList();
+
+    public SamplesRunnerJUnitEngine() throws IOException {
+        tempDir = Files.createTempDirectory("samples-junit-engine").toFile();
+    }
+
+    @Override
+    public String getId() {
+        return "SamplesRunnerJUnitEngine";
+    }
+
+    @Override
+    public TestDescriptor discover(EngineDiscoveryRequest discoveryRequest, UniqueId uniqueId) {
+        EngineDescriptor engineDescriptor = new EngineDescriptor(uniqueId, "Samples Runner JUnit Engine");
+        discoveryRequest.getSelectorsByType(ClassSelector.class)
+                .stream()
+                .map(ClassSelector::getJavaClass).forEach(javaClass -> {
+                    SamplesRoot samplesRoot = javaClass.getAnnotation(SamplesRoot.class);
+                    if (samplesRoot != null) {
+                        SamplesTestDescriptor samplesTestDescriptor = new SamplesTestDescriptor(
+                                uniqueId.append("className", javaClass.getSimpleName()),
+                                javaClass
+                        );
+
+                        for (Sample sample : SamplesDiscovery.externalSamples(getSamplesRootDir(samplesRoot))) {
+                            samplesTestDescriptor.addChild(new SampleTestDescriptor(
+                                    uniqueId.append("sampleId", sample.getId()),
+                                    sample
+                            ));
+                        }
+
+                        engineDescriptor.addChild(samplesTestDescriptor);
+                    }
+                });
+
+        return engineDescriptor;
+    }
+
+    @Override
+    public void execute(ExecutionRequest request) {
+        TestDescriptor engineDescriptor = request.getRootTestDescriptor();
+        EngineExecutionListener listener = request.getEngineExecutionListener();
+
+        listener.executionStarted(engineDescriptor);
+        for (TestDescriptor classDescriptor : engineDescriptor.getChildren()) {
+            listener.executionStarted(classDescriptor);
+            for (TestDescriptor testDescriptor : classDescriptor.getChildren()) {
+                SampleTestDescriptor descriptor = (SampleTestDescriptor) testDescriptor;
+                listener.executionStarted(testDescriptor);
+                try {
+                    runSample(descriptor.sample);
+                    listener.executionFinished(testDescriptor, TestExecutionResult.successful());
+                } catch (Throwable t) {
+                    listener.executionFinished(testDescriptor, TestExecutionResult.failed(t));
+                }
+            }
+
+            listener.executionFinished(classDescriptor, TestExecutionResult.successful());
+        }
+        listener.executionFinished(engineDescriptor, TestExecutionResult.successful());
+    }
+
+    protected File getSamplesRootDir(SamplesRoot samplesRoot) {
+        File samplesRootDir = null;
+        try {
+            if (samplesRoot != null) {
+                samplesRootDir = new File(samplesRoot.value());
+            }
+            if (samplesRootDir == null) {
+                throw new IllegalArgumentException("Samples root directory is not declared. Please annotate your test class with @SamplesRoot(\"path/to/samples\")");
+            }
+            if (!samplesRootDir.exists()) {
+                throw new IllegalArgumentException("Samples root directory " + samplesRootDir.getAbsolutePath() + " does not exist");
+            }
+        } catch (Exception e) {
+            throw new RuntimeException("Could not initialize SamplesRunnerJUnitEngine", e);
+        }
+        return samplesRootDir;
+    }
+
+    private void runSample(Sample sample) throws Exception {
+        final Sample testSpecificSample = initSample(sample);
+        File baseWorkingDir = testSpecificSample.getProjectDir();
+
+        // Execute and verify each command
+        for (Command command : testSpecificSample.getCommands()) {
+            File workingDir = baseWorkingDir;
+
+            if (command.getExecutionSubdirectory() != null) {
+                workingDir = new File(workingDir, command.getExecutionSubdirectory());
+            }
+
+            // This should be some kind of plugable executor rather than hard-coded here
+            if (command.getExecutable().equals("cd")) {
+                baseWorkingDir = new File(baseWorkingDir, command.getArgs().get(0)).getCanonicalFile();
+                continue;
+            }
+
+            CommandExecutionResult result = executeSample(getExecutionMetadata(testSpecificSample.getProjectDir()), workingDir, command);
+
+            if (result.getExitCode() != 0 && !command.isExpectFailure()) {
+                throw new AssertionFailedError(String.format(
+                        "Expected sample invocation to succeed but it failed.%nCommand was: '%s %s'%nWorking directory: '%s'%n[BEGIN OUTPUT]%n%s%n[END OUTPUT]%n",
+                        command.getExecutable(),
+                        StringUtils.join(command.getArgs(), " "),
+                        workingDir.getAbsolutePath(),
+                        result.getOutput()
+                ));
+            } else if (result.getExitCode() == 0 && command.isExpectFailure()) {
+                throw new AssertionFailedError(String.format(
+                        "Expected sample invocation to fail but it succeeded.%nCommand was: '%s %s'%nWorking directory: '%s'%n[BEGIN OUTPUT]%n%s%n[END OUTPUT]%n",
+                        command.getExecutable(),
+                        StringUtils.join(command.getArgs(), " "),
+                        workingDir.getAbsolutePath(),
+                        result.getOutput()
+                ));
+            }
+
+            verifyOutput(command, result);
+        }
+    }
+
+    private void verifyOutput(final Command command, final CommandExecutionResult executionResult) {
+        if (command.getExpectedOutput() == null) {
+            return;
+        }
+
+        String expectedOutput = command.getExpectedOutput();
+        String actualOutput = executionResult.getOutput();
+
+        for (OutputNormalizer normalizer : normalizers) {
+            actualOutput = normalizer.normalize(actualOutput, executionResult.getExecutionMetadata());
+        }
+
+        if (command.isAllowDisorderedOutput()) {
+            new AnyOrderLineSegmentedOutputVerifier().verify(expectedOutput, actualOutput, command.isAllowAdditionalOutput());
+        } else {
+            new StrictOrderLineSegmentedOutputVerifier().verify(expectedOutput, actualOutput, command.isAllowAdditionalOutput());
+        }
+    }
+
+    private CommandExecutionResult executeSample(ExecutionMetadata executionMetadata, File workingDir, Command command) {
+        CommandExecutorExtension commandExecutor = selectExecutor(executionMetadata, workingDir, command);
+        return commandExecutor.execute(command, executionMetadata, workingDir);
+    }
+
+    protected CommandExecutorExtension selectExecutor(ExecutionMetadata executionMetadata, File workingDir, Command command) {
+        String executorName = command.getExecutorName();
+        if("CLI".equals(executorName)){
+            return new CliCommandExecutor();
+        }else if(StringUtils.isNotBlank(executorName)){
+            try
+            {
+                return (CommandExecutorExtension)(Class.forName(executorName).getDeclaredConstructor().newInstance());
+            }catch (Exception e){
+                throw new IllegalArgumentException("Could not init a custom executor: '"+executorName+"', caused by: "+e.getMessage(), e);
+            }
+        }
+        throw new IllegalArgumentException("Unknown executor: '"+ executorName +"' provided.");
+    }
+
+    private ExecutionMetadata getExecutionMetadata(final File tempSampleOutputDir) {
+        Map<String, String> systemProperties = new HashMap<>();
+        for (String systemPropertyKey : SamplesRunner.SAFE_SYSTEM_PROPERTIES) {
+            systemProperties.put(systemPropertyKey, System.getProperty(systemPropertyKey));
+        }
+
+        return new ExecutionMetadata(tempSampleOutputDir, systemProperties);
+    }
+
+    private Sample initSample(final Sample sampleIn) throws IOException {
+        File tmpProjectDir = new File(tempDir, sampleIn.getId());
+        FileUtils.copyDirectory(sampleIn.getProjectDir(), tmpProjectDir);
+        Sample sample = new Sample(sampleIn.getId(), tmpProjectDir, sampleIn.getCommands());
+        for (SampleModifier sampleModifier : sampleModifiers) {
+            sample = sampleModifier.modify(sample);
+        }
+        return sample;
+    }
+
+    private static class SampleTestDescriptor extends AbstractTestDescriptor {
+
+        private final Sample sample;
+
+        private SampleTestDescriptor(UniqueId uniqueId, Sample sample) {
+            super(uniqueId, sample.getId());
+            this.sample = sample;
+        }
+
+        @Override
+        public Type getType() {
+            return Type.TEST;
+        }
+    }
+
+    private static class SamplesTestDescriptor extends AbstractTestDescriptor {
+
+        private SamplesTestDescriptor(UniqueId uniqueId, Class<?> testClass) {
+            super(uniqueId, testClass.getSimpleName(), ClassSource.from(testClass));
+        }
+
+        @Override
+        public Type getType() {
+            return Type.CONTAINER;
+        }
+    }
+}

samples-check/src/main/java/org/gradle/exemplar/test/engine/ValidationExecutor.java

@@ -0,0 +1,8 @@
+package org.gradle.exemplar.test.engine;
+
+import org.gradle.exemplar.model.Sample;
+
+public interface ValidationExecutor {
+
+    void executeValidation(Sample testSpecificSample);
+}

samples-check/src/main/java/org/gradle/exemplar/test/runner/GradleSamplesRunner.java

@@ -59,7 +59,7 @@ protected CommandExecutor selectExecutor(ExecutionMetadata executionMetadata, Fi
         if (command.getExecutable().equals(GRADLE_EXECUTABLE)) {
             return new GradleRunnerCommandExecutor(workingDir, customGradleInstallation, expectFailure);
         }
-        return new CliCommandExecutor(workingDir);
+        return new CliCommandExecutor();
     }
 
     @Nullable

samples-check/src/main/java/org/gradle/exemplar/test/runner/SamplesRunner.java

@@ -213,14 +213,14 @@ private void verifyOutput(final Command command, final CommandExecutionResult ex
     }
 
     private CommandExecutionResult execute(ExecutionMetadata executionMetadata, File workingDir, Command command) {
-        return selectExecutor(executionMetadata, workingDir, command).execute(command, executionMetadata);
+        return selectExecutor(executionMetadata, workingDir, command).execute(command, executionMetadata, workingDir);
     }
 
     /**
      * Allows a subclass to provide a custom {@link CommandExecutor}.
      */
     protected CommandExecutor selectExecutor(ExecutionMetadata executionMetadata, File workingDir, Command command) {
-        return new CliCommandExecutor(workingDir);
+        return new CliCommandExecutor();
     }
 
     private ExecutionMetadata getExecutionMetadata(final File tempSampleOutputDir) {

samples-check/src/main/resources/META-INF/services/org.junit.platform.engine.TestEngine

@@ -0,0 +1 @@
+org.gradle.exemplar.test.engine.SamplesRunnerJUnitEngine