From 7b235a421d8b589396efdb241c4cc71501828c5d Mon Sep 17 00:00:00 2001 From: Przemek Bielicki Date: Wed, 6 Oct 2021 13:38:21 +0200 Subject: [PATCH 1/5] Use JUnit5 platform engine --- buildSrc/src/main/kotlin/Dependencies.kt | 2 - docs/build.gradle.kts | 1 + samples-check/build.gradle.kts | 3 +- ...estAwareGradleRunnerCommandExecutor.groovy | 65 +++++++ .../test/engine/SamplesRunnerJUnitEngine.java | 176 ++++++++++++++++++ .../org.junit.platform.engine.TestEngine | 1 + 6 files changed, 245 insertions(+), 3 deletions(-) create mode 100644 samples-check/src/main/java/org/gradle/exemplar/test/engine/PluginUnderTestAwareGradleRunnerCommandExecutor.groovy create mode 100644 samples-check/src/main/java/org/gradle/exemplar/test/engine/SamplesRunnerJUnitEngine.java create mode 100644 samples-check/src/main/resources/META-INF/services/org.junit.platform.engine.TestEngine diff --git a/buildSrc/src/main/kotlin/Dependencies.kt b/buildSrc/src/main/kotlin/Dependencies.kt index d4fae26..d1f4486 100644 --- a/buildSrc/src/main/kotlin/Dependencies.kt +++ b/buildSrc/src/main/kotlin/Dependencies.kt @@ -4,7 +4,6 @@ object Versions { val COMMONS_LANG3 = "3.7" val JUNIT = "4.12" val JSR305 = "3.0.2" - val SLF4J = "1.7.16" val SPOCK_CORE = "1.2-groovy-2.5" val TYPESAFE_CONFIG = "1.2.1" val CGLIB = "3.2.7" @@ -18,7 +17,6 @@ object Libraries { val COMMONS_LANG3 = "org.apache.commons:commons-lang3:${Versions.COMMONS_LANG3}" val JUNIT = "junit:junit:${Versions.JUNIT}" val JSR305 = "com.google.code.findbugs:jsr305:${Versions.JSR305}" - val SLF4J = "org.slf4j:slf4j-simple:${Versions.SLF4J}" val SPOCK_CORE = "org.spockframework:spock-core:${Versions.SPOCK_CORE}" val TYPESAFE_CONFIG = "com.typesafe:config:${Versions.TYPESAFE_CONFIG}" val CGLIB = "cglib:cglib-nodep:${Versions.CGLIB}" diff --git a/docs/build.gradle.kts b/docs/build.gradle.kts index 330d21d..af952b1 100644 --- a/docs/build.gradle.kts +++ b/docs/build.gradle.kts @@ -4,6 +4,7 @@ plugins { dependencies { implementation(project(":samples-check")) + implementation(Libraries.JUNIT) } tasks.test { diff --git a/samples-check/build.gradle.kts b/samples-check/build.gradle.kts index a1795c0..f94b9c1 100644 --- a/samples-check/build.gradle.kts +++ b/samples-check/build.gradle.kts @@ -6,7 +6,8 @@ plugins { dependencies { api(project(":samples-discovery")) - api(Libraries.JUNIT) + implementation(Libraries.JUNIT) + implementation("org.junit.platform:junit-platform-engine:1.8.1") compileOnly(Libraries.JSR305) implementation(Libraries.COMMONS_IO) implementation(Libraries.COMMONS_LANG3) diff --git a/samples-check/src/main/java/org/gradle/exemplar/test/engine/PluginUnderTestAwareGradleRunnerCommandExecutor.groovy b/samples-check/src/main/java/org/gradle/exemplar/test/engine/PluginUnderTestAwareGradleRunnerCommandExecutor.groovy new file mode 100644 index 0000000..0bc5680 --- /dev/null +++ b/samples-check/src/main/java/org/gradle/exemplar/test/engine/PluginUnderTestAwareGradleRunnerCommandExecutor.groovy @@ -0,0 +1,65 @@ +/* + * Copyright 2020 Benedikt Ritter + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.exemplar.test.engine + +import org.gradle.exemplar.executor.CommandExecutor +import org.gradle.testkit.runner.BuildResult +import org.gradle.testkit.runner.GradleRunner + +import javax.annotation.Nullable + +class PluginUnderTestAwareGradleRunnerCommandExecutor extends CommandExecutor { + private final File workingDir; + private final File customGradleInstallation; + private final boolean expectFailure; + + public PluginUnderTestAwareGradleRunnerCommandExecutor(File workingDir, @Nullable File customGradleInstallation, boolean expectFailure) { + this.workingDir = workingDir; + this.customGradleInstallation = customGradleInstallation; + this.expectFailure = expectFailure; + } + + @Override + protected int run(String executable, List args, List flags, OutputStream output) { + List allArguments = new ArrayList<>(args); + allArguments.addAll(flags); + + println(System.getProperty("java.class.path")) + + GradleRunner gradleRunner = GradleRunner.create() + .withProjectDir(workingDir) + .withArguments(allArguments) + .withPluginClasspath() + .forwardOutput(); + + if (customGradleInstallation != null) { + gradleRunner.withGradleInstallation(customGradleInstallation); + } + + output.withWriter { + BuildResult buildResult; + if (expectFailure) { + buildResult = gradleRunner.buildAndFail(); + } else { + buildResult = gradleRunner.build(); + } + it.write(buildResult.getOutput()); + it.close(); + return expectFailure ? 1 : 0; + } + } +} diff --git a/samples-check/src/main/java/org/gradle/exemplar/test/engine/SamplesRunnerJUnitEngine.java b/samples-check/src/main/java/org/gradle/exemplar/test/engine/SamplesRunnerJUnitEngine.java new file mode 100644 index 0000000..5c3ac1d --- /dev/null +++ b/samples-check/src/main/java/org/gradle/exemplar/test/engine/SamplesRunnerJUnitEngine.java @@ -0,0 +1,176 @@ +package org.gradle.exemplar.test.engine; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.gradle.exemplar.executor.CliCommandExecutor; +import org.gradle.exemplar.executor.CommandExecutionResult; +import org.gradle.exemplar.executor.CommandExecutor; +import org.gradle.exemplar.executor.ExecutionMetadata; +import org.gradle.exemplar.loader.SamplesDiscovery; +import org.gradle.exemplar.model.Command; +import org.gradle.exemplar.model.Sample; +import org.gradle.exemplar.test.normalizer.OutputNormalizer; +import org.gradle.exemplar.test.runner.SampleModifier; +import org.gradle.exemplar.test.runner.SamplesRunner; +import org.gradle.exemplar.test.verifier.AnyOrderLineSegmentedOutputVerifier; +import org.gradle.exemplar.test.verifier.StrictOrderLineSegmentedOutputVerifier; +import org.junit.Assert; +import org.junit.platform.engine.EngineDiscoveryRequest; +import org.junit.platform.engine.EngineExecutionListener; +import org.junit.platform.engine.ExecutionRequest; +import org.junit.platform.engine.TestDescriptor; +import org.junit.platform.engine.TestEngine; +import org.junit.platform.engine.TestExecutionResult; +import org.junit.platform.engine.UniqueId; +import org.junit.platform.engine.support.descriptor.AbstractTestDescriptor; +import org.junit.platform.engine.support.descriptor.EngineDescriptor; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static java.util.Collections.emptyList; + +public class SamplesRunnerJUnitEngine implements TestEngine { + + private final File tempDir; + private final List normalizers = emptyList(); + private final List sampleModifiers = emptyList(); + + public SamplesRunnerJUnitEngine() throws IOException { + tempDir = Files.createTempDirectory("samples-junit-engine").toFile(); + } + + @Override + public String getId() { + return "SamplesRunnerJUnitEngine"; + } + + @Override + public TestDescriptor discover(EngineDiscoveryRequest discoveryRequest, UniqueId uniqueId) { + EngineDescriptor engineDescriptor = new EngineDescriptor(uniqueId, "Samples Runner JUnit Engine"); + for (Sample sample : SamplesDiscovery.externalSamples(new File("src/docs/snippets"))) { + engineDescriptor.addChild(new SampleTestDescriptor(sample)); + } + return engineDescriptor; + } + + @Override + public void execute(ExecutionRequest request) { + TestDescriptor engineDescriptor = request.getRootTestDescriptor(); + EngineExecutionListener listener = request.getEngineExecutionListener(); + + listener.executionStarted(engineDescriptor); + for (TestDescriptor testDescriptor : engineDescriptor.getChildren()) { + SampleTestDescriptor descriptor = (SampleTestDescriptor) testDescriptor; + listener.executionStarted(testDescriptor); + + try { + runSample(descriptor.sample); + listener.executionFinished(testDescriptor, TestExecutionResult.successful()); + } catch (Throwable t) { + listener.executionFinished(testDescriptor, TestExecutionResult.failed(t)); + } + + } + listener.executionFinished(engineDescriptor, TestExecutionResult.successful()); + } + + private void runSample(Sample sample) throws Exception { + final Sample testSpecificSample = initSample(sample); + File baseWorkingDir = testSpecificSample.getProjectDir(); + + // Execute and verify each command + for (Command command : testSpecificSample.getCommands()) { + File workingDir = baseWorkingDir; + + if (command.getExecutionSubdirectory() != null) { + workingDir = new File(workingDir, command.getExecutionSubdirectory()); + } + + // This should be some kind of plugable executor rather than hard-coded here + if (command.getExecutable().equals("cd")) { + baseWorkingDir = new File(baseWorkingDir, command.getArgs().get(0)).getCanonicalFile(); + continue; + } + + CommandExecutionResult result = executeSample(getExecutionMetadata(testSpecificSample.getProjectDir()), workingDir, command); + + if (result.getExitCode() != 0 && !command.isExpectFailure()) { + Assert.fail(String.format("Expected sample invocation to succeed but it failed.%nCommand was: '%s %s'%nWorking directory: '%s'%n[BEGIN OUTPUT]%n%s%n[END OUTPUT]%n", command.getExecutable(), StringUtils.join(command.getArgs(), " "), workingDir.getAbsolutePath(), result.getOutput())); + } else if (result.getExitCode() == 0 && command.isExpectFailure()) { + Assert.fail(String.format("Expected sample invocation to fail but it succeeded.%nCommand was: '%s %s'%nWorking directory: '%s'%n[BEGIN OUTPUT]%n%s%n[END OUTPUT]%n", command.getExecutable(), StringUtils.join(command.getArgs(), " "), workingDir.getAbsolutePath(), result.getOutput())); + } + + verifyOutput(command, result); + } + } + + private void verifyOutput(final Command command, final CommandExecutionResult executionResult) { + if (command.getExpectedOutput() == null) { + return; + } + + String expectedOutput = command.getExpectedOutput(); + String actualOutput = executionResult.getOutput(); + + for (OutputNormalizer normalizer : normalizers) { + actualOutput = normalizer.normalize(actualOutput, executionResult.getExecutionMetadata()); + } + + if (command.isAllowDisorderedOutput()) { + new AnyOrderLineSegmentedOutputVerifier().verify(expectedOutput, actualOutput, command.isAllowAdditionalOutput()); + } else { + new StrictOrderLineSegmentedOutputVerifier().verify(expectedOutput, actualOutput, command.isAllowAdditionalOutput()); + } + } + + private CommandExecutionResult executeSample(ExecutionMetadata executionMetadata, File workingDir, Command command) { + return selectExecutor(executionMetadata, workingDir, command).execute(command, executionMetadata); + } + + protected CommandExecutor selectExecutor(ExecutionMetadata executionMetadata, File workingDir, Command command) { + boolean expectFailure = command.isExpectFailure(); + if (command.getExecutable().equals("gradle")) { + return new PluginUnderTestAwareGradleRunnerCommandExecutor(workingDir, null, expectFailure); + } + return new CliCommandExecutor(workingDir); + } + + private ExecutionMetadata getExecutionMetadata(final File tempSampleOutputDir) { + Map systemProperties = new HashMap<>(); + for (String systemPropertyKey : SamplesRunner.SAFE_SYSTEM_PROPERTIES) { + systemProperties.put(systemPropertyKey, System.getProperty(systemPropertyKey)); + } + + return new ExecutionMetadata(tempSampleOutputDir, systemProperties); + } + + private Sample initSample(final Sample sampleIn) throws IOException { + File tmpProjectDir = new File(tempDir, sampleIn.getId()); + FileUtils.copyDirectory(sampleIn.getProjectDir(), tmpProjectDir); + Sample sample = new Sample(sampleIn.getId(), tmpProjectDir, sampleIn.getCommands()); + for (SampleModifier sampleModifier : sampleModifiers) { + sample = sampleModifier.modify(sample); + } + return sample; + } + + private static class SampleTestDescriptor extends AbstractTestDescriptor { + + private final Sample sample; + + private SampleTestDescriptor(Sample sample) { + super(UniqueId.forEngine("SamplesRunnerJUnitEngine" + sample.getId()), sample.getId()); + this.sample = sample; + } + + @Override + public Type getType() { + return Type.TEST; + } + } +} diff --git a/samples-check/src/main/resources/META-INF/services/org.junit.platform.engine.TestEngine b/samples-check/src/main/resources/META-INF/services/org.junit.platform.engine.TestEngine new file mode 100644 index 0000000..f106d72 --- /dev/null +++ b/samples-check/src/main/resources/META-INF/services/org.junit.platform.engine.TestEngine @@ -0,0 +1 @@ +org.gradle.exemplar.test.engine.SamplesRunnerJUnitEngine \ No newline at end of file From e3a81bbd0e1f1c1fa5a8cf603136268938f823b4 Mon Sep 17 00:00:00 2001 From: Przemek Bielicki Date: Wed, 6 Oct 2021 13:46:45 +0200 Subject: [PATCH 2/5] Test custom PluginUnderTestAwareGradleRunnerCommandExecutor --- ...estAwareGradleRunnerCommandExecutor.groovy | 65 ------------------- ...rTestAwareGradleRunnerCommandExecutor.java | 57 ++++++++++++++++ 2 files changed, 57 insertions(+), 65 deletions(-) delete mode 100644 samples-check/src/main/java/org/gradle/exemplar/test/engine/PluginUnderTestAwareGradleRunnerCommandExecutor.groovy create mode 100644 samples-check/src/main/java/org/gradle/exemplar/test/engine/PluginUnderTestAwareGradleRunnerCommandExecutor.java diff --git a/samples-check/src/main/java/org/gradle/exemplar/test/engine/PluginUnderTestAwareGradleRunnerCommandExecutor.groovy b/samples-check/src/main/java/org/gradle/exemplar/test/engine/PluginUnderTestAwareGradleRunnerCommandExecutor.groovy deleted file mode 100644 index 0bc5680..0000000 --- a/samples-check/src/main/java/org/gradle/exemplar/test/engine/PluginUnderTestAwareGradleRunnerCommandExecutor.groovy +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright 2020 Benedikt Ritter - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.gradle.exemplar.test.engine - -import org.gradle.exemplar.executor.CommandExecutor -import org.gradle.testkit.runner.BuildResult -import org.gradle.testkit.runner.GradleRunner - -import javax.annotation.Nullable - -class PluginUnderTestAwareGradleRunnerCommandExecutor extends CommandExecutor { - private final File workingDir; - private final File customGradleInstallation; - private final boolean expectFailure; - - public PluginUnderTestAwareGradleRunnerCommandExecutor(File workingDir, @Nullable File customGradleInstallation, boolean expectFailure) { - this.workingDir = workingDir; - this.customGradleInstallation = customGradleInstallation; - this.expectFailure = expectFailure; - } - - @Override - protected int run(String executable, List args, List flags, OutputStream output) { - List allArguments = new ArrayList<>(args); - allArguments.addAll(flags); - - println(System.getProperty("java.class.path")) - - GradleRunner gradleRunner = GradleRunner.create() - .withProjectDir(workingDir) - .withArguments(allArguments) - .withPluginClasspath() - .forwardOutput(); - - if (customGradleInstallation != null) { - gradleRunner.withGradleInstallation(customGradleInstallation); - } - - output.withWriter { - BuildResult buildResult; - if (expectFailure) { - buildResult = gradleRunner.buildAndFail(); - } else { - buildResult = gradleRunner.build(); - } - it.write(buildResult.getOutput()); - it.close(); - return expectFailure ? 1 : 0; - } - } -} diff --git a/samples-check/src/main/java/org/gradle/exemplar/test/engine/PluginUnderTestAwareGradleRunnerCommandExecutor.java b/samples-check/src/main/java/org/gradle/exemplar/test/engine/PluginUnderTestAwareGradleRunnerCommandExecutor.java new file mode 100644 index 0000000..57908a3 --- /dev/null +++ b/samples-check/src/main/java/org/gradle/exemplar/test/engine/PluginUnderTestAwareGradleRunnerCommandExecutor.java @@ -0,0 +1,57 @@ +package org.gradle.exemplar.test.engine; + +import org.gradle.exemplar.executor.CommandExecutor; +import org.gradle.testkit.runner.BuildResult; +import org.gradle.testkit.runner.GradleRunner; + +import javax.annotation.Nullable; +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.List; + +public class PluginUnderTestAwareGradleRunnerCommandExecutor extends CommandExecutor { + private final File workingDir; + private final File customGradleInstallation; + private final boolean expectFailure; + + PluginUnderTestAwareGradleRunnerCommandExecutor(File workingDir, @Nullable File customGradleInstallation, boolean expectFailure) { + this.workingDir = workingDir; + this.customGradleInstallation = customGradleInstallation; + this.expectFailure = expectFailure; + } + + @Override + protected int run(String executable, List args, List flags, OutputStream output) { + List allArguments = new ArrayList<>(args); + allArguments.addAll(flags); + + GradleRunner gradleRunner = GradleRunner.create() + .withProjectDir(workingDir) + .withArguments(allArguments) + .withPluginClasspath() + .forwardOutput(); + + if (customGradleInstallation != null) { + gradleRunner.withGradleInstallation(customGradleInstallation); + } + + BuildResult buildResult; + if (expectFailure) { + buildResult = gradleRunner.buildAndFail(); + } else { + buildResult = gradleRunner.build(); + } + + try (OutputStreamWriter writer = new OutputStreamWriter(output)) { + writer.write(buildResult.getOutput()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + + return expectFailure ? 1 : 0; + } +} From 183e18e2d0ac5906a9f94241eae2c82f77ad4c42 Mon Sep 17 00:00:00 2001 From: Przemek Bielicki Date: Wed, 6 Oct 2021 13:52:05 +0200 Subject: [PATCH 3/5] Remove PluginUnderTestAwareGradleRunnerCommandExecutor --- ...rTestAwareGradleRunnerCommandExecutor.java | 57 ------------------- .../test/engine/SamplesRunnerJUnitEngine.java | 4 -- 2 files changed, 61 deletions(-) delete mode 100644 samples-check/src/main/java/org/gradle/exemplar/test/engine/PluginUnderTestAwareGradleRunnerCommandExecutor.java diff --git a/samples-check/src/main/java/org/gradle/exemplar/test/engine/PluginUnderTestAwareGradleRunnerCommandExecutor.java b/samples-check/src/main/java/org/gradle/exemplar/test/engine/PluginUnderTestAwareGradleRunnerCommandExecutor.java deleted file mode 100644 index 57908a3..0000000 --- a/samples-check/src/main/java/org/gradle/exemplar/test/engine/PluginUnderTestAwareGradleRunnerCommandExecutor.java +++ /dev/null @@ -1,57 +0,0 @@ -package org.gradle.exemplar.test.engine; - -import org.gradle.exemplar.executor.CommandExecutor; -import org.gradle.testkit.runner.BuildResult; -import org.gradle.testkit.runner.GradleRunner; - -import javax.annotation.Nullable; -import java.io.File; -import java.io.IOException; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.io.UncheckedIOException; -import java.util.ArrayList; -import java.util.List; - -public class PluginUnderTestAwareGradleRunnerCommandExecutor extends CommandExecutor { - private final File workingDir; - private final File customGradleInstallation; - private final boolean expectFailure; - - PluginUnderTestAwareGradleRunnerCommandExecutor(File workingDir, @Nullable File customGradleInstallation, boolean expectFailure) { - this.workingDir = workingDir; - this.customGradleInstallation = customGradleInstallation; - this.expectFailure = expectFailure; - } - - @Override - protected int run(String executable, List args, List flags, OutputStream output) { - List allArguments = new ArrayList<>(args); - allArguments.addAll(flags); - - GradleRunner gradleRunner = GradleRunner.create() - .withProjectDir(workingDir) - .withArguments(allArguments) - .withPluginClasspath() - .forwardOutput(); - - if (customGradleInstallation != null) { - gradleRunner.withGradleInstallation(customGradleInstallation); - } - - BuildResult buildResult; - if (expectFailure) { - buildResult = gradleRunner.buildAndFail(); - } else { - buildResult = gradleRunner.build(); - } - - try (OutputStreamWriter writer = new OutputStreamWriter(output)) { - writer.write(buildResult.getOutput()); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - - return expectFailure ? 1 : 0; - } -} diff --git a/samples-check/src/main/java/org/gradle/exemplar/test/engine/SamplesRunnerJUnitEngine.java b/samples-check/src/main/java/org/gradle/exemplar/test/engine/SamplesRunnerJUnitEngine.java index 5c3ac1d..d88a180 100644 --- a/samples-check/src/main/java/org/gradle/exemplar/test/engine/SamplesRunnerJUnitEngine.java +++ b/samples-check/src/main/java/org/gradle/exemplar/test/engine/SamplesRunnerJUnitEngine.java @@ -133,10 +133,6 @@ private CommandExecutionResult executeSample(ExecutionMetadata executionMetadata } protected CommandExecutor selectExecutor(ExecutionMetadata executionMetadata, File workingDir, Command command) { - boolean expectFailure = command.isExpectFailure(); - if (command.getExecutable().equals("gradle")) { - return new PluginUnderTestAwareGradleRunnerCommandExecutor(workingDir, null, expectFailure); - } return new CliCommandExecutor(workingDir); } From ca3e2e0caf16823f8111088e2c4285a4a3f56628 Mon Sep 17 00:00:00 2001 From: Przemek Bielicki Date: Wed, 6 Oct 2021 16:48:04 +0200 Subject: [PATCH 4/5] Set test class name correctly --- .../test/engine/SamplesRunnerJUnitEngine.java | 100 +++++++++++++++--- 1 file changed, 83 insertions(+), 17 deletions(-) diff --git a/samples-check/src/main/java/org/gradle/exemplar/test/engine/SamplesRunnerJUnitEngine.java b/samples-check/src/main/java/org/gradle/exemplar/test/engine/SamplesRunnerJUnitEngine.java index d88a180..3317530 100644 --- a/samples-check/src/main/java/org/gradle/exemplar/test/engine/SamplesRunnerJUnitEngine.java +++ b/samples-check/src/main/java/org/gradle/exemplar/test/engine/SamplesRunnerJUnitEngine.java @@ -11,10 +11,10 @@ import org.gradle.exemplar.model.Sample; import org.gradle.exemplar.test.normalizer.OutputNormalizer; import org.gradle.exemplar.test.runner.SampleModifier; +import org.gradle.exemplar.test.runner.SamplesRoot; import org.gradle.exemplar.test.runner.SamplesRunner; import org.gradle.exemplar.test.verifier.AnyOrderLineSegmentedOutputVerifier; import org.gradle.exemplar.test.verifier.StrictOrderLineSegmentedOutputVerifier; -import org.junit.Assert; import org.junit.platform.engine.EngineDiscoveryRequest; import org.junit.platform.engine.EngineExecutionListener; import org.junit.platform.engine.ExecutionRequest; @@ -22,8 +22,11 @@ import org.junit.platform.engine.TestEngine; import org.junit.platform.engine.TestExecutionResult; import org.junit.platform.engine.UniqueId; +import org.junit.platform.engine.discovery.ClassSelector; import org.junit.platform.engine.support.descriptor.AbstractTestDescriptor; +import org.junit.platform.engine.support.descriptor.ClassSource; import org.junit.platform.engine.support.descriptor.EngineDescriptor; +import org.opentest4j.AssertionFailedError; import java.io.File; import java.io.IOException; @@ -52,9 +55,27 @@ public String getId() { @Override public TestDescriptor discover(EngineDiscoveryRequest discoveryRequest, UniqueId uniqueId) { EngineDescriptor engineDescriptor = new EngineDescriptor(uniqueId, "Samples Runner JUnit Engine"); - for (Sample sample : SamplesDiscovery.externalSamples(new File("src/docs/snippets"))) { - engineDescriptor.addChild(new SampleTestDescriptor(sample)); - } + discoveryRequest.getSelectorsByType(ClassSelector.class) + .stream() + .map(ClassSelector::getJavaClass).forEach(javaClass -> { + SamplesRoot samplesRoot = javaClass.getAnnotation(SamplesRoot.class); + if (samplesRoot != null) { + SamplesTestDescriptor samplesTestDescriptor = new SamplesTestDescriptor( + uniqueId.append("className", javaClass.getSimpleName()), + javaClass + ); + + for (Sample sample : SamplesDiscovery.externalSamples(getSamplesRootDir(samplesRoot))) { + samplesTestDescriptor.addChild(new SampleTestDescriptor( + uniqueId.append("sampleId", sample.getId()), + sample + )); + } + + engineDescriptor.addChild(samplesTestDescriptor); + } + }); + return engineDescriptor; } @@ -64,21 +85,42 @@ public void execute(ExecutionRequest request) { EngineExecutionListener listener = request.getEngineExecutionListener(); listener.executionStarted(engineDescriptor); - for (TestDescriptor testDescriptor : engineDescriptor.getChildren()) { - SampleTestDescriptor descriptor = (SampleTestDescriptor) testDescriptor; - listener.executionStarted(testDescriptor); - - try { - runSample(descriptor.sample); - listener.executionFinished(testDescriptor, TestExecutionResult.successful()); - } catch (Throwable t) { - listener.executionFinished(testDescriptor, TestExecutionResult.failed(t)); + for (TestDescriptor classDescriptor : engineDescriptor.getChildren()) { + listener.executionStarted(classDescriptor); + for (TestDescriptor testDescriptor : classDescriptor.getChildren()) { + SampleTestDescriptor descriptor = (SampleTestDescriptor) testDescriptor; + listener.executionStarted(testDescriptor); + try { + runSample(descriptor.sample); + listener.executionFinished(testDescriptor, TestExecutionResult.successful()); + } catch (Throwable t) { + listener.executionFinished(testDescriptor, TestExecutionResult.failed(t)); + } } + listener.executionFinished(classDescriptor, TestExecutionResult.successful()); } listener.executionFinished(engineDescriptor, TestExecutionResult.successful()); } + protected File getSamplesRootDir(SamplesRoot samplesRoot) { + File samplesRootDir = null; + try { + if (samplesRoot != null) { + samplesRootDir = new File(samplesRoot.value()); + } + if (samplesRootDir == null) { + throw new IllegalArgumentException("Samples root directory is not declared. Please annotate your test class with @SamplesRoot(\"path/to/samples\")"); + } + if (!samplesRootDir.exists()) { + throw new IllegalArgumentException("Samples root directory " + samplesRootDir.getAbsolutePath() + " does not exist"); + } + } catch (Exception e) { + throw new RuntimeException("Could not initialize SamplesRunnerJUnitEngine", e); + } + return samplesRootDir; + } + private void runSample(Sample sample) throws Exception { final Sample testSpecificSample = initSample(sample); File baseWorkingDir = testSpecificSample.getProjectDir(); @@ -100,9 +142,21 @@ private void runSample(Sample sample) throws Exception { CommandExecutionResult result = executeSample(getExecutionMetadata(testSpecificSample.getProjectDir()), workingDir, command); if (result.getExitCode() != 0 && !command.isExpectFailure()) { - Assert.fail(String.format("Expected sample invocation to succeed but it failed.%nCommand was: '%s %s'%nWorking directory: '%s'%n[BEGIN OUTPUT]%n%s%n[END OUTPUT]%n", command.getExecutable(), StringUtils.join(command.getArgs(), " "), workingDir.getAbsolutePath(), result.getOutput())); + throw new AssertionFailedError(String.format( + "Expected sample invocation to succeed but it failed.%nCommand was: '%s %s'%nWorking directory: '%s'%n[BEGIN OUTPUT]%n%s%n[END OUTPUT]%n", + command.getExecutable(), + StringUtils.join(command.getArgs(), " "), + workingDir.getAbsolutePath(), + result.getOutput() + )); } else if (result.getExitCode() == 0 && command.isExpectFailure()) { - Assert.fail(String.format("Expected sample invocation to fail but it succeeded.%nCommand was: '%s %s'%nWorking directory: '%s'%n[BEGIN OUTPUT]%n%s%n[END OUTPUT]%n", command.getExecutable(), StringUtils.join(command.getArgs(), " "), workingDir.getAbsolutePath(), result.getOutput())); + throw new AssertionFailedError(String.format( + "Expected sample invocation to fail but it succeeded.%nCommand was: '%s %s'%nWorking directory: '%s'%n[BEGIN OUTPUT]%n%s%n[END OUTPUT]%n", + command.getExecutable(), + StringUtils.join(command.getArgs(), " "), + workingDir.getAbsolutePath(), + result.getOutput() + )); } verifyOutput(command, result); @@ -159,8 +213,8 @@ private static class SampleTestDescriptor extends AbstractTestDescriptor { private final Sample sample; - private SampleTestDescriptor(Sample sample) { - super(UniqueId.forEngine("SamplesRunnerJUnitEngine" + sample.getId()), sample.getId()); + private SampleTestDescriptor(UniqueId uniqueId, Sample sample) { + super(uniqueId, sample.getId()); this.sample = sample; } @@ -169,4 +223,16 @@ public Type getType() { return Type.TEST; } } + + private static class SamplesTestDescriptor extends AbstractTestDescriptor { + + private SamplesTestDescriptor(UniqueId uniqueId, Class testClass) { + super(uniqueId, testClass.getSimpleName(), ClassSource.from(testClass)); + } + + @Override + public Type getType() { + return Type.CONTAINER; + } + } } From 8257daf76421605a636664ea45557513a66cc7b1 Mon Sep 17 00:00:00 2001 From: Piotr Zielezinski Date: Mon, 11 Oct 2021 16:35:36 +0200 Subject: [PATCH 5/5] proposal of introducing custom executors --- .../exemplar/executor/CliCommandExecutor.java | 4 --- .../exemplar/executor/CommandExecutor.java | 14 +++----- .../executor/CommandExecutorExtension.java | 11 ++++++ .../test/engine/SamplesRunnerJUnitEngine.java | 23 ++++++++---- .../test/engine/ValidationExecutor.java | 8 +++++ .../test/runner/GradleSamplesRunner.java | 2 +- .../exemplar/test/runner/SamplesRunner.java | 4 +-- .../exemplar/loader/CommandsParser.java | 23 +++++++----- .../AsciidoctorCommandsDiscovery.java | 3 +- .../AsciidoctorSamplesDiscovery.java | 7 ++-- .../org/gradle/exemplar/model/Command.java | 27 +++++++++++--- .../exemplar/model/CommandsParserTest.groovy | 35 +++++++++++++++++-- 12 files changed, 117 insertions(+), 44 deletions(-) create mode 100644 samples-check/src/main/java/org/gradle/exemplar/executor/CommandExecutorExtension.java create mode 100644 samples-check/src/main/java/org/gradle/exemplar/test/engine/ValidationExecutor.java diff --git a/samples-check/src/main/java/org/gradle/exemplar/executor/CliCommandExecutor.java b/samples-check/src/main/java/org/gradle/exemplar/executor/CliCommandExecutor.java index 87cc2ff..20b507b 100644 --- a/samples-check/src/main/java/org/gradle/exemplar/executor/CliCommandExecutor.java +++ b/samples-check/src/main/java/org/gradle/exemplar/executor/CliCommandExecutor.java @@ -15,15 +15,11 @@ */ package org.gradle.exemplar.executor; -import java.io.File; import java.io.OutputStream; import java.util.ArrayList; import java.util.List; public class CliCommandExecutor extends CommandExecutor { - public CliCommandExecutor(File directory) { - super(directory); - } @Override public int run(final String executable, final List args, final List flags, final OutputStream output) { diff --git a/samples-check/src/main/java/org/gradle/exemplar/executor/CommandExecutor.java b/samples-check/src/main/java/org/gradle/exemplar/executor/CommandExecutor.java index 9a7ea11..373b2d1 100644 --- a/samples-check/src/main/java/org/gradle/exemplar/executor/CommandExecutor.java +++ b/samples-check/src/main/java/org/gradle/exemplar/executor/CommandExecutor.java @@ -25,16 +25,9 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -public abstract class CommandExecutor { - private final File directory; +public abstract class CommandExecutor implements CommandExecutorExtension { - public CommandExecutor() { - this.directory = null; - } - - protected CommandExecutor(final File directory) { - this.directory = directory; - } + private File directory; protected abstract int run(final String executable, final List args, final List flags, final OutputStream output); @@ -159,7 +152,8 @@ private void shutdownExecutor() { } } - public CommandExecutionResult execute(final Command command, final ExecutionMetadata executionMetadata) { + public CommandExecutionResult execute(final Command command, final ExecutionMetadata executionMetadata, File workingDir) { + this.directory = workingDir; final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); final int exitCode = run(command.getExecutable(), command.getArgs(), command.getFlags(), outputStream); diff --git a/samples-check/src/main/java/org/gradle/exemplar/executor/CommandExecutorExtension.java b/samples-check/src/main/java/org/gradle/exemplar/executor/CommandExecutorExtension.java new file mode 100644 index 0000000..f5f0e6a --- /dev/null +++ b/samples-check/src/main/java/org/gradle/exemplar/executor/CommandExecutorExtension.java @@ -0,0 +1,11 @@ +package org.gradle.exemplar.executor; + +import org.gradle.exemplar.model.Command; + +import java.io.File; + +public interface CommandExecutorExtension { + + CommandExecutionResult execute(final Command command, final ExecutionMetadata executionMetadata, File workingDir); + +} diff --git a/samples-check/src/main/java/org/gradle/exemplar/test/engine/SamplesRunnerJUnitEngine.java b/samples-check/src/main/java/org/gradle/exemplar/test/engine/SamplesRunnerJUnitEngine.java index 3317530..c867546 100644 --- a/samples-check/src/main/java/org/gradle/exemplar/test/engine/SamplesRunnerJUnitEngine.java +++ b/samples-check/src/main/java/org/gradle/exemplar/test/engine/SamplesRunnerJUnitEngine.java @@ -2,10 +2,7 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; -import org.gradle.exemplar.executor.CliCommandExecutor; -import org.gradle.exemplar.executor.CommandExecutionResult; -import org.gradle.exemplar.executor.CommandExecutor; -import org.gradle.exemplar.executor.ExecutionMetadata; +import org.gradle.exemplar.executor.*; import org.gradle.exemplar.loader.SamplesDiscovery; import org.gradle.exemplar.model.Command; import org.gradle.exemplar.model.Sample; @@ -183,11 +180,23 @@ private void verifyOutput(final Command command, final CommandExecutionResult ex } private CommandExecutionResult executeSample(ExecutionMetadata executionMetadata, File workingDir, Command command) { - return selectExecutor(executionMetadata, workingDir, command).execute(command, executionMetadata); + CommandExecutorExtension commandExecutor = selectExecutor(executionMetadata, workingDir, command); + return commandExecutor.execute(command, executionMetadata, workingDir); } - protected CommandExecutor selectExecutor(ExecutionMetadata executionMetadata, File workingDir, Command command) { - return new CliCommandExecutor(workingDir); + protected CommandExecutorExtension selectExecutor(ExecutionMetadata executionMetadata, File workingDir, Command command) { + String executorName = command.getExecutorName(); + if("CLI".equals(executorName)){ + return new CliCommandExecutor(); + }else if(StringUtils.isNotBlank(executorName)){ + try + { + return (CommandExecutorExtension)(Class.forName(executorName).getDeclaredConstructor().newInstance()); + }catch (Exception e){ + throw new IllegalArgumentException("Could not init a custom executor: '"+executorName+"', caused by: "+e.getMessage(), e); + } + } + throw new IllegalArgumentException("Unknown executor: '"+ executorName +"' provided."); } private ExecutionMetadata getExecutionMetadata(final File tempSampleOutputDir) { diff --git a/samples-check/src/main/java/org/gradle/exemplar/test/engine/ValidationExecutor.java b/samples-check/src/main/java/org/gradle/exemplar/test/engine/ValidationExecutor.java new file mode 100644 index 0000000..2ced53d --- /dev/null +++ b/samples-check/src/main/java/org/gradle/exemplar/test/engine/ValidationExecutor.java @@ -0,0 +1,8 @@ +package org.gradle.exemplar.test.engine; + +import org.gradle.exemplar.model.Sample; + +public interface ValidationExecutor { + + void executeValidation(Sample testSpecificSample); +} diff --git a/samples-check/src/main/java/org/gradle/exemplar/test/runner/GradleSamplesRunner.java b/samples-check/src/main/java/org/gradle/exemplar/test/runner/GradleSamplesRunner.java index b0923c3..5c5e04c 100644 --- a/samples-check/src/main/java/org/gradle/exemplar/test/runner/GradleSamplesRunner.java +++ b/samples-check/src/main/java/org/gradle/exemplar/test/runner/GradleSamplesRunner.java @@ -59,7 +59,7 @@ protected CommandExecutor selectExecutor(ExecutionMetadata executionMetadata, Fi if (command.getExecutable().equals(GRADLE_EXECUTABLE)) { return new GradleRunnerCommandExecutor(workingDir, customGradleInstallation, expectFailure); } - return new CliCommandExecutor(workingDir); + return new CliCommandExecutor(); } @Nullable diff --git a/samples-check/src/main/java/org/gradle/exemplar/test/runner/SamplesRunner.java b/samples-check/src/main/java/org/gradle/exemplar/test/runner/SamplesRunner.java index 71dd62c..5807b51 100644 --- a/samples-check/src/main/java/org/gradle/exemplar/test/runner/SamplesRunner.java +++ b/samples-check/src/main/java/org/gradle/exemplar/test/runner/SamplesRunner.java @@ -213,14 +213,14 @@ private void verifyOutput(final Command command, final CommandExecutionResult ex } private CommandExecutionResult execute(ExecutionMetadata executionMetadata, File workingDir, Command command) { - return selectExecutor(executionMetadata, workingDir, command).execute(command, executionMetadata); + return selectExecutor(executionMetadata, workingDir, command).execute(command, executionMetadata, workingDir); } /** * Allows a subclass to provide a custom {@link CommandExecutor}. */ protected CommandExecutor selectExecutor(ExecutionMetadata executionMetadata, File workingDir, Command command) { - return new CliCommandExecutor(workingDir); + return new CliCommandExecutor(); } private ExecutionMetadata getExecutionMetadata(final File tempSampleOutputDir) { diff --git a/samples-discovery/src/main/java/org/gradle/exemplar/loader/CommandsParser.java b/samples-discovery/src/main/java/org/gradle/exemplar/loader/CommandsParser.java index 10b8dd2..43c9eda 100644 --- a/samples-discovery/src/main/java/org/gradle/exemplar/loader/CommandsParser.java +++ b/samples-discovery/src/main/java/org/gradle/exemplar/loader/CommandsParser.java @@ -21,7 +21,7 @@ import java.io.File; import java.io.IOException; -import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -38,6 +38,7 @@ public class CommandsParser { private static final String ALLOW_DISORDERED_OUTPUT = "allow-disordered-output"; private static final String EXPECTED_OUTPUT_FILE = "expected-output-file"; private static final String USER_INPUTS = "user-inputs"; + private static final String COMMAND_EXECUTOR = "command-executor"; public static List parse(final File sampleConfigFile) { try { @@ -63,22 +64,26 @@ public static List parse(final File sampleConfigFile) { } private static Command parseCommand(final Config commandConfig, final File sampleProjectDir) { - // NOTE: A user must specify an executable. This prevents unexpected behavior when an empty or unexpected file is accidentally loaded - String executable; + final String executionDirectory = ConfigUtil.string(commandConfig, EXECUTION_SUBDIRECTORY, null); + final String executorName = ConfigUtil.string(commandConfig, COMMAND_EXECUTOR, "CLI"); + String executable = ""; try { executable = commandConfig.getString(EXECUTABLE); } catch (ConfigException e) { - throw new InvalidSampleException("'executable' field cannot be empty", e); + // NOTE: A user must specify an executable for CLI executor. This prevents unexpected behavior when an empty or unexpected file is accidentally loaded. + // Custom executors may not need them. + if("CLI".equals(executorName)){ + throw new InvalidSampleException("'executable' field cannot be empty for CLI executor", e); + } } - final String executionDirectory = ConfigUtil.string(commandConfig, EXECUTION_SUBDIRECTORY, null); - final List commands = ConfigUtil.strings(commandConfig, ARGS, new ArrayList()); - final List flags = ConfigUtil.strings(commandConfig, FLAGS, new ArrayList()); + final List commands = ConfigUtil.strings(commandConfig, ARGS, new ArrayList<>()); + final List flags = ConfigUtil.strings(commandConfig, FLAGS, new ArrayList<>()); String expectedOutput = null; if (commandConfig.hasPath(EXPECTED_OUTPUT_FILE)) { final File expectedOutputFile = new File(sampleProjectDir, commandConfig.getString(EXPECTED_OUTPUT_FILE)); try { final Path path = Paths.get(expectedOutputFile.getAbsolutePath()); - expectedOutput = new String(Files.readAllBytes(path), Charset.forName("UTF-8")); + expectedOutput = new String(Files.readAllBytes(path), StandardCharsets.UTF_8); } catch (IOException e) { throw new InvalidSampleException("Could not read sample output file " + expectedOutputFile.getAbsolutePath(), e); } @@ -89,6 +94,6 @@ private static Command parseCommand(final Config commandConfig, final File sampl final boolean allowDisorderedOutput = ConfigUtil.booleanValue(commandConfig, ALLOW_DISORDERED_OUTPUT, false); final List userInputs = ConfigUtil.strings(commandConfig, USER_INPUTS, Collections.emptyList()); - return new Command(executable, executionDirectory, commands, flags, expectedOutput, expectFailures, allowAdditionalOutput, allowDisorderedOutput, userInputs); + return new Command(executable, executionDirectory, commands, flags, expectedOutput, expectFailures, allowAdditionalOutput, allowDisorderedOutput, userInputs, executorName); } } diff --git a/samples-discovery/src/main/java/org/gradle/exemplar/loader/asciidoctor/AsciidoctorCommandsDiscovery.java b/samples-discovery/src/main/java/org/gradle/exemplar/loader/asciidoctor/AsciidoctorCommandsDiscovery.java index 83229d7..421fe49 100644 --- a/samples-discovery/src/main/java/org/gradle/exemplar/loader/asciidoctor/AsciidoctorCommandsDiscovery.java +++ b/samples-discovery/src/main/java/org/gradle/exemplar/loader/asciidoctor/AsciidoctorCommandsDiscovery.java @@ -123,7 +123,8 @@ private static int parseOneCommand(String[] lines, int pos, Map attributes.containsKey("expect-failure"), attributes.containsKey("allow-additional-output"), attributes.containsKey("allow-disordered-output"), - attributes.containsKey("user-inputs") ? toUserInputs(attributes.get("user-inputs").toString()) : Collections.emptyList()); + attributes.containsKey("user-inputs") ? toUserInputs(attributes.get("user-inputs").toString()) : Collections.emptyList(), + "CLI"); commands.add(command); return nextCommand; } diff --git a/samples-discovery/src/main/java/org/gradle/exemplar/loader/asciidoctor/AsciidoctorSamplesDiscovery.java b/samples-discovery/src/main/java/org/gradle/exemplar/loader/asciidoctor/AsciidoctorSamplesDiscovery.java index 6829f17..17e4056 100644 --- a/samples-discovery/src/main/java/org/gradle/exemplar/loader/asciidoctor/AsciidoctorSamplesDiscovery.java +++ b/samples-discovery/src/main/java/org/gradle/exemplar/loader/asciidoctor/AsciidoctorSamplesDiscovery.java @@ -127,7 +127,7 @@ private static Sample processSampleNode(AbstractBlock node, Path tempDir, List attributes.containsKey("expect-failure"), attributes.containsKey("allow-additional-output"), attributes.containsKey("allow-disordered-output"), - Collections.emptyList()); + Collections.emptyList(), + "CLI"); commands.add(command); return nextCommand; } @@ -214,7 +215,7 @@ private static void extractEmbeddedSampleSources(Block sampleBlock, File tempSam try { Files.write(sampleFile.toPath(), block.getContent().toString().getBytes()); } catch (IOException e) { - throw new IllegalStateException("Could not write sample source file " + sampleFile.toPath().toString()); + throw new IllegalStateException("Could not write sample source file " + sampleFile.toPath()); } } } diff --git a/samples-discovery/src/main/java/org/gradle/exemplar/model/Command.java b/samples-discovery/src/main/java/org/gradle/exemplar/model/Command.java index a66ff27..f663c74 100644 --- a/samples-discovery/src/main/java/org/gradle/exemplar/model/Command.java +++ b/samples-discovery/src/main/java/org/gradle/exemplar/model/Command.java @@ -20,6 +20,7 @@ import java.util.List; public class Command { + private final String executorName; private final String executable; private final String executionSubdirectory; private final List args; @@ -30,7 +31,9 @@ public class Command { private final boolean allowDisorderedOutput; private final List userInputs; - public Command(@Nonnull String executable, @Nullable String executionDirectory, List args, List flags, @Nullable String expectedOutput, boolean expectFailure, boolean allowAdditionalOutput, boolean allowDisorderedOutput, List userInputs) { + public Command(@Nonnull String executable, @Nullable String executionDirectory, List args, List flags, + @Nullable String expectedOutput, boolean expectFailure, boolean allowAdditionalOutput, + boolean allowDisorderedOutput, List userInputs, String executorName) { this.executable = executable; this.executionSubdirectory = executionDirectory; this.args = args; @@ -40,6 +43,12 @@ public Command(@Nonnull String executable, @Nullable String executionDirectory, this.allowAdditionalOutput = allowAdditionalOutput; this.allowDisorderedOutput = allowDisorderedOutput; this.userInputs = userInputs; + this.executorName = executorName; + } + + @Nonnull + public String getExecutorName() { + return executorName; } @Nonnull @@ -102,7 +111,8 @@ public Builder toBuilder() { isExpectFailure(), isAllowAdditionalOutput(), isAllowDisorderedOutput(), - getUserInputs()); + getUserInputs(), + getExecutorName()); } public static class Builder { @@ -115,8 +125,11 @@ public static class Builder { private boolean allowAdditionalOutput; private boolean allowDisorderedOutput; private List userInputs; + private String executorName; - private Builder(String executable, String executionDirectory, List args, List flags, String expectedOutput, boolean expectFailure, boolean allowAdditionalOutput, boolean allowDisorderedOutput, List userInputs) { + private Builder(String executable, String executionDirectory, List args, List flags, + String expectedOutput, boolean expectFailure, boolean allowAdditionalOutput, + boolean allowDisorderedOutput, List userInputs, String executorName) { this.executable = executable; this.executionSubdirectory = executionDirectory; this.args = args; @@ -126,6 +139,7 @@ private Builder(String executable, String executionDirectory, List args, this.allowAdditionalOutput = allowAdditionalOutput; this.allowDisorderedOutput = allowDisorderedOutput; this.userInputs = userInputs; + this.executorName = executorName; } public Builder setExecutable(String executable) { @@ -168,8 +182,13 @@ public Builder setAllowDisorderedOutput(boolean allowDisorderedOutput) { return this; } + public Builder setExecutorName(String executorName){ + this.executorName = executorName; + return this; + } + public Command build() { - return new Command(executable, executionSubdirectory, args, flags, expectedOutput, expectFailure, allowAdditionalOutput, allowDisorderedOutput, userInputs); + return new Command(executable, executionSubdirectory, args, flags, expectedOutput, expectFailure, allowAdditionalOutput, allowDisorderedOutput, userInputs, executorName); } } } diff --git a/samples-discovery/src/test/groovy/org/gradle/exemplar/model/CommandsParserTest.groovy b/samples-discovery/src/test/groovy/org/gradle/exemplar/model/CommandsParserTest.groovy index 274265c..7dd7826 100644 --- a/samples-discovery/src/test/groovy/org/gradle/exemplar/model/CommandsParserTest.groovy +++ b/samples-discovery/src/test/groovy/org/gradle/exemplar/model/CommandsParserTest.groovy @@ -57,10 +57,10 @@ class CommandsParserTest extends Specification { e.cause.message == "A sample must be defined with an 'executable' or 'commands'" } - def "fails fast when no executable for command specified"() { + def "fails fast when no executable for CLI command is specified - default way"() { given: sampleConfigFile << """ - commands: [{ }, { }] + commands: [{ }] """ when: @@ -69,7 +69,36 @@ class CommandsParserTest extends Specification { then: def e = thrown(InvalidSampleException) e.message == "Could not read sample definition from ${sampleConfigFile}." - e.cause.message == "'executable' field cannot be empty" + e.cause.message == "'executable' field cannot be empty for CLI executor" + } + + def "fails fast when no executable for CLI command is specified - explicit way"() { + given: + sampleConfigFile << """ + commands: [{ command-executor: CLI}] + """ + + when: + CommandsParser.parse(sampleConfigFile) + + then: + def e = thrown(InvalidSampleException) + e.message == "Could not read sample definition from ${sampleConfigFile}." + e.cause.message == "'executable' field cannot be empty for CLI executor" + } + + def "succeeds when no executable for non-CLI command is specified"() { + given: + sampleConfigFile << """ + commands: [{ command-executor: my.custom.executor }] + """ + + when: + def config = CommandsParser.parse(sampleConfigFile) + + then: + config.get(0).getExecutorName() == "my.custom.executor" + config.get(0).getExecutable() == "" } def "provides reasonable defaults for command config"() {