From eeb4ad92da42e8e6405c2563769a9efeabce984e Mon Sep 17 00:00:00 2001 From: Przemek Bielicki Date: Wed, 6 Oct 2021 13:38:21 +0200 Subject: [PATCH 1/6] Use JUnit5 platform engine --- docs/build.gradle.kts | 1 + samples-check/build.gradle.kts | 1 + ...estAwareGradleRunnerCommandExecutor.groovy | 65 +++++++ .../test/engine/SamplesRunnerJUnitEngine.java | 176 ++++++++++++++++++ .../org.junit.platform.engine.TestEngine | 1 + 5 files changed, 244 insertions(+) create mode 100644 samples-check/src/main/java/org/gradle/exemplar/test/engine/PluginUnderTestAwareGradleRunnerCommandExecutor.groovy create mode 100644 samples-check/src/main/java/org/gradle/exemplar/test/engine/SamplesRunnerJUnitEngine.java create mode 100644 samples-check/src/main/resources/META-INF/services/org.junit.platform.engine.TestEngine diff --git a/docs/build.gradle.kts b/docs/build.gradle.kts index 87c19d6..27b0ab3 100644 --- a/docs/build.gradle.kts +++ b/docs/build.gradle.kts @@ -4,6 +4,7 @@ plugins { dependencies { implementation(project(":samples-check")) + implementation(Libraries.JUNIT) } tasks.test { diff --git a/samples-check/build.gradle.kts b/samples-check/build.gradle.kts index 987dd0f..be2cb70 100644 --- a/samples-check/build.gradle.kts +++ b/samples-check/build.gradle.kts @@ -9,6 +9,7 @@ dependencies { compileOnly(libs.jsr305) implementation(libs.commons.io) implementation(libs.commons.lang3) + implementation("org.junit.platform:junit-platform-engine:1.8.1") implementation(gradleTestKit()) testImplementation(libs.cglib) testImplementation(libs.groovy) diff --git a/samples-check/src/main/java/org/gradle/exemplar/test/engine/PluginUnderTestAwareGradleRunnerCommandExecutor.groovy b/samples-check/src/main/java/org/gradle/exemplar/test/engine/PluginUnderTestAwareGradleRunnerCommandExecutor.groovy new file mode 100644 index 0000000..0bc5680 --- /dev/null +++ b/samples-check/src/main/java/org/gradle/exemplar/test/engine/PluginUnderTestAwareGradleRunnerCommandExecutor.groovy @@ -0,0 +1,65 @@ +/* + * Copyright 2020 Benedikt Ritter + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.exemplar.test.engine + +import org.gradle.exemplar.executor.CommandExecutor +import org.gradle.testkit.runner.BuildResult +import org.gradle.testkit.runner.GradleRunner + +import javax.annotation.Nullable + +class PluginUnderTestAwareGradleRunnerCommandExecutor extends CommandExecutor { + private final File workingDir; + private final File customGradleInstallation; + private final boolean expectFailure; + + public PluginUnderTestAwareGradleRunnerCommandExecutor(File workingDir, @Nullable File customGradleInstallation, boolean expectFailure) { + this.workingDir = workingDir; + this.customGradleInstallation = customGradleInstallation; + this.expectFailure = expectFailure; + } + + @Override + protected int run(String executable, List args, List flags, OutputStream output) { + List allArguments = new ArrayList<>(args); + allArguments.addAll(flags); + + println(System.getProperty("java.class.path")) + + GradleRunner gradleRunner = GradleRunner.create() + .withProjectDir(workingDir) + .withArguments(allArguments) + .withPluginClasspath() + .forwardOutput(); + + if (customGradleInstallation != null) { + gradleRunner.withGradleInstallation(customGradleInstallation); + } + + output.withWriter { + BuildResult buildResult; + if (expectFailure) { + buildResult = gradleRunner.buildAndFail(); + } else { + buildResult = gradleRunner.build(); + } + it.write(buildResult.getOutput()); + it.close(); + return expectFailure ? 1 : 0; + } + } +} diff --git a/samples-check/src/main/java/org/gradle/exemplar/test/engine/SamplesRunnerJUnitEngine.java b/samples-check/src/main/java/org/gradle/exemplar/test/engine/SamplesRunnerJUnitEngine.java new file mode 100644 index 0000000..5c3ac1d --- /dev/null +++ b/samples-check/src/main/java/org/gradle/exemplar/test/engine/SamplesRunnerJUnitEngine.java @@ -0,0 +1,176 @@ +package org.gradle.exemplar.test.engine; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.gradle.exemplar.executor.CliCommandExecutor; +import org.gradle.exemplar.executor.CommandExecutionResult; +import org.gradle.exemplar.executor.CommandExecutor; +import org.gradle.exemplar.executor.ExecutionMetadata; +import org.gradle.exemplar.loader.SamplesDiscovery; +import org.gradle.exemplar.model.Command; +import org.gradle.exemplar.model.Sample; +import org.gradle.exemplar.test.normalizer.OutputNormalizer; +import org.gradle.exemplar.test.runner.SampleModifier; +import org.gradle.exemplar.test.runner.SamplesRunner; +import org.gradle.exemplar.test.verifier.AnyOrderLineSegmentedOutputVerifier; +import org.gradle.exemplar.test.verifier.StrictOrderLineSegmentedOutputVerifier; +import org.junit.Assert; +import org.junit.platform.engine.EngineDiscoveryRequest; +import org.junit.platform.engine.EngineExecutionListener; +import org.junit.platform.engine.ExecutionRequest; +import org.junit.platform.engine.TestDescriptor; +import org.junit.platform.engine.TestEngine; +import org.junit.platform.engine.TestExecutionResult; +import org.junit.platform.engine.UniqueId; +import org.junit.platform.engine.support.descriptor.AbstractTestDescriptor; +import org.junit.platform.engine.support.descriptor.EngineDescriptor; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static java.util.Collections.emptyList; + +public class SamplesRunnerJUnitEngine implements TestEngine { + + private final File tempDir; + private final List normalizers = emptyList(); + private final List sampleModifiers = emptyList(); + + public SamplesRunnerJUnitEngine() throws IOException { + tempDir = Files.createTempDirectory("samples-junit-engine").toFile(); + } + + @Override + public String getId() { + return "SamplesRunnerJUnitEngine"; + } + + @Override + public TestDescriptor discover(EngineDiscoveryRequest discoveryRequest, UniqueId uniqueId) { + EngineDescriptor engineDescriptor = new EngineDescriptor(uniqueId, "Samples Runner JUnit Engine"); + for (Sample sample : SamplesDiscovery.externalSamples(new File("src/docs/snippets"))) { + engineDescriptor.addChild(new SampleTestDescriptor(sample)); + } + return engineDescriptor; + } + + @Override + public void execute(ExecutionRequest request) { + TestDescriptor engineDescriptor = request.getRootTestDescriptor(); + EngineExecutionListener listener = request.getEngineExecutionListener(); + + listener.executionStarted(engineDescriptor); + for (TestDescriptor testDescriptor : engineDescriptor.getChildren()) { + SampleTestDescriptor descriptor = (SampleTestDescriptor) testDescriptor; + listener.executionStarted(testDescriptor); + + try { + runSample(descriptor.sample); + listener.executionFinished(testDescriptor, TestExecutionResult.successful()); + } catch (Throwable t) { + listener.executionFinished(testDescriptor, TestExecutionResult.failed(t)); + } + + } + listener.executionFinished(engineDescriptor, TestExecutionResult.successful()); + } + + private void runSample(Sample sample) throws Exception { + final Sample testSpecificSample = initSample(sample); + File baseWorkingDir = testSpecificSample.getProjectDir(); + + // Execute and verify each command + for (Command command : testSpecificSample.getCommands()) { + File workingDir = baseWorkingDir; + + if (command.getExecutionSubdirectory() != null) { + workingDir = new File(workingDir, command.getExecutionSubdirectory()); + } + + // This should be some kind of plugable executor rather than hard-coded here + if (command.getExecutable().equals("cd")) { + baseWorkingDir = new File(baseWorkingDir, command.getArgs().get(0)).getCanonicalFile(); + continue; + } + + CommandExecutionResult result = executeSample(getExecutionMetadata(testSpecificSample.getProjectDir()), workingDir, command); + + if (result.getExitCode() != 0 && !command.isExpectFailure()) { + Assert.fail(String.format("Expected sample invocation to succeed but it failed.%nCommand was: '%s %s'%nWorking directory: '%s'%n[BEGIN OUTPUT]%n%s%n[END OUTPUT]%n", command.getExecutable(), StringUtils.join(command.getArgs(), " "), workingDir.getAbsolutePath(), result.getOutput())); + } else if (result.getExitCode() == 0 && command.isExpectFailure()) { + Assert.fail(String.format("Expected sample invocation to fail but it succeeded.%nCommand was: '%s %s'%nWorking directory: '%s'%n[BEGIN OUTPUT]%n%s%n[END OUTPUT]%n", command.getExecutable(), StringUtils.join(command.getArgs(), " "), workingDir.getAbsolutePath(), result.getOutput())); + } + + verifyOutput(command, result); + } + } + + private void verifyOutput(final Command command, final CommandExecutionResult executionResult) { + if (command.getExpectedOutput() == null) { + return; + } + + String expectedOutput = command.getExpectedOutput(); + String actualOutput = executionResult.getOutput(); + + for (OutputNormalizer normalizer : normalizers) { + actualOutput = normalizer.normalize(actualOutput, executionResult.getExecutionMetadata()); + } + + if (command.isAllowDisorderedOutput()) { + new AnyOrderLineSegmentedOutputVerifier().verify(expectedOutput, actualOutput, command.isAllowAdditionalOutput()); + } else { + new StrictOrderLineSegmentedOutputVerifier().verify(expectedOutput, actualOutput, command.isAllowAdditionalOutput()); + } + } + + private CommandExecutionResult executeSample(ExecutionMetadata executionMetadata, File workingDir, Command command) { + return selectExecutor(executionMetadata, workingDir, command).execute(command, executionMetadata); + } + + protected CommandExecutor selectExecutor(ExecutionMetadata executionMetadata, File workingDir, Command command) { + boolean expectFailure = command.isExpectFailure(); + if (command.getExecutable().equals("gradle")) { + return new PluginUnderTestAwareGradleRunnerCommandExecutor(workingDir, null, expectFailure); + } + return new CliCommandExecutor(workingDir); + } + + private ExecutionMetadata getExecutionMetadata(final File tempSampleOutputDir) { + Map systemProperties = new HashMap<>(); + for (String systemPropertyKey : SamplesRunner.SAFE_SYSTEM_PROPERTIES) { + systemProperties.put(systemPropertyKey, System.getProperty(systemPropertyKey)); + } + + return new ExecutionMetadata(tempSampleOutputDir, systemProperties); + } + + private Sample initSample(final Sample sampleIn) throws IOException { + File tmpProjectDir = new File(tempDir, sampleIn.getId()); + FileUtils.copyDirectory(sampleIn.getProjectDir(), tmpProjectDir); + Sample sample = new Sample(sampleIn.getId(), tmpProjectDir, sampleIn.getCommands()); + for (SampleModifier sampleModifier : sampleModifiers) { + sample = sampleModifier.modify(sample); + } + return sample; + } + + private static class SampleTestDescriptor extends AbstractTestDescriptor { + + private final Sample sample; + + private SampleTestDescriptor(Sample sample) { + super(UniqueId.forEngine("SamplesRunnerJUnitEngine" + sample.getId()), sample.getId()); + this.sample = sample; + } + + @Override + public Type getType() { + return Type.TEST; + } + } +} diff --git a/samples-check/src/main/resources/META-INF/services/org.junit.platform.engine.TestEngine b/samples-check/src/main/resources/META-INF/services/org.junit.platform.engine.TestEngine new file mode 100644 index 0000000..f106d72 --- /dev/null +++ b/samples-check/src/main/resources/META-INF/services/org.junit.platform.engine.TestEngine @@ -0,0 +1 @@ +org.gradle.exemplar.test.engine.SamplesRunnerJUnitEngine \ No newline at end of file From 524b48d91a75ab182a4ba86630b978e264fd4cb1 Mon Sep 17 00:00:00 2001 From: Przemek Bielicki Date: Wed, 6 Oct 2021 13:46:45 +0200 Subject: [PATCH 2/6] Test custom PluginUnderTestAwareGradleRunnerCommandExecutor --- ...estAwareGradleRunnerCommandExecutor.groovy | 65 ------------------- ...rTestAwareGradleRunnerCommandExecutor.java | 57 ++++++++++++++++ 2 files changed, 57 insertions(+), 65 deletions(-) delete mode 100644 samples-check/src/main/java/org/gradle/exemplar/test/engine/PluginUnderTestAwareGradleRunnerCommandExecutor.groovy create mode 100644 samples-check/src/main/java/org/gradle/exemplar/test/engine/PluginUnderTestAwareGradleRunnerCommandExecutor.java diff --git a/samples-check/src/main/java/org/gradle/exemplar/test/engine/PluginUnderTestAwareGradleRunnerCommandExecutor.groovy b/samples-check/src/main/java/org/gradle/exemplar/test/engine/PluginUnderTestAwareGradleRunnerCommandExecutor.groovy deleted file mode 100644 index 0bc5680..0000000 --- a/samples-check/src/main/java/org/gradle/exemplar/test/engine/PluginUnderTestAwareGradleRunnerCommandExecutor.groovy +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright 2020 Benedikt Ritter - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.gradle.exemplar.test.engine - -import org.gradle.exemplar.executor.CommandExecutor -import org.gradle.testkit.runner.BuildResult -import org.gradle.testkit.runner.GradleRunner - -import javax.annotation.Nullable - -class PluginUnderTestAwareGradleRunnerCommandExecutor extends CommandExecutor { - private final File workingDir; - private final File customGradleInstallation; - private final boolean expectFailure; - - public PluginUnderTestAwareGradleRunnerCommandExecutor(File workingDir, @Nullable File customGradleInstallation, boolean expectFailure) { - this.workingDir = workingDir; - this.customGradleInstallation = customGradleInstallation; - this.expectFailure = expectFailure; - } - - @Override - protected int run(String executable, List args, List flags, OutputStream output) { - List allArguments = new ArrayList<>(args); - allArguments.addAll(flags); - - println(System.getProperty("java.class.path")) - - GradleRunner gradleRunner = GradleRunner.create() - .withProjectDir(workingDir) - .withArguments(allArguments) - .withPluginClasspath() - .forwardOutput(); - - if (customGradleInstallation != null) { - gradleRunner.withGradleInstallation(customGradleInstallation); - } - - output.withWriter { - BuildResult buildResult; - if (expectFailure) { - buildResult = gradleRunner.buildAndFail(); - } else { - buildResult = gradleRunner.build(); - } - it.write(buildResult.getOutput()); - it.close(); - return expectFailure ? 1 : 0; - } - } -} diff --git a/samples-check/src/main/java/org/gradle/exemplar/test/engine/PluginUnderTestAwareGradleRunnerCommandExecutor.java b/samples-check/src/main/java/org/gradle/exemplar/test/engine/PluginUnderTestAwareGradleRunnerCommandExecutor.java new file mode 100644 index 0000000..57908a3 --- /dev/null +++ b/samples-check/src/main/java/org/gradle/exemplar/test/engine/PluginUnderTestAwareGradleRunnerCommandExecutor.java @@ -0,0 +1,57 @@ +package org.gradle.exemplar.test.engine; + +import org.gradle.exemplar.executor.CommandExecutor; +import org.gradle.testkit.runner.BuildResult; +import org.gradle.testkit.runner.GradleRunner; + +import javax.annotation.Nullable; +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.List; + +public class PluginUnderTestAwareGradleRunnerCommandExecutor extends CommandExecutor { + private final File workingDir; + private final File customGradleInstallation; + private final boolean expectFailure; + + PluginUnderTestAwareGradleRunnerCommandExecutor(File workingDir, @Nullable File customGradleInstallation, boolean expectFailure) { + this.workingDir = workingDir; + this.customGradleInstallation = customGradleInstallation; + this.expectFailure = expectFailure; + } + + @Override + protected int run(String executable, List args, List flags, OutputStream output) { + List allArguments = new ArrayList<>(args); + allArguments.addAll(flags); + + GradleRunner gradleRunner = GradleRunner.create() + .withProjectDir(workingDir) + .withArguments(allArguments) + .withPluginClasspath() + .forwardOutput(); + + if (customGradleInstallation != null) { + gradleRunner.withGradleInstallation(customGradleInstallation); + } + + BuildResult buildResult; + if (expectFailure) { + buildResult = gradleRunner.buildAndFail(); + } else { + buildResult = gradleRunner.build(); + } + + try (OutputStreamWriter writer = new OutputStreamWriter(output)) { + writer.write(buildResult.getOutput()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + + return expectFailure ? 1 : 0; + } +} From ef454b309aa7ba070b7ee336fb57251092d758c3 Mon Sep 17 00:00:00 2001 From: Przemek Bielicki Date: Wed, 6 Oct 2021 13:52:05 +0200 Subject: [PATCH 3/6] Remove PluginUnderTestAwareGradleRunnerCommandExecutor --- ...rTestAwareGradleRunnerCommandExecutor.java | 57 ------------------- .../test/engine/SamplesRunnerJUnitEngine.java | 4 -- 2 files changed, 61 deletions(-) delete mode 100644 samples-check/src/main/java/org/gradle/exemplar/test/engine/PluginUnderTestAwareGradleRunnerCommandExecutor.java diff --git a/samples-check/src/main/java/org/gradle/exemplar/test/engine/PluginUnderTestAwareGradleRunnerCommandExecutor.java b/samples-check/src/main/java/org/gradle/exemplar/test/engine/PluginUnderTestAwareGradleRunnerCommandExecutor.java deleted file mode 100644 index 57908a3..0000000 --- a/samples-check/src/main/java/org/gradle/exemplar/test/engine/PluginUnderTestAwareGradleRunnerCommandExecutor.java +++ /dev/null @@ -1,57 +0,0 @@ -package org.gradle.exemplar.test.engine; - -import org.gradle.exemplar.executor.CommandExecutor; -import org.gradle.testkit.runner.BuildResult; -import org.gradle.testkit.runner.GradleRunner; - -import javax.annotation.Nullable; -import java.io.File; -import java.io.IOException; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.io.UncheckedIOException; -import java.util.ArrayList; -import java.util.List; - -public class PluginUnderTestAwareGradleRunnerCommandExecutor extends CommandExecutor { - private final File workingDir; - private final File customGradleInstallation; - private final boolean expectFailure; - - PluginUnderTestAwareGradleRunnerCommandExecutor(File workingDir, @Nullable File customGradleInstallation, boolean expectFailure) { - this.workingDir = workingDir; - this.customGradleInstallation = customGradleInstallation; - this.expectFailure = expectFailure; - } - - @Override - protected int run(String executable, List args, List flags, OutputStream output) { - List allArguments = new ArrayList<>(args); - allArguments.addAll(flags); - - GradleRunner gradleRunner = GradleRunner.create() - .withProjectDir(workingDir) - .withArguments(allArguments) - .withPluginClasspath() - .forwardOutput(); - - if (customGradleInstallation != null) { - gradleRunner.withGradleInstallation(customGradleInstallation); - } - - BuildResult buildResult; - if (expectFailure) { - buildResult = gradleRunner.buildAndFail(); - } else { - buildResult = gradleRunner.build(); - } - - try (OutputStreamWriter writer = new OutputStreamWriter(output)) { - writer.write(buildResult.getOutput()); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - - return expectFailure ? 1 : 0; - } -} diff --git a/samples-check/src/main/java/org/gradle/exemplar/test/engine/SamplesRunnerJUnitEngine.java b/samples-check/src/main/java/org/gradle/exemplar/test/engine/SamplesRunnerJUnitEngine.java index 5c3ac1d..d88a180 100644 --- a/samples-check/src/main/java/org/gradle/exemplar/test/engine/SamplesRunnerJUnitEngine.java +++ b/samples-check/src/main/java/org/gradle/exemplar/test/engine/SamplesRunnerJUnitEngine.java @@ -133,10 +133,6 @@ private CommandExecutionResult executeSample(ExecutionMetadata executionMetadata } protected CommandExecutor selectExecutor(ExecutionMetadata executionMetadata, File workingDir, Command command) { - boolean expectFailure = command.isExpectFailure(); - if (command.getExecutable().equals("gradle")) { - return new PluginUnderTestAwareGradleRunnerCommandExecutor(workingDir, null, expectFailure); - } return new CliCommandExecutor(workingDir); } From 540f6ab732f9a97ad629041a84cb9d15e4a6ff3c Mon Sep 17 00:00:00 2001 From: Przemek Bielicki Date: Wed, 6 Oct 2021 16:48:04 +0200 Subject: [PATCH 4/6] Set test class name correctly --- .../test/engine/SamplesRunnerJUnitEngine.java | 100 +++++++++++++++--- 1 file changed, 83 insertions(+), 17 deletions(-) diff --git a/samples-check/src/main/java/org/gradle/exemplar/test/engine/SamplesRunnerJUnitEngine.java b/samples-check/src/main/java/org/gradle/exemplar/test/engine/SamplesRunnerJUnitEngine.java index d88a180..3317530 100644 --- a/samples-check/src/main/java/org/gradle/exemplar/test/engine/SamplesRunnerJUnitEngine.java +++ b/samples-check/src/main/java/org/gradle/exemplar/test/engine/SamplesRunnerJUnitEngine.java @@ -11,10 +11,10 @@ import org.gradle.exemplar.model.Sample; import org.gradle.exemplar.test.normalizer.OutputNormalizer; import org.gradle.exemplar.test.runner.SampleModifier; +import org.gradle.exemplar.test.runner.SamplesRoot; import org.gradle.exemplar.test.runner.SamplesRunner; import org.gradle.exemplar.test.verifier.AnyOrderLineSegmentedOutputVerifier; import org.gradle.exemplar.test.verifier.StrictOrderLineSegmentedOutputVerifier; -import org.junit.Assert; import org.junit.platform.engine.EngineDiscoveryRequest; import org.junit.platform.engine.EngineExecutionListener; import org.junit.platform.engine.ExecutionRequest; @@ -22,8 +22,11 @@ import org.junit.platform.engine.TestEngine; import org.junit.platform.engine.TestExecutionResult; import org.junit.platform.engine.UniqueId; +import org.junit.platform.engine.discovery.ClassSelector; import org.junit.platform.engine.support.descriptor.AbstractTestDescriptor; +import org.junit.platform.engine.support.descriptor.ClassSource; import org.junit.platform.engine.support.descriptor.EngineDescriptor; +import org.opentest4j.AssertionFailedError; import java.io.File; import java.io.IOException; @@ -52,9 +55,27 @@ public String getId() { @Override public TestDescriptor discover(EngineDiscoveryRequest discoveryRequest, UniqueId uniqueId) { EngineDescriptor engineDescriptor = new EngineDescriptor(uniqueId, "Samples Runner JUnit Engine"); - for (Sample sample : SamplesDiscovery.externalSamples(new File("src/docs/snippets"))) { - engineDescriptor.addChild(new SampleTestDescriptor(sample)); - } + discoveryRequest.getSelectorsByType(ClassSelector.class) + .stream() + .map(ClassSelector::getJavaClass).forEach(javaClass -> { + SamplesRoot samplesRoot = javaClass.getAnnotation(SamplesRoot.class); + if (samplesRoot != null) { + SamplesTestDescriptor samplesTestDescriptor = new SamplesTestDescriptor( + uniqueId.append("className", javaClass.getSimpleName()), + javaClass + ); + + for (Sample sample : SamplesDiscovery.externalSamples(getSamplesRootDir(samplesRoot))) { + samplesTestDescriptor.addChild(new SampleTestDescriptor( + uniqueId.append("sampleId", sample.getId()), + sample + )); + } + + engineDescriptor.addChild(samplesTestDescriptor); + } + }); + return engineDescriptor; } @@ -64,21 +85,42 @@ public void execute(ExecutionRequest request) { EngineExecutionListener listener = request.getEngineExecutionListener(); listener.executionStarted(engineDescriptor); - for (TestDescriptor testDescriptor : engineDescriptor.getChildren()) { - SampleTestDescriptor descriptor = (SampleTestDescriptor) testDescriptor; - listener.executionStarted(testDescriptor); - - try { - runSample(descriptor.sample); - listener.executionFinished(testDescriptor, TestExecutionResult.successful()); - } catch (Throwable t) { - listener.executionFinished(testDescriptor, TestExecutionResult.failed(t)); + for (TestDescriptor classDescriptor : engineDescriptor.getChildren()) { + listener.executionStarted(classDescriptor); + for (TestDescriptor testDescriptor : classDescriptor.getChildren()) { + SampleTestDescriptor descriptor = (SampleTestDescriptor) testDescriptor; + listener.executionStarted(testDescriptor); + try { + runSample(descriptor.sample); + listener.executionFinished(testDescriptor, TestExecutionResult.successful()); + } catch (Throwable t) { + listener.executionFinished(testDescriptor, TestExecutionResult.failed(t)); + } } + listener.executionFinished(classDescriptor, TestExecutionResult.successful()); } listener.executionFinished(engineDescriptor, TestExecutionResult.successful()); } + protected File getSamplesRootDir(SamplesRoot samplesRoot) { + File samplesRootDir = null; + try { + if (samplesRoot != null) { + samplesRootDir = new File(samplesRoot.value()); + } + if (samplesRootDir == null) { + throw new IllegalArgumentException("Samples root directory is not declared. Please annotate your test class with @SamplesRoot(\"path/to/samples\")"); + } + if (!samplesRootDir.exists()) { + throw new IllegalArgumentException("Samples root directory " + samplesRootDir.getAbsolutePath() + " does not exist"); + } + } catch (Exception e) { + throw new RuntimeException("Could not initialize SamplesRunnerJUnitEngine", e); + } + return samplesRootDir; + } + private void runSample(Sample sample) throws Exception { final Sample testSpecificSample = initSample(sample); File baseWorkingDir = testSpecificSample.getProjectDir(); @@ -100,9 +142,21 @@ private void runSample(Sample sample) throws Exception { CommandExecutionResult result = executeSample(getExecutionMetadata(testSpecificSample.getProjectDir()), workingDir, command); if (result.getExitCode() != 0 && !command.isExpectFailure()) { - Assert.fail(String.format("Expected sample invocation to succeed but it failed.%nCommand was: '%s %s'%nWorking directory: '%s'%n[BEGIN OUTPUT]%n%s%n[END OUTPUT]%n", command.getExecutable(), StringUtils.join(command.getArgs(), " "), workingDir.getAbsolutePath(), result.getOutput())); + throw new AssertionFailedError(String.format( + "Expected sample invocation to succeed but it failed.%nCommand was: '%s %s'%nWorking directory: '%s'%n[BEGIN OUTPUT]%n%s%n[END OUTPUT]%n", + command.getExecutable(), + StringUtils.join(command.getArgs(), " "), + workingDir.getAbsolutePath(), + result.getOutput() + )); } else if (result.getExitCode() == 0 && command.isExpectFailure()) { - Assert.fail(String.format("Expected sample invocation to fail but it succeeded.%nCommand was: '%s %s'%nWorking directory: '%s'%n[BEGIN OUTPUT]%n%s%n[END OUTPUT]%n", command.getExecutable(), StringUtils.join(command.getArgs(), " "), workingDir.getAbsolutePath(), result.getOutput())); + throw new AssertionFailedError(String.format( + "Expected sample invocation to fail but it succeeded.%nCommand was: '%s %s'%nWorking directory: '%s'%n[BEGIN OUTPUT]%n%s%n[END OUTPUT]%n", + command.getExecutable(), + StringUtils.join(command.getArgs(), " "), + workingDir.getAbsolutePath(), + result.getOutput() + )); } verifyOutput(command, result); @@ -159,8 +213,8 @@ private static class SampleTestDescriptor extends AbstractTestDescriptor { private final Sample sample; - private SampleTestDescriptor(Sample sample) { - super(UniqueId.forEngine("SamplesRunnerJUnitEngine" + sample.getId()), sample.getId()); + private SampleTestDescriptor(UniqueId uniqueId, Sample sample) { + super(uniqueId, sample.getId()); this.sample = sample; } @@ -169,4 +223,16 @@ public Type getType() { return Type.TEST; } } + + private static class SamplesTestDescriptor extends AbstractTestDescriptor { + + private SamplesTestDescriptor(UniqueId uniqueId, Class testClass) { + super(uniqueId, testClass.getSimpleName(), ClassSource.from(testClass)); + } + + @Override + public Type getType() { + return Type.CONTAINER; + } + } } From 3b3ecaa4c978af3a56e3925c18a75a68d8536daa Mon Sep 17 00:00:00 2001 From: Przemek Bielicki Date: Tue, 12 Oct 2021 10:39:11 +0200 Subject: [PATCH 5/6] Get rid of JUnit4 completely --- docs/README.adoc | 6 +- .../java/org/gradle/exemplar/ReadmeTest.java | 12 +- .../exemplar/test/NoopOutputNormalizer.java | 11 + .../exemplar/test/NoopRootDirSupplier.java | 11 + .../exemplar/test/NoopSampleModifier.java | 10 + .../test/{runner => }/SampleModifier.java | 2 +- .../org/gradle/exemplar/test/Samples.java | 54 ++++ .../test/engine/CommandExecutorFunction.java | 8 + .../test/engine/CommandExecutorParams.java | 19 ++ .../DefaultCommandExecutorFunction.java | 11 + .../GradleSamplesExtension.java} | 64 ++--- .../test/engine/SamplesRunnerJUnitEngine.java | 157 +++++++++--- .../test/runner/EmbeddedSamplesRunner.java | 46 ---- .../runner/GradleEmbeddedSamplesRunner.java | 41 --- .../exemplar/test/runner/SampleModifiers.java | 29 --- .../test/runner/SamplesOutputNormalizers.java | 33 --- .../exemplar/test/runner/SamplesRoot.java | 52 ---- .../exemplar/test/runner/SamplesRunner.java | 234 ------------------ .../exemplar/test/runner/Transformer.java | 34 --- .../SamplesRunnerIntegrationTest.groovy | 11 +- .../SamplesRunnerSadDayIntegrationTest.groovy | 7 +- .../CliSamplesRunnerIntegrationTest.java | 5 +- .../EmbeddedSamplesRunnerIntegrationTest.java | 11 +- .../GradleSamplesRunnerIntegrationTest.java | 11 +- .../runner/SampleModifierIntegrationTest.java | 11 +- .../ExtraCommandArgumentsSampleModifier.java | 2 +- .../exemplar/loader/SamplesDiscovery.java | 9 +- 27 files changed, 319 insertions(+), 582 deletions(-) create mode 100644 samples-check/src/main/java/org/gradle/exemplar/test/NoopOutputNormalizer.java create mode 100644 samples-check/src/main/java/org/gradle/exemplar/test/NoopRootDirSupplier.java create mode 100644 samples-check/src/main/java/org/gradle/exemplar/test/NoopSampleModifier.java rename samples-check/src/main/java/org/gradle/exemplar/test/{runner => }/SampleModifier.java (95%) create mode 100644 samples-check/src/main/java/org/gradle/exemplar/test/Samples.java create mode 100644 samples-check/src/main/java/org/gradle/exemplar/test/engine/CommandExecutorFunction.java create mode 100644 samples-check/src/main/java/org/gradle/exemplar/test/engine/CommandExecutorParams.java create mode 100644 samples-check/src/main/java/org/gradle/exemplar/test/engine/DefaultCommandExecutorFunction.java rename samples-check/src/main/java/org/gradle/exemplar/test/{runner/GradleSamplesRunner.java => engine/GradleSamplesExtension.java} (51%) delete mode 100644 samples-check/src/main/java/org/gradle/exemplar/test/runner/EmbeddedSamplesRunner.java delete mode 100644 samples-check/src/main/java/org/gradle/exemplar/test/runner/GradleEmbeddedSamplesRunner.java delete mode 100644 samples-check/src/main/java/org/gradle/exemplar/test/runner/SampleModifiers.java delete mode 100644 samples-check/src/main/java/org/gradle/exemplar/test/runner/SamplesOutputNormalizers.java delete mode 100644 samples-check/src/main/java/org/gradle/exemplar/test/runner/SamplesRoot.java delete mode 100644 samples-check/src/main/java/org/gradle/exemplar/test/runner/SamplesRunner.java delete mode 100644 samples-check/src/main/java/org/gradle/exemplar/test/runner/Transformer.java diff --git a/docs/README.adoc b/docs/README.adoc index 971aefa..37a6645 100644 --- a/docs/README.adoc +++ b/docs/README.adoc @@ -135,7 +135,7 @@ You can verify samples either through one of the < { + @Override + public File get() { + return null; + } +} diff --git a/samples-check/src/main/java/org/gradle/exemplar/test/NoopSampleModifier.java b/samples-check/src/main/java/org/gradle/exemplar/test/NoopSampleModifier.java new file mode 100644 index 0000000..70982ef --- /dev/null +++ b/samples-check/src/main/java/org/gradle/exemplar/test/NoopSampleModifier.java @@ -0,0 +1,10 @@ +package org.gradle.exemplar.test; + +import org.gradle.exemplar.model.Sample; + +public class NoopSampleModifier implements SampleModifier { + @Override + public Sample modify(Sample sampleIn) { + return sampleIn; + } +} diff --git a/samples-check/src/main/java/org/gradle/exemplar/test/runner/SampleModifier.java b/samples-check/src/main/java/org/gradle/exemplar/test/SampleModifier.java similarity index 95% rename from samples-check/src/main/java/org/gradle/exemplar/test/runner/SampleModifier.java rename to samples-check/src/main/java/org/gradle/exemplar/test/SampleModifier.java index 0d81af0..e06784b 100644 --- a/samples-check/src/main/java/org/gradle/exemplar/test/runner/SampleModifier.java +++ b/samples-check/src/main/java/org/gradle/exemplar/test/SampleModifier.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.gradle.exemplar.test.runner; +package org.gradle.exemplar.test; import org.gradle.exemplar.model.Sample; diff --git a/samples-check/src/main/java/org/gradle/exemplar/test/Samples.java b/samples-check/src/main/java/org/gradle/exemplar/test/Samples.java new file mode 100644 index 0000000..3ba1981 --- /dev/null +++ b/samples-check/src/main/java/org/gradle/exemplar/test/Samples.java @@ -0,0 +1,54 @@ +/* + * Copyright 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradle.exemplar.test; + +import org.gradle.exemplar.executor.CommandExecutor; +import org.gradle.exemplar.test.engine.CommandExecutorParams; +import org.gradle.exemplar.test.engine.DefaultCommandExecutorFunction; +import org.gradle.exemplar.test.normalizer.OutputNormalizer; + +import java.io.File; +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import java.util.function.Function; +import java.util.function.Supplier; + +@Documented +@Inherited +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +public @interface Samples { + String root(); + + SamplesType samplesType() default SamplesType.DEFAULT; + + Class[] modifiers() default { NoopSampleModifier.class }; + + Class[] outputNormalizers() default { NoopOutputNormalizer.class }; + + Class> implicitRootDirSupplier() default NoopRootDirSupplier.class; + + Class> commandExecutorFunction() default DefaultCommandExecutorFunction.class; + + enum SamplesType { + DEFAULT, + EMBEDDED + } +} diff --git a/samples-check/src/main/java/org/gradle/exemplar/test/engine/CommandExecutorFunction.java b/samples-check/src/main/java/org/gradle/exemplar/test/engine/CommandExecutorFunction.java new file mode 100644 index 0000000..8fa4007 --- /dev/null +++ b/samples-check/src/main/java/org/gradle/exemplar/test/engine/CommandExecutorFunction.java @@ -0,0 +1,8 @@ +package org.gradle.exemplar.test.engine; + +import org.gradle.exemplar.executor.CommandExecutor; + +import java.util.function.Function; + +public interface CommandExecutorFunction extends Function { +} diff --git a/samples-check/src/main/java/org/gradle/exemplar/test/engine/CommandExecutorParams.java b/samples-check/src/main/java/org/gradle/exemplar/test/engine/CommandExecutorParams.java new file mode 100644 index 0000000..79c2566 --- /dev/null +++ b/samples-check/src/main/java/org/gradle/exemplar/test/engine/CommandExecutorParams.java @@ -0,0 +1,19 @@ +package org.gradle.exemplar.test.engine; + +import org.gradle.exemplar.executor.ExecutionMetadata; +import org.gradle.exemplar.model.Command; + +import java.io.File; + +public class CommandExecutorParams { + final ExecutionMetadata executionMetadata; + final File workingDir; + final Command command; + + public CommandExecutorParams(ExecutionMetadata executionMetadata, File workingDir, Command command) { + this.executionMetadata = executionMetadata; + this.workingDir = workingDir; + this.command = command; + } + +} diff --git a/samples-check/src/main/java/org/gradle/exemplar/test/engine/DefaultCommandExecutorFunction.java b/samples-check/src/main/java/org/gradle/exemplar/test/engine/DefaultCommandExecutorFunction.java new file mode 100644 index 0000000..9fcf150 --- /dev/null +++ b/samples-check/src/main/java/org/gradle/exemplar/test/engine/DefaultCommandExecutorFunction.java @@ -0,0 +1,11 @@ +package org.gradle.exemplar.test.engine; + +import org.gradle.exemplar.executor.CliCommandExecutor; +import org.gradle.exemplar.executor.CommandExecutor; + +public class DefaultCommandExecutorFunction implements CommandExecutorFunction { + @Override + public CommandExecutor apply(CommandExecutorParams params) { + return new CliCommandExecutor(params.workingDir); + } +} diff --git a/samples-check/src/main/java/org/gradle/exemplar/test/runner/GradleSamplesRunner.java b/samples-check/src/main/java/org/gradle/exemplar/test/engine/GradleSamplesExtension.java similarity index 51% rename from samples-check/src/main/java/org/gradle/exemplar/test/runner/GradleSamplesRunner.java rename to samples-check/src/main/java/org/gradle/exemplar/test/engine/GradleSamplesExtension.java index ff0aa2b..dfd5b92 100644 --- a/samples-check/src/main/java/org/gradle/exemplar/test/runner/GradleSamplesRunner.java +++ b/samples-check/src/main/java/org/gradle/exemplar/test/engine/GradleSamplesExtension.java @@ -13,75 +13,65 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.gradle.exemplar.test.runner; +package org.gradle.exemplar.test.engine; import org.gradle.api.JavaVersion; import org.gradle.exemplar.executor.CliCommandExecutor; import org.gradle.exemplar.executor.CommandExecutor; -import org.gradle.exemplar.executor.ExecutionMetadata; import org.gradle.exemplar.executor.GradleRunnerCommandExecutor; -import org.gradle.exemplar.model.Command; import org.gradle.exemplar.model.Sample; -import org.junit.Rule; -import org.junit.rules.TemporaryFolder; -import org.junit.runners.model.InitializationError; import javax.annotation.Nullable; import java.io.File; +import java.util.function.Function; +import java.util.function.Supplier; -/** - * A custom implementation of {@link SamplesRunner} that uses the Gradle Tooling API to execute sample builds. - */ -public class GradleSamplesRunner extends SamplesRunner { +public class GradleSamplesExtension { private static final String GRADLE_EXECUTABLE = "gradle"; - @Rule - public TemporaryFolder tempGradleUserHomeDir = new TemporaryFolder(); - private File customGradleInstallation = null; - - public GradleSamplesRunner(Class testClass) throws InitializationError { - super(testClass); - } + private static File customGradleInstallation = null; /** * Gradle samples tests are ignored on Java 7 and below. */ - @Override protected boolean isIgnored(Sample child) { return !JavaVersion.current().isJava8Compatible(); } - @Override - protected CommandExecutor selectExecutor(ExecutionMetadata executionMetadata, File workingDir, Command command) { - boolean expectFailure = command.isExpectFailure(); - if (command.getExecutable().equals(GRADLE_EXECUTABLE)) { - return new GradleRunnerCommandExecutor(workingDir, customGradleInstallation, expectFailure); + public static class GradleCommandExecutorFunction implements CommandExecutorFunction { + @Override + public CommandExecutor apply(CommandExecutorParams params) { + boolean expectFailure = params.command.isExpectFailure(); + if (params.command.getExecutable().equals(GRADLE_EXECUTABLE)) { + return new GradleRunnerCommandExecutor(params.workingDir, customGradleInstallation, expectFailure); + } + return new CliCommandExecutor(params.workingDir); } - return new CliCommandExecutor(workingDir); } - @Nullable - @Override - protected File getImplicitSamplesRootDir() { - String gradleHomeDir = getCustomGradleInstallationFromSystemProperty(); - if (System.getProperty("integTest.samplesdir") != null) { - String samplesRootProperty = System.getProperty("integTest.samplesdir", gradleHomeDir + "/samples"); - return new File(samplesRootProperty); - } else if (customGradleInstallation != null) { - return new File(customGradleInstallation, "samples"); - } else { - return null; + public static class ImplicitSamplesRootDirSupplier implements Supplier { + @Override + public File get() { + String gradleHomeDir = getCustomGradleInstallationFromSystemProperty(); + if (System.getProperty("integTest.samplesdir") != null) { + String samplesRootProperty = System.getProperty("integTest.samplesdir", gradleHomeDir + "/samples"); + return new File(samplesRootProperty); + } else if (customGradleInstallation != null) { + return new File(customGradleInstallation, "samples"); + } else { + return null; + } } } @Nullable - private String getCustomGradleInstallationFromSystemProperty() { + private static String getCustomGradleInstallationFromSystemProperty() { // Allow Gradle installation and samples root dir to be set from a system property // This is to allow Gradle to test Gradle installations during integration testing final String gradleHomeDirProperty = System.getProperty("integTest.gradleHomeDir"); if (gradleHomeDirProperty != null) { File customGradleInstallationDir = new File(gradleHomeDirProperty); if (customGradleInstallationDir.exists()) { - this.customGradleInstallation = customGradleInstallationDir; + customGradleInstallation = customGradleInstallationDir; } else { throw new RuntimeException(String.format("Custom Gradle installation dir at %s was not found", gradleHomeDirProperty)); } diff --git a/samples-check/src/main/java/org/gradle/exemplar/test/engine/SamplesRunnerJUnitEngine.java b/samples-check/src/main/java/org/gradle/exemplar/test/engine/SamplesRunnerJUnitEngine.java index 3317530..825a183 100644 --- a/samples-check/src/main/java/org/gradle/exemplar/test/engine/SamplesRunnerJUnitEngine.java +++ b/samples-check/src/main/java/org/gradle/exemplar/test/engine/SamplesRunnerJUnitEngine.java @@ -10,9 +10,8 @@ import org.gradle.exemplar.model.Command; import org.gradle.exemplar.model.Sample; import org.gradle.exemplar.test.normalizer.OutputNormalizer; -import org.gradle.exemplar.test.runner.SampleModifier; -import org.gradle.exemplar.test.runner.SamplesRoot; -import org.gradle.exemplar.test.runner.SamplesRunner; +import org.gradle.exemplar.test.SampleModifier; +import org.gradle.exemplar.test.Samples; import org.gradle.exemplar.test.verifier.AnyOrderLineSegmentedOutputVerifier; import org.gradle.exemplar.test.verifier.StrictOrderLineSegmentedOutputVerifier; import org.junit.platform.engine.EngineDiscoveryRequest; @@ -30,18 +29,24 @@ import java.io.File; import java.io.IOException; +import java.lang.reflect.InvocationTargetException; import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Function; +import java.util.function.Supplier; import static java.util.Collections.emptyList; public class SamplesRunnerJUnitEngine implements TestEngine { + // See https://docs.oracle.com/javase/tutorial/essential/environment/sysprop.html + public static final List SAFE_SYSTEM_PROPERTIES = Arrays.asList("file.separator", "java.home", "java.vendor", "java.version", "line.separator", "os.arch", "os.name", "os.version", "path.separator", "user.dir", "user.home", "user.name"); + private final File tempDir; - private final List normalizers = emptyList(); - private final List sampleModifiers = emptyList(); public SamplesRunnerJUnitEngine() throws IOException { tempDir = Files.createTempDirectory("samples-junit-engine").toFile(); @@ -58,27 +63,49 @@ public TestDescriptor discover(EngineDiscoveryRequest discoveryRequest, UniqueId discoveryRequest.getSelectorsByType(ClassSelector.class) .stream() .map(ClassSelector::getJavaClass).forEach(javaClass -> { - SamplesRoot samplesRoot = javaClass.getAnnotation(SamplesRoot.class); - if (samplesRoot != null) { - SamplesTestDescriptor samplesTestDescriptor = new SamplesTestDescriptor( - uniqueId.append("className", javaClass.getSimpleName()), - javaClass - ); - - for (Sample sample : SamplesDiscovery.externalSamples(getSamplesRootDir(samplesRoot))) { - samplesTestDescriptor.addChild(new SampleTestDescriptor( - uniqueId.append("sampleId", sample.getId()), - sample - )); - } - - engineDescriptor.addChild(samplesTestDescriptor); - } + filterClass(uniqueId, engineDescriptor, javaClass); }); return engineDescriptor; } + private void filterClass(UniqueId uniqueId, EngineDescriptor engineDescriptor, Class javaClass) { + Samples samplesDef = javaClass.getAnnotation(Samples.class); + if (samplesDef != null) { + UniqueId classUniqueId = uniqueId.append("className", javaClass.getSimpleName()); + SamplesTestDescriptor samplesTestDescriptor = new SamplesTestDescriptor( + classUniqueId, + javaClass + ); + + List samples = samplesDef.samplesType() == Samples.SamplesType.DEFAULT ? + SamplesDiscovery.externalSamples(getSamplesRootDir( + samplesDef.root(), + samplesDef.implicitRootDirSupplier() + )) : + SamplesDiscovery.embeddedSamples(getSamplesRootDir( + samplesDef.root(), + samplesDef.implicitRootDirSupplier() + )); + + List normalizers = instantiateList(samplesDef.outputNormalizers()); + List sampleModifiers = instantiateList(samplesDef.modifiers()); + Function commandExecutorFunction = instantiateObject(samplesDef.commandExecutorFunction()); + + samples.forEach(sample -> + samplesTestDescriptor.addChild(new SampleTestDescriptor( + classUniqueId.append("sampleId", sample.getId()), + sample, + commandExecutorFunction, + normalizers, + sampleModifiers + )) + ); + + engineDescriptor.addChild(samplesTestDescriptor); + } + } + @Override public void execute(ExecutionRequest request) { TestDescriptor engineDescriptor = request.getRootTestDescriptor(); @@ -91,7 +118,7 @@ public void execute(ExecutionRequest request) { SampleTestDescriptor descriptor = (SampleTestDescriptor) testDescriptor; listener.executionStarted(testDescriptor); try { - runSample(descriptor.sample); + runSample(descriptor); listener.executionFinished(testDescriptor, TestExecutionResult.successful()); } catch (Throwable t) { listener.executionFinished(testDescriptor, TestExecutionResult.failed(t)); @@ -103,12 +130,15 @@ public void execute(ExecutionRequest request) { listener.executionFinished(engineDescriptor, TestExecutionResult.successful()); } - protected File getSamplesRootDir(SamplesRoot samplesRoot) { - File samplesRootDir = null; + protected File getSamplesRootDir(String samplesRoot, Class> implicitRootDirSupplier) { + File samplesRootDir; try { if (samplesRoot != null) { - samplesRootDir = new File(samplesRoot.value()); + samplesRootDir = new File(samplesRoot); + } else { + samplesRootDir = supply(implicitRootDirSupplier); } + if (samplesRootDir == null) { throw new IllegalArgumentException("Samples root directory is not declared. Please annotate your test class with @SamplesRoot(\"path/to/samples\")"); } @@ -121,8 +151,37 @@ protected File getSamplesRootDir(SamplesRoot samplesRoot) { return samplesRootDir; } - private void runSample(Sample sample) throws Exception { - final Sample testSpecificSample = initSample(sample); + private static List instantiateList(Class[] classes) { + if (classes == null) { + return emptyList(); + } + + List list = new ArrayList<>(); + for (Class clazz : classes) { + try { + list.add(clazz.getConstructor().newInstance()); + } catch (NoSuchMethodException | InvocationTargetException | InstantiationException | IllegalAccessException e) { + throw new RuntimeException("Could not instantiate " + clazz.getName(), e); + } + } + + return list; + } + + private static T instantiateObject(Class supplierClass) { + try { + return supplierClass.getDeclaredConstructor().newInstance(); + } catch (NoSuchMethodException | InvocationTargetException | InstantiationException | IllegalAccessException e) { + throw new IllegalArgumentException("Unable to supply a value from " + supplierClass + " class", e); + } + } + + private static T supply(Class> supplierClass) { + return instantiateObject(supplierClass).get(); + } + + private void runSample(SampleTestDescriptor descriptor) throws Exception { + final Sample testSpecificSample = initSample(descriptor); File baseWorkingDir = testSpecificSample.getProjectDir(); // Execute and verify each command @@ -139,7 +198,7 @@ private void runSample(Sample sample) throws Exception { continue; } - CommandExecutionResult result = executeSample(getExecutionMetadata(testSpecificSample.getProjectDir()), workingDir, command); + CommandExecutionResult result = executeSample(descriptor, workingDir, command); if (result.getExitCode() != 0 && !command.isExpectFailure()) { throw new AssertionFailedError(String.format( @@ -159,11 +218,15 @@ private void runSample(Sample sample) throws Exception { )); } - verifyOutput(command, result); + verifyOutput(command, result, descriptor.normalizers); } } - private void verifyOutput(final Command command, final CommandExecutionResult executionResult) { + private void verifyOutput( + final Command command, + final CommandExecutionResult executionResult, + final List normalizers + ) { if (command.getExpectedOutput() == null) { return; } @@ -182,28 +245,29 @@ private void verifyOutput(final Command command, final CommandExecutionResult ex } } - private CommandExecutionResult executeSample(ExecutionMetadata executionMetadata, File workingDir, Command command) { - return selectExecutor(executionMetadata, workingDir, command).execute(command, executionMetadata); - } - - protected CommandExecutor selectExecutor(ExecutionMetadata executionMetadata, File workingDir, Command command) { - return new CliCommandExecutor(workingDir); + private CommandExecutionResult executeSample(SampleTestDescriptor descriptor, File workingDir, Command command) { + ExecutionMetadata executionMetadata = getExecutionMetadata(descriptor.sample.getProjectDir()); + CommandExecutorParams commandExecutorParams = new CommandExecutorParams(executionMetadata, workingDir, command); + return descriptor.commandExecutorFunction + .apply(commandExecutorParams) + .execute(command, executionMetadata); } private ExecutionMetadata getExecutionMetadata(final File tempSampleOutputDir) { Map systemProperties = new HashMap<>(); - for (String systemPropertyKey : SamplesRunner.SAFE_SYSTEM_PROPERTIES) { + for (String systemPropertyKey : SAFE_SYSTEM_PROPERTIES) { systemProperties.put(systemPropertyKey, System.getProperty(systemPropertyKey)); } return new ExecutionMetadata(tempSampleOutputDir, systemProperties); } - private Sample initSample(final Sample sampleIn) throws IOException { + private Sample initSample(final SampleTestDescriptor descriptor) throws IOException { + Sample sampleIn = descriptor.sample; File tmpProjectDir = new File(tempDir, sampleIn.getId()); FileUtils.copyDirectory(sampleIn.getProjectDir(), tmpProjectDir); Sample sample = new Sample(sampleIn.getId(), tmpProjectDir, sampleIn.getCommands()); - for (SampleModifier sampleModifier : sampleModifiers) { + for (SampleModifier sampleModifier : descriptor.sampleModifiers) { sample = sampleModifier.modify(sample); } return sample; @@ -212,10 +276,21 @@ private Sample initSample(final Sample sampleIn) throws IOException { private static class SampleTestDescriptor extends AbstractTestDescriptor { private final Sample sample; - - private SampleTestDescriptor(UniqueId uniqueId, Sample sample) { + private final Function commandExecutorFunction; + private final List normalizers; + private final List sampleModifiers; + + private SampleTestDescriptor( + UniqueId uniqueId, + Sample sample, + Function commandExecutorFunction, List normalizers, + List sampleModifiers + ) { super(uniqueId, sample.getId()); this.sample = sample; + this.commandExecutorFunction = commandExecutorFunction; + this.normalizers = normalizers; + this.sampleModifiers = sampleModifiers; } @Override diff --git a/samples-check/src/main/java/org/gradle/exemplar/test/runner/EmbeddedSamplesRunner.java b/samples-check/src/main/java/org/gradle/exemplar/test/runner/EmbeddedSamplesRunner.java deleted file mode 100644 index 7ee342c..0000000 --- a/samples-check/src/main/java/org/gradle/exemplar/test/runner/EmbeddedSamplesRunner.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2016 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.gradle.exemplar.test.runner; - -import org.gradle.exemplar.loader.SamplesDiscovery; -import org.gradle.exemplar.model.Sample; -import org.junit.runners.model.InitializationError; - -import java.io.File; -import java.io.IOException; -import java.util.List; - -public class EmbeddedSamplesRunner extends SamplesRunner { - - /** - * Constructs a new {@code ParentRunner} that will run {@code @TestClass} - * - * @param testClass reference to test class being run - */ - public EmbeddedSamplesRunner(Class testClass) throws InitializationError { - super(testClass); - } - - @Override - protected List getChildren() { - File samplesRootDir = getSamplesRootDir(); - try { - return SamplesDiscovery.embeddedSamples(samplesRootDir); - } catch (IOException e) { - throw new RuntimeException("Could not extract samples from " + samplesRootDir, e); - } - } -} diff --git a/samples-check/src/main/java/org/gradle/exemplar/test/runner/GradleEmbeddedSamplesRunner.java b/samples-check/src/main/java/org/gradle/exemplar/test/runner/GradleEmbeddedSamplesRunner.java deleted file mode 100644 index 59aa067..0000000 --- a/samples-check/src/main/java/org/gradle/exemplar/test/runner/GradleEmbeddedSamplesRunner.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2016 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.gradle.exemplar.test.runner; - -import org.gradle.exemplar.loader.SamplesDiscovery; -import org.gradle.exemplar.model.Sample; -import org.junit.runners.model.InitializationError; - -import java.io.IOException; -import java.util.List; - -/** - * A custom implementation of {@link SamplesRunner} that uses the Gradle Tooling API to execute sample builds. - */ -public class GradleEmbeddedSamplesRunner extends GradleSamplesRunner { - public GradleEmbeddedSamplesRunner(Class testClass) throws InitializationError { - super(testClass); - } - - @Override - protected List getChildren() { - try { - return SamplesDiscovery.embeddedSamples(getSamplesRootDir()); - } catch (IOException e) { - throw new RuntimeException("Could not extract embedded samples", e); - } - } -} diff --git a/samples-check/src/main/java/org/gradle/exemplar/test/runner/SampleModifiers.java b/samples-check/src/main/java/org/gradle/exemplar/test/runner/SampleModifiers.java deleted file mode 100644 index dda196d..0000000 --- a/samples-check/src/main/java/org/gradle/exemplar/test/runner/SampleModifiers.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2016 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.gradle.exemplar.test.runner; - -import java.lang.annotation.*; - -/** - * Specifies execution update classes to invoke before the execution. - */ -@Documented -@Inherited -@Target(ElementType.TYPE) -@Retention(RetentionPolicy.RUNTIME) -public @interface SampleModifiers { - Class[] value(); -} diff --git a/samples-check/src/main/java/org/gradle/exemplar/test/runner/SamplesOutputNormalizers.java b/samples-check/src/main/java/org/gradle/exemplar/test/runner/SamplesOutputNormalizers.java deleted file mode 100644 index 8b6c9b9..0000000 --- a/samples-check/src/main/java/org/gradle/exemplar/test/runner/SamplesOutputNormalizers.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2016 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.gradle.exemplar.test.runner; - -import org.gradle.exemplar.test.normalizer.OutputNormalizer; - -import java.lang.annotation.*; - -/** - * Specifies output normalizer classes to invoke on a given subset of samples. - * - * @see SamplesRunner - */ -@Documented -@Inherited -@Target(ElementType.TYPE) -@Retention(RetentionPolicy.RUNTIME) -public @interface SamplesOutputNormalizers { - Class[] value(); -} diff --git a/samples-check/src/main/java/org/gradle/exemplar/test/runner/SamplesRoot.java b/samples-check/src/main/java/org/gradle/exemplar/test/runner/SamplesRoot.java deleted file mode 100644 index 1a7a500..0000000 --- a/samples-check/src/main/java/org/gradle/exemplar/test/runner/SamplesRoot.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2018 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.gradle.exemplar.test.runner; - -import java.lang.annotation.*; - -/** - * Specifies the directory to find samples, be they external or embedded. - * - * This directory is relative to project where Exemplar is invoked. - * - * For example, given this structure: - * - * 
- * monorepo/
- * ├── build.gradle
- * ├── subprojectBar/
- * │   └── build.gradle
- * │   └── src/
- * │       ├── samples/
- * │       │   └── bar.adoc
- * │       └── test/
- * │           └── java/
- * │               └── DocsSampleTest.java
- * └── subprojectFoo/
- *     └── src/
- *
- * - * ...DocsSampleTest should declare @AsciidocSourcesRoot("src/samples"). - * - * @see SamplesRunner - */ -@Documented -@Inherited -@Target(ElementType.TYPE) -@Retention(RetentionPolicy.RUNTIME) -public @interface SamplesRoot { - String value(); -} diff --git a/samples-check/src/main/java/org/gradle/exemplar/test/runner/SamplesRunner.java b/samples-check/src/main/java/org/gradle/exemplar/test/runner/SamplesRunner.java deleted file mode 100644 index 71dd62c..0000000 --- a/samples-check/src/main/java/org/gradle/exemplar/test/runner/SamplesRunner.java +++ /dev/null @@ -1,234 +0,0 @@ -/* - * Copyright 2016 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.gradle.exemplar.test.runner; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.StringUtils; -import org.gradle.exemplar.executor.CliCommandExecutor; -import org.gradle.exemplar.executor.CommandExecutionResult; -import org.gradle.exemplar.executor.CommandExecutor; -import org.gradle.exemplar.executor.ExecutionMetadata; -import org.gradle.exemplar.loader.SamplesDiscovery; -import org.gradle.exemplar.model.Command; -import org.gradle.exemplar.model.Sample; -import org.gradle.exemplar.test.normalizer.OutputNormalizer; -import org.gradle.exemplar.test.verifier.AnyOrderLineSegmentedOutputVerifier; -import org.gradle.exemplar.test.verifier.StrictOrderLineSegmentedOutputVerifier; -import org.junit.Assert; -import org.junit.Rule; -import org.junit.rules.TemporaryFolder; -import org.junit.runner.Description; -import org.junit.runner.notification.Failure; -import org.junit.runner.notification.RunNotifier; -import org.junit.runners.ParentRunner; -import org.junit.runners.model.InitializationError; - -import javax.annotation.Nullable; -import java.io.File; -import java.io.IOException; -import java.lang.annotation.Annotation; -import java.lang.reflect.InvocationTargetException; -import java.util.*; - -/** - * A JUnit test runner that verifies all samples discovered in the directory specified by the {@link SamplesRoot} annotation. - */ -public class SamplesRunner extends ParentRunner { - // See https://docs.oracle.com/javase/tutorial/essential/environment/sysprop.html - public static final List SAFE_SYSTEM_PROPERTIES = Arrays.asList("file.separator", "java.home", "java.vendor", "java.version", "line.separator", "os.arch", "os.name", "os.version", "path.separator", "user.dir", "user.home", "user.name"); - - private final List normalizers; - - private final List sampleModifiers; - - @Rule - public TemporaryFolder tmpDir = new TemporaryFolder(); - - /** - * Constructs a new {@code ParentRunner} that will run {@code @TestClass} - * - * @param testClass reference to test class being run - */ - public SamplesRunner(Class testClass) throws InitializationError { - super(testClass); - - normalizers = this.instantiateAnnotationClasses(testClass, SamplesOutputNormalizers.class, new Transformer[], SamplesOutputNormalizers>() { - @Override - public Class[] transform(SamplesOutputNormalizers samplesOutputNormalizers) { - return (Class[]) samplesOutputNormalizers.value(); - } - }); - sampleModifiers = instantiateAnnotationClasses(testClass, SampleModifiers.class, new Transformer[], SampleModifiers>() { - @Override - public Class[] transform(SampleModifiers modifiers) { - return (Class[]) modifiers.value(); - } - }); - - try { - tmpDir.create(); - } catch (IOException e) { - throw new RuntimeException("Could not create temporary folder " + tmpDir.getRoot().getAbsolutePath(), e); - } - } - - private List instantiateAnnotationClasses(Class testClass, Class annotationClass, Transformer[], A> transformer) { - A annotation = (A) testClass.getAnnotation(annotationClass); - List ret = new ArrayList<>(); - if (annotation != null) { - for (Class clazz : transformer.transform(annotation)) { - try { - ret.add(clazz.getConstructor().newInstance()); - } catch (NoSuchMethodException | InvocationTargetException | InstantiationException | IllegalAccessException e) { - throw new RuntimeException("Could not instantiate " + clazz.getName(), e); - } - } - } - return ret; - } - - @Override - protected List getChildren() { - return SamplesDiscovery.externalSamples(getSamplesRootDir()); - } - - protected File getSamplesRootDir() { - SamplesRoot samplesRoot = getTestClass().getAnnotation(SamplesRoot.class); - File samplesRootDir; - try { - if (samplesRoot != null) { - samplesRootDir = new File(samplesRoot.value()); - } else { - samplesRootDir = getImplicitSamplesRootDir(); - } - if (samplesRootDir == null) { - throw new IllegalArgumentException("Samples root directory is not declared. Please annotate your test class with @SamplesRoot(\"path/to/samples\")"); - } - if (!samplesRootDir.exists()) { - throw new IllegalArgumentException("Samples root directory " + samplesRootDir.getAbsolutePath() + " does not exist"); - } - } catch (Exception e) { - throw new RuntimeException("Could not initialize SamplesRunner", e); - } - return samplesRootDir; - } - - /** - * Allows a subclass to provide an implicit samples root dir when one is not explicitly defined using {@link SamplesRoot}. - */ - @Nullable - protected File getImplicitSamplesRootDir() { - return null; - } - - @Override - protected Description describeChild(final Sample child) { - return Description.createTestDescription(getTestClass().getJavaClass(), child.getId()); - } - - @Override - protected void runChild(final Sample sample, final RunNotifier notifier) { - Description childDescription = describeChild(sample); - if (isIgnored(sample)) { - notifier.fireTestIgnored(childDescription); - } else { - notifier.fireTestStarted(childDescription); - try { - final Sample testSpecificSample = initSample(sample); - File baseWorkingDir = testSpecificSample.getProjectDir(); - - // Execute and verify each command - for (Command command : testSpecificSample.getCommands()) { - File workingDir = baseWorkingDir; - - if (command.getExecutionSubdirectory() != null) { - workingDir = new File(workingDir, command.getExecutionSubdirectory()); - } - - // This should be some kind of plugable executor rather than hard-coded here - if (command.getExecutable().equals("cd")) { - baseWorkingDir = new File(baseWorkingDir, command.getArgs().get(0)).getCanonicalFile(); - continue; - } - - CommandExecutionResult result = execute(getExecutionMetadata(testSpecificSample.getProjectDir()), workingDir, command); - - if (result.getExitCode() != 0 && !command.isExpectFailure()) { - Assert.fail(String.format("Expected sample invocation to succeed but it failed.%nCommand was: '%s %s'%nWorking directory: '%s'%n[BEGIN OUTPUT]%n%s%n[END OUTPUT]%n", command.getExecutable(), StringUtils.join(command.getArgs(), " "), workingDir.getAbsolutePath(), result.getOutput())); - } else if (result.getExitCode() == 0 && command.isExpectFailure()) { - Assert.fail(String.format("Expected sample invocation to fail but it succeeded.%nCommand was: '%s %s'%nWorking directory: '%s'%n[BEGIN OUTPUT]%n%s%n[END OUTPUT]%n", command.getExecutable(), StringUtils.join(command.getArgs(), " "), workingDir.getAbsolutePath(), result.getOutput())); - } - - verifyOutput(command, result); - } - } catch (Throwable t) { - notifier.fireTestFailure(new Failure(childDescription, t)); - } finally { - notifier.fireTestFinished(childDescription); - } - } - } - - private Sample initSample(final Sample sampleIn) throws IOException { - File tmpProjectDir = new File(tmpDir.getRoot(), sampleIn.getId()); - tmpProjectDir.mkdirs(); - FileUtils.copyDirectory(sampleIn.getProjectDir(), tmpProjectDir); - Sample sample = new Sample(sampleIn.getId(), tmpProjectDir, sampleIn.getCommands()); - for (SampleModifier sampleModifier : sampleModifiers) { - sample = sampleModifier.modify(sample); - } - return sample; - } - - private void verifyOutput(final Command command, final CommandExecutionResult executionResult) { - if (command.getExpectedOutput() == null) { - return; - } - - String expectedOutput = command.getExpectedOutput(); - String actualOutput = executionResult.getOutput(); - - for (OutputNormalizer normalizer : normalizers) { - actualOutput = normalizer.normalize(actualOutput, executionResult.getExecutionMetadata()); - } - - if (command.isAllowDisorderedOutput()) { - new AnyOrderLineSegmentedOutputVerifier().verify(expectedOutput, actualOutput, command.isAllowAdditionalOutput()); - } else { - new StrictOrderLineSegmentedOutputVerifier().verify(expectedOutput, actualOutput, command.isAllowAdditionalOutput()); - } - } - - private CommandExecutionResult execute(ExecutionMetadata executionMetadata, File workingDir, Command command) { - return selectExecutor(executionMetadata, workingDir, command).execute(command, executionMetadata); - } - - /** - * Allows a subclass to provide a custom {@link CommandExecutor}. - */ - protected CommandExecutor selectExecutor(ExecutionMetadata executionMetadata, File workingDir, Command command) { - return new CliCommandExecutor(workingDir); - } - - private ExecutionMetadata getExecutionMetadata(final File tempSampleOutputDir) { - Map systemProperties = new HashMap<>(); - for (String systemPropertyKey : SAFE_SYSTEM_PROPERTIES) { - systemProperties.put(systemPropertyKey, System.getProperty(systemPropertyKey)); - } - - return new ExecutionMetadata(tempSampleOutputDir, systemProperties); - } -} diff --git a/samples-check/src/main/java/org/gradle/exemplar/test/runner/Transformer.java b/samples-check/src/main/java/org/gradle/exemplar/test/runner/Transformer.java deleted file mode 100644 index cc8959a..0000000 --- a/samples-check/src/main/java/org/gradle/exemplar/test/runner/Transformer.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2016 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.gradle.exemplar.test.runner; - -/** - *

A {@code Transformer} transforms objects of type.

- * - *

Implementations are free to return new objects or mutate the incoming value.

- * - * @param The type the value is transformed to. - * @param The type of the value to be transformed. - */ -public interface Transformer { - /** - * Transforms the given object, and returns the transformed value. - * - * @param in The object to update. - * @return The transformed object. - */ - OUT transform(IN in); -} diff --git a/samples-check/src/test/groovy/org/gradle/exemplar/test/runner/SamplesRunnerIntegrationTest.groovy b/samples-check/src/test/groovy/org/gradle/exemplar/test/runner/SamplesRunnerIntegrationTest.groovy index 79cc956..ba29420 100644 --- a/samples-check/src/test/groovy/org/gradle/exemplar/test/runner/SamplesRunnerIntegrationTest.groovy +++ b/samples-check/src/test/groovy/org/gradle/exemplar/test/runner/SamplesRunnerIntegrationTest.groovy @@ -1,8 +1,8 @@ package org.gradle.exemplar.test.runner +import org.gradle.exemplar.test.Samples import org.junit.experimental.categories.Category import org.junit.runner.Request -import org.junit.runner.RunWith import spock.lang.Specification class SamplesRunnerIntegrationTest extends Specification { @@ -23,8 +23,7 @@ class SamplesRunnerIntegrationTest extends Specification { notifier.failures.empty } - @RunWith(SamplesRunner.class) - @SamplesRoot("src/test/samples/cli") + @Samples(root = "src/test/samples/cli") @Category(CoveredByTests) static class HappyDaySamples {} @@ -42,8 +41,7 @@ class SamplesRunnerIntegrationTest extends Specification { notifier.failures.empty } - @RunWith(SamplesRunner.class) - @SamplesRoot("src/test/samples/cli-with-working-directory") + @Samples(root = "src/test/samples/cli-with-working-directory") @Category(CoveredByTests) static class HappyDayWithWorkingDirectorySamples {} @@ -61,8 +59,7 @@ class SamplesRunnerIntegrationTest extends Specification { notifier.failures.empty } - @RunWith(SamplesRunner.class) - @SamplesRoot("src/test/samples/cli-with-working-directory-and-change-directory") + @Samples(root = "src/test/samples/cli-with-working-directory-and-change-directory") @Category(CoveredByTests) static class HappyDayWithWorkingDirectoryAndChangeDirectoryCommandSamples {} } diff --git a/samples-check/src/test/groovy/org/gradle/exemplar/test/runner/SamplesRunnerSadDayIntegrationTest.groovy b/samples-check/src/test/groovy/org/gradle/exemplar/test/runner/SamplesRunnerSadDayIntegrationTest.groovy index a379a7b..e7501d1 100644 --- a/samples-check/src/test/groovy/org/gradle/exemplar/test/runner/SamplesRunnerSadDayIntegrationTest.groovy +++ b/samples-check/src/test/groovy/org/gradle/exemplar/test/runner/SamplesRunnerSadDayIntegrationTest.groovy @@ -1,5 +1,6 @@ package org.gradle.exemplar.test.runner +import org.gradle.exemplar.test.Samples import org.junit.Rule import org.junit.experimental.categories.Category import org.junit.rules.TemporaryFolder @@ -59,13 +60,11 @@ class SamplesRunnerSadDayIntegrationTest extends Specification { """.stripIndent().trim() } - @SamplesRoot("src/test/resources/broken/command") - @RunWith(SamplesRunner) + @Samples(root = "src/test/resources/broken/command") @Category(CoveredByTests) static class HasBadCommand {} - @SamplesRoot("src/test/resources/broken/output") - @RunWith(SamplesRunner) + @Samples(root = "src/test/resources/broken/output") @Category(CoveredByTests) static class HasBadOutput {} } diff --git a/samples-check/src/test/java/org/gradle/exemplar/test/runner/CliSamplesRunnerIntegrationTest.java b/samples-check/src/test/java/org/gradle/exemplar/test/runner/CliSamplesRunnerIntegrationTest.java index 7e5d200..61b4cfd 100644 --- a/samples-check/src/test/java/org/gradle/exemplar/test/runner/CliSamplesRunnerIntegrationTest.java +++ b/samples-check/src/test/java/org/gradle/exemplar/test/runner/CliSamplesRunnerIntegrationTest.java @@ -16,10 +16,9 @@ package org.gradle.exemplar.test.runner; // tag::source[] -import org.junit.runner.RunWith; +import org.gradle.exemplar.test.Samples; -@RunWith(SamplesRunner.class) -@SamplesRoot("src/test/samples/cli") +@Samples(root = "src/test/samples/cli") public class CliSamplesRunnerIntegrationTest { } // end::source[] diff --git a/samples-check/src/test/java/org/gradle/exemplar/test/runner/EmbeddedSamplesRunnerIntegrationTest.java b/samples-check/src/test/java/org/gradle/exemplar/test/runner/EmbeddedSamplesRunnerIntegrationTest.java index a20bbf0..1de77c3 100644 --- a/samples-check/src/test/java/org/gradle/exemplar/test/runner/EmbeddedSamplesRunnerIntegrationTest.java +++ b/samples-check/src/test/java/org/gradle/exemplar/test/runner/EmbeddedSamplesRunnerIntegrationTest.java @@ -15,9 +15,14 @@ */ package org.gradle.exemplar.test.runner; -import org.junit.runner.RunWith; +import org.gradle.exemplar.test.Samples; +import org.gradle.exemplar.test.engine.GradleSamplesExtension; -@RunWith(GradleEmbeddedSamplesRunner.class) -@SamplesRoot("src/test/docs") +@Samples( + root = "src/test/docs", + samplesType = Samples.SamplesType.EMBEDDED, + implicitRootDirSupplier = GradleSamplesExtension.ImplicitSamplesRootDirSupplier.class, + commandExecutorFunction = GradleSamplesExtension.GradleCommandExecutorFunction.class +) public class EmbeddedSamplesRunnerIntegrationTest { } diff --git a/samples-check/src/test/java/org/gradle/exemplar/test/runner/GradleSamplesRunnerIntegrationTest.java b/samples-check/src/test/java/org/gradle/exemplar/test/runner/GradleSamplesRunnerIntegrationTest.java index 08a075a..e7cca9b 100644 --- a/samples-check/src/test/java/org/gradle/exemplar/test/runner/GradleSamplesRunnerIntegrationTest.java +++ b/samples-check/src/test/java/org/gradle/exemplar/test/runner/GradleSamplesRunnerIntegrationTest.java @@ -15,15 +15,20 @@ */ package org.gradle.exemplar.test.runner; +import org.gradle.exemplar.test.Samples; +import org.gradle.exemplar.test.engine.GradleSamplesExtension; import org.gradle.exemplar.test.normalizer.FileSeparatorOutputNormalizer; import org.gradle.exemplar.test.normalizer.GradleOutputNormalizer; import org.gradle.exemplar.test.normalizer.JavaObjectSerializationOutputNormalizer; import org.junit.runner.RunWith; -@RunWith(GradleSamplesRunner.class) -@SamplesRoot("src/test/samples/gradle") +@Samples( + root = "src/test/samples/gradle", + implicitRootDirSupplier = GradleSamplesExtension.ImplicitSamplesRootDirSupplier.class, + commandExecutorFunction = GradleSamplesExtension.GradleCommandExecutorFunction.class, // tag::sample-output-normalizers[] -@SamplesOutputNormalizers({JavaObjectSerializationOutputNormalizer.class, FileSeparatorOutputNormalizer.class, GradleOutputNormalizer.class}) + outputNormalizers = {JavaObjectSerializationOutputNormalizer.class, FileSeparatorOutputNormalizer.class, GradleOutputNormalizer.class} // end::sample-output-normalizers[] +) public class GradleSamplesRunnerIntegrationTest { } diff --git a/samples-check/src/test/java/org/gradle/exemplar/test/runner/SampleModifierIntegrationTest.java b/samples-check/src/test/java/org/gradle/exemplar/test/runner/SampleModifierIntegrationTest.java index 77c962d..ace86f1 100644 --- a/samples-check/src/test/java/org/gradle/exemplar/test/runner/SampleModifierIntegrationTest.java +++ b/samples-check/src/test/java/org/gradle/exemplar/test/runner/SampleModifierIntegrationTest.java @@ -15,11 +15,16 @@ */ package org.gradle.exemplar.test.runner; +import org.gradle.exemplar.test.Samples; +import org.gradle.exemplar.test.engine.GradleSamplesExtension; import org.gradle.exemplar.test.runner.modifiers.ExtraCommandArgumentsSampleModifier; import org.junit.runner.RunWith; -@RunWith(GradleSamplesRunner.class) -@SamplesRoot("src/test/samples/customization") -@SampleModifiers({ExtraCommandArgumentsSampleModifier.class}) +@Samples( + root = "src/test/samples/customization", + implicitRootDirSupplier = GradleSamplesExtension.ImplicitSamplesRootDirSupplier.class, + commandExecutorFunction = GradleSamplesExtension.GradleCommandExecutorFunction.class, + modifiers = {ExtraCommandArgumentsSampleModifier.class} +) public class SampleModifierIntegrationTest { } diff --git a/samples-check/src/test/java/org/gradle/exemplar/test/runner/modifiers/ExtraCommandArgumentsSampleModifier.java b/samples-check/src/test/java/org/gradle/exemplar/test/runner/modifiers/ExtraCommandArgumentsSampleModifier.java index 1ec2447..189ec89 100644 --- a/samples-check/src/test/java/org/gradle/exemplar/test/runner/modifiers/ExtraCommandArgumentsSampleModifier.java +++ b/samples-check/src/test/java/org/gradle/exemplar/test/runner/modifiers/ExtraCommandArgumentsSampleModifier.java @@ -2,7 +2,7 @@ import org.gradle.exemplar.model.Command; import org.gradle.exemplar.model.Sample; -import org.gradle.exemplar.test.runner.SampleModifier; +import org.gradle.exemplar.test.SampleModifier; import java.util.ArrayList; import java.util.List; diff --git a/samples-discovery/src/main/java/org/gradle/exemplar/loader/SamplesDiscovery.java b/samples-discovery/src/main/java/org/gradle/exemplar/loader/SamplesDiscovery.java index b131662..f6b6318 100644 --- a/samples-discovery/src/main/java/org/gradle/exemplar/loader/SamplesDiscovery.java +++ b/samples-discovery/src/main/java/org/gradle/exemplar/loader/SamplesDiscovery.java @@ -23,6 +23,7 @@ import java.io.File; import java.io.IOException; +import java.io.UncheckedIOException; import java.util.*; public class SamplesDiscovery { @@ -49,8 +50,12 @@ public static List filteredExternalSamples(File rootSamplesDir, String[] return samples; } - public static List embeddedSamples(File asciidocSrcDir) throws IOException { - return filteredEmbeddedSamples(asciidocSrcDir, new String[]{"adoc", "asciidoc"}, true); + public static List embeddedSamples(File asciidocSrcDir) { + try { + return filteredEmbeddedSamples(asciidocSrcDir, new String[]{"adoc", "asciidoc"}, true); + } catch (IOException e) { + throw new UncheckedIOException(e); + } } public static List filteredEmbeddedSamples(File rootSamplesDir, String[] fileExtensions, boolean recursive) throws IOException { From 0f85578e7e7db2b97422a044241b55a825b6ad47 Mon Sep 17 00:00:00 2001 From: Przemek Bielicki Date: Tue, 2 Nov 2021 09:30:35 +0100 Subject: [PATCH 6/6] Polish --- docs/build.gradle.kts | 2 - .../java/org/gradle/exemplar/ReadmeTest.java | 4 +- gradle/libs.versions.toml | 3 ++ samples-check/build.gradle.kts | 3 +- .../test/engine/SamplesRunnerJUnitEngine.java | 47 +++++++++++++++++-- .../SamplesRunnerIntegrationTest.groovy | 2 + .../SamplesRunnerSadDayIntegrationTest.groovy | 3 +- 7 files changed, 55 insertions(+), 9 deletions(-) diff --git a/docs/build.gradle.kts b/docs/build.gradle.kts index 27b0ab3..330d21d 100644 --- a/docs/build.gradle.kts +++ b/docs/build.gradle.kts @@ -4,10 +4,8 @@ plugins { dependencies { implementation(project(":samples-check")) - implementation(Libraries.JUNIT) } tasks.test { - useJUnit() inputs.file("README.adoc").withPathSensitivity(PathSensitivity.RELATIVE) } diff --git a/docs/src/test/java/org/gradle/exemplar/ReadmeTest.java b/docs/src/test/java/org/gradle/exemplar/ReadmeTest.java index 98a284c..8b6c6c6 100644 --- a/docs/src/test/java/org/gradle/exemplar/ReadmeTest.java +++ b/docs/src/test/java/org/gradle/exemplar/ReadmeTest.java @@ -18,8 +18,10 @@ import org.gradle.exemplar.test.Samples; import org.gradle.exemplar.test.engine.GradleSamplesExtension; +import static org.gradle.exemplar.test.Samples.SamplesType.EMBEDDED; + @Samples(root = ".", - samplesType = Samples.SamplesType.EMBEDDED, + samplesType = EMBEDDED, implicitRootDirSupplier = GradleSamplesExtension.ImplicitSamplesRootDirSupplier.class, commandExecutorFunction = GradleSamplesExtension.GradleCommandExecutorFunction.class ) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index a745d73..3348a8c 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -1,5 +1,6 @@ [versions] spock = "2.0-groovy-3.0" +junit-platform = "1.8.1" [libraries] asciidoctorj = "org.asciidoctor:asciidoctorj:1.5.8.1" @@ -9,6 +10,8 @@ commons-lang3 = "org.apache.commons:commons-lang3:3.12.0" groovy = "org.codehaus.groovy:groovy:3.0.8" junit = "junit:junit:4.13.2" junit-vintage-engine = "org.junit.vintage:junit-vintage-engine:5.8.1" +junit-platform-engine = { module="org.junit.platform:junit-platform-engine", version.ref="junit-platform" } +junit-platform-launcher = { module="org.junit.platform:junit-platform-launcher", version.ref="junit-platform" } jsr305 = "com.google.code.findbugs:jsr305:3.0.2" objenesis = "org.objenesis:objenesis:3.2" spock-core = { module="org.spockframework:spock-core", version.ref="spock" } diff --git a/samples-check/build.gradle.kts b/samples-check/build.gradle.kts index be2cb70..1a10c35 100644 --- a/samples-check/build.gradle.kts +++ b/samples-check/build.gradle.kts @@ -9,7 +9,8 @@ dependencies { compileOnly(libs.jsr305) implementation(libs.commons.io) implementation(libs.commons.lang3) - implementation("org.junit.platform:junit-platform-engine:1.8.1") + implementation(libs.junit.platform.engine) + implementation(libs.junit.platform.launcher) implementation(gradleTestKit()) testImplementation(libs.cglib) testImplementation(libs.groovy) diff --git a/samples-check/src/main/java/org/gradle/exemplar/test/engine/SamplesRunnerJUnitEngine.java b/samples-check/src/main/java/org/gradle/exemplar/test/engine/SamplesRunnerJUnitEngine.java index 825a183..cd28c7a 100644 --- a/samples-check/src/main/java/org/gradle/exemplar/test/engine/SamplesRunnerJUnitEngine.java +++ b/samples-check/src/main/java/org/gradle/exemplar/test/engine/SamplesRunnerJUnitEngine.java @@ -2,29 +2,32 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; -import org.gradle.exemplar.executor.CliCommandExecutor; import org.gradle.exemplar.executor.CommandExecutionResult; import org.gradle.exemplar.executor.CommandExecutor; import org.gradle.exemplar.executor.ExecutionMetadata; import org.gradle.exemplar.loader.SamplesDiscovery; import org.gradle.exemplar.model.Command; import org.gradle.exemplar.model.Sample; -import org.gradle.exemplar.test.normalizer.OutputNormalizer; import org.gradle.exemplar.test.SampleModifier; import org.gradle.exemplar.test.Samples; +import org.gradle.exemplar.test.normalizer.OutputNormalizer; import org.gradle.exemplar.test.verifier.AnyOrderLineSegmentedOutputVerifier; import org.gradle.exemplar.test.verifier.StrictOrderLineSegmentedOutputVerifier; +import org.junit.experimental.categories.Category; import org.junit.platform.engine.EngineDiscoveryRequest; import org.junit.platform.engine.EngineExecutionListener; import org.junit.platform.engine.ExecutionRequest; import org.junit.platform.engine.TestDescriptor; import org.junit.platform.engine.TestEngine; import org.junit.platform.engine.TestExecutionResult; +import org.junit.platform.engine.TestTag; import org.junit.platform.engine.UniqueId; import org.junit.platform.engine.discovery.ClassSelector; import org.junit.platform.engine.support.descriptor.AbstractTestDescriptor; import org.junit.platform.engine.support.descriptor.ClassSource; import org.junit.platform.engine.support.descriptor.EngineDescriptor; +import org.junit.platform.launcher.LauncherDiscoveryRequest; +import org.junit.platform.launcher.PostDiscoveryFilter; import org.opentest4j.AssertionFailedError; import java.io.File; @@ -34,12 +37,16 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Function; import java.util.function.Supplier; import static java.util.Collections.emptyList; +import static java.util.Collections.emptySet; +import static java.util.Collections.unmodifiableSet; public class SamplesRunnerJUnitEngine implements TestEngine { @@ -60,16 +67,26 @@ public String getId() { @Override public TestDescriptor discover(EngineDiscoveryRequest discoveryRequest, UniqueId uniqueId) { EngineDescriptor engineDescriptor = new EngineDescriptor(uniqueId, "Samples Runner JUnit Engine"); + List postDiscoveryFilters = new ArrayList<>(); + if (discoveryRequest instanceof LauncherDiscoveryRequest) { + postDiscoveryFilters.addAll(((LauncherDiscoveryRequest) discoveryRequest).getPostDiscoveryFilters()); + } + discoveryRequest.getSelectorsByType(ClassSelector.class) .stream() .map(ClassSelector::getJavaClass).forEach(javaClass -> { - filterClass(uniqueId, engineDescriptor, javaClass); + filterClass(uniqueId, postDiscoveryFilters, engineDescriptor, javaClass); }); return engineDescriptor; } - private void filterClass(UniqueId uniqueId, EngineDescriptor engineDescriptor, Class javaClass) { + private void filterClass( + UniqueId uniqueId, + List postDiscoveryFilters, + EngineDescriptor engineDescriptor, + Class javaClass + ) { Samples samplesDef = javaClass.getAnnotation(Samples.class); if (samplesDef != null) { UniqueId classUniqueId = uniqueId.append("className", javaClass.getSimpleName()); @@ -78,6 +95,10 @@ private void filterClass(UniqueId uniqueId, EngineDescriptor engineDescriptor, C javaClass ); + if (postDiscoveryFilters.stream().anyMatch(filter -> filter.apply(samplesTestDescriptor).excluded())) { + return; + } + List samples = samplesDef.samplesType() == Samples.SamplesType.DEFAULT ? SamplesDiscovery.externalSamples(getSamplesRootDir( samplesDef.root(), @@ -301,8 +322,26 @@ public Type getType() { private static class SamplesTestDescriptor extends AbstractTestDescriptor { + private final Set tags; + private SamplesTestDescriptor(UniqueId uniqueId, Class testClass) { super(uniqueId, testClass.getSimpleName(), ClassSource.from(testClass)); + Category category = testClass.getAnnotation(Category.class); + if (category != null) { + Set tmpTags = new HashSet<>(); + for (Class tag : category.value()) { + tmpTags.add(TestTag.create(tag.getName())); + } + + tags = unmodifiableSet(tmpTags); + } else { + tags = emptySet(); + } + } + + @Override + public Set getTags() { + return tags; } @Override diff --git a/samples-check/src/test/groovy/org/gradle/exemplar/test/runner/SamplesRunnerIntegrationTest.groovy b/samples-check/src/test/groovy/org/gradle/exemplar/test/runner/SamplesRunnerIntegrationTest.groovy index ba29420..574b12c 100644 --- a/samples-check/src/test/groovy/org/gradle/exemplar/test/runner/SamplesRunnerIntegrationTest.groovy +++ b/samples-check/src/test/groovy/org/gradle/exemplar/test/runner/SamplesRunnerIntegrationTest.groovy @@ -3,8 +3,10 @@ package org.gradle.exemplar.test.runner import org.gradle.exemplar.test.Samples import org.junit.experimental.categories.Category import org.junit.runner.Request +import spock.lang.Ignore import spock.lang.Specification +@Ignore("Find a way to test the JUnit5 engine instead of JUnit4 runner") class SamplesRunnerIntegrationTest extends Specification { def "runs samples-check CLI samples"() { def notifier = new CollectingNotifier() diff --git a/samples-check/src/test/groovy/org/gradle/exemplar/test/runner/SamplesRunnerSadDayIntegrationTest.groovy b/samples-check/src/test/groovy/org/gradle/exemplar/test/runner/SamplesRunnerSadDayIntegrationTest.groovy index e7501d1..c9782fb 100644 --- a/samples-check/src/test/groovy/org/gradle/exemplar/test/runner/SamplesRunnerSadDayIntegrationTest.groovy +++ b/samples-check/src/test/groovy/org/gradle/exemplar/test/runner/SamplesRunnerSadDayIntegrationTest.groovy @@ -5,9 +5,10 @@ import org.junit.Rule import org.junit.experimental.categories.Category import org.junit.rules.TemporaryFolder import org.junit.runner.Request -import org.junit.runner.RunWith +import spock.lang.Ignore import spock.lang.Specification +@Ignore("Find a way to test the JUnit5 engine instead of JUnit4 runner") class SamplesRunnerSadDayIntegrationTest extends Specification { @Rule TemporaryFolder tmpDir = new TemporaryFolder()