From 18f59dee65882152ae1918d5692710d0e9e03709 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Don=C3=A1t=20Csik=C3=B3s?= Date: Mon, 19 Jan 2026 15:57:08 +0100 Subject: [PATCH 1/6] Use Gradle 9.4 snapshot --- gradle/wrapper/gradle-wrapper.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 23449a2..21c40f7 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-9.2.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions-snapshots/gradle-9.4.0-20251226063043+0000-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME From 9f2c1c7545d41bddde7a2f980478d8574fee0e92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Don=C3=A1t=20Csik=C3=B3s?= Date: Fri, 26 Dec 2025 23:04:57 +0100 Subject: [PATCH 2/6] Implement Exemplar junit test engine --- gradle/libs.versions.toml | 5 +- .../exemplar/loader/SamplesDiscovery.java | 2 +- .../org/gradle/exemplar/model/Sample.java | 15 ++ samples-junit-engine/build.gradle.kts | 39 +++ .../java/test/TestOutputNormalizer.java | 12 + .../java/test/TestSampleModifier.java | 12 + .../exemplar/ExemplarTestDescriptor.java | 50 ++++ .../gradle/exemplar/ExemplarTestEngine.java | 232 ++++++++++++++++++ .../gradle/exemplar/ExemplarTestResolver.java | 33 +++ .../org.junit.platform.engine.TestEngine | 1 + .../my-test-project/output.sample.conf | 3 + .../my-test-project/sample.out | 1 + settings.gradle.kts | 1 + 13 files changed, 404 insertions(+), 2 deletions(-) create mode 100644 samples-junit-engine/build.gradle.kts create mode 100644 samples-junit-engine/src/integTest/java/test/TestOutputNormalizer.java create mode 100644 samples-junit-engine/src/integTest/java/test/TestSampleModifier.java create mode 100644 samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestDescriptor.java create mode 100644 samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestEngine.java create mode 100644 samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestResolver.java create mode 100644 samples-junit-engine/src/main/resources/META-INF/services/org.junit.platform.engine.TestEngine create mode 100644 samples-junit-engine/src/test-definitions/my-test-project/output.sample.conf create mode 100644 samples-junit-engine/src/test-definitions/my-test-project/sample.out diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 42a87b3..a2875a3 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -11,6 +11,9 @@ groovy = "org.apache.groovy:groovy:4.0.29" junit4 = "junit:junit:4.13.2" junit-vintage = { module = "org.junit.vintage:junit-vintage-engine", version.ref = "junit-vintage" } junit-launcher = { module = "org.junit.platform:junit-platform-launcher", version.ref = "junit-launcher" } +junit-engine = { module = "org.junit.platform:junit-platform-engine", version.ref = "junit-launcher" } +junit-jupiter-api = { module = "org.junit.jupiter:junit-jupiter-api", version.ref = "junit-launcher" } +junit-jupiter-engine = { module = "org.junit.jupiter:junit-jupiter-engine", version.ref = "junit-launcher" } jsr305 = "com.google.code.findbugs:jsr305:3.0.2" objenesis = "org.objenesis:objenesis:3.4" spock-core = { module="org.spockframework:spock-core", version.ref="spock" } @@ -18,4 +21,4 @@ spock-junit4 = { module="org.spockframework:spock-junit4", version.ref="spock" } typesafe-config = "com.typesafe:config:1.4.3" [bundles] -spock = ["spock-core", "spock-junit4"] \ No newline at end of file +spock = ["spock-core", "spock-junit4"] diff --git a/samples-discovery/src/main/java/org/gradle/exemplar/loader/SamplesDiscovery.java b/samples-discovery/src/main/java/org/gradle/exemplar/loader/SamplesDiscovery.java index 2ee1a4e..0ece36f 100644 --- a/samples-discovery/src/main/java/org/gradle/exemplar/loader/SamplesDiscovery.java +++ b/samples-discovery/src/main/java/org/gradle/exemplar/loader/SamplesDiscovery.java @@ -43,7 +43,7 @@ public static List filteredExternalSamples(File rootSamplesDir, String[] // FIXME: Currently the temp directory used when running samples-check has a different name. // This causes Gradle project names to differ when one is not explicitly set in settings.gradle. This should be preserved. final File sampleProjectDir = sampleConfigFile.getParentFile(); - samples.add(new Sample(id, sampleProjectDir, commands)); + samples.add(new Sample(id, sampleProjectDir, commands, sampleConfigFile)); } catch (Exception e) { samples.add(new InvalidSample(id, e)); } diff --git a/samples-discovery/src/main/java/org/gradle/exemplar/model/Sample.java b/samples-discovery/src/main/java/org/gradle/exemplar/model/Sample.java index 06c7f43..e1b1f8f 100644 --- a/samples-discovery/src/main/java/org/gradle/exemplar/model/Sample.java +++ b/samples-discovery/src/main/java/org/gradle/exemplar/model/Sample.java @@ -15,6 +15,7 @@ */ package org.gradle.exemplar.model; +import javax.annotation.Nullable; import java.io.File; import java.util.List; @@ -22,11 +23,20 @@ public class Sample { private final String id; private final File projectDir; private final List commands; + private final File configFile; public Sample(String id, File projectDir, List commands) { this.id = id; this.projectDir = projectDir; this.commands = commands; + this.configFile = null; + } + + public Sample(String id, File projectDir, List commands, File configFile) { + this.id = id; + this.projectDir = projectDir; + this.commands = commands; + this.configFile = configFile; } public String getId() { @@ -40,4 +50,9 @@ public File getProjectDir() { public List getCommands() { return commands; } + + @Nullable + public File getConfigFile() { + return configFile; + } } diff --git a/samples-junit-engine/build.gradle.kts b/samples-junit-engine/build.gradle.kts new file mode 100644 index 0000000..9e45fc3 --- /dev/null +++ b/samples-junit-engine/build.gradle.kts @@ -0,0 +1,39 @@ +plugins { + id("exemplar.java-conventions") + id("jvm-test-suite") +} + +dependencies { + implementation(project(":samples-check")) + implementation(project(":samples-discovery")) + implementation(libs.junit.engine) + implementation(libs.commons.io) + implementation(libs.commons.lang3) +} + +testing { + suites { + register("integTest", JvmTestSuite::class) { + useJUnitJupiter() + dependencies { + implementation(project()) + implementation(project(":samples-check")) + } + + targets { + all { + testTask.configure { + mustRunAfter(tasks.named("jar")) + testDefinitionDirs.from("src/test-definitions") + systemProperty("exemplar.sample.modifiers", "test.TestSampleModifier") + systemProperty("exemplar.output.normalizers", "test.TestOutputNormalizer") + } + } + } + } + } +} + +tasks.named("test", Test::class) { + useJUnitPlatform() +} diff --git a/samples-junit-engine/src/integTest/java/test/TestOutputNormalizer.java b/samples-junit-engine/src/integTest/java/test/TestOutputNormalizer.java new file mode 100644 index 0000000..7fd9e56 --- /dev/null +++ b/samples-junit-engine/src/integTest/java/test/TestOutputNormalizer.java @@ -0,0 +1,12 @@ +package test; + +import org.gradle.exemplar.executor.ExecutionMetadata; +import org.gradle.exemplar.test.normalizer.OutputNormalizer; + +@SuppressWarnings("unused") // used by system property +public class TestOutputNormalizer implements OutputNormalizer { + @Override + public String normalize(String commandOutput, ExecutionMetadata executionMetadata) { + return commandOutput; + } +} diff --git a/samples-junit-engine/src/integTest/java/test/TestSampleModifier.java b/samples-junit-engine/src/integTest/java/test/TestSampleModifier.java new file mode 100644 index 0000000..b3c3c06 --- /dev/null +++ b/samples-junit-engine/src/integTest/java/test/TestSampleModifier.java @@ -0,0 +1,12 @@ +package test; + +import org.gradle.exemplar.model.Sample; +import org.gradle.exemplar.test.runner.SampleModifier; + +@SuppressWarnings("unused") // used by system property +public class TestSampleModifier implements SampleModifier { + @Override + public Sample modify(Sample sampleIn) { + return new Sample(sampleIn.getId(), sampleIn.getProjectDir(), sampleIn.getCommands()); + } +} diff --git a/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestDescriptor.java b/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestDescriptor.java new file mode 100644 index 0000000..28f2be1 --- /dev/null +++ b/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestDescriptor.java @@ -0,0 +1,50 @@ +package org.gradle.exemplar; + +import org.gradle.exemplar.model.Sample; +import org.junit.platform.engine.TestSource; +import org.junit.platform.engine.UniqueId; +import org.junit.platform.engine.support.descriptor.AbstractTestDescriptor; +import org.junit.platform.engine.support.descriptor.DirectorySource; + +import java.io.File; +import java.util.Optional; + +public final class ExemplarTestDescriptor extends AbstractTestDescriptor { + private final File file; + private final String name; + private final Sample sample; + + public ExemplarTestDescriptor(UniqueId parentId, File file, String name, Sample sample) { + super( + parentId.append("testDefinitionFile", fileNameWithoutExtension(file)).append("testDefinition", name), + file.getParentFile().getName() + " - " + fileNameWithoutExtension(file), + DirectorySource.from(sample.getProjectDir()) + ); + this.file = file; + this.name = name; + this.sample = sample; + } + + private static String fileNameWithoutExtension(File file) { + String name = file.getName(); + int i = name.indexOf(".sample.conf"); + if (i > 0) { + return name.substring(0, i); + } + return name; + } + + @Override + public Type getType() { + return Type.TEST; + } + + public Sample getSample() { + return sample; + } + + @Override + public String toString() { + return "Sample[file=" + file.getName() + ", name=" + name + "]"; + } +} diff --git a/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestEngine.java b/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestEngine.java new file mode 100644 index 0000000..f1465d7 --- /dev/null +++ b/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestEngine.java @@ -0,0 +1,232 @@ +package org.gradle.exemplar; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.gradle.exemplar.executor.CliCommandExecutor; +import org.gradle.exemplar.executor.CommandExecutionResult; +import org.gradle.exemplar.executor.CommandExecutor; +import org.gradle.exemplar.executor.ExecutionMetadata; +import org.gradle.exemplar.model.Command; +import org.gradle.exemplar.model.InvalidSample; +import org.gradle.exemplar.model.Sample; +import org.gradle.exemplar.test.normalizer.OutputNormalizer; +import org.gradle.exemplar.test.runner.SampleModifier; +import org.gradle.exemplar.test.verifier.AnyOrderLineSegmentedOutputVerifier; +import org.gradle.exemplar.test.verifier.StrictOrderLineSegmentedOutputVerifier; +import org.junit.platform.commons.logging.Logger; +import org.junit.platform.commons.logging.LoggerFactory; +import org.junit.platform.engine.DiscoverySelector; +import org.junit.platform.engine.EngineDiscoveryRequest; +import org.junit.platform.engine.EngineExecutionListener; +import org.junit.platform.engine.ExecutionRequest; +import org.junit.platform.engine.TestDescriptor; +import org.junit.platform.engine.TestEngine; +import org.junit.platform.engine.TestExecutionResult; +import org.junit.platform.engine.UniqueId; +import org.junit.platform.engine.support.descriptor.EngineDescriptor; +import org.junit.platform.engine.support.discovery.EngineDiscoveryRequestResolver; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.*; +import java.util.stream.Collectors; + +public class ExemplarTestEngine implements TestEngine { + public static final Logger LOGGER = LoggerFactory.getLogger(ExemplarTestEngine.class); + + public static final List SAFE_SYSTEM_PROPERTIES = Arrays.asList("file.separator", "java.home", "java.vendor", "java.version", "line.separator", "os.arch", "os.name", "os.version", "path.separator", "user.dir", "user.home", "user.name"); + public static final String ENGINE_ID = "exemplar"; + public static final String ENGINE_NAME = "Exemplar Test Engine"; + + private final Set sampleNormalizers = new LinkedHashSet<>(); + private final Set sampleModifiers = new LinkedHashSet<>(); + + @Override + public String getId() { + return ENGINE_ID; + } + + @Override + public TestDescriptor discover(EngineDiscoveryRequest discoveryRequest, UniqueId uniqueId) { + LOGGER.info(() -> { + String selectorsMsg = discoveryRequest.getSelectorsByType(DiscoverySelector.class).stream() + .map(Object::toString) + .collect(Collectors.joining("\n\t", "\t", "")); + return "Discovering tests with engine: " + uniqueId + " using selectors:\n" + selectorsMsg; + }); + + EngineDescriptor engineDescriptor = new EngineDescriptor(uniqueId, ENGINE_NAME); + + EngineDiscoveryRequestResolver.builder() + .addSelectorResolver(new ExemplarTestResolver()) + .build() + .resolve(discoveryRequest, engineDescriptor); + + return engineDescriptor; + } + + @Override + public void execute(ExecutionRequest executionRequest) { + LOGGER.info(() -> "Executing tests with engine: " + executionRequest.getRootTestDescriptor().getUniqueId()); + + File tmpDir = createTmpDir(); + try { + EngineExecutionListener listener = executionRequest.getEngineExecutionListener(); + executionRequest.getRootTestDescriptor().getChildren().forEach(test -> { + if (test instanceof ExemplarTestDescriptor) { + execute(((ExemplarTestDescriptor) test), tmpDir, executionRequest, listener); + } else { + throw new IllegalStateException("Cannot execute test: " + test + " of type: " + test.getClass().getName()); + } + }); + } finally { + deleteRecursively(tmpDir); + } + } + + private static File createTmpDir() { + try { + Path path = Files.createTempDirectory("exemplar-"); + LOGGER.info(() -> "Testing base directory: " + path.toAbsolutePath()); + return path.toFile(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private void deleteRecursively(File tmpDir) { + if (tmpDir != null && tmpDir.exists()) { + try { + FileUtils.deleteDirectory(tmpDir); + } catch (IOException e) { + throw new RuntimeException("Could not delete temporary directory: ", e); + } + } + } + + private void execute(ExemplarTestDescriptor test, File tmpDir, ExecutionRequest request, EngineExecutionListener listener) { + populateSampleModifiers(request); + populateOutputNormalizers(request); + + Sample sample = test.getSample(); + if (sample instanceof InvalidSample) { + listener.executionFinished(test, TestExecutionResult.failed(((InvalidSample) sample).getException())); + } else { + listener.executionStarted(test); + try { + final Sample testSpecificSample = initSample(sample, tmpDir); + File baseWorkingDir = testSpecificSample.getProjectDir(); + + // Execute and verify each command + for (Command command : testSpecificSample.getCommands()) { + File workingDir = baseWorkingDir; + + if (command.getExecutionSubdirectory() != null) { + workingDir = new File(workingDir, command.getExecutionSubdirectory()); + } + + // This should be some kind of plugable executor rather than hard-coded here + if (command.getExecutable().equals("cd")) { + baseWorkingDir = new File(baseWorkingDir, command.getArgs().get(0)).getCanonicalFile(); + continue; + } + + CommandExecutionResult result = execute(getExecutionMetadata(testSpecificSample.getProjectDir()), workingDir, command); + + if (result.getExitCode() != 0 && !command.isExpectFailure()) { + String message = String.format("Expected sample invocation to succeed but it failed.%nCommand was: '%s %s'%nWorking directory: '%s'%n[BEGIN OUTPUT]%n%s%n[END OUTPUT]%n", command.getExecutable(), StringUtils.join(command.getArgs(), " "), workingDir.getAbsolutePath(), result.getOutput()); + + listener.executionFinished(test, TestExecutionResult.failed(new RuntimeException(message))); + } else if (result.getExitCode() == 0 && command.isExpectFailure()) { + String message = String.format("Expected sample invocation to fail but it succeeded.%nCommand was: '%s %s'%nWorking directory: '%s'%n[BEGIN OUTPUT]%n%s%n[END OUTPUT]%n", command.getExecutable(), StringUtils.join(command.getArgs(), " "), workingDir.getAbsolutePath(), result.getOutput()); + listener.executionFinished(test, TestExecutionResult.failed(new RuntimeException(message))); + } + verifyOutput(command, result); + } + listener.executionFinished(test, TestExecutionResult.successful()); + } catch (Throwable t) { + listener.executionFinished(test, TestExecutionResult.failed(t)); + } + } + } + + private void populateSampleModifiers(ExecutionRequest request) { + populateFromSystemProperty(request, "exemplar.output.modifiers", sampleModifiers); + } + + private void populateOutputNormalizers(ExecutionRequest request) { + populateFromSystemProperty(request, "exemplar.output.normalizers", sampleNormalizers); + } + + private static void populateFromSystemProperty(ExecutionRequest request, String propertyName, Set targetSet) { + Optional values = request.getConfigurationParameters().get(propertyName); + boolean hasValues = values.isPresent() && !values.get().trim().isEmpty(); + if (hasValues) { + String[] classNames = StringUtils.split(values.get(), ','); + for (String className : classNames) { + targetSet.add(newInstance(className)); + } + } + } + + private static T newInstance(String className) { + try { + Class clazz = Class.forName(className); + return (T) clazz.getConstructor().newInstance(); + } catch (ClassNotFoundException | NoSuchMethodException | InvocationTargetException | + InstantiationException | IllegalAccessException e) { + throw new RuntimeException("Could not instantiate class: " + className, e); + } + } + + private Sample initSample(final Sample sampleIn, File tmpDir) throws IOException { + File tmpProjectDir = new File(tmpDir, sampleIn.getId()); + tmpProjectDir.mkdirs(); + FileUtils.copyDirectory(sampleIn.getProjectDir(), tmpProjectDir); + Sample sample = new Sample(sampleIn.getId(), tmpProjectDir, sampleIn.getCommands()); + + for (SampleModifier sampleModifier : sampleModifiers) { + LOGGER.debug(()-> "Modifier: " + sampleModifier.getClass().getName()); + sample = sampleModifier.modify(sample); + } + return sample; + } + + private ExecutionMetadata getExecutionMetadata(final File tempSampleOutputDir) { + Map systemProperties = new HashMap<>(); + for (String systemPropertyKey : SAFE_SYSTEM_PROPERTIES) { + systemProperties.put(systemPropertyKey, System.getProperty(systemPropertyKey)); + } + return new ExecutionMetadata(tempSampleOutputDir, systemProperties); + } + + private CommandExecutionResult execute(ExecutionMetadata executionMetadata, File workingDir, Command command) { + return selectExecutor(executionMetadata, workingDir, command).execute(command, executionMetadata); + } + + protected CommandExecutor selectExecutor(ExecutionMetadata executionMetadata, File workingDir, Command command) { + return new CliCommandExecutor(workingDir); + } + + private void verifyOutput(final Command command, final CommandExecutionResult executionResult) { + if (command.getExpectedOutput() == null) { + return; + } + + String expectedOutput = command.getExpectedOutput(); + String actualOutput = executionResult.getOutput(); + + for (OutputNormalizer normalizer : sampleNormalizers) { + actualOutput = normalizer.normalize(actualOutput, executionResult.getExecutionMetadata()); + } + + if (command.isAllowDisorderedOutput()) { + new AnyOrderLineSegmentedOutputVerifier().verify(expectedOutput, actualOutput, command.isAllowAdditionalOutput()); + } else { + new StrictOrderLineSegmentedOutputVerifier().verify(expectedOutput, actualOutput, command.isAllowAdditionalOutput()); + } + } +} diff --git a/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestResolver.java b/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestResolver.java new file mode 100644 index 0000000..0658c5c --- /dev/null +++ b/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestResolver.java @@ -0,0 +1,33 @@ +package org.gradle.exemplar; + +import org.gradle.exemplar.loader.SamplesDiscovery; +import org.gradle.exemplar.model.Sample; +import org.junit.platform.commons.logging.Logger; +import org.junit.platform.commons.logging.LoggerFactory; +import org.junit.platform.engine.discovery.DirectorySelector; +import org.junit.platform.engine.support.discovery.SelectorResolver; + +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +public class ExemplarTestResolver implements SelectorResolver { + public static final Logger LOGGER = LoggerFactory.getLogger(ExemplarTestResolver.class); + + @Override + public Resolution resolve(DirectorySelector selector, Context context) { + LOGGER.info(() -> "Test specification dir: " + selector.getDirectory().getAbsolutePath()); + List samples = SamplesDiscovery.externalSamples(selector.getDirectory()); + Set tests = samples.stream() + .map(s -> context.addToParent(parent -> Optional.of(new ExemplarTestDescriptor(parent.getUniqueId(), s.getConfigFile(), s.getId(), s)))) + .map(Optional::get) + .map(Match::exact) + .collect(Collectors.toSet()); + + if (!tests.isEmpty()) { + return Resolution.matches(tests); + } + return Resolution.unresolved(); + } +} diff --git a/samples-junit-engine/src/main/resources/META-INF/services/org.junit.platform.engine.TestEngine b/samples-junit-engine/src/main/resources/META-INF/services/org.junit.platform.engine.TestEngine new file mode 100644 index 0000000..2ad62cf --- /dev/null +++ b/samples-junit-engine/src/main/resources/META-INF/services/org.junit.platform.engine.TestEngine @@ -0,0 +1 @@ +org.gradle.exemplar.ExemplarTestEngine diff --git a/samples-junit-engine/src/test-definitions/my-test-project/output.sample.conf b/samples-junit-engine/src/test-definitions/my-test-project/output.sample.conf new file mode 100644 index 0000000..0ce42fa --- /dev/null +++ b/samples-junit-engine/src/test-definitions/my-test-project/output.sample.conf @@ -0,0 +1,3 @@ +executable = echo +args = thing +expected-output-file: sample.out diff --git a/samples-junit-engine/src/test-definitions/my-test-project/sample.out b/samples-junit-engine/src/test-definitions/my-test-project/sample.out new file mode 100644 index 0000000..27ee738 --- /dev/null +++ b/samples-junit-engine/src/test-definitions/my-test-project/sample.out @@ -0,0 +1 @@ +thing diff --git a/settings.gradle.kts b/settings.gradle.kts index 02cd74d..83d45ff 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -7,4 +7,5 @@ rootProject.name = "exemplar" include("samples-discovery") include("samples-check") +include("samples-junit-engine") include("docs") From 0c9fdf15863a0877df4d3cd2af10bdf005a8e191 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Don=C3=A1t=20Csik=C3=B3s?= Date: Fri, 30 Jan 2026 11:45:15 +0100 Subject: [PATCH 3/6] Create test events for each command --- samples-junit-engine/build.gradle.kts | 1 + .../ExemplarTestCommandDescriptor.java | 48 +++++++++++++++++++ .../exemplar/ExemplarTestDescriptor.java | 10 ++-- .../gradle/exemplar/ExemplarTestEngine.java | 19 ++++++-- 4 files changed, 69 insertions(+), 9 deletions(-) create mode 100644 samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestCommandDescriptor.java diff --git a/samples-junit-engine/build.gradle.kts b/samples-junit-engine/build.gradle.kts index 9e45fc3..91d3d0c 100644 --- a/samples-junit-engine/build.gradle.kts +++ b/samples-junit-engine/build.gradle.kts @@ -1,6 +1,7 @@ plugins { id("exemplar.java-conventions") id("jvm-test-suite") + id("exemplar.publishing-conventions") } dependencies { diff --git a/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestCommandDescriptor.java b/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestCommandDescriptor.java new file mode 100644 index 0000000..c5f067c --- /dev/null +++ b/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestCommandDescriptor.java @@ -0,0 +1,48 @@ +package org.gradle.exemplar; + +import org.gradle.exemplar.model.Command; +import org.gradle.exemplar.model.Sample; +import org.junit.platform.engine.UniqueId; +import org.junit.platform.engine.support.descriptor.AbstractTestDescriptor; +import org.junit.platform.engine.support.descriptor.DirectorySource; + +public final class ExemplarTestCommandDescriptor extends AbstractTestDescriptor { + + private final Command command; + + public ExemplarTestCommandDescriptor(ExemplarTestDescriptor exemplarTestDescriptor, Sample sample, Command command) { + super( + uniqueId(exemplarTestDescriptor, command), + displayName(command), + DirectorySource.from(sample.getProjectDir()) + ); + setParent(exemplarTestDescriptor); + this.command = command; + } + + private static UniqueId uniqueId(ExemplarTestDescriptor exemplarTestDescriptor, Command command) { + return exemplarTestDescriptor.getUniqueId().append("commandName", displayName(command)); + } + + private static String displayName(Command command) { + StringBuilder displayName = new StringBuilder(command.getExecutable()); + for (String flag : command.getFlags()) { + displayName.append(" "); + displayName.append(flag); + } + for (String arg : command.getArgs()) { + displayName.append(" "); + displayName.append(arg); + } + return displayName.toString(); + } + + @Override + public Type getType() { + return Type.TEST; + } + + public Command getCommand() { + return command; + } +} diff --git a/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestDescriptor.java b/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestDescriptor.java index 28f2be1..544342a 100644 --- a/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestDescriptor.java +++ b/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestDescriptor.java @@ -1,13 +1,12 @@ package org.gradle.exemplar; +import org.gradle.exemplar.model.Command; import org.gradle.exemplar.model.Sample; -import org.junit.platform.engine.TestSource; import org.junit.platform.engine.UniqueId; import org.junit.platform.engine.support.descriptor.AbstractTestDescriptor; import org.junit.platform.engine.support.descriptor.DirectorySource; import java.io.File; -import java.util.Optional; public final class ExemplarTestDescriptor extends AbstractTestDescriptor { private final File file; @@ -17,12 +16,15 @@ public final class ExemplarTestDescriptor extends AbstractTestDescriptor { public ExemplarTestDescriptor(UniqueId parentId, File file, String name, Sample sample) { super( parentId.append("testDefinitionFile", fileNameWithoutExtension(file)).append("testDefinition", name), - file.getParentFile().getName() + " - " + fileNameWithoutExtension(file), + file.getParentFile().getName(), DirectorySource.from(sample.getProjectDir()) ); this.file = file; this.name = name; this.sample = sample; + for (Command command : sample.getCommands()) { + children.add(new ExemplarTestCommandDescriptor(this, sample, command)); + } } private static String fileNameWithoutExtension(File file) { @@ -36,7 +38,7 @@ private static String fileNameWithoutExtension(File file) { @Override public Type getType() { - return Type.TEST; + return Type.CONTAINER; } public Sample getSample() { diff --git a/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestEngine.java b/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestEngine.java index f1465d7..533b589 100644 --- a/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestEngine.java +++ b/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestEngine.java @@ -23,6 +23,7 @@ import org.junit.platform.engine.TestEngine; import org.junit.platform.engine.TestExecutionResult; import org.junit.platform.engine.UniqueId; +import org.junit.platform.engine.reporting.ReportEntry; import org.junit.platform.engine.support.descriptor.EngineDescriptor; import org.junit.platform.engine.support.discovery.EngineDiscoveryRequestResolver; @@ -121,7 +122,11 @@ private void execute(ExemplarTestDescriptor test, File tmpDir, ExecutionRequest File baseWorkingDir = testSpecificSample.getProjectDir(); // Execute and verify each command - for (Command command : testSpecificSample.getCommands()) { + for (TestDescriptor child : test.getChildren()) { + ExemplarTestCommandDescriptor childDescriptor = (ExemplarTestCommandDescriptor) child; + listener.executionStarted(childDescriptor); + Command command = childDescriptor.getCommand(); + File workingDir = baseWorkingDir; if (command.getExecutionSubdirectory() != null) { @@ -138,13 +143,17 @@ private void execute(ExemplarTestDescriptor test, File tmpDir, ExecutionRequest if (result.getExitCode() != 0 && !command.isExpectFailure()) { String message = String.format("Expected sample invocation to succeed but it failed.%nCommand was: '%s %s'%nWorking directory: '%s'%n[BEGIN OUTPUT]%n%s%n[END OUTPUT]%n", command.getExecutable(), StringUtils.join(command.getArgs(), " "), workingDir.getAbsolutePath(), result.getOutput()); - - listener.executionFinished(test, TestExecutionResult.failed(new RuntimeException(message))); + listener.executionFinished(childDescriptor, TestExecutionResult.failed(new RuntimeException(message))); } else if (result.getExitCode() == 0 && command.isExpectFailure()) { String message = String.format("Expected sample invocation to fail but it succeeded.%nCommand was: '%s %s'%nWorking directory: '%s'%n[BEGIN OUTPUT]%n%s%n[END OUTPUT]%n", command.getExecutable(), StringUtils.join(command.getArgs(), " "), workingDir.getAbsolutePath(), result.getOutput()); - listener.executionFinished(test, TestExecutionResult.failed(new RuntimeException(message))); + listener.executionFinished(childDescriptor, TestExecutionResult.failed(new RuntimeException(message))); + } + try { + verifyOutput(command, result); + listener.executionFinished(childDescriptor, TestExecutionResult.successful()); + } catch (Exception e) { + listener.executionFinished(childDescriptor, TestExecutionResult.failed(e)); } - verifyOutput(command, result); } listener.executionFinished(test, TestExecutionResult.successful()); } catch (Throwable t) { From 77874af863b3e39e6a654f709262eac7a54fa98a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Don=C3=A1t=20Csik=C3=B3s?= Date: Fri, 30 Jan 2026 12:16:51 +0100 Subject: [PATCH 4/6] Make sample working dir configurable --- .../gradle/exemplar/ExemplarTestEngine.java | 44 ++++++++++++------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestEngine.java b/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestEngine.java index 533b589..3c76ea1 100644 --- a/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestEngine.java +++ b/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestEngine.java @@ -72,25 +72,31 @@ public TestDescriptor discover(EngineDiscoveryRequest discoveryRequest, UniqueId @Override public void execute(ExecutionRequest executionRequest) { LOGGER.info(() -> "Executing tests with engine: " + executionRequest.getRootTestDescriptor().getUniqueId()); - - File tmpDir = createTmpDir(); - try { - EngineExecutionListener listener = executionRequest.getEngineExecutionListener(); - executionRequest.getRootTestDescriptor().getChildren().forEach(test -> { - if (test instanceof ExemplarTestDescriptor) { - execute(((ExemplarTestDescriptor) test), tmpDir, executionRequest, listener); - } else { - throw new IllegalStateException("Cannot execute test: " + test + " of type: " + test.getClass().getName()); - } - }); - } finally { - deleteRecursively(tmpDir); - } + File tmpDir = createTmpDir(executionRequest); + EngineExecutionListener listener = executionRequest.getEngineExecutionListener(); + executionRequest.getRootTestDescriptor().getChildren().forEach(test -> { + if (test instanceof ExemplarTestDescriptor) { + execute(((ExemplarTestDescriptor) test), tmpDir, executionRequest, listener); + } else { + throw new IllegalStateException("Cannot execute test: " + test + " of type: " + test.getClass().getName()); + } + }); } - private static File createTmpDir() { + private static File createTmpDir(ExecutionRequest request) { try { - Path path = Files.createTempDirectory("exemplar-"); + Optional s = request.getConfigurationParameters().get("exemplar.tmpdir"); + Path path; + if (s.isPresent()) { + File dir = getFile(s); + if (dir.exists()) { + deleteRecursively(dir); + } + path = dir.toPath(); + Files.createDirectory(path); + } else { + path = Files.createTempDirectory("exemplar-"); + } LOGGER.info(() -> "Testing base directory: " + path.toAbsolutePath()); return path.toFile(); } catch (IOException e) { @@ -98,7 +104,11 @@ private static File createTmpDir() { } } - private void deleteRecursively(File tmpDir) { + private static File getFile(Optional s) { + return new File(s.get()); + } + + private static void deleteRecursively(File tmpDir) { if (tmpDir != null && tmpDir.exists()) { try { FileUtils.deleteDirectory(tmpDir); From 1240de49099b4c1829873a598f8c04278005e600 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Don=C3=A1t=20Csik=C3=B3s?= Date: Fri, 30 Jan 2026 13:31:43 +0100 Subject: [PATCH 5/6] Cleanup --- ...or.java => ExemplarCommandDescriptor.java} | 13 +++++----- ...tor.java => ExemplarSampleDescriptor.java} | 24 +++++++++---------- .../gradle/exemplar/ExemplarTestEngine.java | 10 ++++---- .../gradle/exemplar/ExemplarTestResolver.java | 2 +- 4 files changed, 24 insertions(+), 25 deletions(-) rename samples-junit-engine/src/main/java/org/gradle/exemplar/{ExemplarTestCommandDescriptor.java => ExemplarCommandDescriptor.java} (68%) rename samples-junit-engine/src/main/java/org/gradle/exemplar/{ExemplarTestDescriptor.java => ExemplarSampleDescriptor.java} (61%) diff --git a/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestCommandDescriptor.java b/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarCommandDescriptor.java similarity index 68% rename from samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestCommandDescriptor.java rename to samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarCommandDescriptor.java index c5f067c..0054185 100644 --- a/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestCommandDescriptor.java +++ b/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarCommandDescriptor.java @@ -1,26 +1,25 @@ package org.gradle.exemplar; import org.gradle.exemplar.model.Command; -import org.gradle.exemplar.model.Sample; import org.junit.platform.engine.UniqueId; import org.junit.platform.engine.support.descriptor.AbstractTestDescriptor; import org.junit.platform.engine.support.descriptor.DirectorySource; -public final class ExemplarTestCommandDescriptor extends AbstractTestDescriptor { +public final class ExemplarCommandDescriptor extends AbstractTestDescriptor { private final Command command; - public ExemplarTestCommandDescriptor(ExemplarTestDescriptor exemplarTestDescriptor, Sample sample, Command command) { + public ExemplarCommandDescriptor(ExemplarSampleDescriptor parent, Command command) { super( - uniqueId(exemplarTestDescriptor, command), + uniqueId(parent, command), displayName(command), - DirectorySource.from(sample.getProjectDir()) + DirectorySource.from(parent.getSample().getProjectDir()) ); - setParent(exemplarTestDescriptor); + setParent(parent); this.command = command; } - private static UniqueId uniqueId(ExemplarTestDescriptor exemplarTestDescriptor, Command command) { + private static UniqueId uniqueId(ExemplarSampleDescriptor exemplarTestDescriptor, Command command) { return exemplarTestDescriptor.getUniqueId().append("commandName", displayName(command)); } diff --git a/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestDescriptor.java b/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarSampleDescriptor.java similarity index 61% rename from samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestDescriptor.java rename to samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarSampleDescriptor.java index 544342a..821e1fc 100644 --- a/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestDescriptor.java +++ b/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarSampleDescriptor.java @@ -8,22 +8,22 @@ import java.io.File; -public final class ExemplarTestDescriptor extends AbstractTestDescriptor { - private final File file; - private final String name; +public final class ExemplarSampleDescriptor extends AbstractTestDescriptor { private final Sample sample; - public ExemplarTestDescriptor(UniqueId parentId, File file, String name, Sample sample) { + public ExemplarSampleDescriptor(UniqueId parentId, Sample sample) { super( - parentId.append("testDefinitionFile", fileNameWithoutExtension(file)).append("testDefinition", name), - file.getParentFile().getName(), - DirectorySource.from(sample.getProjectDir()) - ); - this.file = file; - this.name = name; + parentId.append("testDefinitionFile", fileNameWithoutExtension(sample.getConfigFile())) + .append("testDefinition", sample.getId()), + sample.getConfigFile().getParentFile().getName(), + DirectorySource.from(sample.getProjectDir())); this.sample = sample; + defineTests(sample); + } + + private void defineTests(Sample sample) { for (Command command : sample.getCommands()) { - children.add(new ExemplarTestCommandDescriptor(this, sample, command)); + children.add(new ExemplarCommandDescriptor(this, command)); } } @@ -47,6 +47,6 @@ public Sample getSample() { @Override public String toString() { - return "Sample[file=" + file.getName() + ", name=" + name + "]"; + return "Sample[file=" + sample.getConfigFile().getName() + ", name=" + sample.getId() + "]"; } } diff --git a/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestEngine.java b/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestEngine.java index 3c76ea1..db60df8 100644 --- a/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestEngine.java +++ b/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestEngine.java @@ -75,8 +75,8 @@ public void execute(ExecutionRequest executionRequest) { File tmpDir = createTmpDir(executionRequest); EngineExecutionListener listener = executionRequest.getEngineExecutionListener(); executionRequest.getRootTestDescriptor().getChildren().forEach(test -> { - if (test instanceof ExemplarTestDescriptor) { - execute(((ExemplarTestDescriptor) test), tmpDir, executionRequest, listener); + if (test instanceof ExemplarSampleDescriptor) { + execute(((ExemplarSampleDescriptor) test), tmpDir, executionRequest, listener); } else { throw new IllegalStateException("Cannot execute test: " + test + " of type: " + test.getClass().getName()); } @@ -118,7 +118,7 @@ private static void deleteRecursively(File tmpDir) { } } - private void execute(ExemplarTestDescriptor test, File tmpDir, ExecutionRequest request, EngineExecutionListener listener) { + private void execute(ExemplarSampleDescriptor test, File tmpDir, ExecutionRequest request, EngineExecutionListener listener) { populateSampleModifiers(request); populateOutputNormalizers(request); @@ -133,7 +133,7 @@ private void execute(ExemplarTestDescriptor test, File tmpDir, ExecutionRequest // Execute and verify each command for (TestDescriptor child : test.getChildren()) { - ExemplarTestCommandDescriptor childDescriptor = (ExemplarTestCommandDescriptor) child; + ExemplarCommandDescriptor childDescriptor = (ExemplarCommandDescriptor) child; listener.executionStarted(childDescriptor); Command command = childDescriptor.getCommand(); @@ -150,7 +150,7 @@ private void execute(ExemplarTestDescriptor test, File tmpDir, ExecutionRequest } CommandExecutionResult result = execute(getExecutionMetadata(testSpecificSample.getProjectDir()), workingDir, command); - + listener.reportingEntryPublished(childDescriptor, ReportEntry.from("exitCode", String.valueOf(result.getExitCode()))); if (result.getExitCode() != 0 && !command.isExpectFailure()) { String message = String.format("Expected sample invocation to succeed but it failed.%nCommand was: '%s %s'%nWorking directory: '%s'%n[BEGIN OUTPUT]%n%s%n[END OUTPUT]%n", command.getExecutable(), StringUtils.join(command.getArgs(), " "), workingDir.getAbsolutePath(), result.getOutput()); listener.executionFinished(childDescriptor, TestExecutionResult.failed(new RuntimeException(message))); diff --git a/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestResolver.java b/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestResolver.java index 0658c5c..22daab4 100644 --- a/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestResolver.java +++ b/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestResolver.java @@ -20,7 +20,7 @@ public Resolution resolve(DirectorySelector selector, Context context) { LOGGER.info(() -> "Test specification dir: " + selector.getDirectory().getAbsolutePath()); List samples = SamplesDiscovery.externalSamples(selector.getDirectory()); Set tests = samples.stream() - .map(s -> context.addToParent(parent -> Optional.of(new ExemplarTestDescriptor(parent.getUniqueId(), s.getConfigFile(), s.getId(), s)))) + .map(s -> context.addToParent(parent -> Optional.of(new ExemplarSampleDescriptor(parent.getUniqueId(), s)))) .map(Optional::get) .map(Match::exact) .collect(Collectors.toSet()); From 42606dbfef6a78bbe2dcf89210a6ae88a853a8f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Don=C3=A1t=20Csik=C3=B3s?= Date: Fri, 30 Jan 2026 13:42:19 +0100 Subject: [PATCH 6/6] Capture unexpected output failures --- .../java/org/gradle/exemplar/ExemplarTestEngine.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestEngine.java b/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestEngine.java index db60df8..9255a5f 100644 --- a/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestEngine.java +++ b/samples-junit-engine/src/main/java/org/gradle/exemplar/ExemplarTestEngine.java @@ -12,6 +12,7 @@ import org.gradle.exemplar.test.normalizer.OutputNormalizer; import org.gradle.exemplar.test.runner.SampleModifier; import org.gradle.exemplar.test.verifier.AnyOrderLineSegmentedOutputVerifier; +import org.gradle.exemplar.test.verifier.OutputVerifier; import org.gradle.exemplar.test.verifier.StrictOrderLineSegmentedOutputVerifier; import org.junit.platform.commons.logging.Logger; import org.junit.platform.commons.logging.LoggerFactory; @@ -161,7 +162,7 @@ private void execute(ExemplarSampleDescriptor test, File tmpDir, ExecutionReques try { verifyOutput(command, result); listener.executionFinished(childDescriptor, TestExecutionResult.successful()); - } catch (Exception e) { + } catch (Throwable e) { listener.executionFinished(childDescriptor, TestExecutionResult.failed(e)); } } @@ -242,10 +243,7 @@ private void verifyOutput(final Command command, final CommandExecutionResult ex actualOutput = normalizer.normalize(actualOutput, executionResult.getExecutionMetadata()); } - if (command.isAllowDisorderedOutput()) { - new AnyOrderLineSegmentedOutputVerifier().verify(expectedOutput, actualOutput, command.isAllowAdditionalOutput()); - } else { - new StrictOrderLineSegmentedOutputVerifier().verify(expectedOutput, actualOutput, command.isAllowAdditionalOutput()); - } + OutputVerifier verifier = command.isAllowDisorderedOutput() ? new AnyOrderLineSegmentedOutputVerifier() : new StrictOrderLineSegmentedOutputVerifier(); + verifier.verify(expectedOutput, actualOutput, command.isAllowAdditionalOutput()); } }