diff --git a/.gitmodules b/.gitmodules index 3994607a6..a2d25a69c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,7 +4,7 @@ [submodule "providers/flagd/test-harness"] path = providers/flagd/test-harness url = https://github.com/open-feature/test-harness.git - branch = v3.0.1 + branch = feat/add-env-var-tag [submodule "providers/flagd/spec"] path = providers/flagd/spec url = https://github.com/open-feature/spec.git diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/ConfigCucumberTest.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/ConfigCucumberTest.java index f031091da..0bb64dd5c 100644 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/ConfigCucumberTest.java +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/ConfigCucumberTest.java @@ -1,6 +1,7 @@ package dev.openfeature.contrib.providers.flagd; import static io.cucumber.junit.platform.engine.Constants.GLUE_PROPERTY_NAME; +import static io.cucumber.junit.platform.engine.Constants.PARALLEL_EXECUTION_ENABLED_PROPERTY_NAME; import static io.cucumber.junit.platform.engine.Constants.PLUGIN_PROPERTY_NAME; import org.junit.jupiter.api.Order; @@ -17,6 +18,11 @@ @Suite @IncludeEngines("cucumber") @SelectFile("test-harness/gherkin/config.feature") -@ConfigurationParameter(key = PLUGIN_PROPERTY_NAME, value = "pretty") +@ConfigurationParameter(key = PLUGIN_PROPERTY_NAME, value = "summary") @ConfigurationParameter(key = GLUE_PROPERTY_NAME, value = "dev.openfeature.contrib.providers.flagd.e2e.steps.config") +// Config scenarios read System env vars in FlagdOptions.build() and some scenarios also +// mutate them. Parallel execution causes env-var races (e.g. FLAGD_PORT=3456 leaking into +// a "Default Config" scenario that expects 8015). Since the entire suite runs in <0.4s, +// parallelism offers no benefit here — run sequentially for correctness. +@ConfigurationParameter(key = PARALLEL_EXECUTION_ENABLED_PROPERTY_NAME, value = "false") public class ConfigCucumberTest {} diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/ContainerEntry.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/ContainerEntry.java new file mode 100644 index 000000000..820f80869 --- /dev/null +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/ContainerEntry.java @@ -0,0 +1,50 @@ +package dev.openfeature.contrib.providers.flagd.e2e; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.RandomStringUtils; +import org.testcontainers.containers.ComposeContainer; +import org.testcontainers.containers.wait.strategy.Wait; + +/** A single pre-warmed Docker Compose stack (flagd + envoy) and its associated temp directory. */ +public class ContainerEntry { + + public static final int FORBIDDEN_PORT = 9212; + + public final ComposeContainer container; + public final Path tempDir; + + private ContainerEntry(ComposeContainer container, Path tempDir) { + this.container = container; + this.tempDir = tempDir; + } + + /** Start a new container entry. Blocks until all services are ready. */ + public static ContainerEntry start() throws IOException { + Path tempDir = Files.createDirectories( + Paths.get("tmp/" + RandomStringUtils.randomAlphanumeric(8).toLowerCase() + "/")); + + ComposeContainer container = new ComposeContainer(new File("test-harness/docker-compose.yaml")) + .withEnv("FLAGS_DIR", tempDir.toAbsolutePath().toString()) + .withExposedService("flagd", 8013, Wait.forListeningPort()) + .withExposedService("flagd", 8015, Wait.forListeningPort()) + .withExposedService("flagd", 8080, Wait.forListeningPort()) + .withExposedService("envoy", 9211, Wait.forListeningPort()) + .withExposedService("envoy", FORBIDDEN_PORT, Wait.forListeningPort()) + .withStartupTimeout(Duration.ofSeconds(45)); + container.start(); + + return new ContainerEntry(container, tempDir); + } + + /** Stop the container and clean up the temp directory. */ + public void stop() throws IOException { + container.stop(); + FileUtils.deleteDirectory(tempDir.toFile()); + } +} diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/ContainerPool.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/ContainerPool.java new file mode 100644 index 000000000..1eb96b966 --- /dev/null +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/ContainerPool.java @@ -0,0 +1,134 @@ +package dev.openfeature.contrib.providers.flagd.e2e; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.Semaphore; +import java.util.concurrent.atomic.AtomicBoolean; +import lombok.extern.slf4j.Slf4j; + +/** + * A pool of pre-warmed {@link ContainerEntry} instances. + * + *

All containers are started in parallel on the first {@link #acquire()} call, paying the + * Docker Compose startup cost only once per JVM. Scenarios borrow a container via + * {@link #acquire()} and return it via {@link #release(ContainerEntry)} after teardown. + * + *

Cleanup is handled automatically via a JVM shutdown hook — no explicit lifecycle calls are + * needed from test classes. This means multiple test classes (e.g. several {@code @Suite} runners + * or {@code @TestFactory} methods) share the same pool across the entire JVM lifetime without + * redundant container startups. + * + *

Pool size is controlled by the system property {@code flagd.e2e.pool.size} + * (default: min(availableProcessors, 4)). + */ +@Slf4j +public class ContainerPool { + + private static final int POOL_SIZE = Integer.getInteger( + "flagd.e2e.pool.size", Math.min(Runtime.getRuntime().availableProcessors(), 4)); + + private static final BlockingQueue pool = new LinkedBlockingQueue<>(); + private static final List all = new ArrayList<>(); + private static final AtomicBoolean initialized = new AtomicBoolean(false); + + /** + * JVM-wide semaphore that serializes disruptive container operations (stop/restart) across all + * parallel Cucumber engines. Only one scenario at a time may bring a container down, preventing + * cascading initialization timeouts in sibling scenarios that are waiting for a container slot. + */ + private static final Semaphore restartSlot = new Semaphore(1); + + static { + Runtime.getRuntime().addShutdownHook(new Thread(ContainerPool::stopAll, "container-pool-shutdown")); + } + + /** + * Borrow a container from the pool, blocking until one becomes available. + * Initializes the pool on the first call. The caller MUST call + * {@link #release(ContainerEntry)} when done. + */ + public static ContainerEntry acquire() throws Exception { + ensureInitialized(); + return pool.take(); + } + + /** Return a container to the pool so the next scenario can use it. */ + public static void release(ContainerEntry entry) { + pool.add(entry); + } + + /** + * Acquires the JVM-wide restart slot before stopping or restarting a container. + * Must be paired with {@link #releaseRestartSlot()} in the scenario {@code @After} hook. + */ + public static void acquireRestartSlot() throws InterruptedException { + log.debug("Acquiring restart slot..."); + restartSlot.acquire(); + log.debug("Restart slot acquired."); + } + + /** Releases the JVM-wide restart slot acquired by {@link #acquireRestartSlot()}. */ + public static void releaseRestartSlot() { + restartSlot.release(); + log.debug("Restart slot released."); + } + + private static void ensureInitialized() throws Exception { + if (initialized.get()) { + return; + } + synchronized (ContainerPool.class) { + if (!initialized.compareAndSet(false, true)) { + return; + } + log.info("Starting container pool of size {}...", POOL_SIZE); + ExecutorService executor = Executors.newFixedThreadPool(POOL_SIZE); + try { + List> futures = new ArrayList<>(); + for (int i = 0; i < POOL_SIZE; i++) { + futures.add(executor.submit(ContainerEntry::start)); + } + for (Future future : futures) { + ContainerEntry entry = future.get(); + pool.add(entry); + all.add(entry); + } + } catch (Exception e) { + all.forEach(entry -> { + try { + entry.stop(); + } catch (IOException suppressed) { + e.addSuppressed(suppressed); + } + }); + pool.clear(); + all.clear(); + initialized.set(false); + throw e; + } finally { + executor.shutdown(); + } + log.info("Container pool ready ({} containers).", POOL_SIZE); + } + } + + private static void stopAll() { + if (all.isEmpty()) return; + log.info("Shutdown hook — stopping all containers."); + all.forEach(entry -> { + try { + entry.stop(); + } catch (IOException e) { + log.warn("Error stopping container entry", e); + } + }); + pool.clear(); + all.clear(); + } +} diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/ContainerUtil.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/ContainerUtil.java index b63967223..6d28266e5 100644 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/ContainerUtil.java +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/ContainerUtil.java @@ -1,6 +1,9 @@ package dev.openfeature.contrib.providers.flagd.e2e; import dev.openfeature.contrib.providers.flagd.Config; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Socket; import java.util.Optional; import org.testcontainers.containers.ComposeContainer; import org.testcontainers.containers.ContainerState; @@ -29,4 +32,39 @@ public static String getLaunchpadUrl(ComposeContainer container) { }) .orElseThrow(() -> new RuntimeException("Could not find launchpad url")); } + + /** + * Blocks until the given flagd service port accepts TCP connections, or the timeout elapses. + * The launchpad's {@code /start} endpoint polls flagd's HTTP {@code /readyz} before returning, + * but the gRPC ports (8013, 8015) may become available slightly later. Waiting here prevents + * {@code setProviderAndWait} from timing out under parallel load. + */ + public static void waitForGrpcPort(ComposeContainer container, Config.Resolver resolver, long timeoutMs) + throws InterruptedException { + int internalPort; + switch (resolver) { + case RPC: + internalPort = 8013; + break; + case IN_PROCESS: + internalPort = 8015; + break; + default: + return; + } + ContainerState state = container + .getContainerByServiceName("flagd") + .orElseThrow(() -> new RuntimeException("Could not find flagd container")); + String host = state.getHost(); + int mappedPort = state.getMappedPort(internalPort); + long deadline = System.currentTimeMillis() + timeoutMs; + while (System.currentTimeMillis() < deadline) { + try (Socket s = new Socket()) { + s.connect(new InetSocketAddress(host, mappedPort), 100); + return; + } catch (IOException ignored) { + Thread.sleep(50); + } + } + } } diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/CucumberResultListener.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/CucumberResultListener.java new file mode 100644 index 000000000..44b08d2b3 --- /dev/null +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/CucumberResultListener.java @@ -0,0 +1,103 @@ +package dev.openfeature.contrib.providers.flagd.e2e; + +import java.util.Collections; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import lombok.extern.slf4j.Slf4j; +import org.junit.platform.engine.TestExecutionResult; +import org.junit.platform.engine.reporting.ReportEntry; +import org.junit.platform.launcher.TestExecutionListener; +import org.junit.platform.launcher.TestIdentifier; +import org.junit.platform.launcher.TestPlan; + +/** + * Captures the full lifecycle of a JUnit Platform test execution, tracking start, finish, and skip + * events for every node in the test plan (both containers and tests). Results are later replayed as + * JUnit Jupiter {@link org.junit.jupiter.api.DynamicTest} instances to expose the Cucumber scenario + * tree in IDEs. + */ +@Slf4j +class CucumberResultListener implements TestExecutionListener { + + private final Set started = Collections.newSetFromMap(new ConcurrentHashMap<>()); + private final Map results = new ConcurrentHashMap<>(); + private final Map skipped = new ConcurrentHashMap<>(); + + @Override + public void testPlanExecutionStarted(TestPlan testPlan) { + log.debug("Cucumber execution started"); + } + + @Override + public void testPlanExecutionFinished(TestPlan testPlan) { + log.debug( + "Cucumber execution finished — started={}, finished={}, skipped={}", + started.size(), + results.size(), + skipped.size()); + } + + @Override + public void executionStarted(TestIdentifier id) { + log.debug(" START {}", id.getDisplayName()); + started.add(id.getUniqueId()); + } + + @Override + public void executionFinished(TestIdentifier id, TestExecutionResult result) { + results.put(id.getUniqueId(), result); + if (result.getStatus() == TestExecutionResult.Status.FAILED) { + log.debug( + " FAIL {} — {}", + id.getDisplayName(), + result.getThrowable().map(Throwable::getMessage).orElse("(no message)")); + } else { + log.debug(" {} {}", result.getStatus(), id.getDisplayName()); + } + } + + @Override + public void executionSkipped(TestIdentifier id, String reason) { + skipped.put(id.getUniqueId(), reason); + log.debug(" SKIP {} — {}", id.getDisplayName(), reason); + } + + @Override + public void dynamicTestRegistered(TestIdentifier id) { + log.debug(" DYN {}", id.getDisplayName()); + } + + @Override + public void reportingEntryPublished(TestIdentifier id, ReportEntry entry) { + log.debug(" REPORT {} — {}", id.getDisplayName(), entry); + } + + /** Whether the node with the given unique ID had {@code executionStarted} called. */ + boolean wasStarted(String uniqueId) { + return started.contains(uniqueId); + } + + /** Whether the node was skipped before starting. */ + boolean wasSkipped(String uniqueId) { + return skipped.containsKey(uniqueId); + } + + /** The skip reason for a skipped node, or {@code null} if not skipped. */ + String getSkipReason(String uniqueId) { + return skipped.get(uniqueId); + } + + /** Whether a finished result was recorded for the given node. */ + boolean hasResult(String uniqueId) { + return results.containsKey(uniqueId); + } + + /** + * The recorded {@link TestExecutionResult}, or {@code null} if the node never finished. + * Use {@link #hasResult} to distinguish "finished with success" from "never finished". + */ + TestExecutionResult getResult(String uniqueId) { + return results.get(uniqueId); + } +} diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunE2ETests.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunE2ETests.java new file mode 100644 index 000000000..39fdecffd --- /dev/null +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunE2ETests.java @@ -0,0 +1,158 @@ +package dev.openfeature.contrib.providers.flagd.e2e; + +import static io.cucumber.junit.platform.engine.Constants.GLUE_PROPERTY_NAME; +import static io.cucumber.junit.platform.engine.Constants.OBJECT_FACTORY_PROPERTY_NAME; +import static io.cucumber.junit.platform.engine.Constants.PARALLEL_EXECUTION_ENABLED_PROPERTY_NAME; +import static io.cucumber.junit.platform.engine.Constants.PLUGIN_PROPERTY_NAME; +import static org.junit.platform.engine.discovery.DiscoverySelectors.selectDirectory; + +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.DynamicContainer; +import org.junit.jupiter.api.DynamicNode; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.TestFactory; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; +import org.junit.platform.launcher.EngineFilter; +import org.junit.platform.launcher.Launcher; +import org.junit.platform.launcher.LauncherDiscoveryRequest; +import org.junit.platform.launcher.TagFilter; +import org.junit.platform.launcher.TestIdentifier; +import org.junit.platform.launcher.TestPlan; +import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder; +import org.junit.platform.launcher.core.LauncherFactory; + +/** + * Runs all three resolver modes (RPC, in-process, file) concurrently via three + * {@code @TestFactory} methods. Each factory launches a full Cucumber engine run for its + * resolver, captures every scenario result via {@link CucumberResultListener}, then returns + * a {@code Stream} that mirrors the TestPlan hierarchy — giving IDEs a + * fully-expandable tree (Feature → Scenario) with accurate pass/fail/skip per scenario. + * + *

With {@code @Execution(CONCURRENT)} on each factory method and + * {@code junit.jupiter.execution.parallel.enabled=true} in {@code junit-platform.properties}, + * all three Cucumber runs execute simultaneously, so wall-clock time ≈ max(RPC, IN_PROCESS, FILE). + * + *

Each factory method ({@link #rpc()}, {@link #inProcess()}, {@link #file()}) can also be + * run individually from an IDE for targeted single-resolver debugging. + * + *

Run via {@code -Pe2e} from the repo root: + * 

./mvnw -pl providers/flagd -Pe2e test
+ */ +public class RunE2ETests { + + private static final String STEPS = "dev.openfeature.contrib.providers.flagd.e2e.steps"; + + @TestFactory + @Execution(ExecutionMode.CONCURRENT) + Stream rpc() { + return resolverTests(STEPS + ".resolver.rpc", "rpc", "unixsocket", "deprecated"); + } + + @TestFactory + @Execution(ExecutionMode.CONCURRENT) + Stream inProcess() { + // targetURI scenarios are excluded: the retryBackoffMaxMs that controls initial-connection + // throttle also controls post-disconnect reconnect backoff, so they cannot be tuned + // independently. Under parallel load the first getMetadata() call times out (envoy + // upstream not yet ready), the throttle fires for retryBackoffMaxMs, and the retry arrives + // after the waitForInitialization deadline. Tracked in flagd issue #1584 — once + // getMetadata() is removed, these scenarios can be re-enabled. + return resolverTests(STEPS + ".resolver.inprocess", "in-process", "unixsocket", "targetURI", "deprecated"); + } + + @TestFactory + @Execution(ExecutionMode.CONCURRENT) + Stream file() { + return resolverTests( + STEPS + ".resolver.file", + "file", + "unixsocket", + "targetURI", + "reconnect", + "customCert", + "events", + "contextEnrichment", + "deprecated"); + } + + private Stream resolverTests(String resolverGlue, String includeTag, String... excludeTags) { + LauncherDiscoveryRequest request = LauncherDiscoveryRequestBuilder.request() + .selectors(selectDirectory("test-harness/gherkin")) + .filters( + EngineFilter.includeEngines("cucumber"), + TagFilter.includeTags(includeTag), + TagFilter.excludeTags(excludeTags)) + .configurationParameter(GLUE_PROPERTY_NAME, STEPS + "," + resolverGlue) + .configurationParameter(PLUGIN_PROPERTY_NAME, "summary") + .configurationParameter(OBJECT_FACTORY_PROPERTY_NAME, "io.cucumber.picocontainer.PicoFactory") + .configurationParameter(PARALLEL_EXECUTION_ENABLED_PROPERTY_NAME, "true") + .configurationParameter("cucumber.execution.parallel.config.strategy", "dynamic") + .configurationParameter("cucumber.execution.parallel.config.dynamic.factor", "1") + .configurationParameter("cucumber.execution.exclusive-resources.env-var.read-write", "ENV_VARS") + .configurationParameter("cucumber.execution.exclusive-resources.grace.read-write", "CONTAINER_RESTART") + .build(); + + Launcher launcher = LauncherFactory.create(); + TestPlan testPlan = launcher.discover(request); + + // Run the full Cucumber suite synchronously, capturing all lifecycle events. + // Internal Cucumber scenario-parallelism (cucumber.execution.parallel.enabled) still applies. + CucumberResultListener listener = new CucumberResultListener(); + launcher.execute(request, listener); + + // Build a DynamicNode tree mirroring the discovered TestPlan (engine → feature → scenario). + return testPlan.getRoots().stream() + .flatMap(root -> testPlan.getChildren(root).stream()) + .flatMap(node -> buildNodes(testPlan, node, listener)); + } + + private Stream buildNodes(TestPlan plan, TestIdentifier id, CucumberResultListener listener) { + if (id.isTest()) { + String uid = id.getUniqueId(); + return Stream.of(DynamicTest.dynamicTest(id.getDisplayName(), () -> { + if (listener.wasSkipped(uid)) { + Assumptions.assumeTrue(false, listener.getSkipReason(uid)); + return; + } + if (!listener.wasStarted(uid)) { + Assumptions.assumeTrue(false, "Scenario was discovered but not executed"); + return; + } + if (!listener.hasResult(uid)) { + throw new AssertionError("Scenario started but did not complete: " + uid); + } + switch (listener.getResult(uid).getStatus()) { + case FAILED: + Throwable t = listener.getResult(uid) + .getThrowable() + .orElse(new AssertionError("Test failed: " + uid)); + if (t instanceof AssertionError) throw (AssertionError) t; + if (t instanceof RuntimeException) throw (RuntimeException) t; + throw new AssertionError(t); + case ABORTED: + Assumptions.assumeTrue( + false, + listener.getResult(uid) + .getThrowable() + .map(Throwable::getMessage) + .orElse("aborted")); + break; + default: + break; + } + })); + } + Set children = plan.getChildren(id); + if (children.isEmpty()) return Stream.empty(); + List childNodes = children.stream() + .flatMap(child -> buildNodes(plan, child, listener)) + .collect(Collectors.toList()); + if (childNodes.isEmpty()) return Stream.empty(); + return Stream.of(DynamicContainer.dynamicContainer(id.getDisplayName(), childNodes.stream())); + } +} diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunFileTest.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunFileTest.java deleted file mode 100644 index edea71850..000000000 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunFileTest.java +++ /dev/null @@ -1,39 +0,0 @@ -package dev.openfeature.contrib.providers.flagd.e2e; - -import static io.cucumber.junit.platform.engine.Constants.GLUE_PROPERTY_NAME; -import static io.cucumber.junit.platform.engine.Constants.OBJECT_FACTORY_PROPERTY_NAME; -import static io.cucumber.junit.platform.engine.Constants.PLUGIN_PROPERTY_NAME; - -import dev.openfeature.contrib.providers.flagd.Config; -import org.apache.logging.log4j.core.config.Order; -import org.junit.platform.suite.api.BeforeSuite; -import org.junit.platform.suite.api.ConfigurationParameter; -import org.junit.platform.suite.api.ExcludeTags; -import org.junit.platform.suite.api.IncludeEngines; -import org.junit.platform.suite.api.IncludeTags; -import org.junit.platform.suite.api.SelectDirectories; -import org.junit.platform.suite.api.Suite; -import org.testcontainers.junit.jupiter.Testcontainers; - -/** - * Class for running the reconnection tests for the RPC provider - */ -@Order(value = Integer.MAX_VALUE) -@Suite -@IncludeEngines("cucumber") -@SelectDirectories("test-harness/gherkin") -// if you want to run just one feature file, use the following line instead of @SelectDirectories -// @SelectFile("test-harness/gherkin/connection.feature") -@ConfigurationParameter(key = PLUGIN_PROPERTY_NAME, value = "pretty") -@ConfigurationParameter(key = GLUE_PROPERTY_NAME, value = "dev.openfeature.contrib.providers.flagd.e2e.steps") -@ConfigurationParameter(key = OBJECT_FACTORY_PROPERTY_NAME, value = "io.cucumber.picocontainer.PicoFactory") -@IncludeTags("file") -@ExcludeTags({"unixsocket", "targetURI", "reconnect", "customCert", "events", "contextEnrichment", "deprecated"}) -@Testcontainers -public class RunFileTest { - - @BeforeSuite - public static void before() { - State.resolverType = Config.Resolver.FILE; - } -} diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunInProcessTest.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunInProcessTest.java deleted file mode 100644 index 385d4e83c..000000000 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunInProcessTest.java +++ /dev/null @@ -1,39 +0,0 @@ -package dev.openfeature.contrib.providers.flagd.e2e; - -import static io.cucumber.junit.platform.engine.Constants.GLUE_PROPERTY_NAME; -import static io.cucumber.junit.platform.engine.Constants.OBJECT_FACTORY_PROPERTY_NAME; -import static io.cucumber.junit.platform.engine.Constants.PLUGIN_PROPERTY_NAME; - -import dev.openfeature.contrib.providers.flagd.Config; -import org.apache.logging.log4j.core.config.Order; -import org.junit.platform.suite.api.BeforeSuite; -import org.junit.platform.suite.api.ConfigurationParameter; -import org.junit.platform.suite.api.ExcludeTags; -import org.junit.platform.suite.api.IncludeEngines; -import org.junit.platform.suite.api.IncludeTags; -import org.junit.platform.suite.api.SelectDirectories; -import org.junit.platform.suite.api.Suite; -import org.testcontainers.junit.jupiter.Testcontainers; - -/** - * Class for running the reconnection tests for the RPC provider - */ -@Order(value = Integer.MAX_VALUE) -@Suite -@IncludeEngines("cucumber") -@SelectDirectories("test-harness/gherkin") -// if you want to run just one feature file, use the following line instead of @SelectDirectories -// @SelectFile("test-harness/gherkin/selector.feature") -@ConfigurationParameter(key = PLUGIN_PROPERTY_NAME, value = "pretty") -@ConfigurationParameter(key = GLUE_PROPERTY_NAME, value = "dev.openfeature.contrib.providers.flagd.e2e.steps") -@ConfigurationParameter(key = OBJECT_FACTORY_PROPERTY_NAME, value = "io.cucumber.picocontainer.PicoFactory") -@IncludeTags("in-process") -@ExcludeTags({"unixsocket", "deprecated"}) -@Testcontainers -public class RunInProcessTest { - - @BeforeSuite - public static void before() { - State.resolverType = Config.Resolver.IN_PROCESS; - } -} diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunRpcTest.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunRpcTest.java deleted file mode 100644 index d98fb5986..000000000 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/RunRpcTest.java +++ /dev/null @@ -1,39 +0,0 @@ -package dev.openfeature.contrib.providers.flagd.e2e; - -import static io.cucumber.junit.platform.engine.Constants.GLUE_PROPERTY_NAME; -import static io.cucumber.junit.platform.engine.Constants.OBJECT_FACTORY_PROPERTY_NAME; -import static io.cucumber.junit.platform.engine.Constants.PLUGIN_PROPERTY_NAME; - -import dev.openfeature.contrib.providers.flagd.Config; -import org.apache.logging.log4j.core.config.Order; -import org.junit.platform.suite.api.BeforeSuite; -import org.junit.platform.suite.api.ConfigurationParameter; -import org.junit.platform.suite.api.ExcludeTags; -import org.junit.platform.suite.api.IncludeEngines; -import org.junit.platform.suite.api.IncludeTags; -import org.junit.platform.suite.api.SelectDirectories; -import org.junit.platform.suite.api.Suite; -import org.testcontainers.junit.jupiter.Testcontainers; - -/** - * Class for running the reconnection tests for the RPC provider - */ -@Order(value = Integer.MAX_VALUE) -@Suite -@IncludeEngines("cucumber") -@SelectDirectories("test-harness/gherkin") -// if you want to run just one feature file, use the following line instead of @SelectDirectories -// @SelectFile("test-harness/gherkin/rpc-caching.feature") -@ConfigurationParameter(key = PLUGIN_PROPERTY_NAME, value = "pretty") -@ConfigurationParameter(key = GLUE_PROPERTY_NAME, value = "dev.openfeature.contrib.providers.flagd.e2e.steps") -@ConfigurationParameter(key = OBJECT_FACTORY_PROPERTY_NAME, value = "io.cucumber.picocontainer.PicoFactory") -@IncludeTags({"rpc"}) -@ExcludeTags({"unixsocket", "deprecated"}) -@Testcontainers -public class RunRpcTest { - - @BeforeSuite - public static void before() { - State.resolverType = Config.Resolver.RPC; - } -} diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/State.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/State.java index 2d3a227a4..843e65327 100644 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/State.java +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/State.java @@ -16,6 +16,11 @@ public class State { public ProviderType providerType; public Client client; public FeatureProvider provider; + /** The domain name under which this scenario's provider is registered with OpenFeatureAPI. */ + public String providerName; + /** The container borrowed from {@link ContainerPool} for this scenario. */ + public ContainerEntry containerEntry; + public ConcurrentLinkedQueue events = new ConcurrentLinkedQueue<>(); public Optional lastEvent; public FlagSteps.Flag flag; @@ -23,6 +28,8 @@ public class State { public FlagEvaluationDetails evaluation; public FlagdOptions options; public FlagdOptions.FlagdOptionsBuilder builder = FlagdOptions.builder(); - public static Config.Resolver resolverType; + public Config.Resolver resolverType; public boolean hasError; + /** True if this scenario acquired the JVM-wide restart slot via {@link ContainerPool#acquireRestartSlot()}. */ + public boolean restartSlotAcquired; } diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/EventSteps.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/EventSteps.java index dc11bbb6a..6e8222b19 100644 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/EventSteps.java +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/EventSteps.java @@ -60,9 +60,18 @@ public void eventHandlerShouldBeExecutedWithin(String eventType, int ms) { .atMost(ms, MILLISECONDS) .pollInterval(10, MILLISECONDS) .until(() -> state.events.stream().anyMatch(event -> event.type.equals(eventType))); - state.lastEvent = state.events.stream() - .filter(event -> event.type.equals(eventType)) - .findFirst(); - state.events.clear(); + // Drain all events up to and including the first match. This ensures that + // older events (e.g. a READY from before a disconnect) cannot satisfy a + // later assertion that expects a *new* event of the same type, while still + // preserving events that arrived *after* the match for subsequent steps. + Event matched = null; + while (!state.events.isEmpty()) { + Event head = state.events.poll(); + if (head != null && head.type.equals(eventType)) { + matched = head; + break; + } + } + state.lastEvent = java.util.Optional.ofNullable(matched); } } diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/ProviderSteps.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/ProviderSteps.java index 90d082292..123531a98 100644 --- a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/ProviderSteps.java +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/ProviderSteps.java @@ -6,9 +6,12 @@ import dev.openfeature.contrib.providers.flagd.Config; import dev.openfeature.contrib.providers.flagd.FlagdOptions; import dev.openfeature.contrib.providers.flagd.FlagdProvider; +import dev.openfeature.contrib.providers.flagd.e2e.ContainerEntry; +import dev.openfeature.contrib.providers.flagd.e2e.ContainerPool; import dev.openfeature.contrib.providers.flagd.e2e.ContainerUtil; import dev.openfeature.contrib.providers.flagd.e2e.State; import dev.openfeature.sdk.FeatureProvider; +import dev.openfeature.sdk.NoOpProvider; import dev.openfeature.sdk.OpenFeatureAPI; import dev.openfeature.sdk.ProviderState; import io.cucumber.java.After; @@ -18,132 +21,129 @@ import io.cucumber.java.en.Then; import io.cucumber.java.en.When; import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.Duration; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.StringUtils; import org.testcontainers.containers.ComposeContainer; -import org.testcontainers.containers.wait.strategy.Wait; @Slf4j public class ProviderSteps extends AbstractSteps { public static final int UNAVAILABLE_PORT = 9999; - public static final int FORBIDDEN_PORT = 9212; - static ComposeContainer container; - - static Path sharedTempDir; public ProviderSteps(State state) { super(state); } @BeforeAll - public static void beforeAll() throws IOException { - sharedTempDir = Files.createDirectories( - Paths.get("tmp/" + RandomStringUtils.randomAlphanumeric(8).toLowerCase() + "/")); - container = new ComposeContainer(new File("test-harness/docker-compose.yaml")) - .withEnv("FLAGS_DIR", sharedTempDir.toAbsolutePath().toString()) - .withExposedService("flagd", 8013, Wait.forListeningPort()) - .withExposedService("flagd", 8015, Wait.forListeningPort()) - .withExposedService("flagd", 8080, Wait.forListeningPort()) - .withExposedService("envoy", 9211, Wait.forListeningPort()) - .withExposedService("envoy", FORBIDDEN_PORT, Wait.forListeningPort()) - .withStartupTimeout(Duration.ofSeconds(45)); - container.start(); + public static void beforeAll() throws Exception { + // Container pool initializes lazily on first acquire() — nothing to do here. } @AfterAll - public static void afterAll() throws IOException { - container.stop(); - FileUtils.deleteDirectory(sharedTempDir.toFile()); + public static void afterAll() { + // Container pool shuts down via JVM shutdown hook — nothing to do here. } @After public void tearDown() { - if (state.client != null) { - when().post("http://" + ContainerUtil.getLaunchpadUrl(container) + "/stop") - .then() - .statusCode(200); + if (state.restartSlotAcquired) { + ContainerPool.releaseRestartSlot(); + state.restartSlotAcquired = false; + } + if (state.containerEntry != null) { + if (state.client != null) { + when().post("http://" + ContainerUtil.getLaunchpadUrl(state.containerEntry.container) + "/stop") + .then() + .statusCode(200); + } + ContainerPool.release(state.containerEntry); + state.containerEntry = null; + } + // Replace the domain provider with a NoOp through the SDK lifecycle so the SDK + // properly calls detachEventProvider (nulls onEmit) and shuts down the emitter + // executor — neither of which happens when calling provider.shutdown() directly. + if (state.providerName != null) { + OpenFeatureAPI.getInstance().setProvider(state.providerName, new NoOpProvider()); } - OpenFeatureAPI.getInstance().shutdown(); } @Given("a {} flagd provider") - public void setupProvider(String providerType) throws InterruptedException { + public void setupProvider(String providerType) throws Exception { + state.containerEntry = ContainerPool.acquire(); + ComposeContainer container = state.containerEntry.container; + String flagdConfig = "default"; - state.builder.deadline(1000).keepAlive(0).retryGracePeriod(2); + state.builder + .deadline(1000) + .keepAlive(0) + .retryGracePeriod(2) + .retryBackoffMs(500) + .retryBackoffMaxMs(2000); boolean wait = true; switch (providerType) { case "unavailable": this.state.providerType = ProviderType.SOCKET; state.builder.port(UNAVAILABLE_PORT); - if (State.resolverType == Config.Resolver.FILE) { - + if (state.resolverType == Config.Resolver.FILE) { state.builder.offlineFlagSourcePath("not-existing"); } wait = false; break; case "forbidden": - state.builder.port(container.getServicePort("envoy", FORBIDDEN_PORT)); + state.builder.port(container.getServicePort("envoy", ContainerEntry.FORBIDDEN_PORT)); wait = false; break; case "socket": this.state.providerType = ProviderType.SOCKET; - String socketPath = - sharedTempDir.resolve("socket.sock").toAbsolutePath().toString(); + String socketPath = state.containerEntry + .tempDir + .resolve("socket.sock") + .toAbsolutePath() + .toString(); state.builder.socketPath(socketPath); state.builder.port(UNAVAILABLE_PORT); break; case "ssl": String path = "test-harness/ssl/custom-root-cert.crt"; - File file = new File(path); String absolutePath = file.getAbsolutePath(); this.state.providerType = ProviderType.SSL; state.builder - .port(ContainerUtil.getPort(container, State.resolverType)) + .port(ContainerUtil.getPort(container, state.resolverType)) .tls(true) .certPath(absolutePath); flagdConfig = "ssl"; break; case "metadata": flagdConfig = "metadata"; - - if (State.resolverType == Config.Resolver.FILE) { + if (state.resolverType == Config.Resolver.FILE) { FlagdOptions build = state.builder.build(); String selector = build.getSelector(); String replace = selector.replace("rawflags/", ""); - state.builder .port(UNAVAILABLE_PORT) .offlineFlagSourcePath(new File("test-harness/flags/" + replace).getAbsolutePath()); } else { - state.builder.port(ContainerUtil.getPort(container, State.resolverType)); + state.builder.port(ContainerUtil.getPort(container, state.resolverType)); } break; case "syncpayload": flagdConfig = "sync-payload"; - state.builder.port(ContainerUtil.getPort(container, State.resolverType)); + state.builder.port(ContainerUtil.getPort(container, state.resolverType)); break; case "stable": this.state.providerType = ProviderType.DEFAULT; - if (State.resolverType == Config.Resolver.FILE) { - + if (state.resolverType == Config.Resolver.FILE) { state.builder .port(UNAVAILABLE_PORT) - .offlineFlagSourcePath(sharedTempDir + .offlineFlagSourcePath(state.containerEntry + .tempDir .resolve("allFlags.json") .toAbsolutePath() .toString()); } else { - state.builder.port(ContainerUtil.getPort(container, State.resolverType)); + state.builder.port(ContainerUtil.getPort(container, state.resolverType)); } break; default: @@ -162,10 +162,26 @@ public void setupProvider(String providerType) throws InterruptedException { .then() .statusCode(200); - Thread.sleep(50); + // For FILE resolver, the provider reads from a file written by the launchpad. + // Under parallel load the write may not complete within a fixed sleep — poll instead. + if (state.resolverType == Config.Resolver.FILE) { + FlagdOptions built = state.builder.build(); + String filePath = built.getOfflineFlagSourcePath(); + if (filePath != null) { + java.io.File flagFile = new java.io.File(filePath); + long deadline = System.currentTimeMillis() + 5_000; + while ((!flagFile.exists() || flagFile.length() == 0) && System.currentTimeMillis() < deadline) { + Thread.sleep(50); + } + } + } else { + // The launchpad polls /readyz before returning, but gRPC ports may lag slightly. + // Wait for the actual gRPC port to accept connections to prevent init timeouts. + ContainerUtil.waitForGrpcPort(container, state.resolverType, 5_000); + } FeatureProvider provider = - new FlagdProvider(state.builder.resolverType(State.resolverType).build()); + new FlagdProvider(state.builder.resolverType(state.resolverType).build()); String providerName = "Provider " + Math.random(); OpenFeatureAPI api = OpenFeatureAPI.getInstance(); @@ -174,26 +190,35 @@ public void setupProvider(String providerType) throws InterruptedException { } else { api.setProvider(providerName, provider); } + this.state.provider = provider; + this.state.providerName = providerName; this.state.client = api.getClient(providerName); } @When("the connection is lost") - public void the_connection_is_lost() { - when().post("http://" + ContainerUtil.getLaunchpadUrl(container) + "/stop") + public void the_connection_is_lost() throws InterruptedException { + ContainerPool.acquireRestartSlot(); + state.restartSlotAcquired = true; + when().post("http://" + ContainerUtil.getLaunchpadUrl(state.containerEntry.container) + "/stop") .then() .statusCode(200); } @When("the connection is lost for {int}s") - public void the_connection_is_lost_for(int seconds) { - when().post("http://" + ContainerUtil.getLaunchpadUrl(container) + "/restart?seconds={seconds}", seconds) + public void the_connection_is_lost_for(int seconds) throws InterruptedException { + ContainerPool.acquireRestartSlot(); + state.restartSlotAcquired = true; + when().post( + "http://" + ContainerUtil.getLaunchpadUrl(state.containerEntry.container) + + "/restart?seconds={seconds}", + seconds) .then() .statusCode(200); } @When("the flag was modified") public void the_flag_was_modded() { - when().post("http://" + ContainerUtil.getLaunchpadUrl(container) + "/change") + when().post("http://" + ContainerUtil.getLaunchpadUrl(state.containerEntry.container) + "/change") .then() .statusCode(200); } diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/resolver/file/FileSetup.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/resolver/file/FileSetup.java new file mode 100644 index 000000000..3a86c9e84 --- /dev/null +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/resolver/file/FileSetup.java @@ -0,0 +1,19 @@ +package dev.openfeature.contrib.providers.flagd.e2e.steps.resolver.file; + +import dev.openfeature.contrib.providers.flagd.Config; +import dev.openfeature.contrib.providers.flagd.e2e.State; +import io.cucumber.java.Before; + +public class FileSetup { + + private final State state; + + public FileSetup(State state) { + this.state = state; + } + + @Before + public void setup() { + state.resolverType = Config.Resolver.FILE; + } +} diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/resolver/inprocess/InProcessSetup.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/resolver/inprocess/InProcessSetup.java new file mode 100644 index 000000000..5d033b2c1 --- /dev/null +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/resolver/inprocess/InProcessSetup.java @@ -0,0 +1,19 @@ +package dev.openfeature.contrib.providers.flagd.e2e.steps.resolver.inprocess; + +import dev.openfeature.contrib.providers.flagd.Config; +import dev.openfeature.contrib.providers.flagd.e2e.State; +import io.cucumber.java.Before; + +public class InProcessSetup { + + private final State state; + + public InProcessSetup(State state) { + this.state = state; + } + + @Before + public void setup() { + state.resolverType = Config.Resolver.IN_PROCESS; + } +} diff --git a/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/resolver/rpc/RpcSetup.java b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/resolver/rpc/RpcSetup.java new file mode 100644 index 000000000..93a8e740b --- /dev/null +++ b/providers/flagd/src/test/java/dev/openfeature/contrib/providers/flagd/e2e/steps/resolver/rpc/RpcSetup.java @@ -0,0 +1,19 @@ +package dev.openfeature.contrib.providers.flagd.e2e.steps.resolver.rpc; + +import dev.openfeature.contrib.providers.flagd.Config; +import dev.openfeature.contrib.providers.flagd.e2e.State; +import io.cucumber.java.Before; + +public class RpcSetup { + + private final State state; + + public RpcSetup(State state) { + this.state = state; + } + + @Before + public void setup() { + state.resolverType = Config.Resolver.RPC; + } +} diff --git a/providers/flagd/src/test/resources/junit-platform.properties b/providers/flagd/src/test/resources/junit-platform.properties new file mode 100644 index 000000000..0d0be24ee --- /dev/null +++ b/providers/flagd/src/test/resources/junit-platform.properties @@ -0,0 +1,19 @@ +# Enable parallel scenario execution within each suite runner. +# Each scenario borrows its own ContainerEntry from ContainerPool, so +# concurrent scenarios are fully isolated — no shared flagd process. +cucumber.execution.parallel.enabled=true +# Dynamic strategy scales with available CPUs (factor=1.0 → 1 thread per core). +# ContainerPool caps at min(availableProcessors, 4) containers so Docker isn't +# overwhelmed; extra threads simply block waiting for a free container. +# Override pool size via -Dflagd.e2e.pool.size=N if needed. +cucumber.execution.parallel.config.strategy=dynamic +cucumber.execution.parallel.config.dynamic.factor=1 +# Scenarios tagged @env-var mutate System env vars globally. +# Serialise them behind an exclusive resource lock so concurrent scenarios +# don't clobber each other's environment variable state. +cucumber.execution.exclusive-resources.env-var.read-write=ENV_VARS +# Scenarios tagged @grace involve container restart + reconnection timing. +# Running two concurrent restarts under parallel load can push the +# reconnection past the 12-second EVENT_TIMEOUT_MS threshold. Serialise +# them so each restart has the full machine resources to itself. +cucumber.execution.exclusive-resources.grace.read-write=CONTAINER_RESTART diff --git a/providers/flagd/test-harness b/providers/flagd/test-harness index 3bff4b7ea..f2782788e 160000 --- a/providers/flagd/test-harness +++ b/providers/flagd/test-harness @@ -1 +1 @@ -Subproject commit 3bff4b7eaee0efc8cfe60e0ef6fbd77441b370e6 +Subproject commit f2782788e72633e447b024548cd8a2cbf0c2a026