From 4e0ba3dd224b61ff5432526bf408583be11100de Mon Sep 17 00:00:00 2001 From: Dawid Wysakowicz Date: Wed, 4 May 2022 16:46:31 +0200 Subject: [PATCH] [FLINK-27489] Allow users to run dedicated tests in the CI --- src/main/java/com/ververica/Core.java | 56 ++++++++++++++++++- src/main/java/com/ververica/ci/CiActions.java | 3 +- .../ververica/ci/azure/AzureActionsImpl.java | 42 +++++++++++++- 3 files changed, 98 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/ververica/Core.java b/src/main/java/com/ververica/Core.java index 9e719fd..1703bdb 100644 --- a/src/main/java/com/ververica/Core.java +++ b/src/main/java/com/ververica/Core.java @@ -80,8 +80,8 @@ public class Core implements AutoCloseable { "ci_(?<" + REGEX_GROUP_PULL_REQUEST_ID + ">[0-9]+)_(?<" + REGEX_GROUP_COMMIT_HASH + ">[0-9a-f]+)", Pattern.DOTALL); private static final String REGEX_GROUP_COMMAND = "command"; - private static final Pattern REGEX_PATTERN_COMMAND_MENTION = Pattern.compile("@flinkbot run (?<" + REGEX_GROUP_COMMAND + ">[\\w ]+)", Pattern.CASE_INSENSITIVE); + private static final Pattern REGEX_PATTERN_COMMAND_MENTION = Pattern.compile("@flinkbot run (?<" + REGEX_GROUP_COMMAND + ">[\\w .#]+)", Pattern.CASE_INSENSITIVE); private static final DateFormat dateFormat = new SimpleDateFormat("HH:mm:ss"); private final String observedRepository; @@ -427,9 +427,11 @@ private Optional runManualBuild(Trigger trigger, CiReport ciReport) { final String[] command = trigger.getCommand().get().split(" "); final AzureCommand azureCommand = new AzureCommand(); + final TestCommand testCommand = new TestCommand(); JCommander jCommander = new JCommander(); jCommander.addCommand(azureCommand); + jCommander.addCommand(testCommand); try { jCommander.parse(command); @@ -441,6 +443,9 @@ private Optional runManualBuild(Trigger trigger, CiReport ciReport) { switch (jCommander.getParsedCommand()) { case AzureCommand.COMMAND_NAME: return runManualBuild(CiProvider.Azure, ciReport, trigger, azureCommand.args); + case TestCommand.COMMAND_NAME: + runTestBuild(CiProvider.Azure, ciReport, testCommand.testPattern); + return Optional.empty(); default: throw new RuntimeException("Unhandled valid command " + Arrays.toString(command) + " ."); } @@ -480,6 +485,48 @@ private Optional runManualBuild(CiProvider ciProvider, CiReport ciReport, return Optional.empty(); } + private void runTestBuild( + CiProvider ciProvider, + CiReport ciReport, + String testPattern) { + Optional lastBuildOptional = ciReport.getBuilds() + .filter(build -> build.status.map(s -> s.getCiProvider() == ciProvider).orElse(false)) + .reduce((first, second) -> second); + if (!lastBuildOptional.isPresent()) { + LOG.debug("Ignoring {} run command since no build was triggered yet.", ciProvider.getName()); + } else { + Build lastBuild = lastBuildOptional.get(); + if (!lastBuild.status.isPresent()) { + LOG.debug("Ignoring {} run command since no build was triggered yet.", ciProvider.getName()); + } else { + GitHubCheckerStatus gitHubCheckerStatus = lastBuild.status.get(); + + final Optional buildUrl = ciActions.getActionsForProvider(gitHubCheckerStatus.getCiProvider()) + .flatMap(ciAction -> ciAction.buildTest( + gitHubCheckerStatus.getDetailsUrl(), + lastBuild.commitHash, + testPattern) + ); + buildUrl.ifPresent( + url -> { + try { + final int pullRequestID = lastBuild.pullRequestID; + LOG.debug( + "Submitting new test run comment for pull request {}", + formatPullRequestID(pullRequestID)); + gitHubActions.submitComment( + observedRepository, + pullRequestID, + String.format("Submitted test run for pattern %s: %s", testPattern, url)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + ); + } + } + } + public void mirrorPullRequest(int pullRequestID) throws GitException { if (pendingMirrors.getIfPresent(pullRequestID) != null) { LOG.debug("Ignoring mirroring for {} due to being cached.", formatPullRequestID(pullRequestID)); @@ -537,4 +584,11 @@ private static final class AzureCommand { @Parameter private List args = Collections.emptyList(); } + + @Parameters(commandNames = TestCommand.COMMAND_NAME) + private static final class TestCommand { + static final String COMMAND_NAME = "test"; + @Parameter + private String testPattern = null; + } } diff --git a/src/main/java/com/ververica/ci/CiActions.java b/src/main/java/com/ververica/ci/CiActions.java index 0a6e166..79b31ce 100644 --- a/src/main/java/com/ververica/ci/CiActions.java +++ b/src/main/java/com/ververica/ci/CiActions.java @@ -19,7 +19,6 @@ import com.ververica.github.GitHubCheckerStatus; -import java.util.List; import java.util.Optional; public interface CiActions extends AutoCloseable { @@ -31,6 +30,8 @@ public interface CiActions extends AutoCloseable { void retryBuild(String detailsUrl, String branch); + Optional buildTest(String detailsUrl, String branch, String testPattern); + default boolean supportsDirectBuildStatusRetrieval() { return false; } diff --git a/src/main/java/com/ververica/ci/azure/AzureActionsImpl.java b/src/main/java/com/ververica/ci/azure/AzureActionsImpl.java index 78f14ac..ad2ec8c 100644 --- a/src/main/java/com/ververica/ci/azure/AzureActionsImpl.java +++ b/src/main/java/com/ververica/ci/azure/AzureActionsImpl.java @@ -32,7 +32,6 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; import java.util.Base64; import java.util.Optional; @@ -131,6 +130,47 @@ public void retryBuild(String detailsUrl, String branch) { RequestBody.create(MediaType.get("application/json"), "{}")); } + @Override + public Optional buildTest(String detailsUrl, String commitId, String testPattern) { + LOG.debug("Triggering build for test {}.", testPattern); + + final String projectSlug = extractProjectSlug(detailsUrl); + final String buildId = extractBuildId(detailsUrl); + + Optional definitionId = getDefinitionId(projectSlug, buildId); + if (!definitionId.isPresent()) { + LOG.error("Failed to trigger build; could not retrieve definition id."); + return Optional.empty(); + } + return submitRequest( + "https://dev.azure.com/" + projectSlug + "/_apis/pipelines/" + definitionId.get() + + "/runs?api-version=6.0-preview.1", + "POST", + RequestBody.create( + MediaType.get("application/json"), + "{" + + "\"stagesToSkip\":[]," + + "\"resources\":" + + "{\"repositories\":" + + "{\"self\":" + + "{\"version\":\"" + commitId + "\"}}}," + + "\"variables\":" + + "{\"MODE\":{\"value\":\"single-test\",\"isSecret\":false}," + + "\"TEST_PATTERN\":{\"value\":\"" + testPattern + "\",\"isSecret\":false}}}")) + .flatMap( + response -> { + try { + final JsonNode json = OBJECT_MAPPER.readTree(response); + final String url = json.get("_links").get("web").get("href").asText(); + LOG.debug("Running tests for {} with pattern {}: {}", commitId, testPattern, url); + return Optional.ofNullable(url); + } catch (IOException e) { + return Optional.empty(); + } + } + ); + } + @Override public boolean supportsDirectBuildStatusRetrieval() { return true;