Add retry mechanism for workflow runs

culprit_finder/README.md

@@ -57,6 +57,7 @@ culprit-finder --repo <OWNER/REPO> --start <GOOD_SHA> --end <BAD_SHA> --workflow
 - `--start`: The full or short SHA of the last known **good** commit.
 - `--end`: The full or short SHA of the first known **bad** commit.
 - `--workflow`: The filename of the GitHub Actions workflow to run (e.g., `ci.yml`, `tests.yaml`).
+- `--retry`: (Optional) Number of times to retry the workflow run if it fails (default: 0).
 
 ### Example
 

culprit_finder/src/culprit_finder/cli.py

@@ -49,6 +49,13 @@ def main() -> None:
     required=True,
     help="Workflow filename (e.g., build_and_test.yml)",
   )
+  parser.add_argument(
+    "--retry",
+    required=False,
+    help="Number of times to retry the workflow run if it fails (default: 0).",
+    default=0,
+    type=int,
+  )
 
   args = parser.parse_args()
 
@@ -65,6 +72,7 @@ def main() -> None:
   logging.info("Start commit: %s", args.start)
   logging.info("End commit: %s", args.end)
   logging.info("Workflow: %s", args.workflow)
+  logging.info("Retries: %s", args.retry)
 
   has_culprit_finder_workflow = any(
     wf["path"] == ".github/workflows/culprit_finder.yml"
@@ -80,6 +88,7 @@ def main() -> None:
     workflow_file=args.workflow,
     has_culprit_finder_workflow=has_culprit_finder_workflow,
     github_client=gh_client,
+    retries=args.retry,
   )
   culprit_commit = finder.run_bisection()
   if culprit_commit:

culprit_finder/src/culprit_finder/culprit_finder.py

@@ -24,6 +24,7 @@ def __init__(
     workflow_file: str,
     has_culprit_finder_workflow: bool,
     github_client: github.GithubClient,
+    retries: int = 0,
   ):
     """
     Initializes the CulpritFinder instance.
@@ -34,6 +35,7 @@ def __init__(
         end_sha: The SHA of the first known bad commit.
         workflow_file: The name of the workflow file to test (e.g., 'build.yml').
         has_culprit_finder_workflow: Whether the repo being tested has a Culprit Finder workflow.
+        retries: Number of times to retry workflow runs in case of failure.
     """
     self._repo = repo
     self._start_sha = start_sha
@@ -42,6 +44,7 @@ def __init__(
     self._workflow_file = workflow_file
     self._has_culprit_finder_workflow = has_culprit_finder_workflow
     self._gh_client = github_client
+    self._retries = retries
 
   def _wait_for_workflow_completion(
     self,
@@ -137,14 +140,28 @@ def _test_commit(
       inputs,
     )
 
-    run = self._wait_for_workflow_completion(
-      workflow_to_trigger,
-      branch_name,
-      commit_sha,
-      previous_run_id,
-    )
+    run: github.Run | None = None
+    for attempt in range(self._retries + 1):
+      run = self._wait_for_workflow_completion(
+        workflow_to_trigger,
+        branch_name,
+        commit_sha,
+        previous_run_id,
+      )
+
+      if run and run["conclusion"] == "success":
+        return True
+
+      if attempt < self._retries:
+        logging.info(
+          "Retrying workflow for commit %s (attempt %d/%d)",
+          commit_sha,
+          attempt + 1,
+          self._retries,
+        )
+
     if not run:
-      logging.error("Workflow failed to complete")
+      logging.error("Workflow failed to complete for commit %s", commit_sha)
       return False
 
     return run["conclusion"] == "success"

culprit_finder/tests/test_cli.py

@@ -195,6 +195,7 @@ def test_cli_success(
     workflow_file="test.yml",
     has_culprit_finder_workflow=has_culprit_workflow,
     github_client=mock_gh_client_instance,
+    retries=0,
   )
   mock_finder.return_value.run_bisection.assert_called_once()
 

culprit_finder/tests/test_culprit_finder.py

@@ -73,6 +73,36 @@ def test_wait_for_workflow_completion_success(mocker, finder, mock_gh_client):
     assert call_args[0][0] == workflow
 
 
+def test_test_commit_with_retries(mocker, mock_gh_client):
+  """Tests that _test_commit retries the specified number of times on failure."""
+  mocker.patch("culprit_finder.culprit_finder.github")
+
+  finder = culprit_finder.CulpritFinder(
+    repo=REPO,
+    start_sha="start_sha",
+    end_sha="end_sha",
+    workflow_file=WORKFLOW_FILE,
+    has_culprit_finder_workflow=True,
+    github_client=mock_gh_client,
+    retries=2,
+  )
+
+  branch = "test-branch"
+  commit_sha = "sha1"
+
+  mock_wait = mocker.patch.object(finder, "_wait_for_workflow_completion")
+  mock_wait.side_effect = [
+    {"conclusion": "failure"},  # Initial attempt
+    {"conclusion": "failure"},  # First retry
+    {"conclusion": "success"},  # Second retry
+  ]
+
+  is_good = finder._test_commit(commit_sha, branch)
+
+  assert is_good is True
+  assert mock_wait.call_count == 3
+
+
 @pytest.mark.parametrize("finder", [True, False], indirect=True)
 def test_test_commit_success(mocker, finder, mock_gh_client):
   """Tests that _test_commit triggers the workflow and returns True on success."""