diff --git a/bin/pest b/bin/pest index 9e6e703c..83c2600e 100755 --- a/bin/pest +++ b/bin/pest @@ -117,6 +117,16 @@ use Symfony\Component\Console\Output\ConsoleOutput; $arguments[] = '--no-output'; unset($_SERVER['COLLISION_PRINTER']); } + + if (str_contains($value, '--attempts=')) { + unset($arguments[$key]); + } elseif ($value === '--attempts') { + unset($arguments[$key]); + + if (isset($arguments[$key + 1])) { + unset($arguments[$key + 1]); + } + } } // Used when Pest is required using composer. @@ -174,6 +184,10 @@ use Symfony\Component\Console\Output\ConsoleOutput; $testSuite->tests->addTestCaseMethodFilter(new PrTestCaseFilter((int) $pr)); } + if ($attempts = $input->getParameterOption('--attempts')) { + $testSuite->attempts = max(1, (int) $attempts); + } + $isDecorated = $input->getParameterOption('--colors', 'always') !== 'never'; $output = new ConsoleOutput(ConsoleOutput::VERBOSITY_NORMAL, $isDecorated); diff --git a/src/Concerns/Testable.php b/src/Concerns/Testable.php index 767a7c69..2c06a91a 100644 --- a/src/Concerns/Testable.php +++ b/src/Concerns/Testable.php @@ -328,7 +328,58 @@ private function __runTest(Closure $closure, ...$args): mixed $arguments = $this->__resolveTestArguments($args); $this->__ensureDatasetArgumentNameAndNumberMatches($arguments); - return $this->__callClosure($closure, $arguments); + return $this->__attemptTestRun(fn () => $this->__callClosure($closure, $arguments)); + } + + private function __attemptTestRun(Closure $closure) + { + $method = TestSuite::getInstance()->tests->get(self::$__filename)->getMethod($this->name()); + $maxAttempts = max(1, max($method->attempts, TestSuite::getInstance()->attempts)); + + $lastException = null; + $attemptNumber = 1; + + while ($attemptNumber <= $maxAttempts) { + try { + $result = $closure(); + + // If we succeeded on a retry attempt, track it as flaky + if ($attemptNumber > 1) { + \Pest\Support\FlakyTestTracker::getInstance()->track( + $this->name(), + $attemptNumber, + true + ); + } + + return $result; + } catch (Throwable $e) { + $lastException = $e; + $attemptNumber++; + + // If we've exhausted all attempts, track and re-throw + if ($attemptNumber > $maxAttempts) { + if ($maxAttempts > 1) { + \Pest\Support\FlakyTestTracker::getInstance()->track( + $this->name(), + $maxAttempts, + false + ); + } + + throw $e; + } + + // Otherwise, continue to next attempt + } + } + + // This should never be reached, but satisfies the return type + if ($lastException !== null) { + throw $lastException; + } + + return null; } /** diff --git a/src/Factories/TestCaseMethodFactory.php b/src/Factories/TestCaseMethodFactory.php index c88db44d..39cf0b7d 100644 --- a/src/Factories/TestCaseMethodFactory.php +++ b/src/Factories/TestCaseMethodFactory.php @@ -46,6 +46,11 @@ final class TestCaseMethodFactory */ public int $repetitions = 1; + /** + * The test's maximum number of retry attempts. + */ + public int $attempts = 1; + /** * Determines if the test is a "todo". */ diff --git a/src/PendingCalls/TestCall.php b/src/PendingCalls/TestCall.php index a65c8bb5..0da3a8ea 100644 --- a/src/PendingCalls/TestCall.php +++ b/src/PendingCalls/TestCall.php @@ -408,6 +408,20 @@ public function repeat(int $times): self return $this; } + /** + * Sets the maximum number of retry attempts for this test. + */ + public function attempts(int $times): self + { + if ($times < 1) { + throw new InvalidArgumentException('The number of attempts must be greater than 0.'); + } + + $this->testCaseMethod->attempts = $times; + + return $this; + } + /** * Marks the test as "todo". */ diff --git a/src/Support/FlakyTestTracker.php b/src/Support/FlakyTestTracker.php new file mode 100644 index 00000000..9d262914 --- /dev/null +++ b/src/Support/FlakyTestTracker.php @@ -0,0 +1,91 @@ + + */ + private array $flakyTests = []; + + /** + * Private constructor to prevent direct instantiation. + */ + private function __construct() + { + // + } + + /** + * Get the singleton instance. + */ + public static function getInstance(): self + { + if (self::$instance === null) { + self::$instance = new self; + } + + return self::$instance; + } + + /** + * Track a test that required retry attempts. + */ + public function track(string $testName, int $attempts, bool $passed): void + { + // Only track if the test required more than 1 attempt + if ($attempts > 1) { + $this->flakyTests[$testName] = [ + 'attempts' => $attempts, + 'passed' => $passed, + ]; + } + } + + /** + * Get all tracked flaky tests. + * + * @return array + */ + public function getFlakyTests(): array + { + return $this->flakyTests; + } + + /** + * Check if there are any tracked flaky tests. + */ + public function hasFlakyTests(): bool + { + return count($this->flakyTests) > 0; + } + + /** + * Clear all tracked tests. + */ + public function clear(): void + { + $this->flakyTests = []; + } + + /** + * Reset the singleton instance (useful for testing). + */ + public static function reset(): void + { + self::$instance = null; + } +} diff --git a/src/TestSuite.php b/src/TestSuite.php index df17ec2d..99342231 100644 --- a/src/TestSuite.php +++ b/src/TestSuite.php @@ -59,6 +59,11 @@ final class TestSuite */ public string $rootPath; + /** + * Holds the global maximum attempts for tests. + */ + public int $attempts = 1; + /** * Holds an instance of the test suite. */ diff --git a/tests/Features/Attempts.php b/tests/Features/Attempts.php new file mode 100644 index 00000000..03aec634 --- /dev/null +++ b/tests/Features/Attempts.php @@ -0,0 +1,16 @@ +toEqual(2); + if ($attempts == 2) { + unlink($path); + } +})->attempts(2);