From f0ea7d51ebbbcebbab302772b8b721ce476770c0 Mon Sep 17 00:00:00 2001 From: Andreea Neamt Date: Wed, 8 Oct 2025 14:23:15 +0300 Subject: [PATCH 01/19] added github workflow for end to end integration tests --- .github/workflows/e2e-tests.yaml | 68 ++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 .github/workflows/e2e-tests.yaml diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml new file mode 100644 index 000000000..35522332c --- /dev/null +++ b/.github/workflows/e2e-tests.yaml @@ -0,0 +1,68 @@ +name: E2E Tests + +on: + pull_request: + types: [opened, synchronize, reopened] + +jobs: + e2e-tests: + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + - name: Wait for Netlify Preview + uses: jakepartusch/wait-for-netlify-action@v1.4 + id: netlify + with: + site_name: "superhero" + max_timeout: 600 + + - name: Clone keypair-testing repo + uses: actions/checkout@v4 + with: + repository: aeternity/keypair-testing + path: keypair-testing + token: ${{ secrets.KEYPAIR_TESTING_TOKEN }} + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'npm' + cache-dependency-path: keypair-testing/package-lock.json + + - name: Install dependencies + working-directory: keypair-testing + run: npm ci + + - name: Install Playwright browsers + working-directory: keypair-testing + run: npx playwright install --with-deps chromium + + - name: Run Superhero React E2E tests + working-directory: keypair-testing + env: + SUPERHERO_COM_BASE_URL: ${{ steps.netlify.outputs.url }} + run: npx playwright test tests_superhero_react/superhero-react.spec.ts --project="Web Chrome" + + - name: Upload Playwright report + if: always() + uses: actions/upload-artifact@v4 + with: + name: playwright-report-${{ github.event.pull_request.number }} + path: keypair-testing/playwright-report/ + retention-days: 30 + + - name: Comment PR with test results + if: always() + uses: actions/github-script@v7 + with: + script: | + const status = '${{ job.status }}' === 'success' ? 'āœ… E2E Tests PASSED' : 'āŒ E2E Tests FAILED'; + + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: `## ${status}\n\n**Tested URL:** ${{ steps.netlify.outputs.url }}\n\nšŸ“Š [View Full Report](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})` + }); \ No newline at end of file From 3b2b17bde29a5bea2039dc0b723d4908d6e7e527 Mon Sep 17 00:00:00 2001 From: Andreea Neamt Date: Wed, 8 Oct 2025 15:21:27 +0300 Subject: [PATCH 02/19] fix: improve Netlify preview wait logic --- .github/workflows/e2e-tests.yaml | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml index 35522332c..3cc0c1c81 100644 --- a/.github/workflows/e2e-tests.yaml +++ b/.github/workflows/e2e-tests.yaml @@ -7,15 +7,37 @@ on: jobs: e2e-tests: runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 15 steps: - name: Wait for Netlify Preview - uses: jakepartusch/wait-for-netlify-action@v1.4 id: netlify - with: - site_name: "superhero" - max_timeout: 600 + run: | + PR_NUMBER=${{ github.event.pull_request.number }} + DEPLOY_URL="https://deploy-preview-${PR_NUMBER}--superhero.netlify.app" + + echo "Waiting for Netlify deployment at: $DEPLOY_URL" + + SECONDS=0 + MAX_WAIT=600 + WAIT_INTERVAL=15 + + while [ $SECONDS -lt $MAX_WAIT ]; do + HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "$DEPLOY_URL" || echo "000") + + if [ "$HTTP_STATUS" = "200" ]; then + echo "āœ… Netlify deployment is ready!" + echo "url=$DEPLOY_URL" >> $GITHUB_OUTPUT + exit 0 + fi + + echo "Deployment not ready yet (HTTP $HTTP_STATUS). Waiting ${WAIT_INTERVAL}s..." + sleep $WAIT_INTERVAL + SECONDS=$((SECONDS + WAIT_INTERVAL)) + done + + echo "āŒ Deployment did not become ready within ${MAX_WAIT}s" + exit 1 - name: Clone keypair-testing repo uses: actions/checkout@v4 From 6063c49612b6806da13f378058219678edd0a945 Mon Sep 17 00:00:00 2001 From: Andreea Neamt Date: Wed, 8 Oct 2025 15:31:28 +0300 Subject: [PATCH 03/19] added specific link --- .github/workflows/e2e-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml index 3cc0c1c81..15adc0898 100644 --- a/.github/workflows/e2e-tests.yaml +++ b/.github/workflows/e2e-tests.yaml @@ -14,7 +14,7 @@ jobs: id: netlify run: | PR_NUMBER=${{ github.event.pull_request.number }} - DEPLOY_URL="https://deploy-preview-${PR_NUMBER}--superhero.netlify.app" + DEPLOY_URL="https://deploy-preview-${PR_NUMBER}--superhero-c9df18.netlify.app" echo "Waiting for Netlify deployment at: $DEPLOY_URL" From fe247307a4eef0195620b77f82dd3372c1a7bf7f Mon Sep 17 00:00:00 2001 From: Andreea Neamt Date: Wed, 8 Oct 2025 16:05:25 +0300 Subject: [PATCH 04/19] added shown name of failed test --- .github/workflows/e2e-tests.yaml | 36 +++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml index 15adc0898..5a9f7b207 100644 --- a/.github/workflows/e2e-tests.yaml +++ b/.github/workflows/e2e-tests.yaml @@ -62,11 +62,32 @@ jobs: run: npx playwright install --with-deps chromium - name: Run Superhero React E2E tests + id: test_run + continue-on-error: true working-directory: keypair-testing env: SUPERHERO_COM_BASE_URL: ${{ steps.netlify.outputs.url }} - run: npx playwright test tests_superhero_react/superhero-react.spec.ts --project="Web Chrome" + run: | + npx playwright test tests_superhero_react/superhero-react.spec.ts --project="Web Chrome" --retries=1 --config=./playwright.config.ts --workers=3 2>&1 | tee test-output.txt || true + - name: Extract failed tests + if: always() + id: failed_tests + working-directory: keypair-testing + run: | + if [ -f "test-output.txt" ]; then + FAILED=$(grep -E "^\s+[0-9]+\)" test-output.txt | grep "✘" | sed 's/^[[:space:]]*[0-9]*)[[:space:]]*//' | head -10 || echo "") + if [ -n "$FAILED" ]; then + echo "failed<> $GITHUB_OUTPUT + echo "$FAILED" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + else + echo "failed=" >> $GITHUB_OUTPUT + fi + else + echo "failed=" >> $GITHUB_OUTPUT + fi + - name: Upload Playwright report if: always() uses: actions/upload-artifact@v4 @@ -80,11 +101,20 @@ jobs: uses: actions/github-script@v7 with: script: | - const status = '${{ job.status }}' === 'success' ? 'āœ… E2E Tests PASSED' : 'āŒ E2E Tests FAILED'; + const status = '${{ steps.test_run.outcome }}' === 'success' ? 'āœ… E2E Tests PASSED' : 'āŒ E2E Tests FAILED'; + const failedTests = `${{ steps.failed_tests.outputs.failed }}`.trim(); + + let body = `## ${status}\n\n**Tested URL:** ${{ steps.netlify.outputs.url }}\n\n`; + + if (failedTests) { + body += `### āŒ Failed Tests:\n\`\`\`\n${failedTests}\n\`\`\`\n\n`; + } + + body += `šŸ“Š [View Full Report](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})`; github.rest.issues.createComment({ issue_number: context.issue.number, owner: context.repo.owner, repo: context.repo.repo, - body: `## ${status}\n\n**Tested URL:** ${{ steps.netlify.outputs.url }}\n\nšŸ“Š [View Full Report](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})` + body: body }); \ No newline at end of file From 6374af380fb69fc7d76a7ae95faec7e475b96ab3 Mon Sep 17 00:00:00 2001 From: Andreea Neamt Date: Wed, 8 Oct 2025 16:50:41 +0300 Subject: [PATCH 05/19] added shown name of failed test - try2 --- .github/workflows/e2e-tests.yaml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml index 5a9f7b207..ca5390451 100644 --- a/.github/workflows/e2e-tests.yaml +++ b/.github/workflows/e2e-tests.yaml @@ -7,7 +7,7 @@ on: jobs: e2e-tests: runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 20 steps: - name: Wait for Netlify Preview @@ -67,19 +67,20 @@ jobs: working-directory: keypair-testing env: SUPERHERO_COM_BASE_URL: ${{ steps.netlify.outputs.url }} - run: | - npx playwright test tests_superhero_react/superhero-react.spec.ts --project="Web Chrome" --retries=1 --config=./playwright.config.ts --workers=3 2>&1 | tee test-output.txt || true + run: npx playwright test tests_superhero_react/superhero-react.spec.ts --project="Web Chrome" --retries=1 --config=./playwright.config.ts --reporter=html --reporter=list --reporter=json:test-results.json || true - name: Extract failed tests if: always() id: failed_tests working-directory: keypair-testing run: | - if [ -f "test-output.txt" ]; then - FAILED=$(grep -E "^\s+[0-9]+\)" test-output.txt | grep "✘" | sed 's/^[[:space:]]*[0-9]*)[[:space:]]*//' | head -10 || echo "") - if [ -n "$FAILED" ]; then + if [ -f "test-results.json" ]; then + # Extract titles of failed or timedOut tests (limit to 10 for brevity) + FAILED_TESTS=$(jq -r '.suites[]?.specs[]? | select(.tests[]?.results[]?.status == "failed" or .tests[]?.results[]?.status == "timedOut") | .title' test-results.json | head -10) + + if [ -n "$FAILED_TESTS" ]; then echo "failed<> $GITHUB_OUTPUT - echo "$FAILED" >> $GITHUB_OUTPUT + echo "$FAILED_TESTS" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT else echo "failed=" >> $GITHUB_OUTPUT From cde41c83ec6d81a30c032d0984b1a66ecc8bab1f Mon Sep 17 00:00:00 2001 From: Andreea Neamt Date: Wed, 8 Oct 2025 17:01:57 +0300 Subject: [PATCH 06/19] fixed result file json --- .github/workflows/e2e-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml index ca5390451..54ed7bb7a 100644 --- a/.github/workflows/e2e-tests.yaml +++ b/.github/workflows/e2e-tests.yaml @@ -67,7 +67,7 @@ jobs: working-directory: keypair-testing env: SUPERHERO_COM_BASE_URL: ${{ steps.netlify.outputs.url }} - run: npx playwright test tests_superhero_react/superhero-react.spec.ts --project="Web Chrome" --retries=1 --config=./playwright.config.ts --reporter=html --reporter=list --reporter=json:test-results.json || true + run: npx playwright test tests_superhero_react/superhero-react.spec.ts --project="Web Chrome" --retries=1 --config=./playwright.config.ts --reporter=html --reporter=list --reporter=json:outputFile=test-results.json || true - name: Extract failed tests if: always() From c1ed5d4f96a9e5ec871a5000d1f8899412e63593 Mon Sep 17 00:00:00 2001 From: Andreea Neamt Date: Wed, 8 Oct 2025 17:17:31 +0300 Subject: [PATCH 07/19] implemented with list report for shown the failed test names --- .github/workflows/e2e-tests.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml index 54ed7bb7a..5ecb619a3 100644 --- a/.github/workflows/e2e-tests.yaml +++ b/.github/workflows/e2e-tests.yaml @@ -67,16 +67,16 @@ jobs: working-directory: keypair-testing env: SUPERHERO_COM_BASE_URL: ${{ steps.netlify.outputs.url }} - run: npx playwright test tests_superhero_react/superhero-react.spec.ts --project="Web Chrome" --retries=1 --config=./playwright.config.ts --reporter=html --reporter=list --reporter=json:outputFile=test-results.json || true + run: npx playwright test tests_superhero_react/superhero-react.spec.ts --project="Web Chrome" --retries=1 --config=./playwright.config.ts --reporter=html --reporter=list 2>&1 | tee test-output.log || true - name: Extract failed tests if: always() id: failed_tests working-directory: keypair-testing run: | - if [ -f "test-results.json" ]; then - # Extract titles of failed or timedOut tests (limit to 10 for brevity) - FAILED_TESTS=$(jq -r '.suites[]?.specs[]? | select(.tests[]?.results[]?.status == "failed" or .tests[]?.results[]?.status == "timedOut") | .title' test-results.json | head -10) + if [ -f "test-output.log" ]; then + # Extract failed test names from list reporter output + FAILED_TESTS=$(grep "✘" test-output.log | sed 's/^.*\[.*\]//' | sed 's/^[[:space:]]*//' | head -10) if [ -n "$FAILED_TESTS" ]; then echo "failed<> $GITHUB_OUTPUT From d8bd5f70de5dfad1ee74a8af4aeb7a7f2bf4d180 Mon Sep 17 00:00:00 2001 From: Andreea Neamt Date: Wed, 8 Oct 2025 17:37:00 +0300 Subject: [PATCH 08/19] implemented with list report for shown the failed test names -2 --- .github/workflows/e2e-tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml index 5ecb619a3..7b543ec5a 100644 --- a/.github/workflows/e2e-tests.yaml +++ b/.github/workflows/e2e-tests.yaml @@ -57,7 +57,7 @@ jobs: working-directory: keypair-testing run: npm ci - - name: Install Playwright browsers + - name: Install Playwright browsers (Only Chrome) working-directory: keypair-testing run: npx playwright install --with-deps chromium @@ -111,7 +111,7 @@ jobs: body += `### āŒ Failed Tests:\n\`\`\`\n${failedTests}\n\`\`\`\n\n`; } - body += `šŸ“Š [View Full Report](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})`; + body += `šŸ“Š[View Full Report](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})`; github.rest.issues.createComment({ issue_number: context.issue.number, From eaf48b1b544a608186dda68098b70533b676fff9 Mon Sep 17 00:00:00 2001 From: Andreea Neamt Date: Wed, 8 Oct 2025 18:03:29 +0300 Subject: [PATCH 09/19] trigger workflow From cead6a03bd4b3f9e4ddb189cbbd96ce326dcd214 Mon Sep 17 00:00:00 2001 From: Andreea Neamt Date: Wed, 8 Oct 2025 19:26:18 +0300 Subject: [PATCH 10/19] fixed flow to failed if at least one test failed --- .github/workflows/e2e-tests.yaml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml index 7b543ec5a..f634bfba0 100644 --- a/.github/workflows/e2e-tests.yaml +++ b/.github/workflows/e2e-tests.yaml @@ -67,7 +67,9 @@ jobs: working-directory: keypair-testing env: SUPERHERO_COM_BASE_URL: ${{ steps.netlify.outputs.url }} - run: npx playwright test tests_superhero_react/superhero-react.spec.ts --project="Web Chrome" --retries=1 --config=./playwright.config.ts --reporter=html --reporter=list 2>&1 | tee test-output.log || true + run: | + npx playwright test tests_superhero_react/superhero-react.spec.ts --retries=1 --config=./playwright.config.ts --reporter=html --reporter=list 2>&1 | tee test-output.log + echo "exit_code=${PIPESTATUS[0]}" >> $GITHUB_OUTPUT - name: Extract failed tests if: always() @@ -96,13 +98,15 @@ jobs: name: playwright-report-${{ github.event.pull_request.number }} path: keypair-testing/playwright-report/ retention-days: 30 + if-no-files-found: warn - name: Comment PR with test results if: always() uses: actions/github-script@v7 with: script: | - const status = '${{ steps.test_run.outcome }}' === 'success' ? 'āœ… E2E Tests PASSED' : 'āŒ E2E Tests FAILED'; + const exitCode = '${{ steps.test_run.outputs.exit_code }}'; + const status = exitCode === '0' ? 'āœ… E2E Tests PASSED' : 'āŒ E2E Tests FAILED'; const failedTests = `${{ steps.failed_tests.outputs.failed }}`.trim(); let body = `## ${status}\n\n**Tested URL:** ${{ steps.netlify.outputs.url }}\n\n`; @@ -111,11 +115,17 @@ jobs: body += `### āŒ Failed Tests:\n\`\`\`\n${failedTests}\n\`\`\`\n\n`; } - body += `šŸ“Š[View Full Report](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})`; + body += `šŸ“Š [View Full Report](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})`; github.rest.issues.createComment({ issue_number: context.issue.number, owner: context.repo.owner, repo: context.repo.repo, body: body - }); \ No newline at end of file + }); + + - name: Fail workflow if tests failed + if: steps.test_run.outputs.exit_code != '0' + run: | + echo "āŒ E2E tests failed with exit code ${{ steps.test_run.outputs.exit_code }}" + exit 1 \ No newline at end of file From 6d5a88891497c28eec4f515cefb535fe228aed0d Mon Sep 17 00:00:00 2001 From: Andreea Neamt Date: Wed, 8 Oct 2025 19:31:08 +0300 Subject: [PATCH 11/19] fixed flow to failed if at least one test failed --- .github/workflows/e2e-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml index f634bfba0..bbb86fa9d 100644 --- a/.github/workflows/e2e-tests.yaml +++ b/.github/workflows/e2e-tests.yaml @@ -68,7 +68,7 @@ jobs: env: SUPERHERO_COM_BASE_URL: ${{ steps.netlify.outputs.url }} run: | - npx playwright test tests_superhero_react/superhero-react.spec.ts --retries=1 --config=./playwright.config.ts --reporter=html --reporter=list 2>&1 | tee test-output.log + npx playwright test tests_superhero_react/superhero-react.spec.ts --retries=0 --config=./playwright.config.ts --reporter=html --reporter=list 2>&1 | tee test-output.log echo "exit_code=${PIPESTATUS[0]}" >> $GITHUB_OUTPUT - name: Extract failed tests From 75e9bf7b5c0fe0d831cf41f48f40086b9d94a201 Mon Sep 17 00:00:00 2001 From: Andreea Neamt Date: Thu, 9 Oct 2025 09:48:41 +0300 Subject: [PATCH 12/19] debug playwright report --- .github/workflows/e2e-tests.yaml | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml index bbb86fa9d..797b33cd3 100644 --- a/.github/workflows/e2e-tests.yaml +++ b/.github/workflows/e2e-tests.yaml @@ -70,7 +70,37 @@ jobs: run: | npx playwright test tests_superhero_react/superhero-react.spec.ts --retries=0 --config=./playwright.config.ts --reporter=html --reporter=list 2>&1 | tee test-output.log echo "exit_code=${PIPESTATUS[0]}" >> $GITHUB_OUTPUT - + + - name: Debug Playwright Report Directory + if: always() + working-directory: keypair-testing + run: | + echo "Checking playwright-report directory..." + if [ -d "playwright-report" ]; then + echo "Directory exists. Listing contents:" + ls -la playwright-report/ + echo "File count: $(find playwright-report/ -type f | wc -l)" + else + echo "Directory does not exist." + ls -la . | grep report + fi + + - name: Show Playwright Config + if: always() + working-directory: keypair-testing + run: | + echo "Playwright config contents:" + cat playwright.config.ts || echo "Config not found" + + - name: Upload Test Output Log + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-output-log-${{ github.event.pull_request.number }} + path: keypair-testing/test-output.log + retention-days: 30 + if-no-files-found: warn + - name: Extract failed tests if: always() id: failed_tests From 27190d075d1c33e29ee9d5801e63e65c295373c2 Mon Sep 17 00:00:00 2001 From: Andreea Neamt Date: Thu, 9 Oct 2025 10:11:25 +0300 Subject: [PATCH 13/19] another try with url in front or command --- .github/workflows/e2e-tests.yaml | 32 +------------------------------- 1 file changed, 1 insertion(+), 31 deletions(-) diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml index 797b33cd3..eb7a9d466 100644 --- a/.github/workflows/e2e-tests.yaml +++ b/.github/workflows/e2e-tests.yaml @@ -68,39 +68,9 @@ jobs: env: SUPERHERO_COM_BASE_URL: ${{ steps.netlify.outputs.url }} run: | - npx playwright test tests_superhero_react/superhero-react.spec.ts --retries=0 --config=./playwright.config.ts --reporter=html --reporter=list 2>&1 | tee test-output.log + SUPERHERO_COM_BASE_URL=$SUPERHERO_COM_BASE_URL npx playwright test tests_superhero_react/superhero-react.spec.ts --retries=0 --config=./playwright.config.ts echo "exit_code=${PIPESTATUS[0]}" >> $GITHUB_OUTPUT - - name: Debug Playwright Report Directory - if: always() - working-directory: keypair-testing - run: | - echo "Checking playwright-report directory..." - if [ -d "playwright-report" ]; then - echo "Directory exists. Listing contents:" - ls -la playwright-report/ - echo "File count: $(find playwright-report/ -type f | wc -l)" - else - echo "Directory does not exist." - ls -la . | grep report - fi - - - name: Show Playwright Config - if: always() - working-directory: keypair-testing - run: | - echo "Playwright config contents:" - cat playwright.config.ts || echo "Config not found" - - - name: Upload Test Output Log - if: always() - uses: actions/upload-artifact@v4 - with: - name: test-output-log-${{ github.event.pull_request.number }} - path: keypair-testing/test-output.log - retention-days: 30 - if-no-files-found: warn - - name: Extract failed tests if: always() id: failed_tests From 642d99eabb6ec44d11a6c17331ae8216d13bb799 Mon Sep 17 00:00:00 2001 From: Andreea Neamt Date: Thu, 9 Oct 2025 12:22:37 +0300 Subject: [PATCH 14/19] added one retry --- .github/workflows/e2e-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml index eb7a9d466..f2a6f82f6 100644 --- a/.github/workflows/e2e-tests.yaml +++ b/.github/workflows/e2e-tests.yaml @@ -68,7 +68,7 @@ jobs: env: SUPERHERO_COM_BASE_URL: ${{ steps.netlify.outputs.url }} run: | - SUPERHERO_COM_BASE_URL=$SUPERHERO_COM_BASE_URL npx playwright test tests_superhero_react/superhero-react.spec.ts --retries=0 --config=./playwright.config.ts + SUPERHERO_COM_BASE_URL=$SUPERHERO_COM_BASE_URL npx playwright test tests_superhero_react/superhero-react.spec.ts --retries=1 --config=./playwright.config.ts echo "exit_code=${PIPESTATUS[0]}" >> $GITHUB_OUTPUT - name: Extract failed tests From 5e18844363514d6d9c2f2c1dc51a8bbf24c1ab8f Mon Sep 17 00:00:00 2001 From: Andreea Neamt Date: Thu, 9 Oct 2025 14:51:39 +0300 Subject: [PATCH 15/19] update exit code to failed the workflow when at least one test failed --- .github/workflows/e2e-tests.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml index f2a6f82f6..771debbd3 100644 --- a/.github/workflows/e2e-tests.yaml +++ b/.github/workflows/e2e-tests.yaml @@ -69,7 +69,9 @@ jobs: SUPERHERO_COM_BASE_URL: ${{ steps.netlify.outputs.url }} run: | SUPERHERO_COM_BASE_URL=$SUPERHERO_COM_BASE_URL npx playwright test tests_superhero_react/superhero-react.spec.ts --retries=1 --config=./playwright.config.ts - echo "exit_code=${PIPESTATUS[0]}" >> $GITHUB_OUTPUT + TEST_EXIT_CODE=$? + echo "exit_code=$TEST_EXIT_CODE" >> $GITHUB_OUTPUT + exit $TEST_EXIT_CODE - name: Extract failed tests if: always() From 60c4e2647c304998d5ae6aeedcc3db535b4c5388 Mon Sep 17 00:00:00 2001 From: Andreea Neamt Date: Thu, 9 Oct 2025 15:28:58 +0300 Subject: [PATCH 16/19] removed exit code --- .github/workflows/e2e-tests.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml index 771debbd3..0b467c1ac 100644 --- a/.github/workflows/e2e-tests.yaml +++ b/.github/workflows/e2e-tests.yaml @@ -69,9 +69,6 @@ jobs: SUPERHERO_COM_BASE_URL: ${{ steps.netlify.outputs.url }} run: | SUPERHERO_COM_BASE_URL=$SUPERHERO_COM_BASE_URL npx playwright test tests_superhero_react/superhero-react.spec.ts --retries=1 --config=./playwright.config.ts - TEST_EXIT_CODE=$? - echo "exit_code=$TEST_EXIT_CODE" >> $GITHUB_OUTPUT - exit $TEST_EXIT_CODE - name: Extract failed tests if: always() From f9297458b334546e9c26a02be47bc91ad482e3ad Mon Sep 17 00:00:00 2001 From: Andreea Neamt Date: Thu, 9 Oct 2025 15:46:31 +0300 Subject: [PATCH 17/19] removed continue on error --- .github/workflows/e2e-tests.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml index 0b467c1ac..93469fdd8 100644 --- a/.github/workflows/e2e-tests.yaml +++ b/.github/workflows/e2e-tests.yaml @@ -63,7 +63,6 @@ jobs: - name: Run Superhero React E2E tests id: test_run - continue-on-error: true working-directory: keypair-testing env: SUPERHERO_COM_BASE_URL: ${{ steps.netlify.outputs.url }} From 011af398c1d2d7cee1b8c2a8472c24a062cadb55 Mon Sep 17 00:00:00 2001 From: Andreea Neamt Date: Thu, 9 Oct 2025 16:33:25 +0300 Subject: [PATCH 18/19] added comment on PR based on jobStatus --- .github/workflows/e2e-tests.yaml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml index 93469fdd8..01656a09f 100644 --- a/.github/workflows/e2e-tests.yaml +++ b/.github/workflows/e2e-tests.yaml @@ -103,8 +103,8 @@ jobs: uses: actions/github-script@v7 with: script: | - const exitCode = '${{ steps.test_run.outputs.exit_code }}'; - const status = exitCode === '0' ? 'āœ… E2E Tests PASSED' : 'āŒ E2E Tests FAILED'; + const jobStatus = '${{ job.status }}'; + const status = jobStatus === 'success' ? 'āœ… E2E Tests PASSED' : 'āŒ E2E Tests FAILED'; const failedTests = `${{ steps.failed_tests.outputs.failed }}`.trim(); let body = `## ${status}\n\n**Tested URL:** ${{ steps.netlify.outputs.url }}\n\n`; @@ -120,10 +120,4 @@ jobs: owner: context.repo.owner, repo: context.repo.repo, body: body - }); - - - name: Fail workflow if tests failed - if: steps.test_run.outputs.exit_code != '0' - run: | - echo "āŒ E2E tests failed with exit code ${{ steps.test_run.outputs.exit_code }}" - exit 1 \ No newline at end of file + }); \ No newline at end of file From 6657ae0c7d5d305b7de3a4d8dd36eee0e2b838a9 Mon Sep 17 00:00:00 2001 From: Andreea Neamt Date: Thu, 9 Oct 2025 16:51:00 +0300 Subject: [PATCH 19/19] handle github 500 error if appear by adding retries --- .github/workflows/e2e-tests.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml index 01656a09f..de2c01a00 100644 --- a/.github/workflows/e2e-tests.yaml +++ b/.github/workflows/e2e-tests.yaml @@ -102,6 +102,8 @@ jobs: if: always() uses: actions/github-script@v7 with: + retries: 3 + retry-exempt-status-codes: 400,401,403,404 script: | const jobStatus = '${{ job.status }}'; const status = jobStatus === 'success' ? 'āœ… E2E Tests PASSED' : 'āŒ E2E Tests FAILED'; @@ -115,7 +117,7 @@ jobs: body += `šŸ“Š [View Full Report](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})`; - github.rest.issues.createComment({ + await github.rest.issues.createComment({ issue_number: context.issue.number, owner: context.repo.owner, repo: context.repo.repo,