combine deploy and test workflows, handle test failures properly

.github/workflows/e2e-test-commitly.yml → .github/workflows/deploy-playground-and-test.yml

@@ -1,19 +1,110 @@
-name: Commitly End-to-End testing check for Playground Apps
+name: Deploy Playground Apps to Vercel and Test on React
 env:
-  BRANCH_NAME_RAW: ${{ github.event.workflow_run.head_branch || github.ref_name }}
+  VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }}
+  VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }}
+  BRANCH_NAME_RAW: ${{ github.head_ref || github.ref_name }}
 
 on:
-  workflow_dispatch:
-  workflow_run:
-    workflows: ["Deploy Playground Apps to Vercel"]
-    types:
-      - completed
-    conclusion: success
+  pull_request:
+  push:
+    branches:
+      - develop
 
 jobs:
-  e2e-tests:
+  deploy:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+
+      - name: Cache node modules
+        id: cache-npm
+        uses: actions/cache@v3
+        env:
+          cache-name: cache-node-modules
+        with:
+          path: |
+            ~/.npm
+            ./node_modules
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}-force-1
+          restore-keys: |
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+
+      - if: ${{ steps.cache-npm.outputs.cache-hit != 'true' }}
+        name: Install Dependencies
+        run: |
+          npm i
+          npm install lerna
+          npm install vercel@33.2.0
+          npm list
+
+      - name: Check ESLint Errors
+        run: npm run lint
+
+      - name: Check Prettier Formatting
+        run: npm run prettier:check
+
+      - name: Build SDKs
+        run: |
+          npm run build
+          npm run build:bundler:local
+
+      - name: Normalize branch name
+        run: |
+          echo "BRANCH_NAME=$(echo $BRANCH_NAME_RAW | tr '/_' '-')" >> "$GITHUB_ENV"
+        shell: bash
+
+      - name: Deploy react playground to Vercel
+        run: |
+          npx vercel link --yes --project react-playground --scope corbado -t $VERCEL_TOKEN
+          npx vercel pull -t $VERCEL_TOKEN
+          npx vercel build -t $VERCEL_TOKEN
+          url="$(npx vercel deploy --prebuilt -t ${{ secrets.VERCEL_TOKEN }})"
+          npx vercel alias -S corbado -t ${{ secrets.VERCEL_TOKEN }} "$url" $BRANCH_NAME.react.playground.corbado.io
+        env:
+          VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID_REACT_PLAYGROUND }}
+
+      - name: Deploy connect next playground to Vercel
+        run: |
+          npx vercel link --yes --project connect-next-playground --scope corbado -t $VERCEL_TOKEN
+          npx vercel pull --environment=preview -t $VERCEL_TOKEN
+          npx vercel build -t $VERCEL_TOKEN
+          url="$(npx vercel deploy --prebuilt -t ${{ secrets.VERCEL_TOKEN }})"
+          npx vercel alias -S corbado -t ${{ secrets.VERCEL_TOKEN }} "$url" $BRANCH_NAME.connect-next.playground.corbado.io
+        env:
+          VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID_CONNECT_NEXT_PLAYGROUND }}
+
+      - name: Deploy web-js playground to Vercel
+        run: |
+          npx vercel link --yes --project web-js-playground --scope corbado -t $VERCEL_TOKEN
+          npx vercel pull -t $VERCEL_TOKEN
+          npx vercel build -t $VERCEL_TOKEN
+          url="$(npx vercel deploy --prebuilt -t ${{ secrets.VERCEL_TOKEN }})"
+          npx vercel alias -S corbado -t ${{ secrets.VERCEL_TOKEN }} "$url" $BRANCH_NAME.web-js.playground.corbado.io
+        env:
+          VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID_WEB_JS_PLAYGROUND }}
+
+      - name: Deploy web-js-script playground to Vercel
+        run: |
+          npx vercel link --yes --project web-js-script-playground --scope corbado -t $VERCEL_TOKEN
+          npx vercel pull -t $VERCEL_TOKEN
+          npx vercel build -t $VERCEL_TOKEN
+          url="$(npx vercel deploy --prebuilt -t ${{ secrets.VERCEL_TOKEN }})"
+          npx vercel alias -S corbado -t ${{ secrets.VERCEL_TOKEN }} "$url" $BRANCH_NAME.web-js-script.playground.corbado.io
+        env:
+          VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID_WEB_JS_SCRIPT_PLAYGROUND }}
+
+  test:
+    needs: deploy
     timeout-minutes: 60
     runs-on: ubuntu-20.04
+
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -82,7 +173,12 @@ jobs:
           cd packages/tests-e2e
           BRANCH_NAME=$(echo $BRANCH_NAME_RAW | tr '/_' '-')
           export PLAYWRIGHT_TEST_URL="https://$BRANCH_NAME.react.playground.corbado.io"
-          PLAYWRIGHT_TEST_URL=$PLAYWRIGHT_TEST_URL npx playwright test --config=playwright.config.complete.ts
+          set +e
+          npx playwright test --config=playwright.config.complete.ts
+          EXIT_CODE=$?
+          if [ $EXIT_CODE -ne 0 ]; then
+            echo "COMPLETE_REACT_FAILED=true" >> $GITHUB_ENV
+          fi
         env:
           PLAYWRIGHT_NUM_CORES: 4
           PLAYWRIGHT_JWT_TOKEN: ${{ secrets.PLAYWRIGHT_JWT_TOKEN }}
@@ -125,7 +221,12 @@ jobs:
           cd packages/tests-e2e
           BRANCH_NAME=$(echo $BRANCH_NAME_RAW | tr '/_' '-')
           export PLAYWRIGHT_TEST_URL="https://$BRANCH_NAME.connect-next.playground.corbado.io"
-          PLAYWRIGHT_TEST_URL=$PLAYWRIGHT_TEST_URL npx playwright test --config=playwright.config.connect.ts
+          set +e
+          npx playwright test --config=playwright.config.connect.ts
+          EXIT_CODE=$?
+          if [ $EXIT_CODE -ne 0 ]; then
+            echo "CONNECT_REACT_FAILED=true" >> $GITHUB_ENV
+          fi
         env:
           PLAYWRIGHT_NUM_CORES: 4
           BACKEND_API_BASIC_AUTH: ${{ secrets.BACKEND_API_BASIC_AUTH }}
@@ -160,3 +261,26 @@ jobs:
           TIMESTAMP=$(date +%s000)
           LOG_EVENT_JSON="[{\"timestamp\":$TIMESTAMP,\"message\":\"$SUMMARY\"}]"
           aws logs put-log-events --log-group-name "test-results-board" --log-stream-name "$LOG_STREAM_NAME" --log-events "$LOG_EVENT_JSON"
+
+      - name: Aggregate results
+        run: |
+          FAILED_STEPS=""
+          if [ "${COMPLETE_REACT_FAILED}" = "true" ]; then
+            FAILED_STEPS="${FAILED_STEPS} Complete-React"
+          fi
+          if [ "${COMPLETE_WEBJS_FAILED}" = "true" ]; then
+            FAILED_STEPS="${FAILED_STEPS} Complete-WebJs"
+          fi
+          if [ "${COMPLETE_WEBJSSCRIPT_FAILED}" = "true" ]; then
+            FAILED_STEPS="${FAILED_STEPS} Complete-WebJsScript"
+          fi
+          if [ "${CONNECT_REACT_FAILED}" = "true" ]; then
+            FAILED_STEPS="${FAILED_STEPS} Connect-React"
+          fi
+
+          if [ -n "$FAILED_STEPS" ]; then
+            echo "The following test steps have failed: $FAILED_STEPS"
+            exit 1
+          else
+            echo "All tests passed."
+          fi

.github/workflows/e2e-test-nightly.yml → .github/workflows/test-all.yml

@@ -1,4 +1,4 @@
-name: Nightly End-to-End testing check for Playground Apps
+name: Test on All Platforms
 env:
   BRANCH_NAME_RAW: ${{ github.head_ref || github.ref_name }}
 
@@ -10,7 +10,7 @@ on:
     - cron: '30 18 * * *' #  7:30 PM CET / 8:30 PM CEST
 
 jobs:
-  prepare:
+  test:
     timeout-minutes: 60
     runs-on: ubuntu-20.04
     steps:
@@ -81,7 +81,12 @@ jobs:
           cd packages/tests-e2e
           BRANCH_NAME=$(echo $BRANCH_NAME_RAW | tr '/_' '-')
           export PLAYWRIGHT_TEST_URL="https://$BRANCH_NAME.react.playground.corbado.io"
-          PLAYWRIGHT_TEST_URL=$PLAYWRIGHT_TEST_URL npx playwright test --config=playwright.config.complete.ts
+          set +e
+          npx playwright test --config=playwright.config.complete.ts
+          EXIT_CODE=$?
+          if [ $EXIT_CODE -ne 0 ]; then
+            echo "COMPLETE_REACT_FAILED=true" >> $GITHUB_ENV
+          fi
         env:
           PLAYWRIGHT_NUM_CORES: 4
           PLAYWRIGHT_JWT_TOKEN: ${{ secrets.PLAYWRIGHT_JWT_TOKEN }}
@@ -124,7 +129,12 @@ jobs:
           cd packages/tests-e2e
           BRANCH_NAME=$(echo $BRANCH_NAME_RAW | tr '/_' '-')
           export PLAYWRIGHT_TEST_URL="https://$BRANCH_NAME.web-js.playground.corbado.io"
-          PLAYWRIGHT_TEST_URL=$PLAYWRIGHT_TEST_URL npx playwright test --config=playwright.config.complete.ts
+          set +e
+          npx playwright test --config=playwright.config.complete.ts
+          EXIT_CODE=$?
+          if [ $EXIT_CODE -ne 0 ]; then
+            echo "COMPLETE_WEBJS_FAILED=true" >> $GITHUB_ENV
+          fi
         env:
           PLAYWRIGHT_NUM_CORES: 4
           PLAYWRIGHT_JWT_TOKEN: ${{ secrets.PLAYWRIGHT_JWT_TOKEN }}
@@ -167,7 +177,12 @@ jobs:
           cd packages/tests-e2e
           BRANCH_NAME=$(echo $BRANCH_NAME_RAW | tr '/_' '-')
           export PLAYWRIGHT_TEST_URL="https://$BRANCH_NAME.web-js-script.playground.corbado.io"
-          PLAYWRIGHT_TEST_URL=$PLAYWRIGHT_TEST_URL npx playwright test --config=playwright.config.complete.ts
+          set +e
+          npx playwright test --config=playwright.config.complete.ts
+          EXIT_CODE=$?
+          if [ $EXIT_CODE -ne 0 ]; then
+            echo "COMPLETE_WEBJSSCRIPT_FAILED=true" >> $GITHUB_ENV
+          fi
         env:
           PLAYWRIGHT_NUM_CORES: 4
           PLAYWRIGHT_JWT_TOKEN: ${{ secrets.PLAYWRIGHT_JWT_TOKEN }}
@@ -210,7 +225,12 @@ jobs:
           cd packages/tests-e2e
           BRANCH_NAME=$(echo $BRANCH_NAME_RAW | tr '/_' '-')
           export PLAYWRIGHT_TEST_URL="https://$BRANCH_NAME.connect-next.playground.corbado.io"
-          PLAYWRIGHT_TEST_URL=$PLAYWRIGHT_TEST_URL npx playwright test --config=playwright.config.connect.ts
+          set +e
+          npx playwright test --config=playwright.config.connect.ts
+          EXIT_CODE=$?
+          if [ $EXIT_CODE -ne 0 ]; then
+            echo "CONNECT_REACT_FAILED=true" >> $GITHUB_ENV
+          fi
         env:
           PLAYWRIGHT_NUM_CORES: 4
           BACKEND_API_BASIC_AUTH: ${{ secrets.BACKEND_API_BASIC_AUTH }}
@@ -245,3 +265,26 @@ jobs:
           TIMESTAMP=$(date +%s000)
           LOG_EVENT_JSON="[{\"timestamp\":$TIMESTAMP,\"message\":\"$SUMMARY\"}]"
           aws logs put-log-events --log-group-name "test-results-board" --log-stream-name "$LOG_STREAM_NAME" --log-events "$LOG_EVENT_JSON"
+
+      - name: Aggregate results
+        run: |
+          FAILED_STEPS=""
+          if [ "${COMPLETE_REACT_FAILED}" = "true" ]; then
+            FAILED_STEPS="${FAILED_STEPS} Complete-React"
+          fi
+          if [ "${COMPLETE_WEBJS_FAILED}" = "true" ]; then
+            FAILED_STEPS="${FAILED_STEPS} Complete-WebJs"
+          fi
+          if [ "${COMPLETE_WEBJSSCRIPT_FAILED}" = "true" ]; then
+            FAILED_STEPS="${FAILED_STEPS} Complete-WebJsScript"
+          fi
+          if [ "${CONNECT_REACT_FAILED}" = "true" ]; then
+            FAILED_STEPS="${FAILED_STEPS} Connect-React"
+          fi
+
+          if [ -n "$FAILED_STEPS" ]; then
+            echo "The following test steps have failed: $FAILED_STEPS"
+            exit 1
+          else
+            echo "All tests passed."
+          fi

packages/tests-e2e/src/connect/scenarios/login.spec.ts

@@ -77,19 +77,6 @@ test.describe('login component (with invitation token, with passkeys)', () => {
     await model.login.repeatedlyFailPasskeyInput();
   });
 
-  test('inaccessible passkey on login', async ({ model }) => {
-    await model.home.logout();
-    await model.expectScreen(ScreenNames.InitLoginOneTap);
-
-    await model.authenticator.clearCredentials();
-    await model.storage.clearLocalStorageAndCookies();
-    await model.storage.loadInvitationToken();
-    await model.expectScreen(ScreenNames.InitLogin);
-
-    await model.login.submitEmail(model.email, false);
-    await model.expectScreen(ScreenNames.InitLoginFallback);
-  });
-
   test('Corbado FAPI unavailable after authentication', async ({ model }) => {
     await model.home.logout();
     await model.expectScreen(ScreenNames.InitLoginOneTap);