More shell tests

.github/workflows/build-ts.yml

@@ -66,10 +66,6 @@ jobs:
         working-directory: ts
         run: |
           pnpm install --frozen-lockfile --strict-peer-dependencies
-      # - name: Install Playwright Browsers
-      #   if: ${{ runner.os == 'windows' && matrix.version == '22' }}
-      #   run: pnpm exec playwright install --with-deps
-      #   working-directory: ts/packages/shell
       - name: Build
         if: ${{ github.event_name != 'pull_request' || steps.filter.outputs.ts == 'true' }}
         working-directory: ts
@@ -85,43 +81,3 @@ jobs:
         working-directory: ts
         run: |
           npm run lint
-      # - name: Login to Azure
-      #   if: ${{ github.event_name != 'merge_group' }}
-      #   uses: azure/login@v2.2.0
-      #   with:
-      #     client-id: ${{ secrets.AZUREAPPSERVICE_CLIENTID_5B0D2D6BA40F4710B45721D2112356DD }}
-      #     tenant-id: ${{ secrets.AZUREAPPSERVICE_TENANTID_39BB903136F14B6EAD8F53A8AB78E3AA }}
-      #     subscription-id: ${{ secrets.AZUREAPPSERVICE_SUBSCRIPTIONID_F36C1F2C4B2C49CA8DD5C52FAB98FA30 }}
-      # - name: Get Keys
-      #   if: ${{ github.event_name != 'merge_group' }}
-      #   run: |
-      #     node tools/scripts/getKeys.mjs --vault build-pipeline-kv
-      #   working-directory: ts
-      # - name: Test CLI - smoke
-      #   if: ${{ github.event_name != 'merge_group' }}
-      #   run: |
-      #     npm run start:dev 'prompt' 'why is the sky blue'
-      #   working-directory: ts/packages/cli
-      #   continue-on-error: true
-      # - name: Shell Tests - smoke (windows)
-      #   if: ${{ github.event_name != 'merge_group' && runner.os == 'windows' && matrix.version == '22' }}
-      #   timeout-minutes: 60
-      #   run: |
-      #     npx playwright test simple.spec.ts
-      #     rm ../../.env
-      # - name: Shell Tests - smoke (linux)
-      #   if: ${{ github.event_name != 'merge_group' && runner.os == 'Linux' && matrix.version == '22' }}
-      #   timeout-minutes: 60
-      #   run: |
-      #     Xvfb :99 -screen 0 1600x1200x24 & export DISPLAY=:99
-      #     npx playwright test simple.spec.ts
-      #     rm ../../.env          
-      #   working-directory: ts/packages/shell
-      #   continue-on-error: true
-      # - name: Live Tests
-      #   if: ${{ github.event_name != 'merge_group' && runner.os == 'linux' && matrix.version == '22' }}
-      #   timeout-minutes: 60
-      #   run: |
-      #     npm run test:live
-      #   working-directory: ts
-      #   continue-on-error: true

.github/workflows/shell-tests.yml

@@ -28,7 +28,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ["windows-latest"]
+        os: ["windows-latest", "ubuntu-latest"]
         version: [20]
 
     runs-on: ${{ matrix.os }}
@@ -98,19 +98,15 @@ jobs:
         timeout-minutes: 60
         run: |
           npm run shell:test
-          rm ../../.env
         working-directory: ts/packages/shell
         continue-on-error: true
 
       - name: Shell Tests (linux)
         if: ${{ runner.os == 'Linux' }}
         timeout-minutes: 60
-        # https://github.com/microsoft/playwright/issues/34251 - sysctl command
         run: |
-          sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 
           Xvfb :99 -screen 0 1600x1200x24 & export DISPLAY=:99
           npm run shell:test
-          rm ../../.env
         working-directory: ts/packages/shell
         continue-on-error: true
 
@@ -129,3 +125,9 @@ jobs:
           path: ts/packages/shell/playwright-report/
           overwrite: true
           retention-days: 30
+
+      - name: Clean up Keys
+        run: |
+          rm ./.env
+        working-directory: ts
+        if: always()          

.github/workflows/smoke-tests.yml

@@ -9,8 +9,6 @@ on:
   workflow_dispatch:
   push:
     branches: ["main"]
-  # pull_request:
-  #   branches: ["main"]
   pull_request_target:
     branches: ["main"]
   merge_group:
@@ -27,7 +25,9 @@ permissions:
 
 env:
   NODE_OPTIONS: --max_old_space_size=8192
-  DEBUG: pw:browser*
+  #  DEBUG: pw:browser*                       # PlayWright debug messages
+  #  ELECTRON_ENABLE_LOGGING: true            # Electron debug messages
+  #  DEBUG: typeagent:*                       # TypeAgent debug messages
 
 jobs:
   shell_and_cli:
@@ -37,7 +37,6 @@ jobs:
       fail-fast: false
       matrix:
         os: ["ubuntu-latest", "windows-latest"]
-        #os: ["ubuntu-latest"]
         version: [20]
 
     runs-on: ${{ matrix.os }}

ts/packages/shell/test/listAgent.spec.ts

@@ -0,0 +1,64 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import test, {
+    ElectronApplication,
+    Page,
+    _electron,
+    _electron as electron,
+    expect,
+} from "@playwright/test";
+import {
+    exitApplication,
+    getAppPath,
+    sendUserRequestAndWaitForCompletion,
+    getLaunchArgs,
+    startShell,
+    testUserRequest,
+} from "./testHelper";
+
+// Annotate entire file as serial.
+test.describe.configure({ mode: "serial" });
+
+test.describe("List Agent Tests", () => {
+    test("create_update_clear_list", async ({}, testInfo) => {
+        console.log(`Running test '${testInfo.title}`);
+
+        await testUserRequest(
+            [
+                "create a shopping list",
+                "what's on the shopping list?",
+                "add eggs, milk, flour to the shopping list",
+                "what's on the shopping list?",
+                "remove milk from the shopping list",
+                "what's on the shopping list?",
+                "clear the shopping list",
+                "what's on the shopping list?",
+            ],
+            [
+                "Created list: shopping",
+                "List 'shopping' is empty",
+                "Added items: eggs,milk,flour to list shopping",
+                "eggs\nmilk\nflour",
+                "Removed items: milk from list shopping",
+                "eggs\nflour",
+                "Cleared list: shopping",
+                "List 'shopping' is empty",
+            ],
+        );
+    });
+
+    // test("delete_list", async ({}, testInfo) => {
+    //     console.log(`Running test '${testInfo.title}`);
+
+    //     await testUserRequest(
+    //         [
+    //             "delete the shopping list",
+    //             "is there a shopping list?"
+    //         ],
+    //         [
+    //             "Cleared list: shopping",
+    //             "List 'shopping' is empty"
+    //         ]);
+    // });
+});