Group realm server tests into larger shards

.github/workflows/ci.yaml

@@ -419,60 +419,66 @@ jobs:
             -d '{"context":"Matrix Playwright tests report","description":"'"$description"'","target_url":"'"$PLAYWRIGHT_REPORT_URL"'","state":"'"$state"'"}'
 
   realm-server-test:
-    name: Realm Server Tests
+    name: Realm Server Tests (shard ${{ matrix.shard }})
     needs: [change-check, test-web-assets]
     if: needs.change-check.outputs.realm-server == 'true' || github.ref == 'refs/heads/main' || needs.change-check.outputs.run_all == 'true'
     runs-on: ubuntu-latest
     concurrency:
-      group: realm-server-test-${{ matrix.testModule }}-${{ github.head_ref || github.run_id }}
+      group: realm-server-test-${{ matrix.shard }}-${{ github.head_ref || github.run_id }}
       cancel-in-progress: true
     strategy:
       fail-fast: false
       matrix:
-        testModule:
-          [
-            "auth-client-test.ts",
-            "billing-test.ts",
-            "card-dependencies-endpoint-test.ts",
-            "card-endpoints-test.ts",
-            "card-source-endpoints-test.ts",
-            "definition-lookup-test.ts",
-            "file-watcher-events-test.ts",
-            "indexing-test.ts",
-            "transpile-test.ts",
-            "module-syntax-test.ts",
-            "permissions/permission-checker-test.ts",
-            "prerendering-test.ts",
-            "prerender-server-test.ts",
-            "prerender-manager-test.ts",
-            "prerender-proxy-test.ts",
-            "remote-prerenderer-test.ts",
-            "queue-test.ts",
-            "realm-endpoints/dependencies-test.ts",
-            "realm-endpoints/directory-test.ts",
-            "realm-endpoints/info-test.ts",
-            "realm-endpoints/lint-test.ts",
-            "realm-endpoints/mtimes-test.ts",
-            "realm-endpoints/permissions-test.ts",
-            "realm-endpoints/publishability-test.ts",
-            "realm-endpoints/search-test.ts",
-            "realm-endpoints/user-test.ts",
-            "realm-endpoints-test.ts",
-            "search-prerendered-test.ts",
-            "types-endpoint-test.ts",
-            "server-endpoints-test.ts",
-            "server-endpoints/search-test.ts",
-            "virtual-network-test.ts",
-            "atomic-endpoints-test.ts",
-            "request-forward-test.ts",
-            "publish-unpublish-realm-test.ts",
-            "boxel-domain-availability-test.ts",
-            "claim-boxel-domain-test.ts",
-            "delete-boxel-claimed-domain-test.ts",
-            "get-boxel-claimed-domain-test.ts",
-            "realm-auth-test.ts",
-            "queries-test.ts",
-          ]
+        include:
+          - shard: 1
+            testModules:
+              - server-endpoints-test.ts
+              - atomic-endpoints-test.ts
+              - request-forward-test.ts
+              - realm-endpoints/info-test.ts
+              - realm-endpoints/user-test.ts
+              - file-watcher-events-test.ts
+              - types-endpoint-test.ts
+              - prerender-manager-test.ts
+              - prerender-proxy-test.ts
+              - transpile-test.ts
+          - shard: 2
+            testModules:
+              - card-endpoints-test.ts
+              - publish-unpublish-realm-test.ts
+              - realm-endpoints/lint-test.ts
+              - claim-boxel-domain-test.ts
+              - realm-endpoints/publishability-test.ts
+              - boxel-domain-availability-test.ts
+              - realm-endpoints/dependencies-test.ts
+              - queue-test.ts
+              - remote-prerenderer-test.ts
+              - permissions/permission-checker-test.ts
+          - shard: 3
+            testModules:
+              - realm-endpoints-test.ts
+              - prerendering-test.ts
+              - search-prerendered-test.ts
+              - card-dependencies-endpoint-test.ts
+              - delete-boxel-claimed-domain-test.ts
+              - get-boxel-claimed-domain-test.ts
+              - realm-endpoints/mtimes-test.ts
+              - billing-test.ts
+              - virtual-network-test.ts
+              - module-syntax-test.ts
+          - shard: 4
+            testModules:
+              - indexing-test.ts
+              - card-source-endpoints-test.ts
+              - realm-endpoints/search-test.ts
+              - server-endpoints/search-test.ts
+              - realm-endpoints/permissions-test.ts
+              - realm-endpoints/directory-test.ts
+              - definition-lookup-test.ts
+              - prerender-server-test.ts
+              - realm-auth-test.ts
+              - auth-client-test.ts
+              - queries-test.ts
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2
       - uses: ./.github/actions/init
@@ -506,15 +512,15 @@ jobs:
         run: pnpm test:wait-for-servers
         working-directory: packages/realm-server
         env:
-          TEST_MODULE: ${{matrix.testModule}}
+          TEST_MODULES: ${{ join(matrix.testModules, '|') }}
       - name: Print realm server logs
         if: always()
         run: cat /tmp/server.log
       - name: Prepare artifact name
         id: artifact_name
         if: always()
         run: |
-          export SAFE_ARTIFACT_NAME=$(echo ${{ matrix.testModule }} |  sed 's/[/]/_/g')
+          export SAFE_ARTIFACT_NAME=shard-${{ matrix.shard }}
           echo "artifact_name=$SAFE_ARTIFACT_NAME" >> "$GITHUB_OUTPUT"
       - name: Upload realm server log
         uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # 4.6.1

README.md

@@ -98,7 +98,7 @@ You can also use `start:development` if you want the functionality of `start:all
 
 Optional environment variables for `start:development`:
 
-- `USE_EXTERNAL_CATALOG=1` to load `/catalog` from `packages/catalog/contents` (cloned from the `boxel-catalog` repo). 
+- `USE_EXTERNAL_CATALOG=1` to load `/catalog` from `packages/catalog/contents` (cloned from the `boxel-catalog` repo).
 
 ### Card Pre-rendering
 
@@ -394,12 +394,15 @@ The tests are available at `http://localhost:4200/tests`
 
 ### Realm Server Node tests
 
-First make sure to generate the host app's `dist/` output in order to support card pre-rendering by first starting the host app (instructions above). If you want to make the host app's `dist/` output without starting the host app, you can run `pnpm build` in the host app's workspace.
-
 To run the `packages/realm-server/` workspace tests start:
 
-1. `pnpm start:all` in the `packages/realm-server/` to serve _both_ the base realm and the realm that serves the test cards for node.
-2. Run `pnpm test` in the `packages/realm-server/` workspace to run the realm node tests. `TEST_MODULE=realm-endpoints-test.ts pnpm test-module` is an example of how to run a single test module.
+1. The host application on port 4200. You can do this by running `pnpm start` in the `packages/host/` workspace, or if you have a built folder you can serve it with a static server with `pnpm serve:dist`.
+2. The base realm and associated workers, postgres and synapse. You can do this by running `pnpm start:all` in the `packages/realm-server/` workspace, or `pnpm:start-services-for-host-tests` for a more lightweight setup.
+3. Run the realm server tests:
+
+- `pnpm test` in the `packages/realm-server/` workspace to run the realm node tests in full (~1hr).
+- `TEST_MODULES="types-endpoint-test.ts|another-test-module.ts" pnpm test` in the `packages/realm-server/` workspace to run tests for a subset of modules.
+- `TEST_MODULE="types-endpoint-test.ts" pnpm test-module` in the `packages/realm-server/` workspace to run tests for a specific module.
 
 ### Boxel UI
 

packages/realm-server/package.json

@@ -85,7 +85,7 @@
     "start:smtp": "cd ../matrix && pnpm assert-smtp-running",
     "start:pg": "./scripts/start-pg.sh",
     "stop:pg": "./scripts/stop-pg.sh",
-    "test:wait-for-servers": "WAIT_ON_TIMEOUT=900000 NODE_NO_WARNINGS=1 start-server-and-test 'pnpm run wait' 'http-get://localhost:4201/base/_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson' 'pnpm run wait' 'http-get://localhost:4202/node-test/_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson|http://localhost:8008|http://localhost:5001' 'test-module'",
+    "test:wait-for-servers": "WAIT_ON_TIMEOUT=900000 NODE_NO_WARNINGS=1 start-server-and-test 'pnpm run wait' 'http-get://localhost:4201/base/_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson' 'pnpm run wait' 'http-get://localhost:4202/node-test/_readiness-check?acceptHeader=application%2Fvnd.api%2Bjson|http://localhost:8008|http://localhost:5001' 'test'",
     "setup:base-in-deployment": "mkdir -p /persistent/base && rsync --dry-run --itemize-changes --checksum --recursive --delete ../base/. /persistent/base/ && rsync --checksum --recursive --delete ../base/. /persistent/base/",
     "setup:experiments-in-deployment": "mkdir -p /persistent/experiments && rsync --dry-run --itemize-changes --checksum --recursive ../experiments-realm/. /persistent/experiments/ && rsync --checksum --recursive ../experiments-realm/. /persistent/experiments/",
     "setup:catalog-in-deployment": "mkdir -p /persistent/catalog && rsync --dry-run --itemize-changes --checksum --recursive --delete ../catalog-realm/. /persistent/catalog/ && rsync --checksum --recursive --delete ../catalog-realm/. /persistent/catalog/",

packages/realm-server/scripts/lint-test-shards.ts

@@ -19,18 +19,49 @@ function getCiTestModules(yamlFilePath: string) {
     const yamlContent = readFileSync(yamlFilePath, 'utf8');
     const yamlData = yaml.load(yamlContent) as Record<string, any>;
 
-    const shardIndexes: string[] =
-      yamlData?.jobs?.['realm-server-test']?.strategy?.matrix?.testModule;
+    const matrix = yamlData?.jobs?.['realm-server-test']?.strategy?.matrix;
+    const testModules = matrix?.testModule;
 
-    if (!Array.isArray(shardIndexes)) {
+    if (Array.isArray(testModules)) {
+      return testModules;
+    }
+
+    const include = matrix?.include;
+    if (!Array.isArray(include)) {
       throw new Error(
-        `Invalid 'jobs.realm-server-test.strategy.matrix.testModule' format in the YAML file.`,
+        `Invalid 'jobs.realm-server-test.strategy.matrix' format in the YAML file.`,
       );
     }
 
-    return shardIndexes;
+    const modules = new Set<string>();
+    const invalidEntries: number[] = [];
+    include.forEach((entry: Record<string, any>, index: number) => {
+      const entryModules = entry?.testModules;
+      if (Array.isArray(entryModules)) {
+        entryModules.forEach((moduleName: string) => modules.add(moduleName));
+        return;
+      }
+      if (typeof entryModules === 'string') {
+        entryModules
+          .split(/[,\s]+/)
+          .filter(Boolean)
+          .forEach((moduleName) => modules.add(moduleName));
+        return;
+      }
+      invalidEntries.push(index);
+    });
+
+    if (invalidEntries.length > 0) {
+      throw new Error(
+        `Invalid 'jobs.realm-server-test.strategy.matrix.include[*].testModules' entries at indexes: ${invalidEntries.join(', ')}`,
+      );
+    }
+
+    return Array.from(modules);
   } catch (error: any) {
-    console.error(`Error reading shardIndex from YAML file: ${error.message}`);
+    console.error(
+      `Error reading test modules from YAML file: ${error.message}`,
+    );
     process.exit(1);
   }
 }

packages/realm-server/scripts/run-test-modules.js

@@ -0,0 +1,57 @@
+#!/usr/bin/env node
+'use strict';
+
+const { spawnSync } = require('node:child_process');
+
+function buildModuleFilter(modulesToMatch) {
+  const escaped = modulesToMatch
+    .map((moduleName) => escapeRegex(moduleName))
+    .join('|');
+  const pattern = `^(?:${escaped})(?:\\s>\\s|:)`;
+  return `/${pattern}/`;
+}
+
+function escapeRegex(value) {
+  return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&').replace(/\//g, '\\/');
+}
+
+const rawModules = process.env.TEST_MODULES ?? '';
+const cleanedRaw = rawModules.trim();
+
+if (!cleanedRaw) {
+  console.error('TEST_MODULES must be set.');
+  process.exit(1);
+}
+
+const modules = cleanedRaw
+  .split(/[|,]/)
+  .map((value) => value.trim())
+  .filter(Boolean)
+  .map((value) => value.replace(/^['"]+|['"]+$/g, ''));
+
+if (modules.length === 0) {
+  console.error('No module names found in TEST_MODULES.');
+  process.exit(1);
+}
+
+const args = ['--require', 'ts-node/register/transpile-only'];
+
+args.push('--filter', buildModuleFilter(modules));
+
+args.push('tests/index.ts');
+
+const qunitBin = require.resolve('qunit/bin/qunit.js');
+const result = spawnSync(process.execPath, [qunitBin, ...args], {
+  stdio: 'inherit',
+  env: process.env,
+});
+
+if (typeof result.status === 'number') {
+  process.exit(result.status);
+}
+
+if (result.error) {
+  console.error(result.error);
+}
+
+process.exit(1);

packages/realm-server/tests/atomic-endpoints-test.ts

@@ -8,19 +8,13 @@ import {
   type LooseSingleCardDocument,
   SupportedMimeType,
 } from '@cardstack/runtime-common';
-import {
-  setupBaseRealmServer,
-  setupPermissionedRealm,
-  matrixURL,
-  testRealmHref,
-  createJWT,
-} from './helpers';
+import { setupPermissionedRealm, testRealmHref, createJWT } from './helpers';
 import '@cardstack/runtime-common/helpers/code-equality-assertion';
 
 module(basename(__filename), function () {
   module(
     'Realm-specific Endpoints: can make request to post /_atomic',
-    function (hooks) {
+    function () {
       let testRealm: Realm;
       let testRealmAdapter: RealmAdapter;
       let request: SuperTest<Test>;
@@ -37,8 +31,6 @@ module(basename(__filename), function () {
         request = args.request;
       }
 
-      setupBaseRealmServer(hooks, matrixURL);
-
       module('writes', function (hooks) {
         setupPermissionedRealm(hooks, {
           permissions: {

packages/realm-server/tests/boxel-domain-availability-test.ts

@@ -10,7 +10,6 @@ import {
   createVirtualNetwork,
   matrixURL,
   closeServer,
-  setupBaseRealmServer,
 } from './helpers';
 import type { RealmServerTokenClaim } from '../utils/jwt';
 import { createJWT as createRealmServerJWT } from '../utils/jwt';
@@ -25,8 +24,6 @@ const testRealmURL = new URL('http://127.0.0.1:0/test/');
 
 module(basename(__filename), function () {
   module('boxel domain availability endpoint', function (hooks) {
-    setupBaseRealmServer(hooks, matrixURL);
-
     let testRealmServer: Server;
     let request: SuperTest<Test>;
     let dir: DirResult;