perf: speed up CI tests with shared Docker image and more shards (#161)

gricha · web-flow · commit ff5746b8702c · 2026-02-13T14:10:34.000-08:00
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -122,13 +122,84 @@ jobs:
           cp -r dist/agent/web test-install/
           ./test-install/bin/perry --version
 
-  test:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker
+
+      - name: Build base image
+        uses: docker/build-push-action@v6
+        with:
+          context: ./perry
+          file: ./perry/Dockerfile.base
+          load: true
+          tags: perry-base:latest
+
+      - name: Build full image
+        uses: docker/build-push-action@v6
+        with:
+          context: ./perry
+          load: true
+          tags: perry:latest
+          build-args: |
+            BASE_IMAGE=perry-base:latest
+
+      - name: Save Docker image
+        run: docker save perry:latest | gzip > perry-image.tar.gz
+
+      - name: Upload Docker image
+        uses: actions/upload-artifact@v4
+        with:
+          name: docker-image
+          path: perry-image.tar.gz
+          retention-days: 1
+          compression-level: 0
+
+  test-unit:
     runs-on: ubuntu-latest
     needs: build
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Download build artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Set up Bun
+        uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: latest
+
+      - name: Cache bun dependencies
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.bun/install/cache
+            node_modules
+          key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock', '**/package.json') }}
+          restore-keys: |
+            ${{ runner.os }}-bun-
+
+      - name: Install dependencies
+        run: bun install
+
+      - name: Run unit tests
+        run: bun run test:unit
+
+  test:
+    runs-on: ubuntu-latest
+    needs: [build, docker]
     strategy:
       fail-fast: false
       matrix:
-        shard: [1, 2, 3]
+        shard: [1, 2, 3, 4, 5]
     steps:
       - uses: actions/checkout@v4
         with:
@@ -140,6 +211,14 @@ jobs:
           name: dist
           path: dist/
 
+      - name: Download Docker image
+        uses: actions/download-artifact@v4
+        with:
+          name: docker-image
+
+      - name: Load Docker image
+        run: gunzip -c perry-image.tar.gz | docker load
+
       - name: Set up Bun
         uses: oven-sh/setup-bun@v2
         with:
@@ -161,44 +240,15 @@ jobs:
           bun install
           cd web && bun install
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-        with:
-          driver: docker
-
-      - name: Log in to Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Build base image
-        uses: docker/build-push-action@v6
-        with:
-          context: ./perry
-          file: ./perry/Dockerfile.base
-          load: true
-          tags: perry-base:latest
-
-      - name: Build full image
-        uses: docker/build-push-action@v6
-        with:
-          context: ./perry
-          load: true
-          tags: perry:latest
-          build-args: |
-            BASE_IMAGE=perry-base:latest
-
-      - name: Run tests (shard ${{ matrix.shard }}/3)
+      - name: Run tests (shard ${{ matrix.shard }}/5)
         run: |
-          # Skip Docker build in test setup since we already have the image
           export SKIP_DOCKER_BUILD=true
-          bun run test --shard=${{ matrix.shard }}/3
+          export SKIP_AGENT_UPDATES=true
+          bun run test --shard=${{ matrix.shard }}/5
 
   e2e:
     runs-on: ubuntu-latest
-    needs: build
+    needs: [build, docker]
     steps:
       - uses: actions/checkout@v4
         with:
@@ -210,6 +260,14 @@ jobs:
           name: dist
           path: dist/
 
+      - name: Download Docker image
+        uses: actions/download-artifact@v4
+        with:
+          name: docker-image
+
+      - name: Load Docker image
+        run: gunzip -c perry-image.tar.gz | docker load
+
       - name: Set up Bun
         uses: oven-sh/setup-bun@v2
         with:
@@ -250,18 +308,6 @@ jobs:
         if: steps.playwright-cache.outputs.cache-hit == 'true'
         run: cd web && bun x playwright install-deps chromium
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-        with:
-          driver: docker
-
-      - name: Log in to Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
       - name: Check for Dockerfile changes
         id: docker-changes
         run: |
@@ -286,23 +332,6 @@ jobs:
             echo "is_fork=false" >> $GITHUB_OUTPUT
           fi
 
-      - name: Build base image
-        uses: docker/build-push-action@v6
-        with:
-          context: ./perry
-          file: ./perry/Dockerfile.base
-          load: true
-          tags: perry-base:latest
-
-      - name: Build full image
-        uses: docker/build-push-action@v6
-        with:
-          context: ./perry
-          load: true
-          tags: perry:latest
-          build-args: |
-            BASE_IMAGE=perry-base:latest
-
       - name: Configure agent for e2e tests
         run: |
           mkdir -p ~/.config/perry
@@ -322,7 +351,7 @@ jobs:
 
       - name: Start agent
         run: |
-          bun run src/index.ts agent run --port 7391 &
+          SKIP_AGENT_UPDATES=true bun run src/index.ts agent run --port 7391 &
           sleep 5
           curl -s http://localhost:7391/health || (echo "Agent failed to start" && exit 1)
 
diff --git a/perry/Dockerfile.base b/perry/Dockerfile.base
@@ -3,28 +3,6 @@
 # Minimal image with Perry runtime, coding agents, and essential tools.
 # Users can extend this with their own language runtimes and tools.
 
-# Stage 1: Build Neovim from source
-FROM ubuntu:noble AS neovim-builder
-
-ENV DEBIAN_FRONTEND=noninteractive
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    git \
-    ninja-build \
-    gettext \
-    cmake \
-    build-essential \
-    curl \
-    ca-certificates \
-  && rm -rf /var/lib/apt/lists/*
-
-RUN git clone --depth 1 --branch v0.11.4 https://github.com/neovim/neovim.git /tmp/neovim \
-  && cd /tmp/neovim \
-  && make CMAKE_BUILD_TYPE=RelWithDebInfo CMAKE_INSTALL_PREFIX=/usr/local \
-  && make install \
-  && rm -rf /tmp/neovim
-
-# Stage 2: Final image
 FROM ubuntu:noble
 
 ENV DEBIAN_FRONTEND=noninteractive
@@ -72,10 +50,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     fzf \
   && rm -rf /var/lib/apt/lists/*
 
-# Copy Neovim from builder stage
-COPY --from=neovim-builder /usr/local/bin/nvim /usr/local/bin/nvim
-COPY --from=neovim-builder /usr/local/lib/nvim /usr/local/lib/nvim
-COPY --from=neovim-builder /usr/local/share/nvim /usr/local/share/nvim
+# Install Neovim from official prebuilt binary
+RUN curl -fsSL https://github.com/neovim/neovim/releases/download/v0.11.6/nvim-linux-x86_64.tar.gz \
+    | tar -xz -C /opt \
+  && ln -s /opt/nvim-linux-x86_64/bin/nvim /usr/local/bin/nvim
 
 # Install GitHub CLI
 RUN curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg -o /usr/share/keyrings/githubcli-archive-keyring.gpg \
diff --git a/src/client/api.ts b/src/client/api.ts
@@ -42,7 +42,7 @@ export class ApiClient {
     this.token = options.token;
 
     const token = this.token;
-    const timeout = options.timeout || 30000;
+    const timeout = options.timeout || 120000;
     const link = new RPCLink({
       url: `${this.baseUrl}/rpc`,
       fetch: (url, init) => {
diff --git a/src/workspace/manager.ts b/src/workspace/manager.ts
@@ -374,6 +374,10 @@ export class WorkspaceManager {
   }
 
   private async updateAgentBinaries(containerName: string): Promise<void> {
+    if (process.env.SKIP_AGENT_UPDATES === 'true') {
+      return;
+    }
+
     const updates = [
       {
         name: 'claude',
diff --git a/test/helpers/agent.ts b/test/helpers/agent.ts
@@ -344,6 +344,7 @@ export async function startTestAgent(options: TestAgentOptions = {}): Promise<Te
       ...process.env,
       PERRY_CONFIG_DIR: configDir,
       PERRY_PORT: String(port),
+      SKIP_AGENT_UPDATES: 'true',
     },
     stdio: ['pipe', 'pipe', 'pipe'],
   });
diff --git a/vitest.config.js b/vitest.config.js
@@ -2,8 +2,8 @@ import { defineConfig } from 'vitest/config';
 
 export default defineConfig({
   test: {
-    testTimeout: 120000,
-    hookTimeout: 120000,
+    testTimeout: 60000,
+    hookTimeout: 60000,
     pool: 'forks',
     poolOptions: {
       forks: {

Original file line number	Diff line number	Diff line change
`@@ -374,6 +374,10 @@ export class WorkspaceManager {`
`374`	`374`	`}`
`375`	`375`
`376`	`376`	`private async updateAgentBinaries(containerName: string): Promise<void> {`
	`377`	`+ if (process.env.SKIP_AGENT_UPDATES === 'true') {`
	`378`	`+ return;`
	`379`	`+ }`
	`380`	`+`
`377`	`381`	`const updates = [`
`378`	`382`	`{`
`379`	`383`	`name: 'claude',`