[WIP] Reuse package artifacts between CI jobs

.buildkite/commands/build-for-mac.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+BUILD_TYPE=${1:?Expected build type as first parameter (dev|release)}
+ARCH=${2:?Expected architecture as second parameter (x64|arm64)}
+
+if [ "$BUILD_TYPE" != "dev" ] && [ "$BUILD_TYPE" != "release" ]; then
+  echo "Unknown build type: $BUILD_TYPE"
+  exit 1
+fi
+
+if [ "$ARCH" != "x64" ] && [ "$ARCH" != "arm64" ]; then
+  echo "Unknown architecture: $ARCH"
+  exit 1
+fi
+
+.buildkite/commands/prepare-environment.sh
+export FILE_ARCHITECTURE="$ARCH"
+.buildkite/commands/install-node-dependencies.sh
+
+if [ "$BUILD_TYPE" = "dev" ]; then
+  node ./scripts/prepare-dev-build-version.mjs
+  export IS_DEV_BUILD=true
+
+  ARTIFACT_PATTERN="*studio-app-mac-$ARCH.tar.gz"
+  printf 'Artifact search query: <%s>\n' "$ARTIFACT_PATTERN"
+  if ! buildkite-agent artifact download "$ARTIFACT_PATTERN" .; then
+    echo "^^^ +++ Required prebuilt app artifact not found: $ARTIFACT_PATTERN"
+    exit 1
+  fi
+
+  ARTIFACT_FILE=$(find . -type f -name "studio-app-mac-$ARCH.tar.gz" | head -n 1)
+  if [ -z "$ARTIFACT_FILE" ]; then
+    echo "^^^ +++ Downloaded artifact but couldn't locate archive for mac-$ARCH"
+    exit 1
+  fi
+
+  printf 'Resolved downloaded artifact: <%s>\n' "$ARTIFACT_FILE"
+  echo "--- :package: Extracting prebuilt app artifacts (mac-$ARCH) from $ARTIFACT_FILE"
+  tar -xzf "$ARTIFACT_FILE"
+else
+  node ./scripts/confirm-tag-matches-version.mjs
+
+  echo "--- :package: Packaging app"
+  npm run "package:macos-$ARCH"
+fi
+
+echo "--- :node: Building installer artifacts"
+npm run "make:macos-$ARCH"
+
+# Local trial and error show this needs to run before DMG generation,
+# but after the binary has been built.
+echo "--- :hammer: Rebuild fs-attr if necessary before generating DMG"
+case "$ARCH" in
+  x64)
+    echo "Rebuilding fs-xattr for $ARCH architecture"
+    npm rebuild fs-xattr --cpu universal
+    ;;
+  arm64)
+    echo "No need to rebuild fs-xattr because it works out of the box on Apple Silicon"
+    ;;
+esac
+
+echo "--- :node: Packaging in DMG"
+npm run "make:dmg-$ARCH"
+
+echo "--- 📃 Notarizing Binary"
+bundle exec fastlane notarize_binary

.buildkite/commands/build-for-windows.ps1

@@ -32,6 +32,10 @@ if ($Architecture -notin $VALID_ARCHITECTURES) {
 & "setup_windows_code_signing.ps1"
 If ($LastExitCode -ne 0) { Exit $LastExitCode }
 
+# Set architecture environment variable before dependency install so cache key
+# matches the target build architecture.
+$env:FILE_ARCHITECTURE=$Architecture
+
 Write-Host "--- :npm: Installing Node dependencies"
 bash .buildkite/commands/install-node-dependencies.sh
 If ($LastExitCode -ne 0) { Exit $LastExitCode }
@@ -50,10 +54,30 @@ if ($BuildType -eq $BUILD_TYPE_DEV) {
 	If ($LastExitCode -ne 0) { Exit $LastExitCode }
 }
 
-# Set architecture environment variable for AppX packaging
-$env:FILE_ARCHITECTURE=$Architecture
+Write-Host "--- :package: Preparing packaged app for architecture: $Architecture"
+if ($BuildType -eq $BUILD_TYPE_DEV) {
+    $artifactFilePattern = "*studio-app-windows-$Architecture.tar.gz"
+    bash -lc "buildkite-agent artifact download `"$artifactFilePattern`" ."
+    if ($LastExitCode -eq 0) {
+        $artifactFile = Get-ChildItem -Path . -Recurse -File -Filter "studio-app-windows-$Architecture.tar.gz" | Select-Object -First 1 -ExpandProperty FullName
+        if (-not $artifactFile) {
+            Write-Host "Error: downloaded artifact but could not locate studio-app-windows-$Architecture.tar.gz" -ForegroundColor Red
+            Exit 1
+        }
+        Write-Host "Using prebuilt app artifact: $artifactFile"
+        tar -xzf "$artifactFile"
+        If ($LastExitCode -ne 0) { Exit $LastExitCode }
+    } else {
+        Write-Host "Error: required prebuilt app artifact not found: $artifactFilePattern" -ForegroundColor Red
+        Exit 1
+    }
+} else {
+    Write-Host "Release build: packaging app for windows-$Architecture"
+    npm run "package:windows-$Architecture"
+    If ($LastExitCode -ne 0) { Exit $LastExitCode }
+}
 
-Write-Host "Building for architecture: $Architecture"
+Write-Host "--- :node: Building installer artifacts for architecture: $Architecture"
 npm run "make:windows-$Architecture"
 If ($LastExitCode -ne 0) { Exit $LastExitCode }
 

.buildkite/commands/install-node-dependencies.sh

@@ -3,17 +3,22 @@
 set -eu
 
 PLATFORM=$(uname -s)
-ARCHITECTURE=$(uname -m)
+ARCHITECTURE=${FILE_ARCHITECTURE:-$(uname -m)}
 NODE_VERSION=$(node --version)
+NPM_VERSION=$(npm --version)
 PACKAGE_HASH=$(hash_file package-lock.json)
-
-if [ -d patches ]; then
-  PATCHES_HASH=$(hash_directory patches/)
-else
-  PATCHES_HASH=nopatch
+IMAGE_KEY=${IMAGE_ID:-noimage}
+CACHE_FORMAT_VERSION=v2
+
+PATCHES_HASH=nopatch
+if [ -d apps/cli/patches ] || [ -d apps/studio/patches ]; then
+  CLI_PATCHES_HASH=$( [ -d apps/cli/patches ] && hash_directory apps/cli/patches || echo none )
+  STUDIO_PATCHES_HASH=$( [ -d apps/studio/patches ] && hash_directory apps/studio/patches || echo none )
+  PATCHES_HASH=$(echo "${CLI_PATCHES_HASH}-${STUDIO_PATCHES_HASH}" | shasum -a 256 | awk '{print $1}')
 fi
 
-CACHEKEY="$BUILDKITE_PIPELINE_SLUG-npm-$PLATFORM-$ARCHITECTURE-node-$NODE_VERSION-$PACKAGE_HASH-$PATCHES_HASH"
+BASE_CACHE_KEY="$BUILDKITE_PIPELINE_SLUG-$CACHE_FORMAT_VERSION-$PLATFORM-$ARCHITECTURE-image-$IMAGE_KEY-node-$NODE_VERSION-npm-$NPM_VERSION-$PACKAGE_HASH-$PATCHES_HASH"
+NPM_CACHE_KEY="$BASE_CACHE_KEY-npm-cache"
 
 LOCAL_NPM_CACHE=./vendor/npm
 mkdir -p $LOCAL_NPM_CACHE
@@ -22,7 +27,7 @@ npm set cache $LOCAL_NPM_CACHE
 echo "npm cache set to $(npm get cache)"
 
 echo "--- :npm: Restore npm cache if present"
-restore_cache "$CACHEKEY"
+restore_cache "$NPM_CACHE_KEY"
 
 echo "--- :npm: Install Node dependencies"
 
@@ -37,29 +42,17 @@ if [ "$PLATFORM" = "Linux" ]; then
 fi
 
 npm ci \
-  --unsafe-perm \
+  --include=dev \
   --prefer-offline \
   --no-audit \
   --no-progress \
   --maxsockets "$MAX_SOCKETS" \
   "$@"
 
-cd cli
-npm ci \
-  --prefer-offline \
-  --no-audit \
-  --no-progress \
-  --maxsockets "$MAX_SOCKETS"
-cd -
-
 echo "--- :npm: Save cache if necessary"
-# Notice that we don't cache the local node_modules.
-# Those get regenerated by npm ci as per Node reccomendations.
-# What we are caching is the root npm folder, which stores pacakge downloads so they are available if the package.json resolution demands them.
-#
 # npm stores temporary files in ~/.npm that we don't want to extract because they might run into naming conflicts.
 # So, before archiving it, we remove those tmp files.
 #
 # Example: https://buildkite.com/automattic/gutenberg-mobile/builds/8857#018e37eb-7afc-4280-b736-cba76f02f1a3/524
 rm -rf "$LOCAL_NPM_CACHE/_cacache/tmp"
-save_cache "$LOCAL_NPM_CACHE" "$CACHEKEY"
+save_cache "$LOCAL_NPM_CACHE" "$NPM_CACHE_KEY"

.buildkite/commands/package-app-for-ci.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+PLATFORM=${1:?Expected platform to be provided as first parameter}
+ARCH=${2:?Expected architecture to be provided as second parameter}
+
+echo "--- :package: Install deps"
+export FILE_ARCHITECTURE="$ARCH"
+bash .buildkite/commands/install-node-dependencies.sh
+
+export IS_DEV_BUILD=true
+
+case "$PLATFORM" in
+  mac)
+    SCRIPT_PLATFORM="macos"
+    ;;
+  windows)
+    SCRIPT_PLATFORM="windows"
+    ;;
+  *)
+    echo "Unknown platform: $PLATFORM"
+    exit 1
+    ;;
+esac
+
+echo "--- :package: Package app for CI reuse ($PLATFORM-$ARCH)"
+npm run "package:${SCRIPT_PLATFORM}-${ARCH}"
+
+mkdir -p artifacts
+ARTIFACT_FILE="artifacts/studio-app-${PLATFORM}-${ARCH}.tar.gz"
+
+echo "--- :package: Bundle packaged app into $ARTIFACT_FILE"
+tar -czf "$ARTIFACT_FILE" apps/studio/out
+
+if [ "$PLATFORM" = "mac" ] && [ "$ARCH" = "arm64" ] && [ "${BUILDKITE_BRANCH:-}" = "trunk" ]; then
+  echo "--- :package: Caching trunk mac-arm64 package artifact for compare-perf reuse"
+  TRUNK_ARTIFACT_CACHE_PATH="artifacts/trunk-mac-arm64"
+  TRUNK_ARTIFACT_CACHE_KEY="$BUILDKITE_PIPELINE_SLUG-trunk-mac-arm64-package"
+
+  mkdir -p "$TRUNK_ARTIFACT_CACHE_PATH"
+  cp "$ARTIFACT_FILE" "$TRUNK_ARTIFACT_CACHE_PATH/studio-app-trunk-mac-arm64.tar.gz"
+  save_cache "$TRUNK_ARTIFACT_CACHE_PATH" "$TRUNK_ARTIFACT_CACHE_KEY"
+fi

.buildkite/commands/prepare-environment.sh

@@ -1,7 +1,7 @@
 #!/bin/bash -eu
 
 # Prepares the CI environment to successfully build the macOS app.
-# Building the app is done via npm run make:macos-*.
+# Building installer artifacts is done via npm run make:macos-*.
 
 echo "--- :rubygems: Setting up Gems"
 install_gems
@@ -10,4 +10,4 @@ echo "--- :closed_lock_with_key: Installing Secrets"
 bundle exec fastlane run configure_apply
 
 echo "--- :testflight: Fetching Signing Certificates"
-bundle exec fastlane set_up_signing
+bundle exec fastlane set_up_signing

.buildkite/commands/run-e2e-tests.sh

@@ -13,26 +13,19 @@ bash .buildkite/commands/install-node-dependencies.sh
 
 export IS_DEV_BUILD=true
 
-# Map platform names to electron-forge platform values
-case "$PLATFORM" in
-  mac)
-    FORGE_PLATFORM="darwin"
-    ;;
-  windows)
-    FORGE_PLATFORM="win32"
-    ;;
-  *)
-    echo "Unknown platform: $PLATFORM"
-    exit 1
-    ;;
-esac
-
-# Use `electron-forge package` instead of `npm run make:*` for E2E tests.
-# `make` creates signed distributables (installers), which requires code signing setup.
-# `package` creates an unsigned app bundle, sufficient for E2E testing.
-echo "--- :package: Package app for testing ($PLATFORM-$ARCH)"
-npx electron-vite build --outDir=dist
-npx electron-forge package --arch="$ARCH" --platform="$FORGE_PLATFORM"
+ARTIFACT_PATTERN="*studio-app-${PLATFORM}-${ARCH}.tar.gz"
+
+if ! buildkite-agent artifact download "$ARTIFACT_PATTERN" .; then
+  echo "^^^ +++ Required prebuilt app artifact not found: $ARTIFACT_PATTERN"
+  exit 1
+fi
+ARTIFACT_FILE=$(find . -type f -name "studio-app-${PLATFORM}-${ARCH}.tar.gz" | head -n 1)
+if [ -z "$ARTIFACT_FILE" ]; then
+  echo "^^^ +++ Downloaded artifact but couldn't locate archive for ${PLATFORM}-${ARCH}"
+  exit 1
+fi
+echo "--- :package: Extracting prebuilt app artifacts ($PLATFORM-$ARCH) from $ARTIFACT_FILE"
+tar -xzf "$ARTIFACT_FILE"
 
 echo '--- :playwright: Run End To End Tests'
 

.buildkite/commands/run-metrics-tests.sh

@@ -9,38 +9,56 @@ echo '--- :package: Install main dependencies'
 bash .buildkite/commands/install-node-dependencies.sh
 
 echo '--- :package: Install compare-perf dependencies'
-cd scripts/compare-perf
-npm ci
-cd -
+npm -w compare-perf install
 
 export IS_DEV_BUILD=true
-export ARTIFACTS_PATH=${PWD}/artifacts
+export ARTIFACTS_PATH=${PWD}/tools/metrics/artifacts
 export SKIP_WORKER_THREAD_BUILD='true'
+export COMPARE_PERF_PREBUILT_BRANCH=$BUILDKITE_COMMIT
+export COMPARE_PERF_PREBUILT_OUT_DIR=${PWD}/apps/studio/out
+
+echo '--- :package: Downloading prebuilt app artifacts'
+buildkite-agent artifact download "artifacts/studio-app-mac-arm64.tar.gz" .
+tar -xzf artifacts/studio-app-mac-arm64.tar.gz
+
+if [ "${BUILDKITE_PULL_REQUEST}" != "false" ]; then
+  TRUNK_ARTIFACT_CACHE_KEY="$BUILDKITE_PIPELINE_SLUG-trunk-mac-arm64-package"
+  TRUNK_ARTIFACT_CACHE_PATH="artifacts/trunk-mac-arm64"
+  TRUNK_ARTIFACT_FILE="$TRUNK_ARTIFACT_CACHE_PATH/studio-app-trunk-mac-arm64.tar.gz"
+  TRUNK_EXTRACT_PATH="${PWD}/artifacts/trunk-mac-arm64-extract"
+
+  echo '--- :package: Restoring latest cached trunk package artifact'
+  if restore_cache "$TRUNK_ARTIFACT_CACHE_KEY" && [ -f "$TRUNK_ARTIFACT_FILE" ]; then
+    rm -rf "$TRUNK_EXTRACT_PATH"
+    mkdir -p "$TRUNK_EXTRACT_PATH"
+    tar -xzf "$TRUNK_ARTIFACT_FILE" -C "$TRUNK_EXTRACT_PATH"
+    export COMPARE_PERF_PREBUILT_BRANCH_BASE=trunk
+    export COMPARE_PERF_PREBUILT_OUT_DIR_BASE="$TRUNK_EXTRACT_PATH/apps/studio/out"
+  else
+    echo "--- :warning: Could not restore cached trunk package artifact; compare-perf will build trunk"
+  fi
+fi
 
 # Detect if this is a PR or trunk push
 if [ "${BUILDKITE_PULL_REQUEST}" != "false" ]; then
   # PR context - compare against trunk
   echo "--- :chart_with_upwards_trend: Running performance comparison against trunk"
-  cd scripts/compare-perf
-  npm run compare -- perf $BUILDKITE_COMMIT trunk --tests-branch $BUILDKITE_COMMIT --rounds 3
+  npm -w compare-perf run compare -- perf $BUILDKITE_COMMIT trunk --tests-branch $BUILDKITE_COMMIT --rounds 3
 
   echo "--- :github: Posting results to PR"
   # Parse repo from git@github.com:owner/repo.git or https://github.com/owner/repo.git format
   REPO_PATH=$(echo $BUILDKITE_REPO | sed 's|^git@github\.com:||' | sed 's|^https://github\.com/||' | sed 's|\.git$||')
-  npm run post-to-github -- $GITHUB_TOKEN $REPO_PATH $BUILDKITE_PULL_REQUEST trunk $BUILDKITE_COMMIT
-  cd -
+  npm -w compare-perf run post-to-github -- $GITHUB_TOKEN $REPO_PATH $BUILDKITE_PULL_REQUEST trunk $BUILDKITE_COMMIT
 elif [ "${BUILDKITE_BRANCH}" == "trunk" ]; then
   # Trunk push context - compare against baseline
   BASELINE_COMMIT="58c52bfee7e585614ced202f43f217a01f94f029"
 
   echo "--- :chart_with_upwards_trend: Running performance comparison against baseline"
-  cd scripts/compare-perf
-  npm run compare -- perf $BUILDKITE_COMMIT $BASELINE_COMMIT --tests-branch $BUILDKITE_COMMIT --rounds 3
+  npm -w compare-perf run compare -- perf $BUILDKITE_COMMIT $BASELINE_COMMIT --tests-branch $BUILDKITE_COMMIT --rounds 3
 
   echo "--- :bar_chart: Logging metrics to CodeVitals"
   COMMITTED_AT=$(git show -s $BUILDKITE_COMMIT --format="%cI")
-  npm run log-to-codevitals -- $CODEVITALS_AUTH_TOKEN trunk $BUILDKITE_COMMIT $BASELINE_COMMIT $COMMITTED_AT
-  cd -
+  npm -w compare-perf run log-to-codevitals -- $CODEVITALS_AUTH_TOKEN trunk $BUILDKITE_COMMIT $BASELINE_COMMIT $COMMITTED_AT
 else
   # Other branches - skip metrics
   echo "--- :information_source: Skipping metrics for non-trunk branch"

.buildkite/commands/should-skip-job.sh

@@ -10,7 +10,7 @@ set -eu
 # Job types:
 #   - validation: Skip if changes are limited to documentation, config, localization, and non-code files.
 #                 Used for lint, unit tests, and e2e tests.
-#   - metrics: Same as validation, but also skips on test-only changes (e2e/**, *.test.ts).
+#   - metrics: Same as validation, but also skips on test-only changes (apps/studio/e2e/**, *.test.ts).
 #              Since metrics measure app performance, test file changes don't affect them.
 #   - build: Skip if changes are limited to documentation and config files.
 #            Does NOT skip on localization changes since builds should include translation updates.
@@ -53,7 +53,7 @@ COMMON_NON_CODE_PATTERNS=(
 
   # Installer assets and configuration
   "installers/**"
-  "assets/appx/**"
+  "apps/studio/assets/appx/**"
 
   # Claude AI configuration
   ".claude/**"
@@ -62,17 +62,17 @@ COMMON_NON_CODE_PATTERNS=(
 
 # Localization files - changes here don't affect runtime behavior or performance
 LOCALIZATION_PATTERNS=(
-  "common/translations/**"
+  "tools/common/translations/**"
 )
 
 # Test files - changes here don't affect app performance (for metrics)
 TEST_PATTERNS=(
-  "e2e/**"
-  "src/tests/**"
-  "src/**/*.test.ts"
-  "src/**/*.test.tsx"
-  "cli/**/*.test.ts"
-  "common/**/*.test.ts"
+  "apps/studio/e2e/**"
+  "apps/studio/src/tests/**"
+  "apps/studio/src/**/*.test.ts"
+  "apps/studio/src/**/*.test.tsx"
+  "apps/cli/**/*.test.ts"
+  "tools/common/**/*.test.ts"
   "metrics/**"
 )