From 8ca10ae8047295e6d0486e7494691394c58322e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gergely=20Cs=C3=A9csey?= Date: Tue, 17 Feb 2026 16:24:20 +0000 Subject: [PATCH 01/14] Add site editor performance benchmark orchestration script (STU-1290) Implements automated benchmarking across a 10-environment matrix to measure site editor performance impact of having 10 plugins installed. Includes orchestration script that: - Sets up isolated Studio, Playground CLI, and Playground Web environments - Supports bare, multi-worker, and plugin variants - Installs 10 popular plugins via reusable blueprint - Runs site-editor-benchmark test against each environment - Outputs comparison table showing performance metrics across all variants Files: - metrics/tests/site-editor-benchmark.test.ts: Playwright benchmark test (from PR #2368) - scripts/benchmark-site-editor/benchmark.ts: Main orchestration script with environment setup - scripts/benchmark-site-editor/plugins-blueprint.json: Blueprint with 10 plugins - scripts/benchmark-site-editor/{package.json,tsconfig.json,README.md}: Script configuration --- metrics/tests/site-editor-benchmark.test.ts | 371 ++++++++ package-lock.json | 10 + scripts/benchmark-site-editor/.gitignore | 4 + scripts/benchmark-site-editor/README.md | 96 ++ scripts/benchmark-site-editor/benchmark.ts | 820 ++++++++++++++++++ scripts/benchmark-site-editor/package.json | 20 + .../plugins-blueprint.json | 75 ++ scripts/benchmark-site-editor/tsconfig.json | 15 + 8 files changed, 1411 insertions(+) create mode 100644 metrics/tests/site-editor-benchmark.test.ts create mode 100644 scripts/benchmark-site-editor/.gitignore create mode 100644 scripts/benchmark-site-editor/README.md create mode 100644 scripts/benchmark-site-editor/benchmark.ts create mode 100644 scripts/benchmark-site-editor/package.json create mode 100644 scripts/benchmark-site-editor/plugins-blueprint.json create mode 100644 scripts/benchmark-site-editor/tsconfig.json diff --git a/metrics/tests/site-editor-benchmark.test.ts b/metrics/tests/site-editor-benchmark.test.ts new file mode 100644 index 0000000000..a1e45359cc --- /dev/null +++ b/metrics/tests/site-editor-benchmark.test.ts @@ -0,0 +1,371 @@ +import { test, chromium, Page, Frame } from '@playwright/test'; +import { getUrlWithAutoLogin } from '../../e2e/utils'; +import { median } from '../utils'; + +// Helper functions for the benchmark test +function findWordPressFrame( page: Page ): Frame | null { + const frames = page.frames(); + let wordPressFrame = + frames.find( ( frame ) => { + const url = frame.url(); + return ( + url.includes( 'wordpress' ) || + url.includes( 'wp-admin' ) || + url.includes( 'wp-login' ) || + url.includes( 'scope:' ) + ); + } ) || null; + + if ( ! wordPressFrame ) { + // Try searching nested frames + for ( const frame of page.frames() ) { + if ( frame.parentFrame() && frame.url().includes( 'scope:' ) ) { + wordPressFrame = frame; + break; + } + } + } + + return wordPressFrame; +} + +function findEditorCanvasFrame( + page: Page, + isPlaygroundWeb: boolean, + wordPressFrame: Frame | null +): Frame | null { + if ( isPlaygroundWeb && wordPressFrame ) { + // For Playground web, search in child frames + let frame = wordPressFrame.childFrames().find( ( f ) => f.name() === 'editor-canvas' ) || null; + // If not found, try searching all frames + if ( ! frame ) { + const allFrames = page.frames(); + frame = allFrames.find( ( f ) => f.name() === 'editor-canvas' ) || null; + } + return frame; + } + // For regular WordPress, get the frame from the page + return page.frame( { name: 'editor-canvas' } ); +} + +test.describe( 'Site Editor Performance Benchmark', () => { + // Results collection structure - each metric stores array of values from all runs + const results: Record< string, number[] > = { + siteEditorLoad: [], + templatesViewLoad: [], + templateOpen: [], + blockAdd: [], + templateSave: [], + }; + + // Get configuration from environment variables + const targetUrl = process.env.BENCHMARK_URL; + const BENCHMARK_RUNS = parseInt( process.env.BENCHMARK_RUNS || '3', 10 ); + + // Environment detection (parsed once, reused for all iterations) + if ( ! targetUrl ) { + throw new Error( + 'BENCHMARK_URL environment variable is required. Example: BENCHMARK_URL=http://localhost:8888' + ); + } + + // Parse and normalize the URL + let wpAdminUrl = targetUrl; + if ( ! wpAdminUrl.startsWith( 'http' ) ) { + wpAdminUrl = `http://${ wpAdminUrl }`; + } + wpAdminUrl = wpAdminUrl.replace( /\/$/, '' ); + + const isPlaygroundWeb = wpAdminUrl.includes( 'playground.wordpress.net' ); + const isLocalPlaygroundCli = wpAdminUrl.includes( '127.0.0.1' ); + + // Create URL identifier for metric names + const urlIdentifier = isPlaygroundWeb + ? 'playground-web' + : wpAdminUrl + .replace( /^https?:\/\//, '' ) + .replace( /\/$/, '' ) + .replace( /[^a-z0-9]/gi, '-' ); + + test.afterAll( async ( {}, testInfo ) => { + // Calculate medians using existing utility + const medians: Record< string, number > = {}; + Object.entries( results ).forEach( ( [ key, values ] ) => { + if ( values.length > 0 ) { + const medianValue = median( values ); + if ( medianValue !== undefined ) { + medians[ `${ urlIdentifier }-${ key }` ] = medianValue; + } + } + } ); + + // Attach aggregated results (for performance reporter) + await testInfo.attach( 'results', { + body: JSON.stringify( medians, null, 2 ), + contentType: 'application/json', + } ); + + // Attach detailed results (individual runs + aggregated) + const detailedResults = { + url: wpAdminUrl, + runs: BENCHMARK_RUNS, + successfulRuns: results.siteEditorLoad.length, + individual: results, + medians: medians, + }; + + await testInfo.attach( 'benchmark-results-detailed', { + body: JSON.stringify( detailedResults, null, 2 ), + contentType: 'application/json', + } ); + } ); + + test( 'benchmark site editor performance', async () => { + // Run benchmark N times with fresh browser per iteration + for ( let run = 1; run <= BENCHMARK_RUNS; run++ ) { + await test.step( `Run ${ run }/${ BENCHMARK_RUNS }`, async () => { + // Launch fresh browser for this iteration + const browser = await chromium.launch(); + const context = await browser.newContext(); + const page = await context.newPage(); + + let wordPressFrame: Frame | null = null; + let wordPressFrameLocator: ReturnType< Page[ 'frameLocator' ] > | null = null; + + try { + // Environment-specific preparation: Navigate to wp-admin + if ( isPlaygroundWeb ) { + // For Playground web: use the URL from environment variable + await page.goto( wpAdminUrl, { waitUntil: 'networkidle' } ); + + // Get WordPress frame locator + wordPressFrameLocator = page + .frameLocator( 'iframe.playground-viewport' ) + .first() + .frameLocator( 'iframe' ) + .first(); + + // Wait for wp-admin to load + await wordPressFrameLocator + .getByRole( 'link', { name: 'Appearance' } ) + .waitFor( { timeout: 30_000 } ); + + // Get the actual frame object + wordPressFrame = findWordPressFrame( page ); + } else if ( isLocalPlaygroundCli ) { + // For local Playground CLI: navigate directly to wp-admin + // Note: Playground CLI may redirect, so we follow redirects + await page.goto( `${ wpAdminUrl }/wp-admin`, { + waitUntil: 'domcontentloaded', + timeout: 120_000, + } ); + // Wait for page to settle and Appearance link to appear + await page.waitForLoadState( 'networkidle', { timeout: 30_000 } ).catch( () => {} ); + await page.getByRole( 'link', { name: 'Appearance' } ).waitFor( { + state: 'visible', + timeout: 60_000, + } ); + } else { + // For Studio: use auto-login endpoint + await page.goto( getUrlWithAutoLogin( `${ wpAdminUrl }/wp-admin` ), { + waitUntil: 'domcontentloaded', + timeout: 120_000, + } ); + // Wait for page to settle and Appearance link to appear + await page.waitForLoadState( 'networkidle', { timeout: 30_000 } ).catch( () => {} ); + await page.getByRole( 'link', { name: 'Appearance' } ).waitFor( { + state: 'visible', + timeout: 60_000, + } ); + } + + // Get the target for interactions + const target = isPlaygroundWeb && wordPressFrameLocator ? wordPressFrameLocator : page; + + // Step 1: Navigate to site editor from wp-admin using Appearance > Editor + const siteEditorStartTime = Date.now(); + + // Click Appearance menu + await target.getByRole( 'link', { name: 'Appearance' } ).click(); + // Click Editor submenu - use href to be specific (site-editor.php is the site editor) + await target.locator( 'a[href="site-editor.php"]' ).click(); + + // Close welcome modal if it appears + const welcomeDialog = target.getByRole( 'dialog', { + name: /welcome to the site editor/i, + } ); + const isModalVisible = await welcomeDialog + .isVisible( { timeout: 5_000 } ) + .catch( () => false ); + if ( isModalVisible ) { + await target.getByRole( 'button', { name: /get started/i } ).click(); + await welcomeDialog.waitFor( { state: 'hidden', timeout: 5_000 } ).catch( () => {} ); + } + + // Wait for editor canvas iframe to appear + await target.locator( 'iframe[name="editor-canvas"]' ).waitFor( { + state: 'visible', + timeout: 120_000, + } ); + + // Find the editor canvas frame + const frame = findEditorCanvasFrame( page, isPlaygroundWeb, wordPressFrame ); + if ( ! frame ) { + throw new Error( 'Editor canvas frame not found' ); + } + + // Wait for frame to be ready + await frame.waitForLoadState( 'domcontentloaded' ); + // Wait for blocks to be present and rendered (positive indicator that editor is ready) + await frame.waitForSelector( '[data-block]', { timeout: 60_000 } ); + // Ensure at least one block is fully rendered (not just in DOM) + await frame.waitForFunction( + () => { + const blocks = document.querySelectorAll( '[data-block]' ); + return ( + blocks.length > 0 && + Array.from( blocks ).some( ( block ) => block.clientHeight > 0 ) + ); + }, + { timeout: 60_000 } + ); + + const siteEditorEndTime = Date.now(); + results.siteEditorLoad.push( siteEditorEndTime - siteEditorStartTime ); + + // Step 2: Navigate to Templates view by clicking Templates button in sidebar + const templatesViewStartTime = Date.now(); + + // Click the Templates button in the sidebar (works across all environments) + await target.getByRole( 'button', { name: 'Templates' } ).click(); + + // Wait for Templates view to load - wait for heading, grid, and ensure first card is clickable + await target.getByRole( 'heading', { name: 'Templates', level: 2 } ).waitFor( { + timeout: 60_000, + } ); + await target + .locator( '.dataviews-view-grid-items.dataviews-view-grid' ) + .waitFor( { timeout: 60_000 } ); + // Wait for the first template card to be visible and clickable (indicates page is ready) + const firstCard = target.locator( '.dataviews-view-grid__card' ).first(); + await firstCard.waitFor( { state: 'visible', timeout: 60_000 } ); + await firstCard + .getByRole( 'button' ) + .first() + .waitFor( { state: 'visible', timeout: 60_000 } ); + + const templatesViewEndTime = Date.now(); + results.templatesViewLoad.push( templatesViewEndTime - templatesViewStartTime ); + + // Step 3: Open a template + const templateOpenStartTime = Date.now(); + + // Click on the first template card (it's a button, not a link) + await target + .locator( '.dataviews-view-grid__card' ) + .first() + .getByRole( 'button' ) + .first() + .click(); + + // Wait for template editor to load + await target.locator( 'iframe[name="editor-canvas"]' ).waitFor( { + state: 'visible', + timeout: 60_000, + } ); + + // Find the template editor canvas frame + const templateFrame = findEditorCanvasFrame( page, isPlaygroundWeb, wordPressFrame ); + if ( ! templateFrame ) { + throw new Error( 'Template editor frame not found' ); + } + + // Wait for template editor to be ready + await templateFrame.waitForLoadState( 'domcontentloaded' ); + // Wait for blocks to be present and rendered (positive indicator that editor is ready) + await templateFrame.waitForSelector( '[data-block]', { timeout: 60_000 } ); + // Ensure at least one block is fully rendered (not just in DOM) + await templateFrame.waitForFunction( + () => { + const blocks = document.querySelectorAll( '[data-block]' ); + return ( + blocks.length > 0 && + Array.from( blocks ).some( ( block ) => block.clientHeight > 0 ) + ); + }, + { timeout: 60_000 } + ); + + const templateOpenEndTime = Date.now(); + results.templateOpen.push( templateOpenEndTime - templateOpenStartTime ); + + // Step 4: Add blocks + const blockAddStartTime = Date.now(); + + // Close any modals + await page.keyboard.press( 'Escape' ); + + // Open block inserter + await target.getByRole( 'button', { name: /Block Inserter/i } ).click(); + + // Search and insert Paragraph block + const searchInput = target.getByPlaceholder( 'Search' ); + await searchInput.fill( 'Paragraph' ); + await target.getByRole( 'option', { name: 'Paragraph', exact: true } ).click(); + + // Wait for paragraph block to appear + await templateFrame.waitForSelector( 'p[data-block]', { timeout: 15_000 } ); + + // Add Heading block + await searchInput.fill( 'Heading' ); + // Get the block type option (not the pattern) - block types are buttons with class "block-editor-block-types-list__item" + await target + .locator( '.block-editor-block-types-list__item' ) + .filter( { hasText: /^Heading$/ } ) + .click(); + + // Wait for heading block to appear + await templateFrame.waitForSelector( 'h1[data-block], h2[data-block], h3[data-block]', { + timeout: 15_000, + } ); + + const blockAddEndTime = Date.now(); + results.blockAdd.push( blockAddEndTime - blockAddStartTime ); + + // Step 5: Save the template + const templateSaveStartTime = Date.now(); + + await target.getByRole( 'button', { name: 'Save' } ).first().click(); + + // Wait for save confirmation - button text changes to "Saved" + await page.waitForFunction( + () => { + const saveButton = Array.from( document.querySelectorAll( 'button' ) ).find( + ( btn ) => + btn.textContent?.includes( 'Saved' ) || + btn.getAttribute( 'aria-label' )?.toLowerCase().includes( 'saved' ) + ); + return saveButton !== null; + }, + { timeout: 30_000 } + ); + + const templateSaveEndTime = Date.now(); + results.templateSave.push( templateSaveEndTime - templateSaveStartTime ); + + console.log( ` ✓ Run ${ run } completed` ); + } finally { + // Always cleanup browser + await page.close(); + await context.close(); + await browser.close(); + } + + // Small delay between runs + if ( run < BENCHMARK_RUNS ) { + await new Promise( ( resolve ) => setTimeout( resolve, 1000 ) ); + } + } ); + } + } ); +} ); diff --git a/package-lock.json b/package-lock.json index ce38e68f3e..81689b096e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8100,6 +8100,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -8116,6 +8117,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -8132,6 +8134,7 @@ "cpu": [ "arm" ], + "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -8148,6 +8151,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -8164,6 +8168,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -8180,6 +8185,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -8196,6 +8202,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -8212,6 +8219,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -8228,6 +8236,7 @@ "cpu": [ "ia32" ], + "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -8244,6 +8253,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ diff --git a/scripts/benchmark-site-editor/.gitignore b/scripts/benchmark-site-editor/.gitignore new file mode 100644 index 0000000000..2c4d248c80 --- /dev/null +++ b/scripts/benchmark-site-editor/.gitignore @@ -0,0 +1,4 @@ +node_modules/ +package-lock.json +dist/ +results-*.json diff --git a/scripts/benchmark-site-editor/README.md b/scripts/benchmark-site-editor/README.md new file mode 100644 index 0000000000..fbded044ef --- /dev/null +++ b/scripts/benchmark-site-editor/README.md @@ -0,0 +1,96 @@ +# Site Editor Performance Benchmark + +Benchmarks site editor performance across Studio, Playground CLI, and Playground Web environments, with optional plugin and multi-worker configurations. + +## Related Issue + +[STU-1290](https://linear.app/a8c/issue/STU-1290) + +## What It Measures + +The benchmark runs the `site-editor-benchmark` Playwright test against each environment, measuring: + +| Metric | Description | +| --------------------- | ---------------------------------------------------------- | +| **siteEditorLoad** | Time from clicking Appearance > Editor to blocks rendering | +| **templatesViewLoad** | Time to open the Templates view and load template cards | +| **templateOpen** | Time to open a specific template in the editor | +| **blockAdd** | Time to add a paragraph and heading block | +| **templateSave** | Time to save the template | + +## Environment Matrix + +| Environment | Name | Description | +| ----------------------------- | ------------------- | ----------------------------------------------- | +| Studio | `studio` | Bare Studio site | +| Studio + MW | `studio-mw` | Studio with multi-worker support enabled | +| Studio + Plugins | `studio-plugins` | Studio with 10 plugins installed | +| Studio + MW + Plugins | `studio-mw-plugins` | Studio with multi-worker and 10 plugins | +| Playground CLI | `pg-cli` | Bare Playground CLI site | +| Playground CLI + MW | `pg-cli-mw` | Playground CLI with multi-worker | +| Playground CLI + Plugins | `pg-cli-plugins` | Playground CLI with 10 plugins | +| Playground CLI + MW + Plugins | `pg-cli-mw-plugins` | Playground CLI with multi-worker and 10 plugins | +| Playground Web | `pg-web` | playground.wordpress.net (bare) | +| Playground Web + Plugins | `pg-web-plugins` | playground.wordpress.net with 10 plugins | + +## Plugins + +When the "plugins" variant is enabled, these 10 plugins are installed via a blueprint: + +- WooCommerce +- Jetpack +- WP Super Cache +- Jetpack Boost +- Jetpack Protect +- Jetpack Social +- Jetpack VideoPress +- WooCommerce Payments +- Contact Form 7 +- Elementor + +## Usage + +```bash +cd scripts/benchmark-site-editor +npm install +npm run benchmark +``` + +### Options + +``` +--rounds=N Number of benchmark runs per environment (default: 1) +--skip-studio Skip Studio environments +--skip-playground-cli Skip Playground CLI environments +--skip-playground-web Skip Playground web environments +--only= Run only named environments (comma-separated) +--help Show help +``` + +### Examples + +```bash +# Quick test: only Studio bare vs Studio with plugins +npm run benchmark -- --only=studio,studio-plugins + +# Full comparison without Playground Web (faster, no network dependency) +npm run benchmark -- --skip-playground-web --rounds=3 + +# Only Playground CLI environments +npm run benchmark -- --skip-studio --skip-playground-web + +# Single specific environment +npm run benchmark -- --only=studio-mw-plugins --rounds=5 +``` + +## Prerequisites + +- **Studio CLI**: Built automatically if `dist/cli/main.js` doesn't exist (`npm run cli:build`) +- **Playground CLI**: Installed automatically via this script's `npm install` +- **Playwright**: Must be installed at the repo root (`npx playwright install chromium`) + +## Output + +Results are printed as a comparison table and saved to `metrics/artifacts/benchmark-comparison-.json`. + +Individual results per environment are saved to `metrics/artifacts/.results.json`. diff --git a/scripts/benchmark-site-editor/benchmark.ts b/scripts/benchmark-site-editor/benchmark.ts new file mode 100644 index 0000000000..696e8ebe36 --- /dev/null +++ b/scripts/benchmark-site-editor/benchmark.ts @@ -0,0 +1,820 @@ +#!/usr/bin/env tsx +/* eslint-disable no-console */ +/** + * Site Editor Performance Benchmark — Orchestration Script + * + * Runs the site-editor-benchmark Playwright test across a matrix of environments: + * - Studio (bare, multi-worker, plugins, multi-worker+plugins) + * - Playground CLI (bare, multi-worker, plugins, multi-worker+plugins) + * - Playground Web (bare, plugins) + * + * Usage: + * cd scripts/benchmark-site-editor + * npm install + * npm run benchmark + * + * Options: + * --rounds=N Number of benchmark runs per environment (default: 1) + * --skip-studio Skip Studio environments + * --skip-playground-cli Skip Playground CLI environments + * --skip-playground-web Skip Playground web environments + * --only= Run only these environments (comma-separated) + * --help Show help + */ + +import { spawn, execSync, ChildProcess } from 'child_process'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import chalk from 'chalk'; + +// --------------------------------------------------------------------------- +// Configuration +// --------------------------------------------------------------------------- + +const STUDIO_ROOT = path.resolve( import.meta.dirname, '../..' ); +const STUDIO_CLI_PATH = path.resolve( STUDIO_ROOT, 'dist/cli/main.js' ); +const PLAYGROUND_CLI_BIN = + process.platform === 'win32' ? 'wp-playground-cli.cmd' : 'wp-playground-cli'; +const PLAYGROUND_CLI_PATH = path.resolve( + import.meta.dirname, + 'node_modules/.bin', + PLAYGROUND_CLI_BIN +); +const PLUGINS_BLUEPRINT_PATH = path.resolve( import.meta.dirname, 'plugins-blueprint.json' ); +const ARTIFACTS_PATH = path.resolve( STUDIO_ROOT, 'metrics', 'artifacts' ); +const BENCHMARK_TEST_NAME = 'site-editor-benchmark'; +const PLAYWRIGHT_CONFIG = path.resolve( STUDIO_ROOT, 'metrics', 'playwright.metrics.config.ts' ); + +const PLAYGROUND_WEB_BASE_URL = 'https://playground.wordpress.net'; + +// Ports used for local servers — offset to avoid collisions with dev servers +const STUDIO_PORT_BASE = 9400; +const PLAYGROUND_CLI_PORT_BASE = 9500; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +type EnvironmentType = 'studio' | 'playground-cli' | 'playground-web'; + +interface EnvironmentConfig { + name: string; + type: EnvironmentType; + plugins: boolean; + multiWorker: boolean; +} + +interface BenchmarkResult { + environment: string; + metrics: Record< string, number >; +} + +// --------------------------------------------------------------------------- +// Environment matrix +// --------------------------------------------------------------------------- + +const ALL_ENVIRONMENTS: EnvironmentConfig[] = [ + { name: 'studio', type: 'studio', plugins: false, multiWorker: false }, + { name: 'studio-mw', type: 'studio', plugins: false, multiWorker: true }, + { name: 'studio-plugins', type: 'studio', plugins: true, multiWorker: false }, + { name: 'studio-mw-plugins', type: 'studio', plugins: true, multiWorker: true }, + { name: 'pg-cli', type: 'playground-cli', plugins: false, multiWorker: false }, + { name: 'pg-cli-mw', type: 'playground-cli', plugins: false, multiWorker: true }, + { name: 'pg-cli-plugins', type: 'playground-cli', plugins: true, multiWorker: false }, + { + name: 'pg-cli-mw-plugins', + type: 'playground-cli', + plugins: true, + multiWorker: true, + }, + { name: 'pg-web', type: 'playground-web', plugins: false, multiWorker: false }, + { name: 'pg-web-plugins', type: 'playground-web', plugins: true, multiWorker: false }, +]; + +// --------------------------------------------------------------------------- +// Argument parsing +// --------------------------------------------------------------------------- + +interface Options { + rounds: number; + skipStudio: boolean; + skipPlaygroundCli: boolean; + skipPlaygroundWeb: boolean; + only: string[]; +} + +function parseArgs(): Options { + const args = process.argv.slice( 2 ); + const opts: Options = { + rounds: 1, + skipStudio: false, + skipPlaygroundCli: false, + skipPlaygroundWeb: false, + only: [], + }; + + for ( const arg of args ) { + if ( arg.startsWith( '--rounds=' ) ) { + opts.rounds = parseInt( arg.split( '=' )[ 1 ], 10 ); + } else if ( arg === '--skip-studio' ) { + opts.skipStudio = true; + } else if ( arg === '--skip-playground-cli' ) { + opts.skipPlaygroundCli = true; + } else if ( arg === '--skip-playground-web' ) { + opts.skipPlaygroundWeb = true; + } else if ( arg.startsWith( '--only=' ) ) { + opts.only = arg + .split( '=' )[ 1 ] + .split( ',' ) + .map( ( s ) => s.trim() ); + } else if ( arg === '--help' ) { + printHelp(); + process.exit( 0 ); + } + } + + return opts; +} + +function printHelp() { + console.log( ` +Usage: npm run benchmark [options] + +Options: + --rounds=N Number of benchmark runs per environment (default: 1) + --skip-studio Skip Studio environments + --skip-playground-cli Skip Playground CLI environments + --skip-playground-web Skip Playground web environments + --only= Run only named environments (comma-separated) + --help Show this help message + +Environments: ${ ALL_ENVIRONMENTS.map( ( e ) => e.name ).join( ', ' ) } +` ); +} + +// --------------------------------------------------------------------------- +// Utility functions +// --------------------------------------------------------------------------- + +function median( values: number[] ): number { + if ( values.length === 0 ) return 0; + const sorted = [ ...values ].sort( ( a, b ) => a - b ); + const mid = Math.floor( sorted.length / 2 ); + return sorted.length % 2 !== 0 ? sorted[ mid ] : ( sorted[ mid - 1 ] + sorted[ mid ] ) / 2; +} + +function formatDuration( ms: number ): string { + if ( ms < 1000 ) return `${ ms.toFixed( 0 ) }ms`; + return `${ ( ms / 1000 ).toFixed( 2 ) }s`; +} + +function createTempDir( prefix: string ): string { + return fs.mkdtempSync( path.join( os.tmpdir(), `benchmark-${ prefix }-` ) ); +} + +function cleanupDir( dir: string ): void { + if ( fs.existsSync( dir ) ) { + fs.rmSync( dir, { recursive: true, force: true } ); + } +} + +async function waitForServer( url: string, timeoutMs = 60_000 ): Promise< boolean > { + const start = Date.now(); + while ( Date.now() - start < timeoutMs ) { + try { + const response = await fetch( url ); + if ( response.ok || response.status === 302 || response.status === 301 ) { + return true; + } + } catch { + // Server not ready yet + } + await sleep( 1000 ); + } + return false; +} + +async function killProcessOnPort( port: number ): Promise< void > { + return new Promise( ( resolve ) => { + try { + if ( process.platform === 'win32' ) { + execSync( + `for /f "tokens=5" %a in ('netstat -aon ^| find ":${ port }"') do taskkill /F /PID %a`, + { stdio: 'ignore' } + ); + } else { + execSync( `lsof -ti:${ port } | xargs kill -9 2>/dev/null || true`, { + stdio: 'ignore', + } ); + } + } catch { + // Process may not exist + } + setTimeout( resolve, 500 ); + } ); +} + +function sleep( ms: number ): Promise< void > { + return new Promise( ( resolve ) => setTimeout( resolve, ms ) ); +} + +function runCommand( + command: string, + args: string[], + options: { cwd?: string; env?: NodeJS.ProcessEnv; timeout?: number } = {} +): Promise< { stdout: string; stderr: string; exitCode: number } > { + const timeout = options.timeout ?? 300_000; + + return new Promise( ( resolve ) => { + let stdout = ''; + let stderr = ''; + let timedOut = false; + + const proc = spawn( command, args, { + cwd: options.cwd, + stdio: [ 'ignore', 'pipe', 'pipe' ], + env: { ...process.env, ...options.env, FORCE_COLOR: '0' }, + } ); + + const timeoutId = setTimeout( () => { + timedOut = true; + proc.kill( 'SIGKILL' ); + }, timeout ); + + proc.stdout?.on( 'data', ( data ) => { + stdout += data.toString(); + } ); + proc.stderr?.on( 'data', ( data ) => { + stderr += data.toString(); + } ); + + proc.on( 'close', ( code ) => { + clearTimeout( timeoutId ); + if ( timedOut ) { + resolve( { stdout, stderr: 'Timeout', exitCode: 1 } ); + } else { + resolve( { stdout, stderr, exitCode: code ?? 1 } ); + } + } ); + + proc.on( 'error', ( err ) => { + clearTimeout( timeoutId ); + resolve( { stdout, stderr: err.message, exitCode: 1 } ); + } ); + } ); +} + +// --------------------------------------------------------------------------- +// Studio environment helpers +// --------------------------------------------------------------------------- + +function createStudioAppdata( appdataDir: string, multiWorker: boolean ): void { + const studioDir = path.join( appdataDir, 'Studio' ); + fs.mkdirSync( studioDir, { recursive: true } ); + + const appdata = { + version: 1, + sites: [], + snapshots: [], + betaFeatures: { + studioSitesCli: true, + multiWorkerSupport: multiWorker, + }, + }; + + fs.writeFileSync( path.join( studioDir, 'appdata-v1.json' ), JSON.stringify( appdata, null, 2 ) ); + + // Set up server-files with symlinks to the repo's bundled wp-files. + // The CLI expects these at /Studio/server-files/. + const serverFilesDir = path.join( studioDir, 'server-files' ); + const wpFilesDir = path.join( STUDIO_ROOT, 'wp-files' ); + + fs.mkdirSync( path.join( serverFilesDir, 'wordpress-versions' ), { recursive: true } ); + fs.symlinkSync( + path.join( wpFilesDir, 'latest', 'wordpress' ), + path.join( serverFilesDir, 'wordpress-versions', 'latest' ) + ); + fs.symlinkSync( + path.join( wpFilesDir, 'sqlite-database-integration' ), + path.join( serverFilesDir, 'sqlite-database-integration' ) + ); + fs.symlinkSync( + path.join( wpFilesDir, 'sqlite-command' ), + path.join( serverFilesDir, 'sqlite-command' ) + ); + fs.symlinkSync( + path.join( wpFilesDir, 'wp-cli', 'wp-cli.phar' ), + path.join( serverFilesDir, 'wp-cli.phar' ) + ); +} + +function getStudioCliEnv( appdataDir: string ): NodeJS.ProcessEnv { + return { + E2E: 'true', + E2E_APP_DATA_PATH: appdataDir, + }; +} + +async function setupStudioSite( + env: EnvironmentConfig +): Promise< { url: string; siteDir: string; appdataDir: string } > { + const siteDir = createTempDir( env.name ); + const appdataDir = createTempDir( `${ env.name }-appdata` ); + const siteName = `bench-${ env.name }`; + + createStudioAppdata( appdataDir, env.multiWorker ); + + const cliEnv = getStudioCliEnv( appdataDir ); + + // Build the create args + const createArgs = [ + STUDIO_CLI_PATH, + 'site', + 'create', + `--path=${ siteDir }`, + `--name=${ siteName }`, + '--start=true', + '--skip-browser', + ]; + + if ( env.plugins ) { + createArgs.push( `--blueprint=${ PLUGINS_BLUEPRINT_PATH }` ); + } + + console.log( chalk.gray( ` Creating Studio site at ${ siteDir }` ) ); + const result = await runCommand( 'node', createArgs, { + cwd: STUDIO_ROOT, + env: cliEnv, + timeout: 600_000, // 10 min for site creation with plugins + } ); + + if ( result.exitCode !== 0 ) { + // The Studio CLI logs errors to stdout via its Logger, so show both + const errorOutput = ( result.stdout + '\n' + result.stderr ).trim(); + console.error( chalk.red( ` Studio site creation failed:\n${ errorOutput }` ) ); + throw new Error( `Studio site creation failed` ); + } + + // Extract the URL from the appdata (site was created and started) + const appdataPath = path.join( appdataDir, 'Studio', 'appdata-v1.json' ); + const appdata = JSON.parse( fs.readFileSync( appdataPath, 'utf-8' ) ); + const site = appdata.sites?.[ 0 ]; + if ( ! site?.port ) { + throw new Error( 'Studio site was created but no port was assigned' ); + } + const url = site.url || `http://localhost:${ site.port }`; + + console.log( chalk.gray( ` Studio site running at ${ url }` ) ); + return { url, siteDir, appdataDir }; +} + +async function teardownStudioSite( siteDir: string, appdataDir: string ): Promise< void > { + const cliEnv = getStudioCliEnv( appdataDir ); + + // Stop the site + await runCommand( 'node', [ STUDIO_CLI_PATH, 'site', 'stop', `--path=${ siteDir }` ], { + cwd: STUDIO_ROOT, + env: cliEnv, + timeout: 30_000, + } ).catch( () => {} ); + + // Delete the site from appdata + await runCommand( 'node', [ STUDIO_CLI_PATH, 'site', 'delete', `--path=${ siteDir }` ], { + cwd: STUDIO_ROOT, + env: cliEnv, + timeout: 30_000, + } ).catch( () => {} ); + + // Clean up temp dirs + cleanupDir( siteDir ); + cleanupDir( appdataDir ); +} + +// --------------------------------------------------------------------------- +// Playground CLI environment helpers +// --------------------------------------------------------------------------- + +async function setupPlaygroundCliSite( + env: EnvironmentConfig, + port: number +): Promise< { url: string; process: ChildProcess; siteDir: string } > { + const siteDir = createTempDir( env.name ); + + await killProcessOnPort( port ); + + const args = [ 'server', `--port=${ port }`, '--wp=latest', '--php=8.2' ]; + + // Mount the site dir + if ( process.platform === 'win32' ) { + args.push( '--mount-dir-before-install', siteDir, '/wordpress' ); + } else { + args.push( `--mount-before-install=${ siteDir }:/wordpress` ); + } + + if ( env.plugins ) { + args.push( `--blueprint=${ PLUGINS_BLUEPRINT_PATH }` ); + } + + if ( env.multiWorker ) { + const workerCount = Math.max( 1, os.cpus().length - 1 ); + args.push( `--experimental-multi-worker=${ workerCount }` ); + } + + console.log( chalk.gray( ` Starting Playground CLI on port ${ port }` ) ); + + const proc = spawn( PLAYGROUND_CLI_PATH, args, { + stdio: [ 'ignore', 'pipe', 'pipe' ], + env: { ...process.env, FORCE_COLOR: '0' }, + detached: process.platform !== 'win32', + shell: process.platform === 'win32', + } ); + + // Log stderr output for debugging if the server fails to start + let pgStderr = ''; + proc.stderr?.on( 'data', ( data ) => { + pgStderr += data.toString(); + } ); + + const url = `http://127.0.0.1:${ port }`; + + // Wait for server to be ready + const ready = await waitForServer( url, 180_000 ); + if ( ! ready ) { + proc.kill( 'SIGKILL' ); + if ( pgStderr ) { + console.error( chalk.gray( pgStderr.slice( 0, 500 ) ) ); + } + throw new Error( `Playground CLI server failed to start on port ${ port }` ); + } + + console.log( chalk.gray( ` Playground CLI running at ${ url }` ) ); + return { url, process: proc, siteDir }; +} + +async function teardownPlaygroundCliSite( + proc: ChildProcess, + port: number, + siteDir: string +): Promise< void > { + try { + if ( proc.pid ) { + if ( process.platform === 'win32' ) { + execSync( `taskkill /F /T /PID ${ proc.pid }`, { stdio: 'ignore' } ); + } else { + process.kill( -proc.pid, 'SIGTERM' ); + } + } + } catch { + // Process may have already exited + } + await killProcessOnPort( port ); + await sleep( 1000 ); + cleanupDir( siteDir ); +} + +// --------------------------------------------------------------------------- +// Playground Web environment helpers +// --------------------------------------------------------------------------- + +function getPlaygroundWebUrl( env: EnvironmentConfig ): string { + if ( ! env.plugins ) { + return PLAYGROUND_WEB_BASE_URL; + } + + // Pass the blueprint as a JSON fragment in the URL hash + const blueprint = JSON.parse( fs.readFileSync( PLUGINS_BLUEPRINT_PATH, 'utf-8' ) ); + return `${ PLAYGROUND_WEB_BASE_URL }/#${ JSON.stringify( blueprint ) }`; +} + +// --------------------------------------------------------------------------- +// Benchmark runner +// --------------------------------------------------------------------------- + +async function runBenchmarkTest( + benchmarkUrl: string, + resultsId: string, + runs: number +): Promise< Record< string, number > | null > { + console.log( chalk.gray( ` Running benchmark (${ runs } run${ runs > 1 ? 's' : '' })...` ) ); + + const result = await runCommand( + 'npx', + [ 'playwright', 'test', `--config=${ PLAYWRIGHT_CONFIG }`, BENCHMARK_TEST_NAME ], + { + cwd: STUDIO_ROOT, + env: { + BENCHMARK_URL: benchmarkUrl, + BENCHMARK_RUNS: String( runs ), + RESULTS_ID: resultsId, + ARTIFACTS_PATH: ARTIFACTS_PATH, + TIMEOUT: '600000', // 10 min — plugin-heavy environments need more time + }, + timeout: 600_000, + } + ); + + if ( result.exitCode !== 0 ) { + console.error( chalk.red( ` Benchmark failed for ${ resultsId }` ) ); + // Playwright outputs test failures to stdout + const output = ( result.stdout + '\n' + result.stderr ).trim(); + console.error( chalk.gray( output.slice( -2000 ) ) ); + return null; + } + + // Read the results file + const resultsFile = path.join( ARTIFACTS_PATH, `${ resultsId }.results.json` ); + if ( fs.existsSync( resultsFile ) ) { + return JSON.parse( fs.readFileSync( resultsFile, 'utf-8' ) ); + } + + console.warn( chalk.yellow( ` Results file not found: ${ resultsFile }` ) ); + return null; +} + +// --------------------------------------------------------------------------- +// Results formatting +// --------------------------------------------------------------------------- + +/** + * The site-editor-benchmark test prefixes metric keys with a URL-derived identifier + * (e.g., "localhost-9401-siteEditorLoad"). We strip the prefix to get the base metric + * name so results from different environments can be compared in the same row. + */ +function stripMetricPrefix( key: string ): string { + // Known metric base names from the benchmark test + const knownMetrics = [ + 'siteEditorLoad', + 'templatesViewLoad', + 'templateOpen', + 'blockAdd', + 'templateSave', + ]; + + for ( const metric of knownMetrics ) { + if ( key.endsWith( metric ) ) { + return metric; + } + } + + // Fallback: strip everything before the last camelCase segment + const match = key.match( /[a-z][a-zA-Z]+$/ ); + return match ? match[ 0 ] : key; +} + +/** + * Normalize metric keys in a results object by stripping URL-derived prefixes. + */ +function normalizeMetricKeys( metrics: Record< string, number > ): Record< string, number > { + const normalized: Record< string, number > = {}; + for ( const [ key, value ] of Object.entries( metrics ) ) { + normalized[ stripMetricPrefix( key ) ] = value; + } + return normalized; +} + +function printComparisonTable( results: BenchmarkResult[] ): void { + if ( results.length === 0 ) { + console.log( chalk.yellow( '\nNo results to display.' ) ); + return; + } + + // Normalize metric keys for comparison + const normalizedResults = results.map( ( r ) => ( { + ...r, + metrics: normalizeMetricKeys( r.metrics ), + } ) ); + + // Collect all unique metric names across all results + const allMetrics = new Set< string >(); + for ( const r of normalizedResults ) { + Object.keys( r.metrics ).forEach( ( k ) => allMetrics.add( k ) ); + } + const metrics = [ ...allMetrics ].sort(); + + // Calculate column widths + const metricColWidth = Math.max( 20, ...metrics.map( ( m ) => m.length + 2 ) ); + const envColWidth = Math.max( 12, ...normalizedResults.map( ( r ) => r.environment.length + 2 ) ); + + // Header + console.log( chalk.bold( '\n\nResults Comparison' ) ); + console.log( '═'.repeat( metricColWidth + envColWidth * normalizedResults.length ) ); + + const header = + 'Metric'.padEnd( metricColWidth ) + + normalizedResults.map( ( r ) => r.environment.padEnd( envColWidth ) ).join( '' ); + console.log( chalk.bold( header ) ); + console.log( '─'.repeat( metricColWidth + envColWidth * normalizedResults.length ) ); + + // Rows + for ( const metric of metrics ) { + let row = metric.padEnd( metricColWidth ); + + for ( const r of normalizedResults ) { + const value = r.metrics[ metric ]; + if ( value !== undefined ) { + row += formatDuration( value ).padEnd( envColWidth ); + } else { + row += '—'.padEnd( envColWidth ); + } + } + + console.log( row ); + } + + console.log( '═'.repeat( metricColWidth + envColWidth * normalizedResults.length ) ); +} + +function saveResultsSummary( results: BenchmarkResult[] ): void { + const summaryPath = path.join( ARTIFACTS_PATH, `benchmark-comparison-${ Date.now() }.json` ); + const summary = { + date: new Date().toISOString(), + platform: os.platform(), + arch: os.arch(), + nodeVersion: process.version, + cpus: os.cpus().length, + results, + }; + fs.mkdirSync( ARTIFACTS_PATH, { recursive: true } ); + fs.writeFileSync( summaryPath, JSON.stringify( summary, null, 2 ) ); + console.log( chalk.gray( `\nResults saved to: ${ summaryPath }` ) ); +} + +// --------------------------------------------------------------------------- +// Setup checks +// --------------------------------------------------------------------------- + +async function ensureStudioCLIBuilt(): Promise< boolean > { + // Ensure CLI dependencies are installed (required for the Vite build to resolve pm2-axon, etc.) + const cliNodeModules = path.resolve( STUDIO_ROOT, 'cli', 'node_modules' ); + if ( ! fs.existsSync( cliNodeModules ) ) { + console.log( chalk.yellow( ' Installing CLI dependencies...' ) ); + try { + execSync( 'npm install', { + cwd: path.resolve( STUDIO_ROOT, 'cli' ), + stdio: 'inherit', + } ); + } catch { + console.error( chalk.red( ' Failed to install CLI dependencies' ) ); + return false; + } + } + + if ( ! fs.existsSync( STUDIO_CLI_PATH ) ) { + console.log( chalk.yellow( ' Building Studio CLI...' ) ); + try { + execSync( 'npm run cli:build', { cwd: STUDIO_ROOT, stdio: 'inherit' } ); + return true; + } catch { + console.error( chalk.red( ' Failed to build Studio CLI' ) ); + return false; + } + } + return true; +} + +async function ensurePlaygroundCLIInstalled(): Promise< boolean > { + if ( ! fs.existsSync( PLAYGROUND_CLI_PATH ) ) { + console.log( chalk.yellow( ' Installing dependencies (including @wp-playground/cli)...' ) ); + try { + execSync( 'npm install', { cwd: import.meta.dirname, stdio: 'inherit' } ); + return true; + } catch { + console.error( chalk.red( ' Failed to install Playground CLI' ) ); + return false; + } + } + return true; +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +async function main() { + const opts = parseArgs(); + + // Filter environments + let environments = ALL_ENVIRONMENTS; + + if ( opts.only.length > 0 ) { + environments = environments.filter( ( e ) => opts.only.includes( e.name ) ); + } else { + if ( opts.skipStudio ) { + environments = environments.filter( ( e ) => e.type !== 'studio' ); + } + if ( opts.skipPlaygroundCli ) { + environments = environments.filter( ( e ) => e.type !== 'playground-cli' ); + } + if ( opts.skipPlaygroundWeb ) { + environments = environments.filter( ( e ) => e.type !== 'playground-web' ); + } + } + + if ( environments.length === 0 ) { + console.log( chalk.yellow( 'No environments selected. Nothing to do.' ) ); + return; + } + + console.log( chalk.bold.cyan( '\n=== Site Editor Performance Benchmark ===' ) ); + console.log( chalk.gray( `Platform: ${ os.platform() } ${ os.arch() }` ) ); + console.log( chalk.gray( `Node: ${ process.version }` ) ); + console.log( chalk.gray( `CPUs: ${ os.cpus().length }` ) ); + console.log( chalk.gray( `Rounds: ${ opts.rounds }` ) ); + console.log( chalk.gray( `Date: ${ new Date().toISOString() }` ) ); + console.log( + chalk.gray( `Environments: ${ environments.map( ( e ) => e.name ).join( ', ' ) }` ) + ); + + // Setup + console.log( chalk.bold( '\nSetup:' ) ); + + // Ensure Playwright browsers are installed + try { + execSync( 'npx playwright install chromium', { cwd: STUDIO_ROOT, stdio: 'ignore' } ); + console.log( chalk.green( ' Playwright chromium ready' ) ); + } catch { + console.error( chalk.red( ' Failed to install Playwright chromium' ) ); + process.exit( 1 ); + } + + const needsStudio = environments.some( ( e ) => e.type === 'studio' ); + const needsPlaygroundCli = environments.some( ( e ) => e.type === 'playground-cli' ); + + if ( needsStudio ) { + if ( ! ( await ensureStudioCLIBuilt() ) ) { + process.exit( 1 ); + } + console.log( chalk.green( ' Studio CLI ready' ) ); + } + + if ( needsPlaygroundCli ) { + if ( ! ( await ensurePlaygroundCLIInstalled() ) ) { + process.exit( 1 ); + } + console.log( chalk.green( ' Playground CLI ready' ) ); + } + + fs.mkdirSync( ARTIFACTS_PATH, { recursive: true } ); + + // Run benchmarks + const allResults: BenchmarkResult[] = []; + let playgroundCliPortOffset = 0; + + for ( const env of environments ) { + console.log( chalk.bold.cyan( `\n ▶ ${ env.name }` ) ); + const tags = [ env.plugins ? 'plugins' : null, env.multiWorker ? 'multi-worker' : null ] + .filter( Boolean ) + .join( ', ' ); + if ( tags ) { + console.log( chalk.gray( ` (${ tags })` ) ); + } + + let benchmarkUrl: string; + let teardownFn: ( () => Promise< void > ) | null = null; + + try { + // Setup environment + if ( env.type === 'studio' ) { + const setup = await setupStudioSite( env ); + benchmarkUrl = setup.url; + teardownFn = () => teardownStudioSite( setup.siteDir, setup.appdataDir ); + } else if ( env.type === 'playground-cli' ) { + const port = PLAYGROUND_CLI_PORT_BASE + playgroundCliPortOffset++; + const setup = await setupPlaygroundCliSite( env, port ); + benchmarkUrl = setup.url; + teardownFn = () => teardownPlaygroundCliSite( setup.process, port, setup.siteDir ); + } else { + benchmarkUrl = getPlaygroundWebUrl( env ); + console.log( chalk.gray( ` URL: ${ benchmarkUrl.slice( 0, 80 ) }...` ) ); + } + + // Run benchmark + const metrics = await runBenchmarkTest( benchmarkUrl, env.name, opts.rounds ); + + if ( metrics ) { + allResults.push( { environment: env.name, metrics } ); + console.log( chalk.green( ` ✓ Done` ) ); + } else { + console.log( chalk.red( ` ✗ Failed` ) ); + } + } catch ( err ) { + console.error( chalk.red( ` ✗ Error: ${ err }` ) ); + } finally { + // Teardown + if ( teardownFn ) { + console.log( chalk.gray( ' Cleaning up...' ) ); + await teardownFn().catch( () => {} ); + } + } + } + + // Print results + printComparisonTable( allResults ); + saveResultsSummary( allResults ); +} + +main().catch( ( err ) => { + console.error( chalk.red( 'Benchmark failed:' ), err ); + process.exit( 1 ); +} ); diff --git a/scripts/benchmark-site-editor/package.json b/scripts/benchmark-site-editor/package.json new file mode 100644 index 0000000000..70bde78447 --- /dev/null +++ b/scripts/benchmark-site-editor/package.json @@ -0,0 +1,20 @@ +{ + "name": "benchmark-site-editor", + "version": "0.0.1", + "description": "Benchmark site editor performance across Studio, Playground CLI, and Playground Web environments", + "author": "Automattic", + "license": "GPLv2", + "type": "module", + "scripts": { + "benchmark": "tsx benchmark.ts" + }, + "dependencies": { + "@wp-playground/cli": "^3.0.22", + "chalk": "^5.3.0", + "tsx": "^4.7.0" + }, + "devDependencies": { + "@types/node": "^20.0.0", + "typescript": "^5.0.0" + } +} diff --git a/scripts/benchmark-site-editor/plugins-blueprint.json b/scripts/benchmark-site-editor/plugins-blueprint.json new file mode 100644 index 0000000000..1a487ff01b --- /dev/null +++ b/scripts/benchmark-site-editor/plugins-blueprint.json @@ -0,0 +1,75 @@ +{ + "$schema": "https://playground.wordpress.net/blueprint-schema.json", + "steps": [ + { + "step": "installPlugin", + "pluginData": { + "resource": "wordpress.org/plugins", + "slug": "woocommerce" + } + }, + { + "step": "installPlugin", + "pluginData": { + "resource": "wordpress.org/plugins", + "slug": "jetpack" + } + }, + { + "step": "installPlugin", + "pluginData": { + "resource": "wordpress.org/plugins", + "slug": "wp-super-cache" + } + }, + { + "step": "installPlugin", + "pluginData": { + "resource": "wordpress.org/plugins", + "slug": "jetpack-boost" + } + }, + { + "step": "installPlugin", + "pluginData": { + "resource": "wordpress.org/plugins", + "slug": "jetpack-protect" + } + }, + { + "step": "installPlugin", + "pluginData": { + "resource": "wordpress.org/plugins", + "slug": "jetpack-social" + } + }, + { + "step": "installPlugin", + "pluginData": { + "resource": "wordpress.org/plugins", + "slug": "jetpack-videopress" + } + }, + { + "step": "installPlugin", + "pluginData": { + "resource": "wordpress.org/plugins", + "slug": "woocommerce-payments" + } + }, + { + "step": "installPlugin", + "pluginData": { + "resource": "wordpress.org/plugins", + "slug": "contact-form-7" + } + }, + { + "step": "installPlugin", + "pluginData": { + "resource": "wordpress.org/plugins", + "slug": "elementor" + } + } + ] +} diff --git a/scripts/benchmark-site-editor/tsconfig.json b/scripts/benchmark-site-editor/tsconfig.json new file mode 100644 index 0000000000..8f21fe7532 --- /dev/null +++ b/scripts/benchmark-site-editor/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ES2022", + "moduleResolution": "bundler", + "esModuleInterop": true, + "strict": true, + "skipLibCheck": true, + "outDir": "./dist", + "rootDir": ".", + "declaration": false, + "sourceMap": false + }, + "include": [ "*.ts" ] +} From ac29eec9b993ebe6e5e0518e72e57f0f29f5ebea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gergely=20Cs=C3=A9csey?= Date: Tue, 17 Feb 2026 17:25:33 +0000 Subject: [PATCH 02/14] Simplify benchmark by replacing Playwright test with direct measurement module (STU-1290) --- metrics/tests/site-editor-benchmark.test.ts | 371 ------------------ scripts/benchmark-site-editor/README.md | 6 +- scripts/benchmark-site-editor/benchmark.ts | 143 +++---- .../measure-site-editor.ts | 304 ++++++++++++++ scripts/benchmark-site-editor/package.json | 1 + 5 files changed, 363 insertions(+), 462 deletions(-) delete mode 100644 metrics/tests/site-editor-benchmark.test.ts create mode 100644 scripts/benchmark-site-editor/measure-site-editor.ts diff --git a/metrics/tests/site-editor-benchmark.test.ts b/metrics/tests/site-editor-benchmark.test.ts deleted file mode 100644 index a1e45359cc..0000000000 --- a/metrics/tests/site-editor-benchmark.test.ts +++ /dev/null @@ -1,371 +0,0 @@ -import { test, chromium, Page, Frame } from '@playwright/test'; -import { getUrlWithAutoLogin } from '../../e2e/utils'; -import { median } from '../utils'; - -// Helper functions for the benchmark test -function findWordPressFrame( page: Page ): Frame | null { - const frames = page.frames(); - let wordPressFrame = - frames.find( ( frame ) => { - const url = frame.url(); - return ( - url.includes( 'wordpress' ) || - url.includes( 'wp-admin' ) || - url.includes( 'wp-login' ) || - url.includes( 'scope:' ) - ); - } ) || null; - - if ( ! wordPressFrame ) { - // Try searching nested frames - for ( const frame of page.frames() ) { - if ( frame.parentFrame() && frame.url().includes( 'scope:' ) ) { - wordPressFrame = frame; - break; - } - } - } - - return wordPressFrame; -} - -function findEditorCanvasFrame( - page: Page, - isPlaygroundWeb: boolean, - wordPressFrame: Frame | null -): Frame | null { - if ( isPlaygroundWeb && wordPressFrame ) { - // For Playground web, search in child frames - let frame = wordPressFrame.childFrames().find( ( f ) => f.name() === 'editor-canvas' ) || null; - // If not found, try searching all frames - if ( ! frame ) { - const allFrames = page.frames(); - frame = allFrames.find( ( f ) => f.name() === 'editor-canvas' ) || null; - } - return frame; - } - // For regular WordPress, get the frame from the page - return page.frame( { name: 'editor-canvas' } ); -} - -test.describe( 'Site Editor Performance Benchmark', () => { - // Results collection structure - each metric stores array of values from all runs - const results: Record< string, number[] > = { - siteEditorLoad: [], - templatesViewLoad: [], - templateOpen: [], - blockAdd: [], - templateSave: [], - }; - - // Get configuration from environment variables - const targetUrl = process.env.BENCHMARK_URL; - const BENCHMARK_RUNS = parseInt( process.env.BENCHMARK_RUNS || '3', 10 ); - - // Environment detection (parsed once, reused for all iterations) - if ( ! targetUrl ) { - throw new Error( - 'BENCHMARK_URL environment variable is required. Example: BENCHMARK_URL=http://localhost:8888' - ); - } - - // Parse and normalize the URL - let wpAdminUrl = targetUrl; - if ( ! wpAdminUrl.startsWith( 'http' ) ) { - wpAdminUrl = `http://${ wpAdminUrl }`; - } - wpAdminUrl = wpAdminUrl.replace( /\/$/, '' ); - - const isPlaygroundWeb = wpAdminUrl.includes( 'playground.wordpress.net' ); - const isLocalPlaygroundCli = wpAdminUrl.includes( '127.0.0.1' ); - - // Create URL identifier for metric names - const urlIdentifier = isPlaygroundWeb - ? 'playground-web' - : wpAdminUrl - .replace( /^https?:\/\//, '' ) - .replace( /\/$/, '' ) - .replace( /[^a-z0-9]/gi, '-' ); - - test.afterAll( async ( {}, testInfo ) => { - // Calculate medians using existing utility - const medians: Record< string, number > = {}; - Object.entries( results ).forEach( ( [ key, values ] ) => { - if ( values.length > 0 ) { - const medianValue = median( values ); - if ( medianValue !== undefined ) { - medians[ `${ urlIdentifier }-${ key }` ] = medianValue; - } - } - } ); - - // Attach aggregated results (for performance reporter) - await testInfo.attach( 'results', { - body: JSON.stringify( medians, null, 2 ), - contentType: 'application/json', - } ); - - // Attach detailed results (individual runs + aggregated) - const detailedResults = { - url: wpAdminUrl, - runs: BENCHMARK_RUNS, - successfulRuns: results.siteEditorLoad.length, - individual: results, - medians: medians, - }; - - await testInfo.attach( 'benchmark-results-detailed', { - body: JSON.stringify( detailedResults, null, 2 ), - contentType: 'application/json', - } ); - } ); - - test( 'benchmark site editor performance', async () => { - // Run benchmark N times with fresh browser per iteration - for ( let run = 1; run <= BENCHMARK_RUNS; run++ ) { - await test.step( `Run ${ run }/${ BENCHMARK_RUNS }`, async () => { - // Launch fresh browser for this iteration - const browser = await chromium.launch(); - const context = await browser.newContext(); - const page = await context.newPage(); - - let wordPressFrame: Frame | null = null; - let wordPressFrameLocator: ReturnType< Page[ 'frameLocator' ] > | null = null; - - try { - // Environment-specific preparation: Navigate to wp-admin - if ( isPlaygroundWeb ) { - // For Playground web: use the URL from environment variable - await page.goto( wpAdminUrl, { waitUntil: 'networkidle' } ); - - // Get WordPress frame locator - wordPressFrameLocator = page - .frameLocator( 'iframe.playground-viewport' ) - .first() - .frameLocator( 'iframe' ) - .first(); - - // Wait for wp-admin to load - await wordPressFrameLocator - .getByRole( 'link', { name: 'Appearance' } ) - .waitFor( { timeout: 30_000 } ); - - // Get the actual frame object - wordPressFrame = findWordPressFrame( page ); - } else if ( isLocalPlaygroundCli ) { - // For local Playground CLI: navigate directly to wp-admin - // Note: Playground CLI may redirect, so we follow redirects - await page.goto( `${ wpAdminUrl }/wp-admin`, { - waitUntil: 'domcontentloaded', - timeout: 120_000, - } ); - // Wait for page to settle and Appearance link to appear - await page.waitForLoadState( 'networkidle', { timeout: 30_000 } ).catch( () => {} ); - await page.getByRole( 'link', { name: 'Appearance' } ).waitFor( { - state: 'visible', - timeout: 60_000, - } ); - } else { - // For Studio: use auto-login endpoint - await page.goto( getUrlWithAutoLogin( `${ wpAdminUrl }/wp-admin` ), { - waitUntil: 'domcontentloaded', - timeout: 120_000, - } ); - // Wait for page to settle and Appearance link to appear - await page.waitForLoadState( 'networkidle', { timeout: 30_000 } ).catch( () => {} ); - await page.getByRole( 'link', { name: 'Appearance' } ).waitFor( { - state: 'visible', - timeout: 60_000, - } ); - } - - // Get the target for interactions - const target = isPlaygroundWeb && wordPressFrameLocator ? wordPressFrameLocator : page; - - // Step 1: Navigate to site editor from wp-admin using Appearance > Editor - const siteEditorStartTime = Date.now(); - - // Click Appearance menu - await target.getByRole( 'link', { name: 'Appearance' } ).click(); - // Click Editor submenu - use href to be specific (site-editor.php is the site editor) - await target.locator( 'a[href="site-editor.php"]' ).click(); - - // Close welcome modal if it appears - const welcomeDialog = target.getByRole( 'dialog', { - name: /welcome to the site editor/i, - } ); - const isModalVisible = await welcomeDialog - .isVisible( { timeout: 5_000 } ) - .catch( () => false ); - if ( isModalVisible ) { - await target.getByRole( 'button', { name: /get started/i } ).click(); - await welcomeDialog.waitFor( { state: 'hidden', timeout: 5_000 } ).catch( () => {} ); - } - - // Wait for editor canvas iframe to appear - await target.locator( 'iframe[name="editor-canvas"]' ).waitFor( { - state: 'visible', - timeout: 120_000, - } ); - - // Find the editor canvas frame - const frame = findEditorCanvasFrame( page, isPlaygroundWeb, wordPressFrame ); - if ( ! frame ) { - throw new Error( 'Editor canvas frame not found' ); - } - - // Wait for frame to be ready - await frame.waitForLoadState( 'domcontentloaded' ); - // Wait for blocks to be present and rendered (positive indicator that editor is ready) - await frame.waitForSelector( '[data-block]', { timeout: 60_000 } ); - // Ensure at least one block is fully rendered (not just in DOM) - await frame.waitForFunction( - () => { - const blocks = document.querySelectorAll( '[data-block]' ); - return ( - blocks.length > 0 && - Array.from( blocks ).some( ( block ) => block.clientHeight > 0 ) - ); - }, - { timeout: 60_000 } - ); - - const siteEditorEndTime = Date.now(); - results.siteEditorLoad.push( siteEditorEndTime - siteEditorStartTime ); - - // Step 2: Navigate to Templates view by clicking Templates button in sidebar - const templatesViewStartTime = Date.now(); - - // Click the Templates button in the sidebar (works across all environments) - await target.getByRole( 'button', { name: 'Templates' } ).click(); - - // Wait for Templates view to load - wait for heading, grid, and ensure first card is clickable - await target.getByRole( 'heading', { name: 'Templates', level: 2 } ).waitFor( { - timeout: 60_000, - } ); - await target - .locator( '.dataviews-view-grid-items.dataviews-view-grid' ) - .waitFor( { timeout: 60_000 } ); - // Wait for the first template card to be visible and clickable (indicates page is ready) - const firstCard = target.locator( '.dataviews-view-grid__card' ).first(); - await firstCard.waitFor( { state: 'visible', timeout: 60_000 } ); - await firstCard - .getByRole( 'button' ) - .first() - .waitFor( { state: 'visible', timeout: 60_000 } ); - - const templatesViewEndTime = Date.now(); - results.templatesViewLoad.push( templatesViewEndTime - templatesViewStartTime ); - - // Step 3: Open a template - const templateOpenStartTime = Date.now(); - - // Click on the first template card (it's a button, not a link) - await target - .locator( '.dataviews-view-grid__card' ) - .first() - .getByRole( 'button' ) - .first() - .click(); - - // Wait for template editor to load - await target.locator( 'iframe[name="editor-canvas"]' ).waitFor( { - state: 'visible', - timeout: 60_000, - } ); - - // Find the template editor canvas frame - const templateFrame = findEditorCanvasFrame( page, isPlaygroundWeb, wordPressFrame ); - if ( ! templateFrame ) { - throw new Error( 'Template editor frame not found' ); - } - - // Wait for template editor to be ready - await templateFrame.waitForLoadState( 'domcontentloaded' ); - // Wait for blocks to be present and rendered (positive indicator that editor is ready) - await templateFrame.waitForSelector( '[data-block]', { timeout: 60_000 } ); - // Ensure at least one block is fully rendered (not just in DOM) - await templateFrame.waitForFunction( - () => { - const blocks = document.querySelectorAll( '[data-block]' ); - return ( - blocks.length > 0 && - Array.from( blocks ).some( ( block ) => block.clientHeight > 0 ) - ); - }, - { timeout: 60_000 } - ); - - const templateOpenEndTime = Date.now(); - results.templateOpen.push( templateOpenEndTime - templateOpenStartTime ); - - // Step 4: Add blocks - const blockAddStartTime = Date.now(); - - // Close any modals - await page.keyboard.press( 'Escape' ); - - // Open block inserter - await target.getByRole( 'button', { name: /Block Inserter/i } ).click(); - - // Search and insert Paragraph block - const searchInput = target.getByPlaceholder( 'Search' ); - await searchInput.fill( 'Paragraph' ); - await target.getByRole( 'option', { name: 'Paragraph', exact: true } ).click(); - - // Wait for paragraph block to appear - await templateFrame.waitForSelector( 'p[data-block]', { timeout: 15_000 } ); - - // Add Heading block - await searchInput.fill( 'Heading' ); - // Get the block type option (not the pattern) - block types are buttons with class "block-editor-block-types-list__item" - await target - .locator( '.block-editor-block-types-list__item' ) - .filter( { hasText: /^Heading$/ } ) - .click(); - - // Wait for heading block to appear - await templateFrame.waitForSelector( 'h1[data-block], h2[data-block], h3[data-block]', { - timeout: 15_000, - } ); - - const blockAddEndTime = Date.now(); - results.blockAdd.push( blockAddEndTime - blockAddStartTime ); - - // Step 5: Save the template - const templateSaveStartTime = Date.now(); - - await target.getByRole( 'button', { name: 'Save' } ).first().click(); - - // Wait for save confirmation - button text changes to "Saved" - await page.waitForFunction( - () => { - const saveButton = Array.from( document.querySelectorAll( 'button' ) ).find( - ( btn ) => - btn.textContent?.includes( 'Saved' ) || - btn.getAttribute( 'aria-label' )?.toLowerCase().includes( 'saved' ) - ); - return saveButton !== null; - }, - { timeout: 30_000 } - ); - - const templateSaveEndTime = Date.now(); - results.templateSave.push( templateSaveEndTime - templateSaveStartTime ); - - console.log( ` ✓ Run ${ run } completed` ); - } finally { - // Always cleanup browser - await page.close(); - await context.close(); - await browser.close(); - } - - // Small delay between runs - if ( run < BENCHMARK_RUNS ) { - await new Promise( ( resolve ) => setTimeout( resolve, 1000 ) ); - } - } ); - } - } ); -} ); diff --git a/scripts/benchmark-site-editor/README.md b/scripts/benchmark-site-editor/README.md index fbded044ef..ab4cbcbb39 100644 --- a/scripts/benchmark-site-editor/README.md +++ b/scripts/benchmark-site-editor/README.md @@ -8,7 +8,7 @@ Benchmarks site editor performance across Studio, Playground CLI, and Playground ## What It Measures -The benchmark runs the `site-editor-benchmark` Playwright test against each environment, measuring: +The benchmark launches a headless Chromium browser against each environment, measuring: | Metric | Description | | --------------------- | ---------------------------------------------------------- | @@ -87,10 +87,8 @@ npm run benchmark -- --only=studio-mw-plugins --rounds=5 - **Studio CLI**: Built automatically if `dist/cli/main.js` doesn't exist (`npm run cli:build`) - **Playground CLI**: Installed automatically via this script's `npm install` -- **Playwright**: Must be installed at the repo root (`npx playwright install chromium`) +- **Playwright**: Chromium is installed automatically during setup ## Output Results are printed as a comparison table and saved to `metrics/artifacts/benchmark-comparison-.json`. - -Individual results per environment are saved to `metrics/artifacts/.results.json`. diff --git a/scripts/benchmark-site-editor/benchmark.ts b/scripts/benchmark-site-editor/benchmark.ts index 696e8ebe36..e328af2f27 100644 --- a/scripts/benchmark-site-editor/benchmark.ts +++ b/scripts/benchmark-site-editor/benchmark.ts @@ -3,7 +3,7 @@ /** * Site Editor Performance Benchmark — Orchestration Script * - * Runs the site-editor-benchmark Playwright test across a matrix of environments: + * Benchmarks site editor performance across a matrix of environments: * - Studio (bare, multi-worker, plugins, multi-worker+plugins) * - Playground CLI (bare, multi-worker, plugins, multi-worker+plugins) * - Playground Web (bare, plugins) @@ -27,6 +27,7 @@ import fs from 'fs'; import os from 'os'; import path from 'path'; import chalk from 'chalk'; +import { measureSiteEditor, METRIC_NAMES, type MeasurementResult } from './measure-site-editor.js'; // --------------------------------------------------------------------------- // Configuration @@ -43,13 +44,10 @@ const PLAYGROUND_CLI_PATH = path.resolve( ); const PLUGINS_BLUEPRINT_PATH = path.resolve( import.meta.dirname, 'plugins-blueprint.json' ); const ARTIFACTS_PATH = path.resolve( STUDIO_ROOT, 'metrics', 'artifacts' ); -const BENCHMARK_TEST_NAME = 'site-editor-benchmark'; -const PLAYWRIGHT_CONFIG = path.resolve( STUDIO_ROOT, 'metrics', 'playwright.metrics.config.ts' ); const PLAYGROUND_WEB_BASE_URL = 'https://playground.wordpress.net'; -// Ports used for local servers — offset to avoid collisions with dev servers -const STUDIO_PORT_BASE = 9400; +// Port offset for Playground CLI servers — avoids collisions with dev servers const PLAYGROUND_CLI_PORT_BASE = 9500; // --------------------------------------------------------------------------- @@ -491,126 +489,97 @@ function getPlaygroundWebUrl( env: EnvironmentConfig ): string { // Benchmark runner // --------------------------------------------------------------------------- -async function runBenchmarkTest( +const MEASUREMENT_TIMEOUT = 600_000; // 10 min per measurement + +async function runBenchmark( benchmarkUrl: string, - resultsId: string, - runs: number + env: EnvironmentConfig, + rounds: number ): Promise< Record< string, number > | null > { - console.log( chalk.gray( ` Running benchmark (${ runs } run${ runs > 1 ? 's' : '' })...` ) ); - - const result = await runCommand( - 'npx', - [ 'playwright', 'test', `--config=${ PLAYWRIGHT_CONFIG }`, BENCHMARK_TEST_NAME ], - { - cwd: STUDIO_ROOT, - env: { - BENCHMARK_URL: benchmarkUrl, - BENCHMARK_RUNS: String( runs ), - RESULTS_ID: resultsId, - ARTIFACTS_PATH: ARTIFACTS_PATH, - TIMEOUT: '600000', // 10 min — plugin-heavy environments need more time - }, - timeout: 600_000, - } + console.log( + chalk.gray( ` Running benchmark (${ rounds } round${ rounds > 1 ? 's' : '' })...` ) ); - if ( result.exitCode !== 0 ) { - console.error( chalk.red( ` Benchmark failed for ${ resultsId }` ) ); - // Playwright outputs test failures to stdout - const output = ( result.stdout + '\n' + result.stderr ).trim(); - console.error( chalk.gray( output.slice( -2000 ) ) ); + const isPlaygroundWeb = benchmarkUrl.includes( 'playground.wordpress.net' ); + const isPlaygroundCli = benchmarkUrl.includes( '127.0.0.1' ); + const allMeasurements: MeasurementResult[] = []; + + for ( let round = 1; round <= rounds; round++ ) { + if ( rounds > 1 ) { + console.log( chalk.gray( ` Round ${ round }/${ rounds }...` ) ); + } + try { + const result = await Promise.race( [ + measureSiteEditor( { url: benchmarkUrl, isPlaygroundWeb, isPlaygroundCli } ), + sleep( MEASUREMENT_TIMEOUT ).then( () => { + throw new Error( 'Measurement timed out' ); + } ), + ] ); + allMeasurements.push( result ); + } catch ( err ) { + console.warn( chalk.yellow( ` Round ${ round } failed: ${ err }` ) ); + } + + // Small delay between rounds + if ( round < rounds ) { + await sleep( 1000 ); + } + } + + if ( allMeasurements.length === 0 ) { return null; } - // Read the results file - const resultsFile = path.join( ARTIFACTS_PATH, `${ resultsId }.results.json` ); - if ( fs.existsSync( resultsFile ) ) { - return JSON.parse( fs.readFileSync( resultsFile, 'utf-8' ) ); + // Calculate medians across rounds + const medians: Record< string, number > = {}; + for ( const metric of METRIC_NAMES ) { + const values = allMeasurements + .map( ( m ) => m[ metric ] ) + .filter( ( v ): v is number => v !== undefined ); + if ( values.length > 0 ) { + medians[ metric ] = median( values ); + } } - console.warn( chalk.yellow( ` Results file not found: ${ resultsFile }` ) ); - return null; + return medians; } // --------------------------------------------------------------------------- // Results formatting // --------------------------------------------------------------------------- -/** - * The site-editor-benchmark test prefixes metric keys with a URL-derived identifier - * (e.g., "localhost-9401-siteEditorLoad"). We strip the prefix to get the base metric - * name so results from different environments can be compared in the same row. - */ -function stripMetricPrefix( key: string ): string { - // Known metric base names from the benchmark test - const knownMetrics = [ - 'siteEditorLoad', - 'templatesViewLoad', - 'templateOpen', - 'blockAdd', - 'templateSave', - ]; - - for ( const metric of knownMetrics ) { - if ( key.endsWith( metric ) ) { - return metric; - } - } - - // Fallback: strip everything before the last camelCase segment - const match = key.match( /[a-z][a-zA-Z]+$/ ); - return match ? match[ 0 ] : key; -} - -/** - * Normalize metric keys in a results object by stripping URL-derived prefixes. - */ -function normalizeMetricKeys( metrics: Record< string, number > ): Record< string, number > { - const normalized: Record< string, number > = {}; - for ( const [ key, value ] of Object.entries( metrics ) ) { - normalized[ stripMetricPrefix( key ) ] = value; - } - return normalized; -} - function printComparisonTable( results: BenchmarkResult[] ): void { if ( results.length === 0 ) { console.log( chalk.yellow( '\nNo results to display.' ) ); return; } - // Normalize metric keys for comparison - const normalizedResults = results.map( ( r ) => ( { - ...r, - metrics: normalizeMetricKeys( r.metrics ), - } ) ); - // Collect all unique metric names across all results const allMetrics = new Set< string >(); - for ( const r of normalizedResults ) { + for ( const r of results ) { Object.keys( r.metrics ).forEach( ( k ) => allMetrics.add( k ) ); } const metrics = [ ...allMetrics ].sort(); // Calculate column widths const metricColWidth = Math.max( 20, ...metrics.map( ( m ) => m.length + 2 ) ); - const envColWidth = Math.max( 12, ...normalizedResults.map( ( r ) => r.environment.length + 2 ) ); + const envColWidth = Math.max( 12, ...results.map( ( r ) => r.environment.length + 2 ) ); // Header console.log( chalk.bold( '\n\nResults Comparison' ) ); - console.log( '═'.repeat( metricColWidth + envColWidth * normalizedResults.length ) ); + console.log( '═'.repeat( metricColWidth + envColWidth * results.length ) ); const header = 'Metric'.padEnd( metricColWidth ) + - normalizedResults.map( ( r ) => r.environment.padEnd( envColWidth ) ).join( '' ); + results.map( ( r ) => r.environment.padEnd( envColWidth ) ).join( '' ); console.log( chalk.bold( header ) ); - console.log( '─'.repeat( metricColWidth + envColWidth * normalizedResults.length ) ); + console.log( '─'.repeat( metricColWidth + envColWidth * results.length ) ); // Rows for ( const metric of metrics ) { let row = metric.padEnd( metricColWidth ); - for ( const r of normalizedResults ) { + for ( const r of results ) { const value = r.metrics[ metric ]; if ( value !== undefined ) { row += formatDuration( value ).padEnd( envColWidth ); @@ -622,7 +591,7 @@ function printComparisonTable( results: BenchmarkResult[] ): void { console.log( row ); } - console.log( '═'.repeat( metricColWidth + envColWidth * normalizedResults.length ) ); + console.log( '═'.repeat( metricColWidth + envColWidth * results.length ) ); } function saveResultsSummary( results: BenchmarkResult[] ): void { @@ -731,7 +700,7 @@ async function main() { // Ensure Playwright browsers are installed try { - execSync( 'npx playwright install chromium', { cwd: STUDIO_ROOT, stdio: 'ignore' } ); + execSync( 'npx playwright install chromium', { cwd: import.meta.dirname, stdio: 'ignore' } ); console.log( chalk.green( ' Playwright chromium ready' ) ); } catch { console.error( chalk.red( ' Failed to install Playwright chromium' ) ); @@ -790,7 +759,7 @@ async function main() { } // Run benchmark - const metrics = await runBenchmarkTest( benchmarkUrl, env.name, opts.rounds ); + const metrics = await runBenchmark( benchmarkUrl, env, opts.rounds ); if ( metrics ) { allResults.push( { environment: env.name, metrics } ); diff --git a/scripts/benchmark-site-editor/measure-site-editor.ts b/scripts/benchmark-site-editor/measure-site-editor.ts new file mode 100644 index 0000000000..1636d30a84 --- /dev/null +++ b/scripts/benchmark-site-editor/measure-site-editor.ts @@ -0,0 +1,304 @@ +/** + * Site editor performance measurement module. + * + * Runs a single benchmark pass against a WordPress site: launches Chromium, + * navigates through the site editor, and returns raw timing values for each metric. + */ + +import { chromium, type Page, type Frame } from 'playwright'; + +// --------------------------------------------------------------------------- +// Metric names and types +// --------------------------------------------------------------------------- + +export const METRIC_NAMES = [ + 'siteEditorLoad', + 'templatesViewLoad', + 'templateOpen', + 'blockAdd', + 'templateSave', +] as const; + +export type MetricName = ( typeof METRIC_NAMES )[ number ]; + +/** A single measurement result: one timing value per metric. A metric may be missing if that step failed. */ +export type MeasurementResult = Partial< Record< MetricName, number > >; + +export interface MeasureOptions { + /** The base URL of the WordPress site to benchmark. */ + url: string; + /** Whether this is a Playground Web URL (playground.wordpress.net). Affects iframe handling. */ + isPlaygroundWeb: boolean; + /** Whether this is a local Playground CLI site (127.0.0.1). Affects login flow. */ + isPlaygroundCli: boolean; +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function getUrlWithAutoLogin( destinationUrl: string ): string { + const parsedUrl = new URL( destinationUrl ); + const baseUrl = `${ parsedUrl.protocol }//${ parsedUrl.hostname }:${ parsedUrl.port }`; + return `${ baseUrl }/studio-auto-login?redirect_to=${ encodeURIComponent( destinationUrl ) }`; +} + +function findWordPressFrame( page: Page ): Frame | null { + const frames = page.frames(); + let wordPressFrame = + frames.find( ( frame ) => { + const url = frame.url(); + return ( + url.includes( 'wordpress' ) || + url.includes( 'wp-admin' ) || + url.includes( 'wp-login' ) || + url.includes( 'scope:' ) + ); + } ) || null; + + if ( ! wordPressFrame ) { + for ( const frame of page.frames() ) { + if ( frame.parentFrame() && frame.url().includes( 'scope:' ) ) { + wordPressFrame = frame; + break; + } + } + } + + return wordPressFrame; +} + +function findEditorCanvasFrame( + page: Page, + isPlaygroundWeb: boolean, + wordPressFrame: Frame | null +): Frame | null { + if ( isPlaygroundWeb && wordPressFrame ) { + let frame = + wordPressFrame.childFrames().find( ( f ) => f.name() === 'editor-canvas' ) || null; + if ( ! frame ) { + frame = page.frames().find( ( f ) => f.name() === 'editor-canvas' ) || null; + } + return frame; + } + return page.frame( { name: 'editor-canvas' } ); +} + +// --------------------------------------------------------------------------- +// Measurement +// --------------------------------------------------------------------------- + +/** + * Runs a single benchmark measurement pass against the given URL. + * + * Launches a fresh Chromium browser, navigates to the site editor, + * performs the measurement steps, and returns timing results. + * The browser is always closed, even on error. + */ +export async function measureSiteEditor( options: MeasureOptions ): Promise< MeasurementResult > { + const { isPlaygroundWeb, isPlaygroundCli } = options; + + // Normalize URL + let wpAdminUrl = options.url; + if ( ! wpAdminUrl.startsWith( 'http' ) ) { + wpAdminUrl = `http://${ wpAdminUrl }`; + } + wpAdminUrl = wpAdminUrl.replace( /\/$/, '' ); + + const result: MeasurementResult = {}; + + const browser = await chromium.launch(); + const context = await browser.newContext(); + const page = await context.newPage(); + + let wordPressFrame: Frame | null = null; + let wordPressFrameLocator: ReturnType< Page[ 'frameLocator' ] > | null = null; + + try { + // Navigate to wp-admin (environment-specific) + if ( isPlaygroundWeb ) { + await page.goto( wpAdminUrl, { waitUntil: 'networkidle' } ); + + wordPressFrameLocator = page + .frameLocator( 'iframe.playground-viewport' ) + .first() + .frameLocator( 'iframe' ) + .first(); + + await wordPressFrameLocator + .getByRole( 'link', { name: 'Appearance' } ) + .waitFor( { timeout: 30_000 } ); + + wordPressFrame = findWordPressFrame( page ); + } else if ( isPlaygroundCli ) { + await page.goto( `${ wpAdminUrl }/wp-admin`, { + waitUntil: 'domcontentloaded', + timeout: 120_000, + } ); + await page.waitForLoadState( 'networkidle', { timeout: 30_000 } ).catch( () => {} ); + await page.getByRole( 'link', { name: 'Appearance' } ).waitFor( { + state: 'visible', + timeout: 60_000, + } ); + } else { + // Studio: use auto-login endpoint + await page.goto( getUrlWithAutoLogin( `${ wpAdminUrl }/wp-admin` ), { + waitUntil: 'domcontentloaded', + timeout: 120_000, + } ); + await page.waitForLoadState( 'networkidle', { timeout: 30_000 } ).catch( () => {} ); + await page.getByRole( 'link', { name: 'Appearance' } ).waitFor( { + state: 'visible', + timeout: 60_000, + } ); + } + + const target = isPlaygroundWeb && wordPressFrameLocator ? wordPressFrameLocator : page; + + // Step 1: Navigate to site editor + const siteEditorStart = Date.now(); + + await target.getByRole( 'link', { name: 'Appearance' } ).click(); + await target.locator( 'a[href="site-editor.php"]' ).click(); + + // Close welcome modal if it appears + const welcomeDialog = target.getByRole( 'dialog', { + name: /welcome to the site editor/i, + } ); + const isModalVisible = await welcomeDialog + .isVisible( { timeout: 5_000 } ) + .catch( () => false ); + if ( isModalVisible ) { + await target.getByRole( 'button', { name: /get started/i } ).click(); + await welcomeDialog.waitFor( { state: 'hidden', timeout: 5_000 } ).catch( () => {} ); + } + + await target.locator( 'iframe[name="editor-canvas"]' ).waitFor( { + state: 'visible', + timeout: 120_000, + } ); + + const frame = findEditorCanvasFrame( page, isPlaygroundWeb, wordPressFrame ); + if ( ! frame ) { + throw new Error( 'Editor canvas frame not found' ); + } + + await frame.waitForLoadState( 'domcontentloaded' ); + await frame.waitForSelector( '[data-block]', { timeout: 60_000 } ); + await frame.waitForFunction( + () => { + const blocks = document.querySelectorAll( '[data-block]' ); + return ( + blocks.length > 0 && + Array.from( blocks ).some( ( block ) => block.clientHeight > 0 ) + ); + }, + { timeout: 60_000 } + ); + + result.siteEditorLoad = Date.now() - siteEditorStart; + + // Step 2: Navigate to Templates view + const templatesViewStart = Date.now(); + + await target.getByRole( 'button', { name: 'Templates' } ).click(); + + await target.getByRole( 'heading', { name: 'Templates', level: 2 } ).waitFor( { + timeout: 60_000, + } ); + await target + .locator( '.dataviews-view-grid-items.dataviews-view-grid' ) + .waitFor( { timeout: 60_000 } ); + const firstCard = target.locator( '.dataviews-view-grid__card' ).first(); + await firstCard.waitFor( { state: 'visible', timeout: 60_000 } ); + await firstCard + .getByRole( 'button' ) + .first() + .waitFor( { state: 'visible', timeout: 60_000 } ); + + result.templatesViewLoad = Date.now() - templatesViewStart; + + // Step 3: Open a template + const templateOpenStart = Date.now(); + + await target + .locator( '.dataviews-view-grid__card' ) + .first() + .getByRole( 'button' ) + .first() + .click(); + + await target.locator( 'iframe[name="editor-canvas"]' ).waitFor( { + state: 'visible', + timeout: 60_000, + } ); + + const templateFrame = findEditorCanvasFrame( page, isPlaygroundWeb, wordPressFrame ); + if ( ! templateFrame ) { + throw new Error( 'Template editor frame not found' ); + } + + await templateFrame.waitForLoadState( 'domcontentloaded' ); + await templateFrame.waitForSelector( '[data-block]', { timeout: 60_000 } ); + await templateFrame.waitForFunction( + () => { + const blocks = document.querySelectorAll( '[data-block]' ); + return ( + blocks.length > 0 && + Array.from( blocks ).some( ( block ) => block.clientHeight > 0 ) + ); + }, + { timeout: 60_000 } + ); + + result.templateOpen = Date.now() - templateOpenStart; + + // Step 4: Add blocks + const blockAddStart = Date.now(); + + await page.keyboard.press( 'Escape' ); + + await target.getByRole( 'button', { name: /Block Inserter/i } ).click(); + + const searchInput = target.getByPlaceholder( 'Search' ); + await searchInput.fill( 'Paragraph' ); + await target.getByRole( 'option', { name: 'Paragraph', exact: true } ).click(); + await templateFrame.waitForSelector( 'p[data-block]', { timeout: 15_000 } ); + + await searchInput.fill( 'Heading' ); + await target + .locator( '.block-editor-block-types-list__item' ) + .filter( { hasText: /^Heading$/ } ) + .click(); + await templateFrame.waitForSelector( 'h1[data-block], h2[data-block], h3[data-block]', { + timeout: 15_000, + } ); + + result.blockAdd = Date.now() - blockAddStart; + + // Step 5: Save the template + const templateSaveStart = Date.now(); + + await target.getByRole( 'button', { name: 'Save' } ).first().click(); + + await page.waitForFunction( + () => { + const saveButton = Array.from( document.querySelectorAll( 'button' ) ).find( + ( btn ) => + btn.textContent?.includes( 'Saved' ) || + btn.getAttribute( 'aria-label' )?.toLowerCase().includes( 'saved' ) + ); + return saveButton !== null; + }, + { timeout: 30_000 } + ); + + result.templateSave = Date.now() - templateSaveStart; + } finally { + await page.close(); + await context.close(); + await browser.close(); + } + + return result; +} diff --git a/scripts/benchmark-site-editor/package.json b/scripts/benchmark-site-editor/package.json index 70bde78447..c6024ce753 100644 --- a/scripts/benchmark-site-editor/package.json +++ b/scripts/benchmark-site-editor/package.json @@ -11,6 +11,7 @@ "dependencies": { "@wp-playground/cli": "^3.0.22", "chalk": "^5.3.0", + "playwright": "^1.58.1", "tsx": "^4.7.0" }, "devDependencies": { From 3497ebe36c7d224ee17728930db5b7fe35f85b90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gergely=20Cs=C3=A9csey?= Date: Thu, 19 Feb 2026 12:31:17 +0000 Subject: [PATCH 03/14] Rewrite Local plugin installer to use WordPress REST API Replace WP-CLI binary approach with HTTP-based REST API for cross-platform simplicity and proper activation hook firing. Also improves login flow timeout handling for Local sites and ensures teardown cleanup even if plugin install fails. - install-plugins-local.ts: Use cookie-based auth + nonce extraction + POST /wp-json/wp/v2/plugins - benchmark.ts: Wrap plugin install in try/catch, fix variable ordering, improve teardown guarantees - measure-site-editor.ts: Use Promise.all for click+navigation to prevent timeout on redirects --- package-lock.json | 10 - scripts/benchmark-site-editor/README.md | 16 +- scripts/benchmark-site-editor/benchmark.ts | 96 ++++- .../install-plugins-local.ts | 180 ++++++++ .../benchmark-site-editor/local-graphql.ts | 384 ++++++++++++++++++ .../measure-site-editor.ts | 39 +- 6 files changed, 698 insertions(+), 27 deletions(-) create mode 100644 scripts/benchmark-site-editor/install-plugins-local.ts create mode 100644 scripts/benchmark-site-editor/local-graphql.ts diff --git a/package-lock.json b/package-lock.json index ddc7645136..c0692175ce 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8439,7 +8439,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -8456,7 +8455,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -8473,7 +8471,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "Apache-2.0", "optional": true, "os": [ @@ -8490,7 +8487,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -8507,7 +8503,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -8524,7 +8519,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -8541,7 +8535,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -8558,7 +8551,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -8575,7 +8567,6 @@ "cpu": [ "ia32" ], - "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ @@ -8592,7 +8583,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "Apache-2.0 AND MIT", "optional": true, "os": [ diff --git a/scripts/benchmark-site-editor/README.md b/scripts/benchmark-site-editor/README.md index ab4cbcbb39..83331bedcc 100644 --- a/scripts/benchmark-site-editor/README.md +++ b/scripts/benchmark-site-editor/README.md @@ -1,6 +1,6 @@ # Site Editor Performance Benchmark -Benchmarks site editor performance across Studio, Playground CLI, and Playground Web environments, with optional plugin and multi-worker configurations. +Benchmarks site editor performance across Studio, Playground CLI, Playground Web, and Local by Flywheel environments, with optional plugin and multi-worker configurations. ## Related Issue @@ -32,6 +32,8 @@ The benchmark launches a headless Chromium browser against each environment, mea | Playground CLI + MW + Plugins | `pg-cli-mw-plugins` | Playground CLI with multi-worker and 10 plugins | | Playground Web | `pg-web` | playground.wordpress.net (bare) | | Playground Web + Plugins | `pg-web-plugins` | playground.wordpress.net with 10 plugins | +| Local | `local` | Local by Flywheel site (nginx+PHP+MySQL) | +| Local + Plugins | `local-plugins` | Local with 10 plugins installed | ## Plugins @@ -63,6 +65,7 @@ npm run benchmark --skip-studio Skip Studio environments --skip-playground-cli Skip Playground CLI environments --skip-playground-web Skip Playground web environments +--include-local Include Local by Flywheel environments (requires Local running) --only= Run only named environments (comma-separated) --help Show help ``` @@ -81,13 +84,24 @@ npm run benchmark -- --skip-studio --skip-playground-web # Single specific environment npm run benchmark -- --only=studio-mw-plugins --rounds=5 + +# Include Local by Flywheel (must be running) +npm run benchmark -- --include-local --only=local,local-plugins + +# Compare Studio vs Local +npm run benchmark -- --only=studio,local --rounds=3 ``` +> **Note:** When using `--only` with Local environments, the `--include-local` flag is not needed — Local environments are automatically included when explicitly named. + ## Prerequisites - **Studio CLI**: Built automatically if `dist/cli/main.js` doesn't exist (`npm run cli:build`) - **Playground CLI**: Installed automatically via this script's `npm install` - **Playwright**: Chromium is installed automatically during setup +- **Local by Flywheel**: Must be installed and running (GUI app). Use `--include-local` to enable. The script connects to Local's GraphQL API to create and manage benchmark sites. + - macOS: `brew install --cask local` + - Windows: `winget install Flywheel.Local` ## Output diff --git a/scripts/benchmark-site-editor/benchmark.ts b/scripts/benchmark-site-editor/benchmark.ts index e328af2f27..c6be0a88e6 100644 --- a/scripts/benchmark-site-editor/benchmark.ts +++ b/scripts/benchmark-site-editor/benchmark.ts @@ -7,6 +7,7 @@ * - Studio (bare, multi-worker, plugins, multi-worker+plugins) * - Playground CLI (bare, multi-worker, plugins, multi-worker+plugins) * - Playground Web (bare, plugins) + * - Local by Flywheel (bare, plugins) — opt-in via --include-local * * Usage: * cd scripts/benchmark-site-editor @@ -18,6 +19,7 @@ * --skip-studio Skip Studio environments * --skip-playground-cli Skip Playground CLI environments * --skip-playground-web Skip Playground web environments + * --include-local Include Local by Flywheel environments (requires Local running) * --only= Run only these environments (comma-separated) * --help Show help */ @@ -28,6 +30,8 @@ import os from 'os'; import path from 'path'; import chalk from 'chalk'; import { measureSiteEditor, METRIC_NAMES, type MeasurementResult } from './measure-site-editor.js'; +import { LocalGraphQLClient, getLocalSitesDir } from './local-graphql.js'; +import { installPluginsForLocalSite } from './install-plugins-local.js'; // --------------------------------------------------------------------------- // Configuration @@ -54,7 +58,7 @@ const PLAYGROUND_CLI_PORT_BASE = 9500; // Types // --------------------------------------------------------------------------- -type EnvironmentType = 'studio' | 'playground-cli' | 'playground-web'; +type EnvironmentType = 'studio' | 'playground-cli' | 'playground-web' | 'local'; interface EnvironmentConfig { name: string; @@ -88,6 +92,8 @@ const ALL_ENVIRONMENTS: EnvironmentConfig[] = [ }, { name: 'pg-web', type: 'playground-web', plugins: false, multiWorker: false }, { name: 'pg-web-plugins', type: 'playground-web', plugins: true, multiWorker: false }, + { name: 'local', type: 'local', plugins: false, multiWorker: false }, + { name: 'local-plugins', type: 'local', plugins: true, multiWorker: false }, ]; // --------------------------------------------------------------------------- @@ -99,6 +105,7 @@ interface Options { skipStudio: boolean; skipPlaygroundCli: boolean; skipPlaygroundWeb: boolean; + includeLocal: boolean; only: string[]; } @@ -109,6 +116,7 @@ function parseArgs(): Options { skipStudio: false, skipPlaygroundCli: false, skipPlaygroundWeb: false, + includeLocal: false, only: [], }; @@ -121,6 +129,8 @@ function parseArgs(): Options { opts.skipPlaygroundCli = true; } else if ( arg === '--skip-playground-web' ) { opts.skipPlaygroundWeb = true; + } else if ( arg === '--include-local' ) { + opts.includeLocal = true; } else if ( arg.startsWith( '--only=' ) ) { opts.only = arg .split( '=' )[ 1 ] @@ -144,6 +154,7 @@ Options: --skip-studio Skip Studio environments --skip-playground-cli Skip Playground CLI environments --skip-playground-web Skip Playground web environments + --include-local Include Local by Flywheel environments (requires Local running) --only= Run only named environments (comma-separated) --help Show this help message @@ -485,6 +496,71 @@ function getPlaygroundWebUrl( env: EnvironmentConfig ): string { return `${ PLAYGROUND_WEB_BASE_URL }/#${ JSON.stringify( blueprint ) }`; } +// --------------------------------------------------------------------------- +// Local by Flywheel environment helpers +// --------------------------------------------------------------------------- + +async function ensureLocalRunning(): Promise< LocalGraphQLClient > { + try { + return await LocalGraphQLClient.connect(); + } catch ( err ) { + console.error( chalk.red( ` Local is not running or not reachable.\n ${ err }` ) ); + process.exit( 1 ); + } +} + +async function setupLocalSite( + env: EnvironmentConfig, + client: LocalGraphQLClient +): Promise< { url: string; siteId: string; siteDir: string } > { + const siteName = `bench-${ env.name }`; + const domain = `bench-${ env.name }.local`; + const siteDir = path.join( getLocalSitesDir(), siteName ); + + console.log( chalk.gray( ` Creating Local site "${ siteName }"...` ) ); + + const site = await client.createSite( { + name: siteName, + domain, + path: siteDir, + wpAdminPassword: 'password', + wpAdminUsername: 'admin', + } ); + + const url = site.url || `http://localhost:${ site.httpPort }`; + + // Plugin install is best-effort — we still want to return site info + // so the caller can set up teardown even if plugin install fails. + if ( env.plugins ) { + console.log( chalk.gray( ` Installing plugins via REST API...` ) ); + try { + await installPluginsForLocalSite( url, PLUGINS_BLUEPRINT_PATH ); + } catch ( err ) { + console.warn( chalk.yellow( ` Plugin installation failed: ${ err }` ) ); + } + } + console.log( chalk.gray( ` Local site running at ${ url }` ) ); + return { url, siteId: site.id, siteDir }; +} + +async function teardownLocalSite( + siteId: string, + client: LocalGraphQLClient, + siteDir: string +): Promise< void > { + try { + await client.stopSite( siteId ); + } catch { + // Site may already be stopped + } + try { + await client.deleteSite( siteId ); + } catch { + // Best effort cleanup + } + cleanupDir( siteDir ); +} + // --------------------------------------------------------------------------- // Benchmark runner // --------------------------------------------------------------------------- @@ -502,6 +578,7 @@ async function runBenchmark( const isPlaygroundWeb = benchmarkUrl.includes( 'playground.wordpress.net' ); const isPlaygroundCli = benchmarkUrl.includes( '127.0.0.1' ); + const isLocal = env.type === 'local'; const allMeasurements: MeasurementResult[] = []; for ( let round = 1; round <= rounds; round++ ) { @@ -510,7 +587,7 @@ async function runBenchmark( } try { const result = await Promise.race( [ - measureSiteEditor( { url: benchmarkUrl, isPlaygroundWeb, isPlaygroundCli } ), + measureSiteEditor( { url: benchmarkUrl, isPlaygroundWeb, isPlaygroundCli, isLocal } ), sleep( MEASUREMENT_TIMEOUT ).then( () => { throw new Error( 'Measurement timed out' ); } ), @@ -678,6 +755,10 @@ async function main() { if ( opts.skipPlaygroundWeb ) { environments = environments.filter( ( e ) => e.type !== 'playground-web' ); } + // Local environments are excluded by default — they require the GUI app running + if ( ! opts.includeLocal ) { + environments = environments.filter( ( e ) => e.type !== 'local' ); + } } if ( environments.length === 0 ) { @@ -709,6 +790,7 @@ async function main() { const needsStudio = environments.some( ( e ) => e.type === 'studio' ); const needsPlaygroundCli = environments.some( ( e ) => e.type === 'playground-cli' ); + const needsLocal = environments.some( ( e ) => e.type === 'local' ); if ( needsStudio ) { if ( ! ( await ensureStudioCLIBuilt() ) ) { @@ -724,6 +806,12 @@ async function main() { console.log( chalk.green( ' Playground CLI ready' ) ); } + let localClient: LocalGraphQLClient | null = null; + if ( needsLocal ) { + localClient = await ensureLocalRunning(); + console.log( chalk.green( ' Local ready' ) ); + } + fs.mkdirSync( ARTIFACTS_PATH, { recursive: true } ); // Run benchmarks @@ -753,6 +841,10 @@ async function main() { const setup = await setupPlaygroundCliSite( env, port ); benchmarkUrl = setup.url; teardownFn = () => teardownPlaygroundCliSite( setup.process, port, setup.siteDir ); + } else if ( env.type === 'local' ) { + const setup = await setupLocalSite( env, localClient! ); + benchmarkUrl = setup.url; + teardownFn = () => teardownLocalSite( setup.siteId, localClient!, setup.siteDir ); } else { benchmarkUrl = getPlaygroundWebUrl( env ); console.log( chalk.gray( ` URL: ${ benchmarkUrl.slice( 0, 80 ) }...` ) ); diff --git a/scripts/benchmark-site-editor/install-plugins-local.ts b/scripts/benchmark-site-editor/install-plugins-local.ts new file mode 100644 index 0000000000..a4fd1936f9 --- /dev/null +++ b/scripts/benchmark-site-editor/install-plugins-local.ts @@ -0,0 +1,180 @@ +/** + * Plugin installer for Local by Flywheel benchmark sites. + * + * Uses the WordPress REST API to install and activate plugins. + * Reads plugin slugs from the shared plugins-blueprint.json + * (single source of truth for all environments). + * + * Flow: wp-login.php → nonce extraction → POST /wp-json/wp/v2/plugins + */ + +/* eslint-disable no-console */ + +import fs from 'fs'; +import chalk from 'chalk'; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +interface BlueprintStep { + step: string; + pluginData?: { + resource: string; + slug: string; + }; +} + +interface Blueprint { + steps: BlueprintStep[]; +} + +// --------------------------------------------------------------------------- +// Blueprint parsing +// --------------------------------------------------------------------------- + +/** + * Parse the blueprint JSON to extract plugin slugs. + * This is the single source of truth for which plugins to install across + * all environments (Studio, Playground CLI, Playground Web, Local). + */ +export function getPluginSlugsFromBlueprint( blueprintPath: string ): string[] { + const blueprint: Blueprint = JSON.parse( fs.readFileSync( blueprintPath, 'utf-8' ) ); + return blueprint.steps + .filter( ( step ) => step.step === 'installPlugin' && step.pluginData?.slug ) + .map( ( step ) => step.pluginData!.slug ); +} + +// --------------------------------------------------------------------------- +// WordPress REST API helpers +// --------------------------------------------------------------------------- + +/** + * Log in to WordPress via wp-login.php and return the auth cookies. + */ +async function wpLogin( siteUrl: string, username: string, password: string ): Promise< string > { + const loginUrl = `${ siteUrl }/wp-login.php`; + const body = new URLSearchParams( { + log: username, + pwd: password, + 'wp-submit': 'Log In', + redirect_to: '/wp-admin/', + testcookie: '1', + } ); + + const response = await fetch( loginUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + Cookie: 'wordpress_test_cookie=WP%20Cookie%20check', + }, + body: body.toString(), + redirect: 'manual', + } ); + + const setCookieHeaders = response.headers.getSetCookie(); + const cookies = setCookieHeaders + .map( ( header ) => header.split( ';' )[ 0 ] ) + .filter( ( c ) => c.startsWith( 'wordpress_' ) ); + + if ( cookies.length === 0 ) { + throw new Error( + `WordPress login failed: no auth cookies returned (HTTP ${ response.status })` + ); + } + + return cookies.join( '; ' ); +} + +/** + * Extract the WP REST API nonce from the wp-admin page. + */ +async function extractNonce( siteUrl: string, cookies: string ): Promise< string > { + const response = await fetch( `${ siteUrl }/wp-admin/`, { + headers: { Cookie: cookies }, + redirect: 'follow', + } ); + + const html = await response.text(); + const match = html.match( /var\s+wpApiSettings\s*=\s*(\{[^;]+\});/ ); + if ( ! match ) { + throw new Error( 'Could not extract wpApiSettings nonce from wp-admin' ); + } + + const settings = JSON.parse( match[ 1 ] ); + if ( ! settings.nonce ) { + throw new Error( 'wpApiSettings found but nonce is missing' ); + } + + return settings.nonce as string; +} + +/** + * Install and activate a single plugin via the WP REST API. + */ +async function installPlugin( + siteUrl: string, + cookies: string, + nonce: string, + slug: string +): Promise< void > { + const response = await fetch( `${ siteUrl }/wp-json/wp/v2/plugins`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Cookie: cookies, + 'X-WP-Nonce': nonce, + }, + body: JSON.stringify( { slug, status: 'active' } ), + } ); + + if ( ! response.ok ) { + const body = await response.text(); + throw new Error( `HTTP ${ response.status }: ${ body.slice( 0, 200 ) }` ); + } +} + +// --------------------------------------------------------------------------- +// Plugin installation +// --------------------------------------------------------------------------- + +/** + * Installs and activates plugins on a Local site using the WordPress REST API. + * + * Reads plugin slugs from the provided blueprint file (the same file used + * for Studio and Playground environments), logs in via wp-login.php, + * extracts a REST API nonce, and installs each plugin via + * POST /wp-json/wp/v2/plugins. + * + * @param siteUrl - The Local site's base URL (e.g., http://localhost:10003) + * @param blueprintPath - Path to plugins-blueprint.json + */ +export async function installPluginsForLocalSite( + siteUrl: string, + blueprintPath: string +): Promise< void > { + const slugs = getPluginSlugsFromBlueprint( blueprintPath ); + if ( slugs.length === 0 ) { + console.log( chalk.yellow( ' No plugins found in blueprint' ) ); + return; + } + + console.log( chalk.gray( ` Logging in to WordPress...` ) ); + const cookies = await wpLogin( siteUrl, 'admin', 'password' ); + + console.log( chalk.gray( ` Extracting REST API nonce...` ) ); + const nonce = await extractNonce( siteUrl, cookies ); + + console.log( chalk.gray( ` Installing ${ slugs.length } plugins via REST API...` ) ); + + for ( const slug of slugs ) { + try { + console.log( chalk.gray( ` ${ slug }...` ) ); + await installPlugin( siteUrl, cookies, nonce, slug ); + } catch ( err ) { + const message = err instanceof Error ? err.message : String( err ); + console.warn( chalk.yellow( ` Failed to install ${ slug }: ${ message }` ) ); + // Continue with remaining plugins + } + } +} diff --git a/scripts/benchmark-site-editor/local-graphql.ts b/scripts/benchmark-site-editor/local-graphql.ts new file mode 100644 index 0000000000..4ddb047c89 --- /dev/null +++ b/scripts/benchmark-site-editor/local-graphql.ts @@ -0,0 +1,384 @@ +/** + * GraphQL client for Local by Flywheel's API. + * + * Local exposes a GraphQL endpoint for programmatic site management. + * Connection details are stored in `graphql-connection-info.json` within + * Local's platform-specific data directory. + */ + +/* eslint-disable no-console */ + +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export interface LocalSite { + id: string; + name: string; + domain: string; + path: string; + status: string; + httpPort: number | null; + url: string; +} + +interface GraphQLConnectionInfo { + port: number; + authToken: string; + url: string; + subscriptionUrl?: string; +} + +interface GraphQLResponse< T > { + data?: T; + errors?: Array< { message: string } >; +} + +// --------------------------------------------------------------------------- +// Platform-specific paths +// --------------------------------------------------------------------------- + +/** Returns the platform-specific Local data directory. */ +export function getLocalDataDir(): string { + if ( process.platform === 'darwin' ) { + return path.join( os.homedir(), 'Library/Application Support/Local' ); + } + if ( process.platform === 'win32' ) { + const appdata = process.env.APPDATA || path.join( os.homedir(), 'AppData', 'Roaming' ); + // Local may use either "Local" or "Local by Flywheel" on Windows + const primary = path.join( appdata, 'Local' ); + const fallback = path.join( appdata, 'Local by Flywheel' ); + if ( fs.existsSync( primary ) ) return primary; + if ( fs.existsSync( fallback ) ) return fallback; + return primary; // Default; will fail later with a clear message + } + // Linux (limited Local support) + return path.join( os.homedir(), '.config', 'Local' ); +} + +/** Returns the default directory where Local creates sites. */ +export function getLocalSitesDir(): string { + return path.join( os.homedir(), 'Local Sites' ); +} + +// --------------------------------------------------------------------------- +// GraphQL client +// --------------------------------------------------------------------------- + +export class LocalGraphQLClient { + private url: string; + private token: string; + + private constructor( url: string, token: string ) { + this.url = url; + this.token = token; + } + + /** + * Connect to a running Local instance. + * Reads graphql-connection-info.json and verifies the API is responsive. + * Throws with a clear message if Local isn't running. + */ + static async connect(): Promise< LocalGraphQLClient > { + const dataDir = getLocalDataDir(); + const connectionInfoPath = path.join( dataDir, 'graphql-connection-info.json' ); + + if ( ! fs.existsSync( connectionInfoPath ) ) { + throw new Error( + `Local connection info not found at ${ connectionInfoPath }.\n` + + 'Please ensure Local is installed and running.' + ); + } + + let connectionInfo: GraphQLConnectionInfo; + try { + connectionInfo = JSON.parse( fs.readFileSync( connectionInfoPath, 'utf-8' ) ); + } catch { + throw new Error( + `Failed to parse ${ connectionInfoPath }.\n` + + 'The file may be corrupted. Try restarting Local.' + ); + } + + const client = new LocalGraphQLClient( connectionInfo.url, connectionInfo.authToken ); + + // Verify the API is responsive + try { + await client.query< { sites: unknown[] } >( '{ sites { id } }' ); + } catch ( err ) { + throw new Error( + `Cannot connect to Local's GraphQL API at ${ connectionInfo.url }.\n` + + `Is Local running? Error: ${ err }` + ); + } + + return client; + } + + /** + * Execute a GraphQL query/mutation. + */ + private async query< T >( query: string, variables?: Record< string, unknown > ): Promise< T > { + const response = await fetch( this.url, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${ this.token }`, + }, + body: JSON.stringify( { query, variables } ), + } ); + + if ( ! response.ok ) { + throw new Error( `GraphQL request failed: HTTP ${ response.status }` ); + } + + const json = ( await response.json() ) as GraphQLResponse< T >; + + if ( json.errors?.length ) { + throw new Error( `GraphQL error: ${ json.errors.map( ( e ) => e.message ).join( ', ' ) }` ); + } + + if ( ! json.data ) { + throw new Error( 'GraphQL response missing data' ); + } + + return json.data; + } + + /** + * Create a site. Returns when the job completes and the site is running. + * + * Local's `addSite` mutation returns a Job (async), not a Site directly. + * We poll the job until it completes, then query sites to find the new one. + */ + async createSite( options: { + name: string; + domain: string; + path: string; + wpAdminEmail?: string; + wpAdminPassword?: string; + wpAdminUsername?: string; + phpVersion?: string; + } ): Promise< LocalSite > { + const { + name, + domain, + path: sitePath, + wpAdminEmail = 'admin@benchmark.local', + wpAdminPassword = 'password', + wpAdminUsername = 'admin', + phpVersion = '8.2.0', + } = options; + + // Create the site via addSite mutation + const createResult = await this.query< { addSite: { id: string } } >( + `mutation AddSite($input: AddSiteInput!) { + addSite(input: $input) { + id + } + }`, + { + input: { + name, + domain, + path: sitePath, + wpAdminEmail, + wpAdminPassword, + wpAdminUsername, + phpVersion, + database: 'mysql', + environment: 'preferred', + blueprint: null, + }, + } + ); + + const jobId = createResult.addSite.id; + + // Poll the job until it completes + await this.waitForJob( jobId, 300_000 ); // 5 minute timeout + + // Find the new site by name + const site = await this.findSiteByName( name ); + if ( ! site ) { + throw new Error( `Site "${ name }" was not found after creation job completed` ); + } + + // Wait for site to be running + await this.waitForSiteStatus( site.id, 'running', 120_000 ); + + // Re-fetch to get the port + const runningSite = await this.getSite( site.id ); + if ( ! runningSite ) { + throw new Error( `Site "${ name }" disappeared after starting` ); + } + + return runningSite; + } + + /** + * Poll a job until it reaches 'successful' status. + */ + private async waitForJob( jobId: string, timeoutMs: number ): Promise< void > { + const start = Date.now(); + const pollInterval = 2000; + + while ( Date.now() - start < timeoutMs ) { + try { + const result = await this.query< { job: { id: string; status: string; error?: string } } >( + `query Job($id: ID!) { + job(id: $id) { + id + status + error + } + }`, + { id: jobId } + ); + + const job = result.job; + if ( job.status === 'successful' ) { + return; + } + if ( job.status === 'failed' ) { + throw new Error( `Job failed: ${ job.error || 'unknown error' }` ); + } + } catch ( err ) { + if ( err instanceof Error && err.message.startsWith( 'Job failed' ) ) { + throw err; + } + // GraphQL query itself might fail during setup; keep polling + } + + await new Promise( ( resolve ) => setTimeout( resolve, pollInterval ) ); + } + + throw new Error( `Job ${ jobId } timed out after ${ timeoutMs / 1000 }s` ); + } + + /** + * Wait for a site to reach a specific status. + */ + private async waitForSiteStatus( + siteId: string, + targetStatus: string, + timeoutMs: number + ): Promise< void > { + const start = Date.now(); + const pollInterval = 2000; + + while ( Date.now() - start < timeoutMs ) { + const site = await this.getSite( siteId ); + if ( site && site.status === targetStatus ) { + return; + } + await new Promise( ( resolve ) => setTimeout( resolve, pollInterval ) ); + } + + throw new Error( + `Site ${ siteId } did not reach "${ targetStatus }" status within ${ timeoutMs / 1000 }s` + ); + } + + /** + * Find a site by name. + */ + private async findSiteByName( name: string ): Promise< LocalSite | null > { + const result = await this.query< { + sites: Array< { + id: string; + name: string; + domain: string; + path: string; + status: string; + httpPort: number | null; + } >; + } >( + `{ + sites { + id + name + domain + path + status + httpPort + } + }` + ); + + const site = result.sites.find( ( s ) => s.name === name ); + if ( ! site ) return null; + + return { + ...site, + url: site.httpPort ? `http://localhost:${ site.httpPort }` : '', + }; + } + + /** + * Get a site by ID. + */ + async getSite( id: string ): Promise< LocalSite | null > { + const result = await this.query< { + site: { + id: string; + name: string; + domain: string; + path: string; + status: string; + httpPort: number | null; + } | null; + } >( + `query Site($id: ID!) { + site(id: $id) { + id + name + domain + path + status + httpPort + } + }`, + { id } + ); + + const site = result.site; + if ( ! site ) return null; + + return { + ...site, + url: site.httpPort ? `http://localhost:${ site.httpPort }` : '', + }; + } + + /** + * Stop a running site. + */ + async stopSite( id: string ): Promise< void > { + await this.query( + `mutation StopSite($id: ID!) { + stopSite(id: $id) { + id + status + } + }`, + { id } + ); + } + + /** + * Delete a site. Uses deleteSitesFromGroups since there's no single deleteSite mutation. + */ + async deleteSite( id: string ): Promise< void > { + await this.query( + `mutation DeleteSites($ids: [ID!]!) { + deleteSitesFromGroups(ids: $ids) + }`, + { ids: [ id ] } + ); + } +} diff --git a/scripts/benchmark-site-editor/measure-site-editor.ts b/scripts/benchmark-site-editor/measure-site-editor.ts index 1636d30a84..de071de65d 100644 --- a/scripts/benchmark-site-editor/measure-site-editor.ts +++ b/scripts/benchmark-site-editor/measure-site-editor.ts @@ -31,6 +31,8 @@ export interface MeasureOptions { isPlaygroundWeb: boolean; /** Whether this is a local Playground CLI site (127.0.0.1). Affects login flow. */ isPlaygroundCli: boolean; + /** Whether this is a Local by Flywheel site. Uses wp-login.php credentials. */ + isLocal: boolean; } // --------------------------------------------------------------------------- @@ -74,8 +76,7 @@ function findEditorCanvasFrame( wordPressFrame: Frame | null ): Frame | null { if ( isPlaygroundWeb && wordPressFrame ) { - let frame = - wordPressFrame.childFrames().find( ( f ) => f.name() === 'editor-canvas' ) || null; + let frame = wordPressFrame.childFrames().find( ( f ) => f.name() === 'editor-canvas' ) || null; if ( ! frame ) { frame = page.frames().find( ( f ) => f.name() === 'editor-canvas' ) || null; } @@ -96,7 +97,7 @@ function findEditorCanvasFrame( * The browser is always closed, even on error. */ export async function measureSiteEditor( options: MeasureOptions ): Promise< MeasurementResult > { - const { isPlaygroundWeb, isPlaygroundCli } = options; + const { isPlaygroundWeb, isPlaygroundCli, isLocal } = options; // Normalize URL let wpAdminUrl = options.url; @@ -140,6 +141,23 @@ export async function measureSiteEditor( options: MeasureOptions ): Promise< Mea state: 'visible', timeout: 60_000, } ); + } else if ( isLocal ) { + // Local sites require wp-login.php with credentials + await page.goto( `${ wpAdminUrl }/wp-login.php`, { + waitUntil: 'domcontentloaded', + timeout: 120_000, + } ); + await page.waitForLoadState( 'networkidle', { timeout: 30_000 } ).catch( () => {} ); + await page.fill( '#user_login', 'admin' ); + await page.fill( '#user_pass', 'password' ); + await Promise.all( [ + page.waitForURL( '**/wp-admin/**', { timeout: 60_000 } ), + page.click( '#wp-submit' ), + ] ); + await page.getByRole( 'link', { name: 'Appearance' } ).waitFor( { + state: 'visible', + timeout: 60_000, + } ); } else { // Studio: use auto-login endpoint await page.goto( getUrlWithAutoLogin( `${ wpAdminUrl }/wp-admin` ), { @@ -165,9 +183,7 @@ export async function measureSiteEditor( options: MeasureOptions ): Promise< Mea const welcomeDialog = target.getByRole( 'dialog', { name: /welcome to the site editor/i, } ); - const isModalVisible = await welcomeDialog - .isVisible( { timeout: 5_000 } ) - .catch( () => false ); + const isModalVisible = await welcomeDialog.isVisible( { timeout: 5_000 } ).catch( () => false ); if ( isModalVisible ) { await target.getByRole( 'button', { name: /get started/i } ).click(); await welcomeDialog.waitFor( { state: 'hidden', timeout: 5_000 } ).catch( () => {} ); @@ -189,8 +205,7 @@ export async function measureSiteEditor( options: MeasureOptions ): Promise< Mea () => { const blocks = document.querySelectorAll( '[data-block]' ); return ( - blocks.length > 0 && - Array.from( blocks ).some( ( block ) => block.clientHeight > 0 ) + blocks.length > 0 && Array.from( blocks ).some( ( block ) => block.clientHeight > 0 ) ); }, { timeout: 60_000 } @@ -211,10 +226,7 @@ export async function measureSiteEditor( options: MeasureOptions ): Promise< Mea .waitFor( { timeout: 60_000 } ); const firstCard = target.locator( '.dataviews-view-grid__card' ).first(); await firstCard.waitFor( { state: 'visible', timeout: 60_000 } ); - await firstCard - .getByRole( 'button' ) - .first() - .waitFor( { state: 'visible', timeout: 60_000 } ); + await firstCard.getByRole( 'button' ).first().waitFor( { state: 'visible', timeout: 60_000 } ); result.templatesViewLoad = Date.now() - templatesViewStart; @@ -244,8 +256,7 @@ export async function measureSiteEditor( options: MeasureOptions ): Promise< Mea () => { const blocks = document.querySelectorAll( '[data-block]' ); return ( - blocks.length > 0 && - Array.from( blocks ).some( ( block ) => block.clientHeight > 0 ) + blocks.length > 0 && Array.from( blocks ).some( ( block ) => block.clientHeight > 0 ) ); }, { timeout: 60_000 } From c783a1641e475b2f75def04033d4f88898b4033e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gergely=20Cs=C3=A9csey?= Date: Thu, 19 Feb 2026 12:45:04 +0000 Subject: [PATCH 04/14] move benchmark-site-editor to tools --- scripts/benchmark-site-editor/README.md | 108 --- scripts/benchmark-site-editor/benchmark.ts | 881 ------------------ .../measure-site-editor.ts | 315 ------- scripts/benchmark-site-editor/package.json | 21 - .../plugins-blueprint.json | 75 -- scripts/benchmark-site-editor/tsconfig.json | 15 - .../benchmark-site-editor/.gitignore | 0 tools/benchmark-site-editor/README.md | 16 +- tools/benchmark-site-editor/benchmark.ts | 129 ++- .../install-plugins-local.ts | 0 .../benchmark-site-editor/local-graphql.ts | 0 .../measure-site-editor.ts | 39 +- tools/benchmark-site-editor/package.json | 6 +- .../plugins-blueprint.json | 2 +- tools/benchmark-site-editor/tsconfig.json | 22 +- 15 files changed, 165 insertions(+), 1464 deletions(-) delete mode 100644 scripts/benchmark-site-editor/README.md delete mode 100644 scripts/benchmark-site-editor/benchmark.ts delete mode 100644 scripts/benchmark-site-editor/measure-site-editor.ts delete mode 100644 scripts/benchmark-site-editor/package.json delete mode 100644 scripts/benchmark-site-editor/plugins-blueprint.json delete mode 100644 scripts/benchmark-site-editor/tsconfig.json rename {scripts => tools}/benchmark-site-editor/.gitignore (100%) rename {scripts => tools}/benchmark-site-editor/install-plugins-local.ts (100%) rename {scripts => tools}/benchmark-site-editor/local-graphql.ts (100%) diff --git a/scripts/benchmark-site-editor/README.md b/scripts/benchmark-site-editor/README.md deleted file mode 100644 index 83331bedcc..0000000000 --- a/scripts/benchmark-site-editor/README.md +++ /dev/null @@ -1,108 +0,0 @@ -# Site Editor Performance Benchmark - -Benchmarks site editor performance across Studio, Playground CLI, Playground Web, and Local by Flywheel environments, with optional plugin and multi-worker configurations. - -## Related Issue - -[STU-1290](https://linear.app/a8c/issue/STU-1290) - -## What It Measures - -The benchmark launches a headless Chromium browser against each environment, measuring: - -| Metric | Description | -| --------------------- | ---------------------------------------------------------- | -| **siteEditorLoad** | Time from clicking Appearance > Editor to blocks rendering | -| **templatesViewLoad** | Time to open the Templates view and load template cards | -| **templateOpen** | Time to open a specific template in the editor | -| **blockAdd** | Time to add a paragraph and heading block | -| **templateSave** | Time to save the template | - -## Environment Matrix - -| Environment | Name | Description | -| ----------------------------- | ------------------- | ----------------------------------------------- | -| Studio | `studio` | Bare Studio site | -| Studio + MW | `studio-mw` | Studio with multi-worker support enabled | -| Studio + Plugins | `studio-plugins` | Studio with 10 plugins installed | -| Studio + MW + Plugins | `studio-mw-plugins` | Studio with multi-worker and 10 plugins | -| Playground CLI | `pg-cli` | Bare Playground CLI site | -| Playground CLI + MW | `pg-cli-mw` | Playground CLI with multi-worker | -| Playground CLI + Plugins | `pg-cli-plugins` | Playground CLI with 10 plugins | -| Playground CLI + MW + Plugins | `pg-cli-mw-plugins` | Playground CLI with multi-worker and 10 plugins | -| Playground Web | `pg-web` | playground.wordpress.net (bare) | -| Playground Web + Plugins | `pg-web-plugins` | playground.wordpress.net with 10 plugins | -| Local | `local` | Local by Flywheel site (nginx+PHP+MySQL) | -| Local + Plugins | `local-plugins` | Local with 10 plugins installed | - -## Plugins - -When the "plugins" variant is enabled, these 10 plugins are installed via a blueprint: - -- WooCommerce -- Jetpack -- WP Super Cache -- Jetpack Boost -- Jetpack Protect -- Jetpack Social -- Jetpack VideoPress -- WooCommerce Payments -- Contact Form 7 -- Elementor - -## Usage - -```bash -cd scripts/benchmark-site-editor -npm install -npm run benchmark -``` - -### Options - -``` ---rounds=N Number of benchmark runs per environment (default: 1) ---skip-studio Skip Studio environments ---skip-playground-cli Skip Playground CLI environments ---skip-playground-web Skip Playground web environments ---include-local Include Local by Flywheel environments (requires Local running) ---only= Run only named environments (comma-separated) ---help Show help -``` - -### Examples - -```bash -# Quick test: only Studio bare vs Studio with plugins -npm run benchmark -- --only=studio,studio-plugins - -# Full comparison without Playground Web (faster, no network dependency) -npm run benchmark -- --skip-playground-web --rounds=3 - -# Only Playground CLI environments -npm run benchmark -- --skip-studio --skip-playground-web - -# Single specific environment -npm run benchmark -- --only=studio-mw-plugins --rounds=5 - -# Include Local by Flywheel (must be running) -npm run benchmark -- --include-local --only=local,local-plugins - -# Compare Studio vs Local -npm run benchmark -- --only=studio,local --rounds=3 -``` - -> **Note:** When using `--only` with Local environments, the `--include-local` flag is not needed — Local environments are automatically included when explicitly named. - -## Prerequisites - -- **Studio CLI**: Built automatically if `dist/cli/main.js` doesn't exist (`npm run cli:build`) -- **Playground CLI**: Installed automatically via this script's `npm install` -- **Playwright**: Chromium is installed automatically during setup -- **Local by Flywheel**: Must be installed and running (GUI app). Use `--include-local` to enable. The script connects to Local's GraphQL API to create and manage benchmark sites. - - macOS: `brew install --cask local` - - Windows: `winget install Flywheel.Local` - -## Output - -Results are printed as a comparison table and saved to `metrics/artifacts/benchmark-comparison-.json`. diff --git a/scripts/benchmark-site-editor/benchmark.ts b/scripts/benchmark-site-editor/benchmark.ts deleted file mode 100644 index c6be0a88e6..0000000000 --- a/scripts/benchmark-site-editor/benchmark.ts +++ /dev/null @@ -1,881 +0,0 @@ -#!/usr/bin/env tsx -/* eslint-disable no-console */ -/** - * Site Editor Performance Benchmark — Orchestration Script - * - * Benchmarks site editor performance across a matrix of environments: - * - Studio (bare, multi-worker, plugins, multi-worker+plugins) - * - Playground CLI (bare, multi-worker, plugins, multi-worker+plugins) - * - Playground Web (bare, plugins) - * - Local by Flywheel (bare, plugins) — opt-in via --include-local - * - * Usage: - * cd scripts/benchmark-site-editor - * npm install - * npm run benchmark - * - * Options: - * --rounds=N Number of benchmark runs per environment (default: 1) - * --skip-studio Skip Studio environments - * --skip-playground-cli Skip Playground CLI environments - * --skip-playground-web Skip Playground web environments - * --include-local Include Local by Flywheel environments (requires Local running) - * --only= Run only these environments (comma-separated) - * --help Show help - */ - -import { spawn, execSync, ChildProcess } from 'child_process'; -import fs from 'fs'; -import os from 'os'; -import path from 'path'; -import chalk from 'chalk'; -import { measureSiteEditor, METRIC_NAMES, type MeasurementResult } from './measure-site-editor.js'; -import { LocalGraphQLClient, getLocalSitesDir } from './local-graphql.js'; -import { installPluginsForLocalSite } from './install-plugins-local.js'; - -// --------------------------------------------------------------------------- -// Configuration -// --------------------------------------------------------------------------- - -const STUDIO_ROOT = path.resolve( import.meta.dirname, '../..' ); -const STUDIO_CLI_PATH = path.resolve( STUDIO_ROOT, 'dist/cli/main.js' ); -const PLAYGROUND_CLI_BIN = - process.platform === 'win32' ? 'wp-playground-cli.cmd' : 'wp-playground-cli'; -const PLAYGROUND_CLI_PATH = path.resolve( - import.meta.dirname, - 'node_modules/.bin', - PLAYGROUND_CLI_BIN -); -const PLUGINS_BLUEPRINT_PATH = path.resolve( import.meta.dirname, 'plugins-blueprint.json' ); -const ARTIFACTS_PATH = path.resolve( STUDIO_ROOT, 'metrics', 'artifacts' ); - -const PLAYGROUND_WEB_BASE_URL = 'https://playground.wordpress.net'; - -// Port offset for Playground CLI servers — avoids collisions with dev servers -const PLAYGROUND_CLI_PORT_BASE = 9500; - -// --------------------------------------------------------------------------- -// Types -// --------------------------------------------------------------------------- - -type EnvironmentType = 'studio' | 'playground-cli' | 'playground-web' | 'local'; - -interface EnvironmentConfig { - name: string; - type: EnvironmentType; - plugins: boolean; - multiWorker: boolean; -} - -interface BenchmarkResult { - environment: string; - metrics: Record< string, number >; -} - -// --------------------------------------------------------------------------- -// Environment matrix -// --------------------------------------------------------------------------- - -const ALL_ENVIRONMENTS: EnvironmentConfig[] = [ - { name: 'studio', type: 'studio', plugins: false, multiWorker: false }, - { name: 'studio-mw', type: 'studio', plugins: false, multiWorker: true }, - { name: 'studio-plugins', type: 'studio', plugins: true, multiWorker: false }, - { name: 'studio-mw-plugins', type: 'studio', plugins: true, multiWorker: true }, - { name: 'pg-cli', type: 'playground-cli', plugins: false, multiWorker: false }, - { name: 'pg-cli-mw', type: 'playground-cli', plugins: false, multiWorker: true }, - { name: 'pg-cli-plugins', type: 'playground-cli', plugins: true, multiWorker: false }, - { - name: 'pg-cli-mw-plugins', - type: 'playground-cli', - plugins: true, - multiWorker: true, - }, - { name: 'pg-web', type: 'playground-web', plugins: false, multiWorker: false }, - { name: 'pg-web-plugins', type: 'playground-web', plugins: true, multiWorker: false }, - { name: 'local', type: 'local', plugins: false, multiWorker: false }, - { name: 'local-plugins', type: 'local', plugins: true, multiWorker: false }, -]; - -// --------------------------------------------------------------------------- -// Argument parsing -// --------------------------------------------------------------------------- - -interface Options { - rounds: number; - skipStudio: boolean; - skipPlaygroundCli: boolean; - skipPlaygroundWeb: boolean; - includeLocal: boolean; - only: string[]; -} - -function parseArgs(): Options { - const args = process.argv.slice( 2 ); - const opts: Options = { - rounds: 1, - skipStudio: false, - skipPlaygroundCli: false, - skipPlaygroundWeb: false, - includeLocal: false, - only: [], - }; - - for ( const arg of args ) { - if ( arg.startsWith( '--rounds=' ) ) { - opts.rounds = parseInt( arg.split( '=' )[ 1 ], 10 ); - } else if ( arg === '--skip-studio' ) { - opts.skipStudio = true; - } else if ( arg === '--skip-playground-cli' ) { - opts.skipPlaygroundCli = true; - } else if ( arg === '--skip-playground-web' ) { - opts.skipPlaygroundWeb = true; - } else if ( arg === '--include-local' ) { - opts.includeLocal = true; - } else if ( arg.startsWith( '--only=' ) ) { - opts.only = arg - .split( '=' )[ 1 ] - .split( ',' ) - .map( ( s ) => s.trim() ); - } else if ( arg === '--help' ) { - printHelp(); - process.exit( 0 ); - } - } - - return opts; -} - -function printHelp() { - console.log( ` -Usage: npm run benchmark [options] - -Options: - --rounds=N Number of benchmark runs per environment (default: 1) - --skip-studio Skip Studio environments - --skip-playground-cli Skip Playground CLI environments - --skip-playground-web Skip Playground web environments - --include-local Include Local by Flywheel environments (requires Local running) - --only= Run only named environments (comma-separated) - --help Show this help message - -Environments: ${ ALL_ENVIRONMENTS.map( ( e ) => e.name ).join( ', ' ) } -` ); -} - -// --------------------------------------------------------------------------- -// Utility functions -// --------------------------------------------------------------------------- - -function median( values: number[] ): number { - if ( values.length === 0 ) return 0; - const sorted = [ ...values ].sort( ( a, b ) => a - b ); - const mid = Math.floor( sorted.length / 2 ); - return sorted.length % 2 !== 0 ? sorted[ mid ] : ( sorted[ mid - 1 ] + sorted[ mid ] ) / 2; -} - -function formatDuration( ms: number ): string { - if ( ms < 1000 ) return `${ ms.toFixed( 0 ) }ms`; - return `${ ( ms / 1000 ).toFixed( 2 ) }s`; -} - -function createTempDir( prefix: string ): string { - return fs.mkdtempSync( path.join( os.tmpdir(), `benchmark-${ prefix }-` ) ); -} - -function cleanupDir( dir: string ): void { - if ( fs.existsSync( dir ) ) { - fs.rmSync( dir, { recursive: true, force: true } ); - } -} - -async function waitForServer( url: string, timeoutMs = 60_000 ): Promise< boolean > { - const start = Date.now(); - while ( Date.now() - start < timeoutMs ) { - try { - const response = await fetch( url ); - if ( response.ok || response.status === 302 || response.status === 301 ) { - return true; - } - } catch { - // Server not ready yet - } - await sleep( 1000 ); - } - return false; -} - -async function killProcessOnPort( port: number ): Promise< void > { - return new Promise( ( resolve ) => { - try { - if ( process.platform === 'win32' ) { - execSync( - `for /f "tokens=5" %a in ('netstat -aon ^| find ":${ port }"') do taskkill /F /PID %a`, - { stdio: 'ignore' } - ); - } else { - execSync( `lsof -ti:${ port } | xargs kill -9 2>/dev/null || true`, { - stdio: 'ignore', - } ); - } - } catch { - // Process may not exist - } - setTimeout( resolve, 500 ); - } ); -} - -function sleep( ms: number ): Promise< void > { - return new Promise( ( resolve ) => setTimeout( resolve, ms ) ); -} - -function runCommand( - command: string, - args: string[], - options: { cwd?: string; env?: NodeJS.ProcessEnv; timeout?: number } = {} -): Promise< { stdout: string; stderr: string; exitCode: number } > { - const timeout = options.timeout ?? 300_000; - - return new Promise( ( resolve ) => { - let stdout = ''; - let stderr = ''; - let timedOut = false; - - const proc = spawn( command, args, { - cwd: options.cwd, - stdio: [ 'ignore', 'pipe', 'pipe' ], - env: { ...process.env, ...options.env, FORCE_COLOR: '0' }, - } ); - - const timeoutId = setTimeout( () => { - timedOut = true; - proc.kill( 'SIGKILL' ); - }, timeout ); - - proc.stdout?.on( 'data', ( data ) => { - stdout += data.toString(); - } ); - proc.stderr?.on( 'data', ( data ) => { - stderr += data.toString(); - } ); - - proc.on( 'close', ( code ) => { - clearTimeout( timeoutId ); - if ( timedOut ) { - resolve( { stdout, stderr: 'Timeout', exitCode: 1 } ); - } else { - resolve( { stdout, stderr, exitCode: code ?? 1 } ); - } - } ); - - proc.on( 'error', ( err ) => { - clearTimeout( timeoutId ); - resolve( { stdout, stderr: err.message, exitCode: 1 } ); - } ); - } ); -} - -// --------------------------------------------------------------------------- -// Studio environment helpers -// --------------------------------------------------------------------------- - -function createStudioAppdata( appdataDir: string, multiWorker: boolean ): void { - const studioDir = path.join( appdataDir, 'Studio' ); - fs.mkdirSync( studioDir, { recursive: true } ); - - const appdata = { - version: 1, - sites: [], - snapshots: [], - betaFeatures: { - studioSitesCli: true, - multiWorkerSupport: multiWorker, - }, - }; - - fs.writeFileSync( path.join( studioDir, 'appdata-v1.json' ), JSON.stringify( appdata, null, 2 ) ); - - // Set up server-files with symlinks to the repo's bundled wp-files. - // The CLI expects these at /Studio/server-files/. - const serverFilesDir = path.join( studioDir, 'server-files' ); - const wpFilesDir = path.join( STUDIO_ROOT, 'wp-files' ); - - fs.mkdirSync( path.join( serverFilesDir, 'wordpress-versions' ), { recursive: true } ); - fs.symlinkSync( - path.join( wpFilesDir, 'latest', 'wordpress' ), - path.join( serverFilesDir, 'wordpress-versions', 'latest' ) - ); - fs.symlinkSync( - path.join( wpFilesDir, 'sqlite-database-integration' ), - path.join( serverFilesDir, 'sqlite-database-integration' ) - ); - fs.symlinkSync( - path.join( wpFilesDir, 'sqlite-command' ), - path.join( serverFilesDir, 'sqlite-command' ) - ); - fs.symlinkSync( - path.join( wpFilesDir, 'wp-cli', 'wp-cli.phar' ), - path.join( serverFilesDir, 'wp-cli.phar' ) - ); -} - -function getStudioCliEnv( appdataDir: string ): NodeJS.ProcessEnv { - return { - E2E: 'true', - E2E_APP_DATA_PATH: appdataDir, - }; -} - -async function setupStudioSite( - env: EnvironmentConfig -): Promise< { url: string; siteDir: string; appdataDir: string } > { - const siteDir = createTempDir( env.name ); - const appdataDir = createTempDir( `${ env.name }-appdata` ); - const siteName = `bench-${ env.name }`; - - createStudioAppdata( appdataDir, env.multiWorker ); - - const cliEnv = getStudioCliEnv( appdataDir ); - - // Build the create args - const createArgs = [ - STUDIO_CLI_PATH, - 'site', - 'create', - `--path=${ siteDir }`, - `--name=${ siteName }`, - '--start=true', - '--skip-browser', - ]; - - if ( env.plugins ) { - createArgs.push( `--blueprint=${ PLUGINS_BLUEPRINT_PATH }` ); - } - - console.log( chalk.gray( ` Creating Studio site at ${ siteDir }` ) ); - const result = await runCommand( 'node', createArgs, { - cwd: STUDIO_ROOT, - env: cliEnv, - timeout: 600_000, // 10 min for site creation with plugins - } ); - - if ( result.exitCode !== 0 ) { - // The Studio CLI logs errors to stdout via its Logger, so show both - const errorOutput = ( result.stdout + '\n' + result.stderr ).trim(); - console.error( chalk.red( ` Studio site creation failed:\n${ errorOutput }` ) ); - throw new Error( `Studio site creation failed` ); - } - - // Extract the URL from the appdata (site was created and started) - const appdataPath = path.join( appdataDir, 'Studio', 'appdata-v1.json' ); - const appdata = JSON.parse( fs.readFileSync( appdataPath, 'utf-8' ) ); - const site = appdata.sites?.[ 0 ]; - if ( ! site?.port ) { - throw new Error( 'Studio site was created but no port was assigned' ); - } - const url = site.url || `http://localhost:${ site.port }`; - - console.log( chalk.gray( ` Studio site running at ${ url }` ) ); - return { url, siteDir, appdataDir }; -} - -async function teardownStudioSite( siteDir: string, appdataDir: string ): Promise< void > { - const cliEnv = getStudioCliEnv( appdataDir ); - - // Stop the site - await runCommand( 'node', [ STUDIO_CLI_PATH, 'site', 'stop', `--path=${ siteDir }` ], { - cwd: STUDIO_ROOT, - env: cliEnv, - timeout: 30_000, - } ).catch( () => {} ); - - // Delete the site from appdata - await runCommand( 'node', [ STUDIO_CLI_PATH, 'site', 'delete', `--path=${ siteDir }` ], { - cwd: STUDIO_ROOT, - env: cliEnv, - timeout: 30_000, - } ).catch( () => {} ); - - // Clean up temp dirs - cleanupDir( siteDir ); - cleanupDir( appdataDir ); -} - -// --------------------------------------------------------------------------- -// Playground CLI environment helpers -// --------------------------------------------------------------------------- - -async function setupPlaygroundCliSite( - env: EnvironmentConfig, - port: number -): Promise< { url: string; process: ChildProcess; siteDir: string } > { - const siteDir = createTempDir( env.name ); - - await killProcessOnPort( port ); - - const args = [ 'server', `--port=${ port }`, '--wp=latest', '--php=8.2' ]; - - // Mount the site dir - if ( process.platform === 'win32' ) { - args.push( '--mount-dir-before-install', siteDir, '/wordpress' ); - } else { - args.push( `--mount-before-install=${ siteDir }:/wordpress` ); - } - - if ( env.plugins ) { - args.push( `--blueprint=${ PLUGINS_BLUEPRINT_PATH }` ); - } - - if ( env.multiWorker ) { - const workerCount = Math.max( 1, os.cpus().length - 1 ); - args.push( `--experimental-multi-worker=${ workerCount }` ); - } - - console.log( chalk.gray( ` Starting Playground CLI on port ${ port }` ) ); - - const proc = spawn( PLAYGROUND_CLI_PATH, args, { - stdio: [ 'ignore', 'pipe', 'pipe' ], - env: { ...process.env, FORCE_COLOR: '0' }, - detached: process.platform !== 'win32', - shell: process.platform === 'win32', - } ); - - // Log stderr output for debugging if the server fails to start - let pgStderr = ''; - proc.stderr?.on( 'data', ( data ) => { - pgStderr += data.toString(); - } ); - - const url = `http://127.0.0.1:${ port }`; - - // Wait for server to be ready - const ready = await waitForServer( url, 180_000 ); - if ( ! ready ) { - proc.kill( 'SIGKILL' ); - if ( pgStderr ) { - console.error( chalk.gray( pgStderr.slice( 0, 500 ) ) ); - } - throw new Error( `Playground CLI server failed to start on port ${ port }` ); - } - - console.log( chalk.gray( ` Playground CLI running at ${ url }` ) ); - return { url, process: proc, siteDir }; -} - -async function teardownPlaygroundCliSite( - proc: ChildProcess, - port: number, - siteDir: string -): Promise< void > { - try { - if ( proc.pid ) { - if ( process.platform === 'win32' ) { - execSync( `taskkill /F /T /PID ${ proc.pid }`, { stdio: 'ignore' } ); - } else { - process.kill( -proc.pid, 'SIGTERM' ); - } - } - } catch { - // Process may have already exited - } - await killProcessOnPort( port ); - await sleep( 1000 ); - cleanupDir( siteDir ); -} - -// --------------------------------------------------------------------------- -// Playground Web environment helpers -// --------------------------------------------------------------------------- - -function getPlaygroundWebUrl( env: EnvironmentConfig ): string { - if ( ! env.plugins ) { - return PLAYGROUND_WEB_BASE_URL; - } - - // Pass the blueprint as a JSON fragment in the URL hash - const blueprint = JSON.parse( fs.readFileSync( PLUGINS_BLUEPRINT_PATH, 'utf-8' ) ); - return `${ PLAYGROUND_WEB_BASE_URL }/#${ JSON.stringify( blueprint ) }`; -} - -// --------------------------------------------------------------------------- -// Local by Flywheel environment helpers -// --------------------------------------------------------------------------- - -async function ensureLocalRunning(): Promise< LocalGraphQLClient > { - try { - return await LocalGraphQLClient.connect(); - } catch ( err ) { - console.error( chalk.red( ` Local is not running or not reachable.\n ${ err }` ) ); - process.exit( 1 ); - } -} - -async function setupLocalSite( - env: EnvironmentConfig, - client: LocalGraphQLClient -): Promise< { url: string; siteId: string; siteDir: string } > { - const siteName = `bench-${ env.name }`; - const domain = `bench-${ env.name }.local`; - const siteDir = path.join( getLocalSitesDir(), siteName ); - - console.log( chalk.gray( ` Creating Local site "${ siteName }"...` ) ); - - const site = await client.createSite( { - name: siteName, - domain, - path: siteDir, - wpAdminPassword: 'password', - wpAdminUsername: 'admin', - } ); - - const url = site.url || `http://localhost:${ site.httpPort }`; - - // Plugin install is best-effort — we still want to return site info - // so the caller can set up teardown even if plugin install fails. - if ( env.plugins ) { - console.log( chalk.gray( ` Installing plugins via REST API...` ) ); - try { - await installPluginsForLocalSite( url, PLUGINS_BLUEPRINT_PATH ); - } catch ( err ) { - console.warn( chalk.yellow( ` Plugin installation failed: ${ err }` ) ); - } - } - console.log( chalk.gray( ` Local site running at ${ url }` ) ); - return { url, siteId: site.id, siteDir }; -} - -async function teardownLocalSite( - siteId: string, - client: LocalGraphQLClient, - siteDir: string -): Promise< void > { - try { - await client.stopSite( siteId ); - } catch { - // Site may already be stopped - } - try { - await client.deleteSite( siteId ); - } catch { - // Best effort cleanup - } - cleanupDir( siteDir ); -} - -// --------------------------------------------------------------------------- -// Benchmark runner -// --------------------------------------------------------------------------- - -const MEASUREMENT_TIMEOUT = 600_000; // 10 min per measurement - -async function runBenchmark( - benchmarkUrl: string, - env: EnvironmentConfig, - rounds: number -): Promise< Record< string, number > | null > { - console.log( - chalk.gray( ` Running benchmark (${ rounds } round${ rounds > 1 ? 's' : '' })...` ) - ); - - const isPlaygroundWeb = benchmarkUrl.includes( 'playground.wordpress.net' ); - const isPlaygroundCli = benchmarkUrl.includes( '127.0.0.1' ); - const isLocal = env.type === 'local'; - const allMeasurements: MeasurementResult[] = []; - - for ( let round = 1; round <= rounds; round++ ) { - if ( rounds > 1 ) { - console.log( chalk.gray( ` Round ${ round }/${ rounds }...` ) ); - } - try { - const result = await Promise.race( [ - measureSiteEditor( { url: benchmarkUrl, isPlaygroundWeb, isPlaygroundCli, isLocal } ), - sleep( MEASUREMENT_TIMEOUT ).then( () => { - throw new Error( 'Measurement timed out' ); - } ), - ] ); - allMeasurements.push( result ); - } catch ( err ) { - console.warn( chalk.yellow( ` Round ${ round } failed: ${ err }` ) ); - } - - // Small delay between rounds - if ( round < rounds ) { - await sleep( 1000 ); - } - } - - if ( allMeasurements.length === 0 ) { - return null; - } - - // Calculate medians across rounds - const medians: Record< string, number > = {}; - for ( const metric of METRIC_NAMES ) { - const values = allMeasurements - .map( ( m ) => m[ metric ] ) - .filter( ( v ): v is number => v !== undefined ); - if ( values.length > 0 ) { - medians[ metric ] = median( values ); - } - } - - return medians; -} - -// --------------------------------------------------------------------------- -// Results formatting -// --------------------------------------------------------------------------- - -function printComparisonTable( results: BenchmarkResult[] ): void { - if ( results.length === 0 ) { - console.log( chalk.yellow( '\nNo results to display.' ) ); - return; - } - - // Collect all unique metric names across all results - const allMetrics = new Set< string >(); - for ( const r of results ) { - Object.keys( r.metrics ).forEach( ( k ) => allMetrics.add( k ) ); - } - const metrics = [ ...allMetrics ].sort(); - - // Calculate column widths - const metricColWidth = Math.max( 20, ...metrics.map( ( m ) => m.length + 2 ) ); - const envColWidth = Math.max( 12, ...results.map( ( r ) => r.environment.length + 2 ) ); - - // Header - console.log( chalk.bold( '\n\nResults Comparison' ) ); - console.log( '═'.repeat( metricColWidth + envColWidth * results.length ) ); - - const header = - 'Metric'.padEnd( metricColWidth ) + - results.map( ( r ) => r.environment.padEnd( envColWidth ) ).join( '' ); - console.log( chalk.bold( header ) ); - console.log( '─'.repeat( metricColWidth + envColWidth * results.length ) ); - - // Rows - for ( const metric of metrics ) { - let row = metric.padEnd( metricColWidth ); - - for ( const r of results ) { - const value = r.metrics[ metric ]; - if ( value !== undefined ) { - row += formatDuration( value ).padEnd( envColWidth ); - } else { - row += '—'.padEnd( envColWidth ); - } - } - - console.log( row ); - } - - console.log( '═'.repeat( metricColWidth + envColWidth * results.length ) ); -} - -function saveResultsSummary( results: BenchmarkResult[] ): void { - const summaryPath = path.join( ARTIFACTS_PATH, `benchmark-comparison-${ Date.now() }.json` ); - const summary = { - date: new Date().toISOString(), - platform: os.platform(), - arch: os.arch(), - nodeVersion: process.version, - cpus: os.cpus().length, - results, - }; - fs.mkdirSync( ARTIFACTS_PATH, { recursive: true } ); - fs.writeFileSync( summaryPath, JSON.stringify( summary, null, 2 ) ); - console.log( chalk.gray( `\nResults saved to: ${ summaryPath }` ) ); -} - -// --------------------------------------------------------------------------- -// Setup checks -// --------------------------------------------------------------------------- - -async function ensureStudioCLIBuilt(): Promise< boolean > { - // Ensure CLI dependencies are installed (required for the Vite build to resolve pm2-axon, etc.) - const cliNodeModules = path.resolve( STUDIO_ROOT, 'cli', 'node_modules' ); - if ( ! fs.existsSync( cliNodeModules ) ) { - console.log( chalk.yellow( ' Installing CLI dependencies...' ) ); - try { - execSync( 'npm install', { - cwd: path.resolve( STUDIO_ROOT, 'cli' ), - stdio: 'inherit', - } ); - } catch { - console.error( chalk.red( ' Failed to install CLI dependencies' ) ); - return false; - } - } - - if ( ! fs.existsSync( STUDIO_CLI_PATH ) ) { - console.log( chalk.yellow( ' Building Studio CLI...' ) ); - try { - execSync( 'npm run cli:build', { cwd: STUDIO_ROOT, stdio: 'inherit' } ); - return true; - } catch { - console.error( chalk.red( ' Failed to build Studio CLI' ) ); - return false; - } - } - return true; -} - -async function ensurePlaygroundCLIInstalled(): Promise< boolean > { - if ( ! fs.existsSync( PLAYGROUND_CLI_PATH ) ) { - console.log( chalk.yellow( ' Installing dependencies (including @wp-playground/cli)...' ) ); - try { - execSync( 'npm install', { cwd: import.meta.dirname, stdio: 'inherit' } ); - return true; - } catch { - console.error( chalk.red( ' Failed to install Playground CLI' ) ); - return false; - } - } - return true; -} - -// --------------------------------------------------------------------------- -// Main -// --------------------------------------------------------------------------- - -async function main() { - const opts = parseArgs(); - - // Filter environments - let environments = ALL_ENVIRONMENTS; - - if ( opts.only.length > 0 ) { - environments = environments.filter( ( e ) => opts.only.includes( e.name ) ); - } else { - if ( opts.skipStudio ) { - environments = environments.filter( ( e ) => e.type !== 'studio' ); - } - if ( opts.skipPlaygroundCli ) { - environments = environments.filter( ( e ) => e.type !== 'playground-cli' ); - } - if ( opts.skipPlaygroundWeb ) { - environments = environments.filter( ( e ) => e.type !== 'playground-web' ); - } - // Local environments are excluded by default — they require the GUI app running - if ( ! opts.includeLocal ) { - environments = environments.filter( ( e ) => e.type !== 'local' ); - } - } - - if ( environments.length === 0 ) { - console.log( chalk.yellow( 'No environments selected. Nothing to do.' ) ); - return; - } - - console.log( chalk.bold.cyan( '\n=== Site Editor Performance Benchmark ===' ) ); - console.log( chalk.gray( `Platform: ${ os.platform() } ${ os.arch() }` ) ); - console.log( chalk.gray( `Node: ${ process.version }` ) ); - console.log( chalk.gray( `CPUs: ${ os.cpus().length }` ) ); - console.log( chalk.gray( `Rounds: ${ opts.rounds }` ) ); - console.log( chalk.gray( `Date: ${ new Date().toISOString() }` ) ); - console.log( - chalk.gray( `Environments: ${ environments.map( ( e ) => e.name ).join( ', ' ) }` ) - ); - - // Setup - console.log( chalk.bold( '\nSetup:' ) ); - - // Ensure Playwright browsers are installed - try { - execSync( 'npx playwright install chromium', { cwd: import.meta.dirname, stdio: 'ignore' } ); - console.log( chalk.green( ' Playwright chromium ready' ) ); - } catch { - console.error( chalk.red( ' Failed to install Playwright chromium' ) ); - process.exit( 1 ); - } - - const needsStudio = environments.some( ( e ) => e.type === 'studio' ); - const needsPlaygroundCli = environments.some( ( e ) => e.type === 'playground-cli' ); - const needsLocal = environments.some( ( e ) => e.type === 'local' ); - - if ( needsStudio ) { - if ( ! ( await ensureStudioCLIBuilt() ) ) { - process.exit( 1 ); - } - console.log( chalk.green( ' Studio CLI ready' ) ); - } - - if ( needsPlaygroundCli ) { - if ( ! ( await ensurePlaygroundCLIInstalled() ) ) { - process.exit( 1 ); - } - console.log( chalk.green( ' Playground CLI ready' ) ); - } - - let localClient: LocalGraphQLClient | null = null; - if ( needsLocal ) { - localClient = await ensureLocalRunning(); - console.log( chalk.green( ' Local ready' ) ); - } - - fs.mkdirSync( ARTIFACTS_PATH, { recursive: true } ); - - // Run benchmarks - const allResults: BenchmarkResult[] = []; - let playgroundCliPortOffset = 0; - - for ( const env of environments ) { - console.log( chalk.bold.cyan( `\n ▶ ${ env.name }` ) ); - const tags = [ env.plugins ? 'plugins' : null, env.multiWorker ? 'multi-worker' : null ] - .filter( Boolean ) - .join( ', ' ); - if ( tags ) { - console.log( chalk.gray( ` (${ tags })` ) ); - } - - let benchmarkUrl: string; - let teardownFn: ( () => Promise< void > ) | null = null; - - try { - // Setup environment - if ( env.type === 'studio' ) { - const setup = await setupStudioSite( env ); - benchmarkUrl = setup.url; - teardownFn = () => teardownStudioSite( setup.siteDir, setup.appdataDir ); - } else if ( env.type === 'playground-cli' ) { - const port = PLAYGROUND_CLI_PORT_BASE + playgroundCliPortOffset++; - const setup = await setupPlaygroundCliSite( env, port ); - benchmarkUrl = setup.url; - teardownFn = () => teardownPlaygroundCliSite( setup.process, port, setup.siteDir ); - } else if ( env.type === 'local' ) { - const setup = await setupLocalSite( env, localClient! ); - benchmarkUrl = setup.url; - teardownFn = () => teardownLocalSite( setup.siteId, localClient!, setup.siteDir ); - } else { - benchmarkUrl = getPlaygroundWebUrl( env ); - console.log( chalk.gray( ` URL: ${ benchmarkUrl.slice( 0, 80 ) }...` ) ); - } - - // Run benchmark - const metrics = await runBenchmark( benchmarkUrl, env, opts.rounds ); - - if ( metrics ) { - allResults.push( { environment: env.name, metrics } ); - console.log( chalk.green( ` ✓ Done` ) ); - } else { - console.log( chalk.red( ` ✗ Failed` ) ); - } - } catch ( err ) { - console.error( chalk.red( ` ✗ Error: ${ err }` ) ); - } finally { - // Teardown - if ( teardownFn ) { - console.log( chalk.gray( ' Cleaning up...' ) ); - await teardownFn().catch( () => {} ); - } - } - } - - // Print results - printComparisonTable( allResults ); - saveResultsSummary( allResults ); -} - -main().catch( ( err ) => { - console.error( chalk.red( 'Benchmark failed:' ), err ); - process.exit( 1 ); -} ); diff --git a/scripts/benchmark-site-editor/measure-site-editor.ts b/scripts/benchmark-site-editor/measure-site-editor.ts deleted file mode 100644 index de071de65d..0000000000 --- a/scripts/benchmark-site-editor/measure-site-editor.ts +++ /dev/null @@ -1,315 +0,0 @@ -/** - * Site editor performance measurement module. - * - * Runs a single benchmark pass against a WordPress site: launches Chromium, - * navigates through the site editor, and returns raw timing values for each metric. - */ - -import { chromium, type Page, type Frame } from 'playwright'; - -// --------------------------------------------------------------------------- -// Metric names and types -// --------------------------------------------------------------------------- - -export const METRIC_NAMES = [ - 'siteEditorLoad', - 'templatesViewLoad', - 'templateOpen', - 'blockAdd', - 'templateSave', -] as const; - -export type MetricName = ( typeof METRIC_NAMES )[ number ]; - -/** A single measurement result: one timing value per metric. A metric may be missing if that step failed. */ -export type MeasurementResult = Partial< Record< MetricName, number > >; - -export interface MeasureOptions { - /** The base URL of the WordPress site to benchmark. */ - url: string; - /** Whether this is a Playground Web URL (playground.wordpress.net). Affects iframe handling. */ - isPlaygroundWeb: boolean; - /** Whether this is a local Playground CLI site (127.0.0.1). Affects login flow. */ - isPlaygroundCli: boolean; - /** Whether this is a Local by Flywheel site. Uses wp-login.php credentials. */ - isLocal: boolean; -} - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -function getUrlWithAutoLogin( destinationUrl: string ): string { - const parsedUrl = new URL( destinationUrl ); - const baseUrl = `${ parsedUrl.protocol }//${ parsedUrl.hostname }:${ parsedUrl.port }`; - return `${ baseUrl }/studio-auto-login?redirect_to=${ encodeURIComponent( destinationUrl ) }`; -} - -function findWordPressFrame( page: Page ): Frame | null { - const frames = page.frames(); - let wordPressFrame = - frames.find( ( frame ) => { - const url = frame.url(); - return ( - url.includes( 'wordpress' ) || - url.includes( 'wp-admin' ) || - url.includes( 'wp-login' ) || - url.includes( 'scope:' ) - ); - } ) || null; - - if ( ! wordPressFrame ) { - for ( const frame of page.frames() ) { - if ( frame.parentFrame() && frame.url().includes( 'scope:' ) ) { - wordPressFrame = frame; - break; - } - } - } - - return wordPressFrame; -} - -function findEditorCanvasFrame( - page: Page, - isPlaygroundWeb: boolean, - wordPressFrame: Frame | null -): Frame | null { - if ( isPlaygroundWeb && wordPressFrame ) { - let frame = wordPressFrame.childFrames().find( ( f ) => f.name() === 'editor-canvas' ) || null; - if ( ! frame ) { - frame = page.frames().find( ( f ) => f.name() === 'editor-canvas' ) || null; - } - return frame; - } - return page.frame( { name: 'editor-canvas' } ); -} - -// --------------------------------------------------------------------------- -// Measurement -// --------------------------------------------------------------------------- - -/** - * Runs a single benchmark measurement pass against the given URL. - * - * Launches a fresh Chromium browser, navigates to the site editor, - * performs the measurement steps, and returns timing results. - * The browser is always closed, even on error. - */ -export async function measureSiteEditor( options: MeasureOptions ): Promise< MeasurementResult > { - const { isPlaygroundWeb, isPlaygroundCli, isLocal } = options; - - // Normalize URL - let wpAdminUrl = options.url; - if ( ! wpAdminUrl.startsWith( 'http' ) ) { - wpAdminUrl = `http://${ wpAdminUrl }`; - } - wpAdminUrl = wpAdminUrl.replace( /\/$/, '' ); - - const result: MeasurementResult = {}; - - const browser = await chromium.launch(); - const context = await browser.newContext(); - const page = await context.newPage(); - - let wordPressFrame: Frame | null = null; - let wordPressFrameLocator: ReturnType< Page[ 'frameLocator' ] > | null = null; - - try { - // Navigate to wp-admin (environment-specific) - if ( isPlaygroundWeb ) { - await page.goto( wpAdminUrl, { waitUntil: 'networkidle' } ); - - wordPressFrameLocator = page - .frameLocator( 'iframe.playground-viewport' ) - .first() - .frameLocator( 'iframe' ) - .first(); - - await wordPressFrameLocator - .getByRole( 'link', { name: 'Appearance' } ) - .waitFor( { timeout: 30_000 } ); - - wordPressFrame = findWordPressFrame( page ); - } else if ( isPlaygroundCli ) { - await page.goto( `${ wpAdminUrl }/wp-admin`, { - waitUntil: 'domcontentloaded', - timeout: 120_000, - } ); - await page.waitForLoadState( 'networkidle', { timeout: 30_000 } ).catch( () => {} ); - await page.getByRole( 'link', { name: 'Appearance' } ).waitFor( { - state: 'visible', - timeout: 60_000, - } ); - } else if ( isLocal ) { - // Local sites require wp-login.php with credentials - await page.goto( `${ wpAdminUrl }/wp-login.php`, { - waitUntil: 'domcontentloaded', - timeout: 120_000, - } ); - await page.waitForLoadState( 'networkidle', { timeout: 30_000 } ).catch( () => {} ); - await page.fill( '#user_login', 'admin' ); - await page.fill( '#user_pass', 'password' ); - await Promise.all( [ - page.waitForURL( '**/wp-admin/**', { timeout: 60_000 } ), - page.click( '#wp-submit' ), - ] ); - await page.getByRole( 'link', { name: 'Appearance' } ).waitFor( { - state: 'visible', - timeout: 60_000, - } ); - } else { - // Studio: use auto-login endpoint - await page.goto( getUrlWithAutoLogin( `${ wpAdminUrl }/wp-admin` ), { - waitUntil: 'domcontentloaded', - timeout: 120_000, - } ); - await page.waitForLoadState( 'networkidle', { timeout: 30_000 } ).catch( () => {} ); - await page.getByRole( 'link', { name: 'Appearance' } ).waitFor( { - state: 'visible', - timeout: 60_000, - } ); - } - - const target = isPlaygroundWeb && wordPressFrameLocator ? wordPressFrameLocator : page; - - // Step 1: Navigate to site editor - const siteEditorStart = Date.now(); - - await target.getByRole( 'link', { name: 'Appearance' } ).click(); - await target.locator( 'a[href="site-editor.php"]' ).click(); - - // Close welcome modal if it appears - const welcomeDialog = target.getByRole( 'dialog', { - name: /welcome to the site editor/i, - } ); - const isModalVisible = await welcomeDialog.isVisible( { timeout: 5_000 } ).catch( () => false ); - if ( isModalVisible ) { - await target.getByRole( 'button', { name: /get started/i } ).click(); - await welcomeDialog.waitFor( { state: 'hidden', timeout: 5_000 } ).catch( () => {} ); - } - - await target.locator( 'iframe[name="editor-canvas"]' ).waitFor( { - state: 'visible', - timeout: 120_000, - } ); - - const frame = findEditorCanvasFrame( page, isPlaygroundWeb, wordPressFrame ); - if ( ! frame ) { - throw new Error( 'Editor canvas frame not found' ); - } - - await frame.waitForLoadState( 'domcontentloaded' ); - await frame.waitForSelector( '[data-block]', { timeout: 60_000 } ); - await frame.waitForFunction( - () => { - const blocks = document.querySelectorAll( '[data-block]' ); - return ( - blocks.length > 0 && Array.from( blocks ).some( ( block ) => block.clientHeight > 0 ) - ); - }, - { timeout: 60_000 } - ); - - result.siteEditorLoad = Date.now() - siteEditorStart; - - // Step 2: Navigate to Templates view - const templatesViewStart = Date.now(); - - await target.getByRole( 'button', { name: 'Templates' } ).click(); - - await target.getByRole( 'heading', { name: 'Templates', level: 2 } ).waitFor( { - timeout: 60_000, - } ); - await target - .locator( '.dataviews-view-grid-items.dataviews-view-grid' ) - .waitFor( { timeout: 60_000 } ); - const firstCard = target.locator( '.dataviews-view-grid__card' ).first(); - await firstCard.waitFor( { state: 'visible', timeout: 60_000 } ); - await firstCard.getByRole( 'button' ).first().waitFor( { state: 'visible', timeout: 60_000 } ); - - result.templatesViewLoad = Date.now() - templatesViewStart; - - // Step 3: Open a template - const templateOpenStart = Date.now(); - - await target - .locator( '.dataviews-view-grid__card' ) - .first() - .getByRole( 'button' ) - .first() - .click(); - - await target.locator( 'iframe[name="editor-canvas"]' ).waitFor( { - state: 'visible', - timeout: 60_000, - } ); - - const templateFrame = findEditorCanvasFrame( page, isPlaygroundWeb, wordPressFrame ); - if ( ! templateFrame ) { - throw new Error( 'Template editor frame not found' ); - } - - await templateFrame.waitForLoadState( 'domcontentloaded' ); - await templateFrame.waitForSelector( '[data-block]', { timeout: 60_000 } ); - await templateFrame.waitForFunction( - () => { - const blocks = document.querySelectorAll( '[data-block]' ); - return ( - blocks.length > 0 && Array.from( blocks ).some( ( block ) => block.clientHeight > 0 ) - ); - }, - { timeout: 60_000 } - ); - - result.templateOpen = Date.now() - templateOpenStart; - - // Step 4: Add blocks - const blockAddStart = Date.now(); - - await page.keyboard.press( 'Escape' ); - - await target.getByRole( 'button', { name: /Block Inserter/i } ).click(); - - const searchInput = target.getByPlaceholder( 'Search' ); - await searchInput.fill( 'Paragraph' ); - await target.getByRole( 'option', { name: 'Paragraph', exact: true } ).click(); - await templateFrame.waitForSelector( 'p[data-block]', { timeout: 15_000 } ); - - await searchInput.fill( 'Heading' ); - await target - .locator( '.block-editor-block-types-list__item' ) - .filter( { hasText: /^Heading$/ } ) - .click(); - await templateFrame.waitForSelector( 'h1[data-block], h2[data-block], h3[data-block]', { - timeout: 15_000, - } ); - - result.blockAdd = Date.now() - blockAddStart; - - // Step 5: Save the template - const templateSaveStart = Date.now(); - - await target.getByRole( 'button', { name: 'Save' } ).first().click(); - - await page.waitForFunction( - () => { - const saveButton = Array.from( document.querySelectorAll( 'button' ) ).find( - ( btn ) => - btn.textContent?.includes( 'Saved' ) || - btn.getAttribute( 'aria-label' )?.toLowerCase().includes( 'saved' ) - ); - return saveButton !== null; - }, - { timeout: 30_000 } - ); - - result.templateSave = Date.now() - templateSaveStart; - } finally { - await page.close(); - await context.close(); - await browser.close(); - } - - return result; -} diff --git a/scripts/benchmark-site-editor/package.json b/scripts/benchmark-site-editor/package.json deleted file mode 100644 index c6024ce753..0000000000 --- a/scripts/benchmark-site-editor/package.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "name": "benchmark-site-editor", - "version": "0.0.1", - "description": "Benchmark site editor performance across Studio, Playground CLI, and Playground Web environments", - "author": "Automattic", - "license": "GPLv2", - "type": "module", - "scripts": { - "benchmark": "tsx benchmark.ts" - }, - "dependencies": { - "@wp-playground/cli": "^3.0.22", - "chalk": "^5.3.0", - "playwright": "^1.58.1", - "tsx": "^4.7.0" - }, - "devDependencies": { - "@types/node": "^20.0.0", - "typescript": "^5.0.0" - } -} diff --git a/scripts/benchmark-site-editor/plugins-blueprint.json b/scripts/benchmark-site-editor/plugins-blueprint.json deleted file mode 100644 index 1a487ff01b..0000000000 --- a/scripts/benchmark-site-editor/plugins-blueprint.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "$schema": "https://playground.wordpress.net/blueprint-schema.json", - "steps": [ - { - "step": "installPlugin", - "pluginData": { - "resource": "wordpress.org/plugins", - "slug": "woocommerce" - } - }, - { - "step": "installPlugin", - "pluginData": { - "resource": "wordpress.org/plugins", - "slug": "jetpack" - } - }, - { - "step": "installPlugin", - "pluginData": { - "resource": "wordpress.org/plugins", - "slug": "wp-super-cache" - } - }, - { - "step": "installPlugin", - "pluginData": { - "resource": "wordpress.org/plugins", - "slug": "jetpack-boost" - } - }, - { - "step": "installPlugin", - "pluginData": { - "resource": "wordpress.org/plugins", - "slug": "jetpack-protect" - } - }, - { - "step": "installPlugin", - "pluginData": { - "resource": "wordpress.org/plugins", - "slug": "jetpack-social" - } - }, - { - "step": "installPlugin", - "pluginData": { - "resource": "wordpress.org/plugins", - "slug": "jetpack-videopress" - } - }, - { - "step": "installPlugin", - "pluginData": { - "resource": "wordpress.org/plugins", - "slug": "woocommerce-payments" - } - }, - { - "step": "installPlugin", - "pluginData": { - "resource": "wordpress.org/plugins", - "slug": "contact-form-7" - } - }, - { - "step": "installPlugin", - "pluginData": { - "resource": "wordpress.org/plugins", - "slug": "elementor" - } - } - ] -} diff --git a/scripts/benchmark-site-editor/tsconfig.json b/scripts/benchmark-site-editor/tsconfig.json deleted file mode 100644 index 8f21fe7532..0000000000 --- a/scripts/benchmark-site-editor/tsconfig.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2022", - "module": "ES2022", - "moduleResolution": "bundler", - "esModuleInterop": true, - "strict": true, - "skipLibCheck": true, - "outDir": "./dist", - "rootDir": ".", - "declaration": false, - "sourceMap": false - }, - "include": [ "*.ts" ] -} diff --git a/scripts/benchmark-site-editor/.gitignore b/tools/benchmark-site-editor/.gitignore similarity index 100% rename from scripts/benchmark-site-editor/.gitignore rename to tools/benchmark-site-editor/.gitignore diff --git a/tools/benchmark-site-editor/README.md b/tools/benchmark-site-editor/README.md index ab4cbcbb39..83331bedcc 100644 --- a/tools/benchmark-site-editor/README.md +++ b/tools/benchmark-site-editor/README.md @@ -1,6 +1,6 @@ # Site Editor Performance Benchmark -Benchmarks site editor performance across Studio, Playground CLI, and Playground Web environments, with optional plugin and multi-worker configurations. +Benchmarks site editor performance across Studio, Playground CLI, Playground Web, and Local by Flywheel environments, with optional plugin and multi-worker configurations. ## Related Issue @@ -32,6 +32,8 @@ The benchmark launches a headless Chromium browser against each environment, mea | Playground CLI + MW + Plugins | `pg-cli-mw-plugins` | Playground CLI with multi-worker and 10 plugins | | Playground Web | `pg-web` | playground.wordpress.net (bare) | | Playground Web + Plugins | `pg-web-plugins` | playground.wordpress.net with 10 plugins | +| Local | `local` | Local by Flywheel site (nginx+PHP+MySQL) | +| Local + Plugins | `local-plugins` | Local with 10 plugins installed | ## Plugins @@ -63,6 +65,7 @@ npm run benchmark --skip-studio Skip Studio environments --skip-playground-cli Skip Playground CLI environments --skip-playground-web Skip Playground web environments +--include-local Include Local by Flywheel environments (requires Local running) --only= Run only named environments (comma-separated) --help Show help ``` @@ -81,13 +84,24 @@ npm run benchmark -- --skip-studio --skip-playground-web # Single specific environment npm run benchmark -- --only=studio-mw-plugins --rounds=5 + +# Include Local by Flywheel (must be running) +npm run benchmark -- --include-local --only=local,local-plugins + +# Compare Studio vs Local +npm run benchmark -- --only=studio,local --rounds=3 ``` +> **Note:** When using `--only` with Local environments, the `--include-local` flag is not needed — Local environments are automatically included when explicitly named. + ## Prerequisites - **Studio CLI**: Built automatically if `dist/cli/main.js` doesn't exist (`npm run cli:build`) - **Playground CLI**: Installed automatically via this script's `npm install` - **Playwright**: Chromium is installed automatically during setup +- **Local by Flywheel**: Must be installed and running (GUI app). Use `--include-local` to enable. The script connects to Local's GraphQL API to create and manage benchmark sites. + - macOS: `brew install --cask local` + - Windows: `winget install Flywheel.Local` ## Output diff --git a/tools/benchmark-site-editor/benchmark.ts b/tools/benchmark-site-editor/benchmark.ts index 7fb11d097f..c6be0a88e6 100644 --- a/tools/benchmark-site-editor/benchmark.ts +++ b/tools/benchmark-site-editor/benchmark.ts @@ -1,5 +1,5 @@ #!/usr/bin/env tsx - +/* eslint-disable no-console */ /** * Site Editor Performance Benchmark — Orchestration Script * @@ -7,6 +7,7 @@ * - Studio (bare, multi-worker, plugins, multi-worker+plugins) * - Playground CLI (bare, multi-worker, plugins, multi-worker+plugins) * - Playground Web (bare, plugins) + * - Local by Flywheel (bare, plugins) — opt-in via --include-local * * Usage: * cd scripts/benchmark-site-editor @@ -18,6 +19,7 @@ * --skip-studio Skip Studio environments * --skip-playground-cli Skip Playground CLI environments * --skip-playground-web Skip Playground web environments + * --include-local Include Local by Flywheel environments (requires Local running) * --only= Run only these environments (comma-separated) * --help Show help */ @@ -27,14 +29,16 @@ import fs from 'fs'; import os from 'os'; import path from 'path'; import chalk from 'chalk'; -import { measureSiteEditor, METRIC_NAMES, type MeasurementResult } from './measure-site-editor.ts'; +import { measureSiteEditor, METRIC_NAMES, type MeasurementResult } from './measure-site-editor.js'; +import { LocalGraphQLClient, getLocalSitesDir } from './local-graphql.js'; +import { installPluginsForLocalSite } from './install-plugins-local.js'; // --------------------------------------------------------------------------- // Configuration // --------------------------------------------------------------------------- const STUDIO_ROOT = path.resolve( import.meta.dirname, '../..' ); -const STUDIO_CLI_PATH = path.resolve( STUDIO_ROOT, 'apps/cli/dist/cli/main.js' ); +const STUDIO_CLI_PATH = path.resolve( STUDIO_ROOT, 'dist/cli/main.js' ); const PLAYGROUND_CLI_BIN = process.platform === 'win32' ? 'wp-playground-cli.cmd' : 'wp-playground-cli'; const PLAYGROUND_CLI_PATH = path.resolve( @@ -54,7 +58,7 @@ const PLAYGROUND_CLI_PORT_BASE = 9500; // Types // --------------------------------------------------------------------------- -type EnvironmentType = 'studio' | 'playground-cli' | 'playground-web'; +type EnvironmentType = 'studio' | 'playground-cli' | 'playground-web' | 'local'; interface EnvironmentConfig { name: string; @@ -88,6 +92,8 @@ const ALL_ENVIRONMENTS: EnvironmentConfig[] = [ }, { name: 'pg-web', type: 'playground-web', plugins: false, multiWorker: false }, { name: 'pg-web-plugins', type: 'playground-web', plugins: true, multiWorker: false }, + { name: 'local', type: 'local', plugins: false, multiWorker: false }, + { name: 'local-plugins', type: 'local', plugins: true, multiWorker: false }, ]; // --------------------------------------------------------------------------- @@ -99,8 +105,8 @@ interface Options { skipStudio: boolean; skipPlaygroundCli: boolean; skipPlaygroundWeb: boolean; + includeLocal: boolean; only: string[]; - headed: boolean; } function parseArgs(): Options { @@ -110,8 +116,8 @@ function parseArgs(): Options { skipStudio: false, skipPlaygroundCli: false, skipPlaygroundWeb: false, + includeLocal: false, only: [], - headed: false, }; for ( const arg of args ) { @@ -123,13 +129,13 @@ function parseArgs(): Options { opts.skipPlaygroundCli = true; } else if ( arg === '--skip-playground-web' ) { opts.skipPlaygroundWeb = true; + } else if ( arg === '--include-local' ) { + opts.includeLocal = true; } else if ( arg.startsWith( '--only=' ) ) { opts.only = arg .split( '=' )[ 1 ] .split( ',' ) .map( ( s ) => s.trim() ); - } else if ( arg === '--headed' ) { - opts.headed = true; } else if ( arg === '--help' ) { printHelp(); process.exit( 0 ); @@ -148,8 +154,8 @@ Options: --skip-studio Skip Studio environments --skip-playground-cli Skip Playground CLI environments --skip-playground-web Skip Playground web environments + --include-local Include Local by Flywheel environments (requires Local running) --only= Run only named environments (comma-separated) - --headed Launch browser in headed mode for debugging --help Show this help message Environments: ${ ALL_ENVIRONMENTS.map( ( e ) => e.name ).join( ', ' ) } @@ -490,6 +496,71 @@ function getPlaygroundWebUrl( env: EnvironmentConfig ): string { return `${ PLAYGROUND_WEB_BASE_URL }/#${ JSON.stringify( blueprint ) }`; } +// --------------------------------------------------------------------------- +// Local by Flywheel environment helpers +// --------------------------------------------------------------------------- + +async function ensureLocalRunning(): Promise< LocalGraphQLClient > { + try { + return await LocalGraphQLClient.connect(); + } catch ( err ) { + console.error( chalk.red( ` Local is not running or not reachable.\n ${ err }` ) ); + process.exit( 1 ); + } +} + +async function setupLocalSite( + env: EnvironmentConfig, + client: LocalGraphQLClient +): Promise< { url: string; siteId: string; siteDir: string } > { + const siteName = `bench-${ env.name }`; + const domain = `bench-${ env.name }.local`; + const siteDir = path.join( getLocalSitesDir(), siteName ); + + console.log( chalk.gray( ` Creating Local site "${ siteName }"...` ) ); + + const site = await client.createSite( { + name: siteName, + domain, + path: siteDir, + wpAdminPassword: 'password', + wpAdminUsername: 'admin', + } ); + + const url = site.url || `http://localhost:${ site.httpPort }`; + + // Plugin install is best-effort — we still want to return site info + // so the caller can set up teardown even if plugin install fails. + if ( env.plugins ) { + console.log( chalk.gray( ` Installing plugins via REST API...` ) ); + try { + await installPluginsForLocalSite( url, PLUGINS_BLUEPRINT_PATH ); + } catch ( err ) { + console.warn( chalk.yellow( ` Plugin installation failed: ${ err }` ) ); + } + } + console.log( chalk.gray( ` Local site running at ${ url }` ) ); + return { url, siteId: site.id, siteDir }; +} + +async function teardownLocalSite( + siteId: string, + client: LocalGraphQLClient, + siteDir: string +): Promise< void > { + try { + await client.stopSite( siteId ); + } catch { + // Site may already be stopped + } + try { + await client.deleteSite( siteId ); + } catch { + // Best effort cleanup + } + cleanupDir( siteDir ); +} + // --------------------------------------------------------------------------- // Benchmark runner // --------------------------------------------------------------------------- @@ -499,8 +570,7 @@ const MEASUREMENT_TIMEOUT = 600_000; // 10 min per measurement async function runBenchmark( benchmarkUrl: string, env: EnvironmentConfig, - rounds: number, - headed: boolean + rounds: number ): Promise< Record< string, number > | null > { console.log( chalk.gray( ` Running benchmark (${ rounds } round${ rounds > 1 ? 's' : '' })...` ) @@ -508,6 +578,7 @@ async function runBenchmark( const isPlaygroundWeb = benchmarkUrl.includes( 'playground.wordpress.net' ); const isPlaygroundCli = benchmarkUrl.includes( '127.0.0.1' ); + const isLocal = env.type === 'local'; const allMeasurements: MeasurementResult[] = []; for ( let round = 1; round <= rounds; round++ ) { @@ -516,7 +587,7 @@ async function runBenchmark( } try { const result = await Promise.race( [ - measureSiteEditor( { url: benchmarkUrl, isPlaygroundWeb, isPlaygroundCli, headed } ), + measureSiteEditor( { url: benchmarkUrl, isPlaygroundWeb, isPlaygroundCli, isLocal } ), sleep( MEASUREMENT_TIMEOUT ).then( () => { throw new Error( 'Measurement timed out' ); } ), @@ -620,6 +691,21 @@ function saveResultsSummary( results: BenchmarkResult[] ): void { // --------------------------------------------------------------------------- async function ensureStudioCLIBuilt(): Promise< boolean > { + // Ensure CLI dependencies are installed (required for the Vite build to resolve pm2-axon, etc.) + const cliNodeModules = path.resolve( STUDIO_ROOT, 'cli', 'node_modules' ); + if ( ! fs.existsSync( cliNodeModules ) ) { + console.log( chalk.yellow( ' Installing CLI dependencies...' ) ); + try { + execSync( 'npm install', { + cwd: path.resolve( STUDIO_ROOT, 'cli' ), + stdio: 'inherit', + } ); + } catch { + console.error( chalk.red( ' Failed to install CLI dependencies' ) ); + return false; + } + } + if ( ! fs.existsSync( STUDIO_CLI_PATH ) ) { console.log( chalk.yellow( ' Building Studio CLI...' ) ); try { @@ -669,6 +755,10 @@ async function main() { if ( opts.skipPlaygroundWeb ) { environments = environments.filter( ( e ) => e.type !== 'playground-web' ); } + // Local environments are excluded by default — they require the GUI app running + if ( ! opts.includeLocal ) { + environments = environments.filter( ( e ) => e.type !== 'local' ); + } } if ( environments.length === 0 ) { @@ -700,6 +790,7 @@ async function main() { const needsStudio = environments.some( ( e ) => e.type === 'studio' ); const needsPlaygroundCli = environments.some( ( e ) => e.type === 'playground-cli' ); + const needsLocal = environments.some( ( e ) => e.type === 'local' ); if ( needsStudio ) { if ( ! ( await ensureStudioCLIBuilt() ) ) { @@ -715,6 +806,12 @@ async function main() { console.log( chalk.green( ' Playground CLI ready' ) ); } + let localClient: LocalGraphQLClient | null = null; + if ( needsLocal ) { + localClient = await ensureLocalRunning(); + console.log( chalk.green( ' Local ready' ) ); + } + fs.mkdirSync( ARTIFACTS_PATH, { recursive: true } ); // Run benchmarks @@ -744,13 +841,17 @@ async function main() { const setup = await setupPlaygroundCliSite( env, port ); benchmarkUrl = setup.url; teardownFn = () => teardownPlaygroundCliSite( setup.process, port, setup.siteDir ); + } else if ( env.type === 'local' ) { + const setup = await setupLocalSite( env, localClient! ); + benchmarkUrl = setup.url; + teardownFn = () => teardownLocalSite( setup.siteId, localClient!, setup.siteDir ); } else { benchmarkUrl = getPlaygroundWebUrl( env ); console.log( chalk.gray( ` URL: ${ benchmarkUrl.slice( 0, 80 ) }...` ) ); } // Run benchmark - const metrics = await runBenchmark( benchmarkUrl, env, opts.rounds, opts.headed ); + const metrics = await runBenchmark( benchmarkUrl, env, opts.rounds ); if ( metrics ) { allResults.push( { environment: env.name, metrics } ); @@ -772,8 +873,6 @@ async function main() { // Print results printComparisonTable( allResults ); saveResultsSummary( allResults ); - - process.exit( 0 ); } main().catch( ( err ) => { diff --git a/scripts/benchmark-site-editor/install-plugins-local.ts b/tools/benchmark-site-editor/install-plugins-local.ts similarity index 100% rename from scripts/benchmark-site-editor/install-plugins-local.ts rename to tools/benchmark-site-editor/install-plugins-local.ts diff --git a/scripts/benchmark-site-editor/local-graphql.ts b/tools/benchmark-site-editor/local-graphql.ts similarity index 100% rename from scripts/benchmark-site-editor/local-graphql.ts rename to tools/benchmark-site-editor/local-graphql.ts diff --git a/tools/benchmark-site-editor/measure-site-editor.ts b/tools/benchmark-site-editor/measure-site-editor.ts index 52a935c101..de071de65d 100644 --- a/tools/benchmark-site-editor/measure-site-editor.ts +++ b/tools/benchmark-site-editor/measure-site-editor.ts @@ -31,8 +31,8 @@ export interface MeasureOptions { isPlaygroundWeb: boolean; /** Whether this is a local Playground CLI site (127.0.0.1). Affects login flow. */ isPlaygroundCli: boolean; - /** Launch browser in headed mode for debugging. */ - headed?: boolean; + /** Whether this is a Local by Flywheel site. Uses wp-login.php credentials. */ + isLocal: boolean; } // --------------------------------------------------------------------------- @@ -97,7 +97,7 @@ function findEditorCanvasFrame( * The browser is always closed, even on error. */ export async function measureSiteEditor( options: MeasureOptions ): Promise< MeasurementResult > { - const { isPlaygroundWeb, isPlaygroundCli } = options; + const { isPlaygroundWeb, isPlaygroundCli, isLocal } = options; // Normalize URL let wpAdminUrl = options.url; @@ -108,7 +108,7 @@ export async function measureSiteEditor( options: MeasureOptions ): Promise< Mea const result: MeasurementResult = {}; - const browser = await chromium.launch( { headless: ! options.headed } ); + const browser = await chromium.launch(); const context = await browser.newContext(); const page = await context.newPage(); @@ -126,11 +126,6 @@ export async function measureSiteEditor( options: MeasureOptions ): Promise< Mea .frameLocator( 'iframe' ) .first(); - await wordPressFrameLocator - .getByRole( 'menuitem', { name: 'My WordPress Website' } ) - .first() - .click( { timeout: 30_000 } ); - await wordPressFrameLocator .getByRole( 'link', { name: 'Appearance' } ) .waitFor( { timeout: 30_000 } ); @@ -142,15 +137,23 @@ export async function measureSiteEditor( options: MeasureOptions ): Promise< Mea timeout: 120_000, } ); await page.waitForLoadState( 'networkidle', { timeout: 30_000 } ).catch( () => {} ); - - // Playground CLI may redirect to wp-login.php — handle login - if ( page.url().includes( 'wp-login.php' ) ) { - await page.fill( '#user_login', 'admin' ); - await page.fill( '#user_pass', 'password' ); - await page.click( '#wp-submit' ); - await page.waitForLoadState( 'networkidle', { timeout: 30_000 } ).catch( () => {} ); - } - + await page.getByRole( 'link', { name: 'Appearance' } ).waitFor( { + state: 'visible', + timeout: 60_000, + } ); + } else if ( isLocal ) { + // Local sites require wp-login.php with credentials + await page.goto( `${ wpAdminUrl }/wp-login.php`, { + waitUntil: 'domcontentloaded', + timeout: 120_000, + } ); + await page.waitForLoadState( 'networkidle', { timeout: 30_000 } ).catch( () => {} ); + await page.fill( '#user_login', 'admin' ); + await page.fill( '#user_pass', 'password' ); + await Promise.all( [ + page.waitForURL( '**/wp-admin/**', { timeout: 60_000 } ), + page.click( '#wp-submit' ), + ] ); await page.getByRole( 'link', { name: 'Appearance' } ).waitFor( { state: 'visible', timeout: 60_000, diff --git a/tools/benchmark-site-editor/package.json b/tools/benchmark-site-editor/package.json index a9a95881cb..c6024ce753 100644 --- a/tools/benchmark-site-editor/package.json +++ b/tools/benchmark-site-editor/package.json @@ -6,13 +6,13 @@ "license": "GPLv2", "type": "module", "scripts": { - "benchmark": "ts-node ./benchmark.ts" + "benchmark": "tsx benchmark.ts" }, "dependencies": { - "@wp-playground/cli": "3.1.1", + "@wp-playground/cli": "^3.0.22", "chalk": "^5.3.0", "playwright": "^1.58.1", - "ts-node": "^10.9.2" + "tsx": "^4.7.0" }, "devDependencies": { "@types/node": "^20.0.0", diff --git a/tools/benchmark-site-editor/plugins-blueprint.json b/tools/benchmark-site-editor/plugins-blueprint.json index 51d52ed7a6..1a487ff01b 100644 --- a/tools/benchmark-site-editor/plugins-blueprint.json +++ b/tools/benchmark-site-editor/plugins-blueprint.json @@ -68,7 +68,7 @@ "step": "installPlugin", "pluginData": { "resource": "wordpress.org/plugins", - "slug": "coblocks" + "slug": "elementor" } } ] diff --git a/tools/benchmark-site-editor/tsconfig.json b/tools/benchmark-site-editor/tsconfig.json index 776319fecb..8f21fe7532 100644 --- a/tools/benchmark-site-editor/tsconfig.json +++ b/tools/benchmark-site-editor/tsconfig.json @@ -1,15 +1,15 @@ { - "extends": "../../tsconfig.base.json", "compilerOptions": { - "composite": true, - "baseUrl": "../..", - "module": "NodeNext", - "moduleResolution": "NodeNext", - "allowImportingTsExtensions": true, - "outDir": "dist", - "declaration": true, - "emitDeclarationOnly": true + "target": "ES2022", + "module": "ES2022", + "moduleResolution": "bundler", + "esModuleInterop": true, + "strict": true, + "skipLibCheck": true, + "outDir": "./dist", + "rootDir": ".", + "declaration": false, + "sourceMap": false }, - "include": [ "**/*" ], - "exclude": [ "**/__mocks__/**/*", "**/node_modules/**/*", "**/dist/**/*", "**/out/**/*" ] + "include": [ "*.ts" ] } From 4c6226673c3115ea2e037f9e1bfc1db69c9a3350 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gergely=20Cs=C3=A9csey?= Date: Thu, 19 Feb 2026 12:46:55 +0000 Subject: [PATCH 05/14] replace elementor with coblocks --- tools/benchmark-site-editor/plugins-blueprint.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/benchmark-site-editor/plugins-blueprint.json b/tools/benchmark-site-editor/plugins-blueprint.json index 1a487ff01b..51d52ed7a6 100644 --- a/tools/benchmark-site-editor/plugins-blueprint.json +++ b/tools/benchmark-site-editor/plugins-blueprint.json @@ -68,7 +68,7 @@ "step": "installPlugin", "pluginData": { "resource": "wordpress.org/plugins", - "slug": "elementor" + "slug": "coblocks" } } ] From 8aacabfdd798a793fd3ff8733b6bdb98cea8fa4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gergely=20Cs=C3=A9csey?= Date: Thu, 19 Feb 2026 12:48:43 +0000 Subject: [PATCH 06/14] update comments and docs references --- tools/benchmark-site-editor/README.md | 4 ++-- tools/benchmark-site-editor/benchmark.ts | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/benchmark-site-editor/README.md b/tools/benchmark-site-editor/README.md index 83331bedcc..3352e55da5 100644 --- a/tools/benchmark-site-editor/README.md +++ b/tools/benchmark-site-editor/README.md @@ -48,12 +48,12 @@ When the "plugins" variant is enabled, these 10 plugins are installed via a blue - Jetpack VideoPress - WooCommerce Payments - Contact Form 7 -- Elementor +- CoBlocks ## Usage ```bash -cd scripts/benchmark-site-editor +cd tools/benchmark-site-editor npm install npm run benchmark ``` diff --git a/tools/benchmark-site-editor/benchmark.ts b/tools/benchmark-site-editor/benchmark.ts index c6be0a88e6..0daeaee789 100644 --- a/tools/benchmark-site-editor/benchmark.ts +++ b/tools/benchmark-site-editor/benchmark.ts @@ -1,5 +1,5 @@ #!/usr/bin/env tsx -/* eslint-disable no-console */ + /** * Site Editor Performance Benchmark — Orchestration Script * @@ -10,7 +10,7 @@ * - Local by Flywheel (bare, plugins) — opt-in via --include-local * * Usage: - * cd scripts/benchmark-site-editor + * cd tools/benchmark-site-editor * npm install * npm run benchmark * @@ -29,9 +29,9 @@ import fs from 'fs'; import os from 'os'; import path from 'path'; import chalk from 'chalk'; -import { measureSiteEditor, METRIC_NAMES, type MeasurementResult } from './measure-site-editor.js'; -import { LocalGraphQLClient, getLocalSitesDir } from './local-graphql.js'; import { installPluginsForLocalSite } from './install-plugins-local.js'; +import { LocalGraphQLClient, getLocalSitesDir } from './local-graphql.js'; +import { measureSiteEditor, METRIC_NAMES, type MeasurementResult } from './measure-site-editor.js'; // --------------------------------------------------------------------------- // Configuration From 5cf9f0f3059728d8073789b8de23126428c765ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gergely=20Cs=C3=A9csey?= Date: Thu, 19 Feb 2026 12:50:03 +0000 Subject: [PATCH 07/14] exit after printing results --- tools/benchmark-site-editor/benchmark.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/benchmark-site-editor/benchmark.ts b/tools/benchmark-site-editor/benchmark.ts index 0daeaee789..916c31cc3c 100644 --- a/tools/benchmark-site-editor/benchmark.ts +++ b/tools/benchmark-site-editor/benchmark.ts @@ -873,6 +873,8 @@ async function main() { // Print results printComparisonTable( allResults ); saveResultsSummary( allResults ); + + process.exit( 0 ); } main().catch( ( err ) => { From 32f6552ba67d870e8cc93f248a25dd1e01b144d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gergely=20Cs=C3=A9csey?= Date: Thu, 19 Feb 2026 12:54:14 +0000 Subject: [PATCH 08/14] Restore --headed option removed during merge --- tools/benchmark-site-editor/benchmark.ts | 12 +++++++++--- tools/benchmark-site-editor/measure-site-editor.ts | 4 +++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/tools/benchmark-site-editor/benchmark.ts b/tools/benchmark-site-editor/benchmark.ts index 916c31cc3c..e0205a7080 100644 --- a/tools/benchmark-site-editor/benchmark.ts +++ b/tools/benchmark-site-editor/benchmark.ts @@ -107,6 +107,7 @@ interface Options { skipPlaygroundWeb: boolean; includeLocal: boolean; only: string[]; + headed: boolean; } function parseArgs(): Options { @@ -118,6 +119,7 @@ function parseArgs(): Options { skipPlaygroundWeb: false, includeLocal: false, only: [], + headed: false, }; for ( const arg of args ) { @@ -136,6 +138,8 @@ function parseArgs(): Options { .split( '=' )[ 1 ] .split( ',' ) .map( ( s ) => s.trim() ); + } else if ( arg === '--headed' ) { + opts.headed = true; } else if ( arg === '--help' ) { printHelp(); process.exit( 0 ); @@ -156,6 +160,7 @@ Options: --skip-playground-web Skip Playground web environments --include-local Include Local by Flywheel environments (requires Local running) --only= Run only named environments (comma-separated) + --headed Launch browser in headed mode for debugging --help Show this help message Environments: ${ ALL_ENVIRONMENTS.map( ( e ) => e.name ).join( ', ' ) } @@ -570,7 +575,8 @@ const MEASUREMENT_TIMEOUT = 600_000; // 10 min per measurement async function runBenchmark( benchmarkUrl: string, env: EnvironmentConfig, - rounds: number + rounds: number, + headed: boolean ): Promise< Record< string, number > | null > { console.log( chalk.gray( ` Running benchmark (${ rounds } round${ rounds > 1 ? 's' : '' })...` ) @@ -587,7 +593,7 @@ async function runBenchmark( } try { const result = await Promise.race( [ - measureSiteEditor( { url: benchmarkUrl, isPlaygroundWeb, isPlaygroundCli, isLocal } ), + measureSiteEditor( { url: benchmarkUrl, isPlaygroundWeb, isPlaygroundCli, isLocal, headed } ), sleep( MEASUREMENT_TIMEOUT ).then( () => { throw new Error( 'Measurement timed out' ); } ), @@ -851,7 +857,7 @@ async function main() { } // Run benchmark - const metrics = await runBenchmark( benchmarkUrl, env, opts.rounds ); + const metrics = await runBenchmark( benchmarkUrl, env, opts.rounds, opts.headed ); if ( metrics ) { allResults.push( { environment: env.name, metrics } ); diff --git a/tools/benchmark-site-editor/measure-site-editor.ts b/tools/benchmark-site-editor/measure-site-editor.ts index de071de65d..c3cce7119d 100644 --- a/tools/benchmark-site-editor/measure-site-editor.ts +++ b/tools/benchmark-site-editor/measure-site-editor.ts @@ -33,6 +33,8 @@ export interface MeasureOptions { isPlaygroundCli: boolean; /** Whether this is a Local by Flywheel site. Uses wp-login.php credentials. */ isLocal: boolean; + /** Launch browser in headed mode for debugging. */ + headed?: boolean; } // --------------------------------------------------------------------------- @@ -108,7 +110,7 @@ export async function measureSiteEditor( options: MeasureOptions ): Promise< Mea const result: MeasurementResult = {}; - const browser = await chromium.launch(); + const browser = await chromium.launch( { headless: ! options.headed } ); const context = await browser.newContext(); const page = await context.newPage(); From 5f5c2e43f636383b291fe6a5e0202db7ad964131 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gergely=20Cs=C3=A9csey?= Date: Thu, 19 Feb 2026 13:02:30 +0000 Subject: [PATCH 09/14] update CLI paths to monorepo structure --- tools/benchmark-site-editor/benchmark.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/benchmark-site-editor/benchmark.ts b/tools/benchmark-site-editor/benchmark.ts index e0205a7080..6536c97cd1 100644 --- a/tools/benchmark-site-editor/benchmark.ts +++ b/tools/benchmark-site-editor/benchmark.ts @@ -38,7 +38,7 @@ import { measureSiteEditor, METRIC_NAMES, type MeasurementResult } from './measu // --------------------------------------------------------------------------- const STUDIO_ROOT = path.resolve( import.meta.dirname, '../..' ); -const STUDIO_CLI_PATH = path.resolve( STUDIO_ROOT, 'dist/cli/main.js' ); +const STUDIO_CLI_PATH = path.resolve( STUDIO_ROOT, 'apps/cli/dist/cli/main.js' ); const PLAYGROUND_CLI_BIN = process.platform === 'win32' ? 'wp-playground-cli.cmd' : 'wp-playground-cli'; const PLAYGROUND_CLI_PATH = path.resolve( @@ -698,12 +698,12 @@ function saveResultsSummary( results: BenchmarkResult[] ): void { async function ensureStudioCLIBuilt(): Promise< boolean > { // Ensure CLI dependencies are installed (required for the Vite build to resolve pm2-axon, etc.) - const cliNodeModules = path.resolve( STUDIO_ROOT, 'cli', 'node_modules' ); + const cliNodeModules = path.resolve( STUDIO_ROOT, 'apps/cli', 'node_modules' ); if ( ! fs.existsSync( cliNodeModules ) ) { console.log( chalk.yellow( ' Installing CLI dependencies...' ) ); try { execSync( 'npm install', { - cwd: path.resolve( STUDIO_ROOT, 'cli' ), + cwd: path.resolve( STUDIO_ROOT, 'apps/cli' ), stdio: 'inherit', } ); } catch { From dab08fd93e2af103aca5c612fb9396b8e80bf303 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gergely=20Cs=C3=A9csey?= Date: Thu, 19 Feb 2026 13:06:35 +0000 Subject: [PATCH 10/14] clean up leftover Local sites before benchmarking --- tools/benchmark-site-editor/benchmark.ts | 11 +++++++++++ tools/benchmark-site-editor/local-graphql.ts | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/tools/benchmark-site-editor/benchmark.ts b/tools/benchmark-site-editor/benchmark.ts index 6536c97cd1..1da24cbe52 100644 --- a/tools/benchmark-site-editor/benchmark.ts +++ b/tools/benchmark-site-editor/benchmark.ts @@ -522,6 +522,17 @@ async function setupLocalSite( const domain = `bench-${ env.name }.local`; const siteDir = path.join( getLocalSitesDir(), siteName ); + // Clean up any leftover site from a previous run + const existing = await client.findSiteByName( siteName ); + if ( existing ) { + console.log( chalk.gray( ` Removing leftover site "${ siteName }"...` ) ); + await client.stopSite( existing.id ).catch( () => {} ); + await client.deleteSite( existing.id ).catch( () => {} ); + } + if ( fs.existsSync( siteDir ) ) { + cleanupDir( siteDir ); + } + console.log( chalk.gray( ` Creating Local site "${ siteName }"...` ) ); const site = await client.createSite( { diff --git a/tools/benchmark-site-editor/local-graphql.ts b/tools/benchmark-site-editor/local-graphql.ts index 4ddb047c89..d8f4a3da37 100644 --- a/tools/benchmark-site-editor/local-graphql.ts +++ b/tools/benchmark-site-editor/local-graphql.ts @@ -287,7 +287,7 @@ export class LocalGraphQLClient { /** * Find a site by name. */ - private async findSiteByName( name: string ): Promise< LocalSite | null > { + async findSiteByName( name: string ): Promise< LocalSite | null > { const result = await this.query< { sites: Array< { id: string; From 1dd371a302d989f4e7434757a883f76ada1b6151 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gergely=20Cs=C3=A9csey?= Date: Thu, 19 Feb 2026 13:18:55 +0000 Subject: [PATCH 11/14] clean up all leftover sites by name --- tools/benchmark-site-editor/benchmark.ts | 12 +++--- tools/benchmark-site-editor/local-graphql.ts | 45 +++++++++++++++----- 2 files changed, 41 insertions(+), 16 deletions(-) diff --git a/tools/benchmark-site-editor/benchmark.ts b/tools/benchmark-site-editor/benchmark.ts index 1da24cbe52..b392cbabc8 100644 --- a/tools/benchmark-site-editor/benchmark.ts +++ b/tools/benchmark-site-editor/benchmark.ts @@ -522,12 +522,12 @@ async function setupLocalSite( const domain = `bench-${ env.name }.local`; const siteDir = path.join( getLocalSitesDir(), siteName ); - // Clean up any leftover site from a previous run - const existing = await client.findSiteByName( siteName ); - if ( existing ) { - console.log( chalk.gray( ` Removing leftover site "${ siteName }"...` ) ); - await client.stopSite( existing.id ).catch( () => {} ); - await client.deleteSite( existing.id ).catch( () => {} ); + // Clean up any leftover sites from previous runs + const existing = await client.findAllSitesByName( siteName ); + for ( const old of existing ) { + console.log( chalk.gray( ` Removing leftover site "${ siteName }" (${ old.id })...` ) ); + await client.stopSite( old.id ).catch( () => {} ); + await client.deleteSite( old.id ).catch( () => {} ); } if ( fs.existsSync( siteDir ) ) { cleanupDir( siteDir ); diff --git a/tools/benchmark-site-editor/local-graphql.ts b/tools/benchmark-site-editor/local-graphql.ts index d8f4a3da37..3698a1fbe4 100644 --- a/tools/benchmark-site-editor/local-graphql.ts +++ b/tools/benchmark-site-editor/local-graphql.ts @@ -208,8 +208,11 @@ export class LocalGraphQLClient { throw new Error( `Site "${ name }" was not found after creation job completed` ); } - // Wait for site to be running - await this.waitForSiteStatus( site.id, 'running', 120_000 ); + // Explicitly start the site — addSite may not auto-start it + if ( site.status !== 'running' ) { + await this.startSite( site.id ); + await this.waitForSiteStatus( site.id, 'running', 120_000 ); + } // Re-fetch to get the port const runningSite = await this.getSite( site.id ); @@ -285,9 +288,9 @@ export class LocalGraphQLClient { } /** - * Find a site by name. + * Find all sites matching a name. */ - async findSiteByName( name: string ): Promise< LocalSite | null > { + async findAllSitesByName( name: string ): Promise< LocalSite[] > { const result = await this.query< { sites: Array< { id: string; @@ -310,13 +313,20 @@ export class LocalGraphQLClient { }` ); - const site = result.sites.find( ( s ) => s.name === name ); - if ( ! site ) return null; + return result.sites + .filter( ( s ) => s.name === name ) + .map( ( s ) => ( { + ...s, + url: s.httpPort ? `http://localhost:${ s.httpPort }` : '', + } ) ); + } - return { - ...site, - url: site.httpPort ? `http://localhost:${ site.httpPort }` : '', - }; + /** + * Find a site by name. + */ + async findSiteByName( name: string ): Promise< LocalSite | null > { + const matches = await this.findAllSitesByName( name ); + return matches[ 0 ] ?? null; } /** @@ -355,6 +365,21 @@ export class LocalGraphQLClient { }; } + /** + * Start a site. + */ + async startSite( id: string ): Promise< void > { + await this.query( + `mutation StartSite($id: ID!) { + startSite(id: $id) { + id + status + } + }`, + { id } + ); + } + /** * Stop a running site. */ From 7501b3e0266bb31f8af403d26cb255f23479fd47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gergely=20Cs=C3=A9csey?= Date: Thu, 19 Feb 2026 13:50:03 +0000 Subject: [PATCH 12/14] move dependencies to devdependencies, update package-lock --- package-lock.json | 554 ++++++++++++++++++++++- tools/benchmark-site-editor/package.json | 8 +- 2 files changed, 551 insertions(+), 11 deletions(-) diff --git a/package-lock.json b/package-lock.json index c0692175ce..dc2cdae53c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8444,6 +8444,7 @@ "os": [ "darwin" ], + "peer": true, "engines": { "node": ">=10" } @@ -8460,6 +8461,7 @@ "os": [ "darwin" ], + "peer": true, "engines": { "node": ">=10" } @@ -8476,6 +8478,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">=10" } @@ -8492,6 +8495,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">=10" } @@ -8508,6 +8512,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">=10" } @@ -8524,6 +8529,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">=10" } @@ -8540,6 +8546,7 @@ "os": [ "linux" ], + "peer": true, "engines": { "node": ">=10" } @@ -8556,6 +8563,7 @@ "os": [ "win32" ], + "peer": true, "engines": { "node": ">=10" } @@ -8572,6 +8580,7 @@ "os": [ "win32" ], + "peer": true, "engines": { "node": ">=10" } @@ -8588,6 +8597,7 @@ "os": [ "win32" ], + "peer": true, "engines": { "node": ">=10" } @@ -21408,6 +21418,7 @@ "version": "1.58.2", "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.58.2.tgz", "integrity": "sha512-vA30H8Nvkq/cPBnNw4Q8TWz1EJyqgpuinBcHET0YVJVFldr8JDNiU9LaWAE1KqSkRYazuaBhTpB5ZzShOezQ6A==", + "dev": true, "license": "Apache-2.0", "dependencies": { "playwright-core": "1.58.2" @@ -21426,6 +21437,7 @@ "version": "1.58.2", "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.58.2.tgz", "integrity": "sha512-yZkEtftgwS8CsfYo7nm0KE8jsvm6i/PTgVtB8DL726wNf6H2IMsDuxCpJj59KDaxCtSnrWan2AeDqM7JBaultg==", + "dev": true, "license": "Apache-2.0", "bin": { "playwright-core": "cli.js" @@ -21438,6 +21450,7 @@ "version": "2.3.2", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, "hasInstallScript": true, "license": "MIT", "optional": true, @@ -25140,6 +25153,536 @@ "version": "2.8.1", "license": "0BSD" }, + "node_modules/tsx": { + "version": "4.21.0", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.21.0.tgz", + "integrity": "sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "~0.27.0", + "get-tsconfig": "^4.7.5" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, + "node_modules/tsx/node_modules/@esbuild/aix-ppc64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.3.tgz", + "integrity": "sha512-9fJMTNFTWZMh5qwrBItuziu834eOCUcEqymSH7pY+zoMVEZg3gcPuBNxH1EvfVYe9h0x/Ptw8KBzv7qxb7l8dg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/android-arm": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.3.tgz", + "integrity": "sha512-i5D1hPY7GIQmXlXhs2w8AWHhenb00+GxjxRncS2ZM7YNVGNfaMxgzSGuO8o8SJzRc/oZwU2bcScvVERk03QhzA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/android-arm64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.3.tgz", + "integrity": "sha512-YdghPYUmj/FX2SYKJ0OZxf+iaKgMsKHVPF1MAq/P8WirnSpCStzKJFjOjzsW0QQ7oIAiccHdcqjbHmJxRb/dmg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/android-x64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.3.tgz", + "integrity": "sha512-IN/0BNTkHtk8lkOM8JWAYFg4ORxBkZQf9zXiEOfERX/CzxW3Vg1ewAhU7QSWQpVIzTW+b8Xy+lGzdYXV6UZObQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/darwin-arm64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.3.tgz", + "integrity": "sha512-Re491k7ByTVRy0t3EKWajdLIr0gz2kKKfzafkth4Q8A5n1xTHrkqZgLLjFEHVD+AXdUGgQMq+Godfq45mGpCKg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/darwin-x64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.3.tgz", + "integrity": "sha512-vHk/hA7/1AckjGzRqi6wbo+jaShzRowYip6rt6q7VYEDX4LEy1pZfDpdxCBnGtl+A5zq8iXDcyuxwtv3hNtHFg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/freebsd-arm64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.3.tgz", + "integrity": "sha512-ipTYM2fjt3kQAYOvo6vcxJx3nBYAzPjgTCk7QEgZG8AUO3ydUhvelmhrbOheMnGOlaSFUoHXB6un+A7q4ygY9w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/freebsd-x64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.3.tgz", + "integrity": "sha512-dDk0X87T7mI6U3K9VjWtHOXqwAMJBNN2r7bejDsc+j03SEjtD9HrOl8gVFByeM0aJksoUuUVU9TBaZa2rgj0oA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-arm": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.3.tgz", + "integrity": "sha512-s6nPv2QkSupJwLYyfS+gwdirm0ukyTFNl3KTgZEAiJDd+iHZcbTPPcWCcRYH+WlNbwChgH2QkE9NSlNrMT8Gfw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-arm64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.3.tgz", + "integrity": "sha512-sZOuFz/xWnZ4KH3YfFrKCf1WyPZHakVzTiqji3WDc0BCl2kBwiJLCXpzLzUBLgmp4veFZdvN5ChW4Eq/8Fc2Fg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-ia32": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.3.tgz", + "integrity": "sha512-yGlQYjdxtLdh0a3jHjuwOrxQjOZYD/C9PfdbgJJF3TIZWnm/tMd/RcNiLngiu4iwcBAOezdnSLAwQDPqTmtTYg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-loong64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.3.tgz", + "integrity": "sha512-WO60Sn8ly3gtzhyjATDgieJNet/KqsDlX5nRC5Y3oTFcS1l0KWba+SEa9Ja1GfDqSF1z6hif/SkpQJbL63cgOA==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-mips64el": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.3.tgz", + "integrity": "sha512-APsymYA6sGcZ4pD6k+UxbDjOFSvPWyZhjaiPyl/f79xKxwTnrn5QUnXR5prvetuaSMsb4jgeHewIDCIWljrSxw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-ppc64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.3.tgz", + "integrity": "sha512-eizBnTeBefojtDb9nSh4vvVQ3V9Qf9Df01PfawPcRzJH4gFSgrObw+LveUyDoKU3kxi5+9RJTCWlj4FjYXVPEA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-riscv64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.3.tgz", + "integrity": "sha512-3Emwh0r5wmfm3ssTWRQSyVhbOHvqegUDRd0WhmXKX2mkHJe1SFCMJhagUleMq+Uci34wLSipf8Lagt4LlpRFWQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-s390x": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.3.tgz", + "integrity": "sha512-pBHUx9LzXWBc7MFIEEL0yD/ZVtNgLytvx60gES28GcWMqil8ElCYR4kvbV2BDqsHOvVDRrOxGySBM9Fcv744hw==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-x64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.3.tgz", + "integrity": "sha512-Czi8yzXUWIQYAtL/2y6vogER8pvcsOsk5cpwL4Gk5nJqH5UZiVByIY8Eorm5R13gq+DQKYg0+JyQoytLQas4dA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/netbsd-arm64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.3.tgz", + "integrity": "sha512-sDpk0RgmTCR/5HguIZa9n9u+HVKf40fbEUt+iTzSnCaGvY9kFP0YKBWZtJaraonFnqef5SlJ8/TiPAxzyS+UoA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/netbsd-x64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.3.tgz", + "integrity": "sha512-P14lFKJl/DdaE00LItAukUdZO5iqNH7+PjoBm+fLQjtxfcfFE20Xf5CrLsmZdq5LFFZzb5JMZ9grUwvtVYzjiA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/openbsd-arm64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.3.tgz", + "integrity": "sha512-AIcMP77AvirGbRl/UZFTq5hjXK+2wC7qFRGoHSDrZ5v5b8DK/GYpXW3CPRL53NkvDqb9D+alBiC/dV0Fb7eJcw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/openbsd-x64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.3.tgz", + "integrity": "sha512-DnW2sRrBzA+YnE70LKqnM3P+z8vehfJWHXECbwBmH/CU51z6FiqTQTHFenPlHmo3a8UgpLyH3PT+87OViOh1AQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/openharmony-arm64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.3.tgz", + "integrity": "sha512-NinAEgr/etERPTsZJ7aEZQvvg/A6IsZG/LgZy+81wON2huV7SrK3e63dU0XhyZP4RKGyTm7aOgmQk0bGp0fy2g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/sunos-x64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.3.tgz", + "integrity": "sha512-PanZ+nEz+eWoBJ8/f8HKxTTD172SKwdXebZ0ndd953gt1HRBbhMsaNqjTyYLGLPdoWHy4zLU7bDVJztF5f3BHA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/win32-arm64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.3.tgz", + "integrity": "sha512-B2t59lWWYrbRDw/tjiWOuzSsFh1Y/E95ofKz7rIVYSQkUYBjfSgf6oeYPNWHToFRr2zx52JKApIcAS/D5TUBnA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/win32-ia32": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.3.tgz", + "integrity": "sha512-QLKSFeXNS8+tHW7tZpMtjlNb7HKau0QDpwm49u0vUp9y1WOF+PEzkU84y9GqYaAVW8aH8f3GcBck26jh54cX4Q==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/win32-x64": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.3.tgz", + "integrity": "sha512-4uJGhsxuptu3OcpVAzli+/gWusVGwZZHTlS63hh++ehExkVT8SgiEf7/uC/PclrPPkLhZqGgCTjd0VWLo6xMqA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/esbuild": { + "version": "0.27.3", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.3.tgz", + "integrity": "sha512-8VwMnyGCONIs6cWue2IdpHxHnAjzxnw2Zr7MkVxB2vjmQ2ivqGFb4LEG3SMnv0Gb2F/G/2yA8zUaiL1gywDCCg==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.27.3", + "@esbuild/android-arm": "0.27.3", + "@esbuild/android-arm64": "0.27.3", + "@esbuild/android-x64": "0.27.3", + "@esbuild/darwin-arm64": "0.27.3", + "@esbuild/darwin-x64": "0.27.3", + "@esbuild/freebsd-arm64": "0.27.3", + "@esbuild/freebsd-x64": "0.27.3", + "@esbuild/linux-arm": "0.27.3", + "@esbuild/linux-arm64": "0.27.3", + "@esbuild/linux-ia32": "0.27.3", + "@esbuild/linux-loong64": "0.27.3", + "@esbuild/linux-mips64el": "0.27.3", + "@esbuild/linux-ppc64": "0.27.3", + "@esbuild/linux-riscv64": "0.27.3", + "@esbuild/linux-s390x": "0.27.3", + "@esbuild/linux-x64": "0.27.3", + "@esbuild/netbsd-arm64": "0.27.3", + "@esbuild/netbsd-x64": "0.27.3", + "@esbuild/openbsd-arm64": "0.27.3", + "@esbuild/openbsd-x64": "0.27.3", + "@esbuild/openharmony-arm64": "0.27.3", + "@esbuild/sunos-x64": "0.27.3", + "@esbuild/win32-arm64": "0.27.3", + "@esbuild/win32-ia32": "0.27.3", + "@esbuild/win32-x64": "0.27.3" + } + }, "node_modules/tunnel-agent": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", @@ -27314,14 +27857,12 @@ "tools/benchmark-site-editor": { "version": "0.0.1", "license": "GPLv2", - "dependencies": { - "@wp-playground/cli": "3.1.1", - "chalk": "^5.3.0", - "playwright": "^1.58.1", - "ts-node": "^10.9.2" - }, "devDependencies": { "@types/node": "^20.0.0", + "@wp-playground/cli": "^3.0.22", + "chalk": "^5.3.0", + "playwright": "^1.58.1", + "tsx": "^4.7.0", "typescript": "^5.0.0" } }, @@ -27339,6 +27880,7 @@ "version": "5.6.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.6.2.tgz", "integrity": "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==", + "dev": true, "license": "MIT", "engines": { "node": "^12.17.0 || ^14.13 || >=16.0.0" diff --git a/tools/benchmark-site-editor/package.json b/tools/benchmark-site-editor/package.json index c6024ce753..37da9cb19c 100644 --- a/tools/benchmark-site-editor/package.json +++ b/tools/benchmark-site-editor/package.json @@ -8,14 +8,12 @@ "scripts": { "benchmark": "tsx benchmark.ts" }, - "dependencies": { + "devDependencies": { + "@types/node": "^20.0.0", "@wp-playground/cli": "^3.0.22", "chalk": "^5.3.0", "playwright": "^1.58.1", - "tsx": "^4.7.0" - }, - "devDependencies": { - "@types/node": "^20.0.0", + "tsx": "^4.7.0", "typescript": "^5.0.0" } } From 51bad107fe7ef04079f2267596a607b3a607721a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gergely=20Cs=C3=A9csey?= Date: Fri, 20 Feb 2026 13:30:19 +0000 Subject: [PATCH 13/14] Replace Local by Flywheel harness with generic custom environment support Remove the Local GraphQL client and site lifecycle management (~500 lines). Instead, accept any running WordPress site via repeatable --custom flags with URL and credentials. Plugin installation uses the standard WP REST API and gracefully skips already-installed plugins. --- tools/benchmark-site-editor/README.md | 58 ++- tools/benchmark-site-editor/benchmark.ts | 231 +++++----- ...ll-plugins-local.ts => install-plugins.ts} | 29 +- tools/benchmark-site-editor/local-graphql.ts | 409 ------------------ .../measure-site-editor.ts | 14 +- 5 files changed, 184 insertions(+), 557 deletions(-) rename tools/benchmark-site-editor/{install-plugins-local.ts => install-plugins.ts} (86%) delete mode 100644 tools/benchmark-site-editor/local-graphql.ts diff --git a/tools/benchmark-site-editor/README.md b/tools/benchmark-site-editor/README.md index 3352e55da5..ea235c94e9 100644 --- a/tools/benchmark-site-editor/README.md +++ b/tools/benchmark-site-editor/README.md @@ -1,6 +1,6 @@ # Site Editor Performance Benchmark -Benchmarks site editor performance across Studio, Playground CLI, Playground Web, and Local by Flywheel environments, with optional plugin and multi-worker configurations. +Benchmarks site editor performance across Studio, Playground CLI, Playground Web, and custom WordPress environments, with optional plugin and multi-worker configurations. ## Related Issue @@ -32,8 +32,7 @@ The benchmark launches a headless Chromium browser against each environment, mea | Playground CLI + MW + Plugins | `pg-cli-mw-plugins` | Playground CLI with multi-worker and 10 plugins | | Playground Web | `pg-web` | playground.wordpress.net (bare) | | Playground Web + Plugins | `pg-web-plugins` | playground.wordpress.net with 10 plugins | -| Local | `local` | Local by Flywheel site (nginx+PHP+MySQL) | -| Local + Plugins | `local-plugins` | Local with 10 plugins installed | +| Custom | user-defined | Any running WordPress site via `--custom` | ## Plugins @@ -61,13 +60,17 @@ npm run benchmark ### Options ``` ---rounds=N Number of benchmark runs per environment (default: 1) ---skip-studio Skip Studio environments ---skip-playground-cli Skip Playground CLI environments ---skip-playground-web Skip Playground web environments ---include-local Include Local by Flywheel environments (requires Local running) ---only= Run only named environments (comma-separated) ---help Show help +--rounds=N Number of benchmark runs per environment (default: 1) +--skip-studio Skip Studio environments +--skip-playground-cli Skip Playground CLI environments +--skip-playground-web Skip Playground web environments +--custom=,[,,] Add a custom WordPress site (repeatable) + user defaults to "admin", password to "password" +--install-plugins Install blueprint plugins on ALL custom environments +--install-plugins=, Install blueprint plugins on specific custom environments +--only= Run only named environments (comma-separated) +--headed Launch browser in headed mode for debugging +--help Show help ``` ### Examples @@ -85,23 +88,42 @@ npm run benchmark -- --skip-studio --skip-playground-web # Single specific environment npm run benchmark -- --only=studio-mw-plugins --rounds=5 -# Include Local by Flywheel (must be running) -npm run benchmark -- --include-local --only=local,local-plugins +# Benchmark a single custom WordPress site +npm run benchmark -- --custom=my-site,http://localhost:10003 -# Compare Studio vs Local -npm run benchmark -- --only=studio,local --rounds=3 +# Two custom sites: bare vs with plugins +npm run benchmark -- \ + --custom=local-bare,http://localhost:10003 \ + --custom=local-plugins,http://localhost:10004 \ + --install-plugins=local-plugins + +# Custom site with non-default credentials +npm run benchmark -- --custom=my-site,http://localhost:10003,admin,secret + +# Compare Studio vs a custom site +npm run benchmark -- --only=studio,my-site --custom=my-site,http://localhost:10003 --rounds=3 ``` -> **Note:** When using `--only` with Local environments, the `--include-local` flag is not needed — Local environments are automatically included when explicitly named. +## Custom Environments + +Use `--custom` to add any running WordPress site to the benchmark. The flag is repeatable, so you can benchmark multiple custom sites in a single run. Each site must be accessible and have a WordPress admin account. + +Format: `--custom=,[,,]` + +- **name** — Label for the environment in the results table +- **url** — Base URL of the running WordPress site +- **user** — WordPress admin username (default: `admin`) +- **password** — WordPress admin password (default: `password`) + +The benchmark logs in via `wp-login.php` using the provided credentials. + +With `--install-plugins`, the benchmark installs the same set of plugins used by the built-in environments via the WordPress REST API before measuring. Use `--install-plugins` to install on all custom environments, or `--install-plugins=name1,name2` to target specific ones. ## Prerequisites - **Studio CLI**: Built automatically if `dist/cli/main.js` doesn't exist (`npm run cli:build`) - **Playground CLI**: Installed automatically via this script's `npm install` - **Playwright**: Chromium is installed automatically during setup -- **Local by Flywheel**: Must be installed and running (GUI app). Use `--include-local` to enable. The script connects to Local's GraphQL API to create and manage benchmark sites. - - macOS: `brew install --cask local` - - Windows: `winget install Flywheel.Local` ## Output diff --git a/tools/benchmark-site-editor/benchmark.ts b/tools/benchmark-site-editor/benchmark.ts index b392cbabc8..2ad7dd3d3b 100644 --- a/tools/benchmark-site-editor/benchmark.ts +++ b/tools/benchmark-site-editor/benchmark.ts @@ -7,7 +7,7 @@ * - Studio (bare, multi-worker, plugins, multi-worker+plugins) * - Playground CLI (bare, multi-worker, plugins, multi-worker+plugins) * - Playground Web (bare, plugins) - * - Local by Flywheel (bare, plugins) — opt-in via --include-local + * - Custom (any running WordPress site via --custom-url) * * Usage: * cd tools/benchmark-site-editor @@ -19,7 +19,8 @@ * --skip-studio Skip Studio environments * --skip-playground-cli Skip Playground CLI environments * --skip-playground-web Skip Playground web environments - * --include-local Include Local by Flywheel environments (requires Local running) + * --custom=,[,,] Add a custom WordPress site (repeatable) + * --install-plugins[=,] Install plugins from blueprint via WP REST API * --only= Run only these environments (comma-separated) * --help Show help */ @@ -29,8 +30,7 @@ import fs from 'fs'; import os from 'os'; import path from 'path'; import chalk from 'chalk'; -import { installPluginsForLocalSite } from './install-plugins-local.js'; -import { LocalGraphQLClient, getLocalSitesDir } from './local-graphql.js'; +import { installPlugins } from './install-plugins.js'; import { measureSiteEditor, METRIC_NAMES, type MeasurementResult } from './measure-site-editor.js'; // --------------------------------------------------------------------------- @@ -58,13 +58,17 @@ const PLAYGROUND_CLI_PORT_BASE = 9500; // Types // --------------------------------------------------------------------------- -type EnvironmentType = 'studio' | 'playground-cli' | 'playground-web' | 'local'; +type EnvironmentType = 'studio' | 'playground-cli' | 'playground-web' | 'custom'; interface EnvironmentConfig { name: string; type: EnvironmentType; plugins: boolean; multiWorker: boolean; + /** Base URL for custom environments. */ + customUrl?: string; + /** WordPress admin credentials for custom environments. */ + credentials?: { username: string; password: string }; } interface BenchmarkResult { @@ -92,22 +96,45 @@ const ALL_ENVIRONMENTS: EnvironmentConfig[] = [ }, { name: 'pg-web', type: 'playground-web', plugins: false, multiWorker: false }, { name: 'pg-web-plugins', type: 'playground-web', plugins: true, multiWorker: false }, - { name: 'local', type: 'local', plugins: false, multiWorker: false }, - { name: 'local-plugins', type: 'local', plugins: true, multiWorker: false }, ]; // --------------------------------------------------------------------------- // Argument parsing // --------------------------------------------------------------------------- +interface CustomEnvInput { + name: string; + url: string; + user: string; + password: string; +} + interface Options { rounds: number; skipStudio: boolean; skipPlaygroundCli: boolean; skipPlaygroundWeb: boolean; - includeLocal: boolean; only: string[]; headed: boolean; + customs: CustomEnvInput[]; + /** Names of custom environments to install plugins on, or 'all'. */ + installPlugins: string[] | 'all' | false; +} + +function parseCustomFlag( value: string ): CustomEnvInput { + const parts = value.split( ',' ); + if ( parts.length < 2 ) { + console.error( + `Invalid --custom value: "${ value }"\nExpected: --custom=name,url[,user,password]` + ); + process.exit( 1 ); + } + return { + name: parts[ 0 ], + url: parts[ 1 ], + user: parts[ 2 ] || 'admin', + password: parts[ 3 ] || 'password', + }; } function parseArgs(): Options { @@ -117,9 +144,10 @@ function parseArgs(): Options { skipStudio: false, skipPlaygroundCli: false, skipPlaygroundWeb: false, - includeLocal: false, only: [], headed: false, + customs: [], + installPlugins: false, }; for ( const arg of args ) { @@ -131,8 +159,15 @@ function parseArgs(): Options { opts.skipPlaygroundCli = true; } else if ( arg === '--skip-playground-web' ) { opts.skipPlaygroundWeb = true; - } else if ( arg === '--include-local' ) { - opts.includeLocal = true; + } else if ( arg.startsWith( '--custom=' ) ) { + opts.customs.push( parseCustomFlag( arg.split( '=' ).slice( 1 ).join( '=' ) ) ); + } else if ( arg === '--install-plugins' ) { + opts.installPlugins = 'all'; + } else if ( arg.startsWith( '--install-plugins=' ) ) { + opts.installPlugins = arg + .split( '=' )[ 1 ] + .split( ',' ) + .map( ( s ) => s.trim() ); } else if ( arg.startsWith( '--only=' ) ) { opts.only = arg .split( '=' )[ 1 ] @@ -154,16 +189,35 @@ function printHelp() { Usage: npm run benchmark [options] Options: - --rounds=N Number of benchmark runs per environment (default: 1) - --skip-studio Skip Studio environments - --skip-playground-cli Skip Playground CLI environments - --skip-playground-web Skip Playground web environments - --include-local Include Local by Flywheel environments (requires Local running) - --only= Run only named environments (comma-separated) - --headed Launch browser in headed mode for debugging - --help Show this help message - -Environments: ${ ALL_ENVIRONMENTS.map( ( e ) => e.name ).join( ', ' ) } + --rounds=N Number of benchmark runs per environment (default: 1) + --skip-studio Skip Studio environments + --skip-playground-cli Skip Playground CLI environments + --skip-playground-web Skip Playground web environments + --custom=,[,,] Add a custom WordPress site (repeatable) + user defaults to "admin", password to "password" + --install-plugins Install blueprint plugins on ALL custom environments + --install-plugins=, Install blueprint plugins on specific custom environments + --only= Run only named environments (comma-separated) + --headed Launch browser in headed mode for debugging + --help Show this help message + +Built-in environments: ${ ALL_ENVIRONMENTS.map( ( e ) => e.name ).join( ', ' ) } + +Examples: + # Benchmark a single custom WordPress site + npm run benchmark -- --custom=my-site,http://localhost:10003 + + # Two custom sites: bare vs with plugins + npm run benchmark -- \\ + --custom=local-bare,http://localhost:10003 \\ + --custom=local-plugins,http://localhost:10004 \\ + --install-plugins=local-plugins + + # Custom site with non-default credentials + npm run benchmark -- --custom=my-site,http://localhost:10003,admin,secret + + # Compare Studio vs a custom site + npm run benchmark -- --only=studio,my-site --custom=my-site,http://localhost:10003 --rounds=3 ` ); } @@ -501,82 +555,6 @@ function getPlaygroundWebUrl( env: EnvironmentConfig ): string { return `${ PLAYGROUND_WEB_BASE_URL }/#${ JSON.stringify( blueprint ) }`; } -// --------------------------------------------------------------------------- -// Local by Flywheel environment helpers -// --------------------------------------------------------------------------- - -async function ensureLocalRunning(): Promise< LocalGraphQLClient > { - try { - return await LocalGraphQLClient.connect(); - } catch ( err ) { - console.error( chalk.red( ` Local is not running or not reachable.\n ${ err }` ) ); - process.exit( 1 ); - } -} - -async function setupLocalSite( - env: EnvironmentConfig, - client: LocalGraphQLClient -): Promise< { url: string; siteId: string; siteDir: string } > { - const siteName = `bench-${ env.name }`; - const domain = `bench-${ env.name }.local`; - const siteDir = path.join( getLocalSitesDir(), siteName ); - - // Clean up any leftover sites from previous runs - const existing = await client.findAllSitesByName( siteName ); - for ( const old of existing ) { - console.log( chalk.gray( ` Removing leftover site "${ siteName }" (${ old.id })...` ) ); - await client.stopSite( old.id ).catch( () => {} ); - await client.deleteSite( old.id ).catch( () => {} ); - } - if ( fs.existsSync( siteDir ) ) { - cleanupDir( siteDir ); - } - - console.log( chalk.gray( ` Creating Local site "${ siteName }"...` ) ); - - const site = await client.createSite( { - name: siteName, - domain, - path: siteDir, - wpAdminPassword: 'password', - wpAdminUsername: 'admin', - } ); - - const url = site.url || `http://localhost:${ site.httpPort }`; - - // Plugin install is best-effort — we still want to return site info - // so the caller can set up teardown even if plugin install fails. - if ( env.plugins ) { - console.log( chalk.gray( ` Installing plugins via REST API...` ) ); - try { - await installPluginsForLocalSite( url, PLUGINS_BLUEPRINT_PATH ); - } catch ( err ) { - console.warn( chalk.yellow( ` Plugin installation failed: ${ err }` ) ); - } - } - console.log( chalk.gray( ` Local site running at ${ url }` ) ); - return { url, siteId: site.id, siteDir }; -} - -async function teardownLocalSite( - siteId: string, - client: LocalGraphQLClient, - siteDir: string -): Promise< void > { - try { - await client.stopSite( siteId ); - } catch { - // Site may already be stopped - } - try { - await client.deleteSite( siteId ); - } catch { - // Best effort cleanup - } - cleanupDir( siteDir ); -} - // --------------------------------------------------------------------------- // Benchmark runner // --------------------------------------------------------------------------- @@ -595,7 +573,6 @@ async function runBenchmark( const isPlaygroundWeb = benchmarkUrl.includes( 'playground.wordpress.net' ); const isPlaygroundCli = benchmarkUrl.includes( '127.0.0.1' ); - const isLocal = env.type === 'local'; const allMeasurements: MeasurementResult[] = []; for ( let round = 1; round <= rounds; round++ ) { @@ -604,7 +581,13 @@ async function runBenchmark( } try { const result = await Promise.race( [ - measureSiteEditor( { url: benchmarkUrl, isPlaygroundWeb, isPlaygroundCli, isLocal, headed } ), + measureSiteEditor( { + url: benchmarkUrl, + isPlaygroundWeb, + isPlaygroundCli, + credentials: env.credentials, + headed, + } ), sleep( MEASUREMENT_TIMEOUT ).then( () => { throw new Error( 'Measurement timed out' ); } ), @@ -758,7 +741,23 @@ async function main() { const opts = parseArgs(); // Filter environments - let environments = ALL_ENVIRONMENTS; + let environments = [ ...ALL_ENVIRONMENTS ]; + + // Add custom environments from --custom flags + for ( const custom of opts.customs ) { + const shouldInstallPlugins = + opts.installPlugins === 'all' || + ( Array.isArray( opts.installPlugins ) && opts.installPlugins.includes( custom.name ) ); + + environments.push( { + name: custom.name, + type: 'custom', + plugins: shouldInstallPlugins, + multiWorker: false, + customUrl: custom.url, + credentials: { username: custom.user, password: custom.password }, + } ); + } if ( opts.only.length > 0 ) { environments = environments.filter( ( e ) => opts.only.includes( e.name ) ); @@ -772,10 +771,6 @@ async function main() { if ( opts.skipPlaygroundWeb ) { environments = environments.filter( ( e ) => e.type !== 'playground-web' ); } - // Local environments are excluded by default — they require the GUI app running - if ( ! opts.includeLocal ) { - environments = environments.filter( ( e ) => e.type !== 'local' ); - } } if ( environments.length === 0 ) { @@ -807,7 +802,6 @@ async function main() { const needsStudio = environments.some( ( e ) => e.type === 'studio' ); const needsPlaygroundCli = environments.some( ( e ) => e.type === 'playground-cli' ); - const needsLocal = environments.some( ( e ) => e.type === 'local' ); if ( needsStudio ) { if ( ! ( await ensureStudioCLIBuilt() ) ) { @@ -823,12 +817,6 @@ async function main() { console.log( chalk.green( ' Playground CLI ready' ) ); } - let localClient: LocalGraphQLClient | null = null; - if ( needsLocal ) { - localClient = await ensureLocalRunning(); - console.log( chalk.green( ' Local ready' ) ); - } - fs.mkdirSync( ARTIFACTS_PATH, { recursive: true } ); // Run benchmarks @@ -858,10 +846,23 @@ async function main() { const setup = await setupPlaygroundCliSite( env, port ); benchmarkUrl = setup.url; teardownFn = () => teardownPlaygroundCliSite( setup.process, port, setup.siteDir ); - } else if ( env.type === 'local' ) { - const setup = await setupLocalSite( env, localClient! ); - benchmarkUrl = setup.url; - teardownFn = () => teardownLocalSite( setup.siteId, localClient!, setup.siteDir ); + } else if ( env.type === 'custom' ) { + benchmarkUrl = env.customUrl!; + console.log( chalk.gray( ` URL: ${ benchmarkUrl }` ) ); + + if ( env.plugins && env.credentials ) { + console.log( chalk.gray( ` Installing plugins via REST API...` ) ); + try { + await installPlugins( + benchmarkUrl, + PLUGINS_BLUEPRINT_PATH, + env.credentials.username, + env.credentials.password + ); + } catch ( err ) { + console.warn( chalk.yellow( ` Plugin installation failed: ${ err }` ) ); + } + } } else { benchmarkUrl = getPlaygroundWebUrl( env ); console.log( chalk.gray( ` URL: ${ benchmarkUrl.slice( 0, 80 ) }...` ) ); diff --git a/tools/benchmark-site-editor/install-plugins-local.ts b/tools/benchmark-site-editor/install-plugins.ts similarity index 86% rename from tools/benchmark-site-editor/install-plugins-local.ts rename to tools/benchmark-site-editor/install-plugins.ts index a4fd1936f9..e0176c382d 100644 --- a/tools/benchmark-site-editor/install-plugins-local.ts +++ b/tools/benchmark-site-editor/install-plugins.ts @@ -1,7 +1,6 @@ /** - * Plugin installer for Local by Flywheel benchmark sites. + * Plugin installer for benchmark sites via the WordPress REST API. * - * Uses the WordPress REST API to install and activate plugins. * Reads plugin slugs from the shared plugins-blueprint.json * (single source of truth for all environments). * @@ -36,7 +35,7 @@ interface Blueprint { /** * Parse the blueprint JSON to extract plugin slugs. * This is the single source of truth for which plugins to install across - * all environments (Studio, Playground CLI, Playground Web, Local). + * all environments (Studio, Playground CLI, Playground Web, custom). */ export function getPluginSlugsFromBlueprint( blueprintPath: string ): string[] { const blueprint: Blueprint = JSON.parse( fs.readFileSync( blueprintPath, 'utf-8' ) ); @@ -130,6 +129,16 @@ async function installPlugin( if ( ! response.ok ) { const body = await response.text(); + let json: { code?: string } | null = null; + try { + json = JSON.parse( body ); + } catch { + // Not JSON, fall through to throw + } + if ( json?.code === 'folder_exists' ) { + console.log( chalk.gray( ` ${ slug } already installed, skipping` ) ); + return; + } throw new Error( `HTTP ${ response.status }: ${ body.slice( 0, 200 ) }` ); } } @@ -139,19 +148,23 @@ async function installPlugin( // --------------------------------------------------------------------------- /** - * Installs and activates plugins on a Local site using the WordPress REST API. + * Installs and activates plugins on a WordPress site using the REST API. * * Reads plugin slugs from the provided blueprint file (the same file used * for Studio and Playground environments), logs in via wp-login.php, * extracts a REST API nonce, and installs each plugin via * POST /wp-json/wp/v2/plugins. * - * @param siteUrl - The Local site's base URL (e.g., http://localhost:10003) + * @param siteUrl - The site's base URL (e.g., http://localhost:10003) * @param blueprintPath - Path to plugins-blueprint.json + * @param username - WordPress admin username + * @param password - WordPress admin password */ -export async function installPluginsForLocalSite( +export async function installPlugins( siteUrl: string, - blueprintPath: string + blueprintPath: string, + username: string, + password: string ): Promise< void > { const slugs = getPluginSlugsFromBlueprint( blueprintPath ); if ( slugs.length === 0 ) { @@ -160,7 +173,7 @@ export async function installPluginsForLocalSite( } console.log( chalk.gray( ` Logging in to WordPress...` ) ); - const cookies = await wpLogin( siteUrl, 'admin', 'password' ); + const cookies = await wpLogin( siteUrl, username, password ); console.log( chalk.gray( ` Extracting REST API nonce...` ) ); const nonce = await extractNonce( siteUrl, cookies ); diff --git a/tools/benchmark-site-editor/local-graphql.ts b/tools/benchmark-site-editor/local-graphql.ts deleted file mode 100644 index 3698a1fbe4..0000000000 --- a/tools/benchmark-site-editor/local-graphql.ts +++ /dev/null @@ -1,409 +0,0 @@ -/** - * GraphQL client for Local by Flywheel's API. - * - * Local exposes a GraphQL endpoint for programmatic site management. - * Connection details are stored in `graphql-connection-info.json` within - * Local's platform-specific data directory. - */ - -/* eslint-disable no-console */ - -import fs from 'fs'; -import os from 'os'; -import path from 'path'; - -// --------------------------------------------------------------------------- -// Types -// --------------------------------------------------------------------------- - -export interface LocalSite { - id: string; - name: string; - domain: string; - path: string; - status: string; - httpPort: number | null; - url: string; -} - -interface GraphQLConnectionInfo { - port: number; - authToken: string; - url: string; - subscriptionUrl?: string; -} - -interface GraphQLResponse< T > { - data?: T; - errors?: Array< { message: string } >; -} - -// --------------------------------------------------------------------------- -// Platform-specific paths -// --------------------------------------------------------------------------- - -/** Returns the platform-specific Local data directory. */ -export function getLocalDataDir(): string { - if ( process.platform === 'darwin' ) { - return path.join( os.homedir(), 'Library/Application Support/Local' ); - } - if ( process.platform === 'win32' ) { - const appdata = process.env.APPDATA || path.join( os.homedir(), 'AppData', 'Roaming' ); - // Local may use either "Local" or "Local by Flywheel" on Windows - const primary = path.join( appdata, 'Local' ); - const fallback = path.join( appdata, 'Local by Flywheel' ); - if ( fs.existsSync( primary ) ) return primary; - if ( fs.existsSync( fallback ) ) return fallback; - return primary; // Default; will fail later with a clear message - } - // Linux (limited Local support) - return path.join( os.homedir(), '.config', 'Local' ); -} - -/** Returns the default directory where Local creates sites. */ -export function getLocalSitesDir(): string { - return path.join( os.homedir(), 'Local Sites' ); -} - -// --------------------------------------------------------------------------- -// GraphQL client -// --------------------------------------------------------------------------- - -export class LocalGraphQLClient { - private url: string; - private token: string; - - private constructor( url: string, token: string ) { - this.url = url; - this.token = token; - } - - /** - * Connect to a running Local instance. - * Reads graphql-connection-info.json and verifies the API is responsive. - * Throws with a clear message if Local isn't running. - */ - static async connect(): Promise< LocalGraphQLClient > { - const dataDir = getLocalDataDir(); - const connectionInfoPath = path.join( dataDir, 'graphql-connection-info.json' ); - - if ( ! fs.existsSync( connectionInfoPath ) ) { - throw new Error( - `Local connection info not found at ${ connectionInfoPath }.\n` + - 'Please ensure Local is installed and running.' - ); - } - - let connectionInfo: GraphQLConnectionInfo; - try { - connectionInfo = JSON.parse( fs.readFileSync( connectionInfoPath, 'utf-8' ) ); - } catch { - throw new Error( - `Failed to parse ${ connectionInfoPath }.\n` + - 'The file may be corrupted. Try restarting Local.' - ); - } - - const client = new LocalGraphQLClient( connectionInfo.url, connectionInfo.authToken ); - - // Verify the API is responsive - try { - await client.query< { sites: unknown[] } >( '{ sites { id } }' ); - } catch ( err ) { - throw new Error( - `Cannot connect to Local's GraphQL API at ${ connectionInfo.url }.\n` + - `Is Local running? Error: ${ err }` - ); - } - - return client; - } - - /** - * Execute a GraphQL query/mutation. - */ - private async query< T >( query: string, variables?: Record< string, unknown > ): Promise< T > { - const response = await fetch( this.url, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${ this.token }`, - }, - body: JSON.stringify( { query, variables } ), - } ); - - if ( ! response.ok ) { - throw new Error( `GraphQL request failed: HTTP ${ response.status }` ); - } - - const json = ( await response.json() ) as GraphQLResponse< T >; - - if ( json.errors?.length ) { - throw new Error( `GraphQL error: ${ json.errors.map( ( e ) => e.message ).join( ', ' ) }` ); - } - - if ( ! json.data ) { - throw new Error( 'GraphQL response missing data' ); - } - - return json.data; - } - - /** - * Create a site. Returns when the job completes and the site is running. - * - * Local's `addSite` mutation returns a Job (async), not a Site directly. - * We poll the job until it completes, then query sites to find the new one. - */ - async createSite( options: { - name: string; - domain: string; - path: string; - wpAdminEmail?: string; - wpAdminPassword?: string; - wpAdminUsername?: string; - phpVersion?: string; - } ): Promise< LocalSite > { - const { - name, - domain, - path: sitePath, - wpAdminEmail = 'admin@benchmark.local', - wpAdminPassword = 'password', - wpAdminUsername = 'admin', - phpVersion = '8.2.0', - } = options; - - // Create the site via addSite mutation - const createResult = await this.query< { addSite: { id: string } } >( - `mutation AddSite($input: AddSiteInput!) { - addSite(input: $input) { - id - } - }`, - { - input: { - name, - domain, - path: sitePath, - wpAdminEmail, - wpAdminPassword, - wpAdminUsername, - phpVersion, - database: 'mysql', - environment: 'preferred', - blueprint: null, - }, - } - ); - - const jobId = createResult.addSite.id; - - // Poll the job until it completes - await this.waitForJob( jobId, 300_000 ); // 5 minute timeout - - // Find the new site by name - const site = await this.findSiteByName( name ); - if ( ! site ) { - throw new Error( `Site "${ name }" was not found after creation job completed` ); - } - - // Explicitly start the site — addSite may not auto-start it - if ( site.status !== 'running' ) { - await this.startSite( site.id ); - await this.waitForSiteStatus( site.id, 'running', 120_000 ); - } - - // Re-fetch to get the port - const runningSite = await this.getSite( site.id ); - if ( ! runningSite ) { - throw new Error( `Site "${ name }" disappeared after starting` ); - } - - return runningSite; - } - - /** - * Poll a job until it reaches 'successful' status. - */ - private async waitForJob( jobId: string, timeoutMs: number ): Promise< void > { - const start = Date.now(); - const pollInterval = 2000; - - while ( Date.now() - start < timeoutMs ) { - try { - const result = await this.query< { job: { id: string; status: string; error?: string } } >( - `query Job($id: ID!) { - job(id: $id) { - id - status - error - } - }`, - { id: jobId } - ); - - const job = result.job; - if ( job.status === 'successful' ) { - return; - } - if ( job.status === 'failed' ) { - throw new Error( `Job failed: ${ job.error || 'unknown error' }` ); - } - } catch ( err ) { - if ( err instanceof Error && err.message.startsWith( 'Job failed' ) ) { - throw err; - } - // GraphQL query itself might fail during setup; keep polling - } - - await new Promise( ( resolve ) => setTimeout( resolve, pollInterval ) ); - } - - throw new Error( `Job ${ jobId } timed out after ${ timeoutMs / 1000 }s` ); - } - - /** - * Wait for a site to reach a specific status. - */ - private async waitForSiteStatus( - siteId: string, - targetStatus: string, - timeoutMs: number - ): Promise< void > { - const start = Date.now(); - const pollInterval = 2000; - - while ( Date.now() - start < timeoutMs ) { - const site = await this.getSite( siteId ); - if ( site && site.status === targetStatus ) { - return; - } - await new Promise( ( resolve ) => setTimeout( resolve, pollInterval ) ); - } - - throw new Error( - `Site ${ siteId } did not reach "${ targetStatus }" status within ${ timeoutMs / 1000 }s` - ); - } - - /** - * Find all sites matching a name. - */ - async findAllSitesByName( name: string ): Promise< LocalSite[] > { - const result = await this.query< { - sites: Array< { - id: string; - name: string; - domain: string; - path: string; - status: string; - httpPort: number | null; - } >; - } >( - `{ - sites { - id - name - domain - path - status - httpPort - } - }` - ); - - return result.sites - .filter( ( s ) => s.name === name ) - .map( ( s ) => ( { - ...s, - url: s.httpPort ? `http://localhost:${ s.httpPort }` : '', - } ) ); - } - - /** - * Find a site by name. - */ - async findSiteByName( name: string ): Promise< LocalSite | null > { - const matches = await this.findAllSitesByName( name ); - return matches[ 0 ] ?? null; - } - - /** - * Get a site by ID. - */ - async getSite( id: string ): Promise< LocalSite | null > { - const result = await this.query< { - site: { - id: string; - name: string; - domain: string; - path: string; - status: string; - httpPort: number | null; - } | null; - } >( - `query Site($id: ID!) { - site(id: $id) { - id - name - domain - path - status - httpPort - } - }`, - { id } - ); - - const site = result.site; - if ( ! site ) return null; - - return { - ...site, - url: site.httpPort ? `http://localhost:${ site.httpPort }` : '', - }; - } - - /** - * Start a site. - */ - async startSite( id: string ): Promise< void > { - await this.query( - `mutation StartSite($id: ID!) { - startSite(id: $id) { - id - status - } - }`, - { id } - ); - } - - /** - * Stop a running site. - */ - async stopSite( id: string ): Promise< void > { - await this.query( - `mutation StopSite($id: ID!) { - stopSite(id: $id) { - id - status - } - }`, - { id } - ); - } - - /** - * Delete a site. Uses deleteSitesFromGroups since there's no single deleteSite mutation. - */ - async deleteSite( id: string ): Promise< void > { - await this.query( - `mutation DeleteSites($ids: [ID!]!) { - deleteSitesFromGroups(ids: $ids) - }`, - { ids: [ id ] } - ); - } -} diff --git a/tools/benchmark-site-editor/measure-site-editor.ts b/tools/benchmark-site-editor/measure-site-editor.ts index c3cce7119d..b2bfce6fd6 100644 --- a/tools/benchmark-site-editor/measure-site-editor.ts +++ b/tools/benchmark-site-editor/measure-site-editor.ts @@ -31,8 +31,8 @@ export interface MeasureOptions { isPlaygroundWeb: boolean; /** Whether this is a local Playground CLI site (127.0.0.1). Affects login flow. */ isPlaygroundCli: boolean; - /** Whether this is a Local by Flywheel site. Uses wp-login.php credentials. */ - isLocal: boolean; + /** WordPress admin credentials for wp-login.php authentication. When provided, logs in via the standard WordPress login form. */ + credentials?: { username: string; password: string }; /** Launch browser in headed mode for debugging. */ headed?: boolean; } @@ -99,7 +99,7 @@ function findEditorCanvasFrame( * The browser is always closed, even on error. */ export async function measureSiteEditor( options: MeasureOptions ): Promise< MeasurementResult > { - const { isPlaygroundWeb, isPlaygroundCli, isLocal } = options; + const { isPlaygroundWeb, isPlaygroundCli, credentials } = options; // Normalize URL let wpAdminUrl = options.url; @@ -143,15 +143,15 @@ export async function measureSiteEditor( options: MeasureOptions ): Promise< Mea state: 'visible', timeout: 60_000, } ); - } else if ( isLocal ) { - // Local sites require wp-login.php with credentials + } else if ( credentials ) { + // Standard WordPress login via wp-login.php await page.goto( `${ wpAdminUrl }/wp-login.php`, { waitUntil: 'domcontentloaded', timeout: 120_000, } ); await page.waitForLoadState( 'networkidle', { timeout: 30_000 } ).catch( () => {} ); - await page.fill( '#user_login', 'admin' ); - await page.fill( '#user_pass', 'password' ); + await page.fill( '#user_login', credentials.username ); + await page.fill( '#user_pass', credentials.password ); await Promise.all( [ page.waitForURL( '**/wp-admin/**', { timeout: 60_000 } ), page.click( '#wp-submit' ), From c0ccf926a6a481f9d5727800c22023ad9e4b9f17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gergely=20Cs=C3=A9csey?= Date: Fri, 20 Feb 2026 13:32:56 +0000 Subject: [PATCH 14/14] Navigate to wp-admin after login to bypass plugin setup wizards --- tools/benchmark-site-editor/measure-site-editor.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tools/benchmark-site-editor/measure-site-editor.ts b/tools/benchmark-site-editor/measure-site-editor.ts index b2bfce6fd6..c61920aa36 100644 --- a/tools/benchmark-site-editor/measure-site-editor.ts +++ b/tools/benchmark-site-editor/measure-site-editor.ts @@ -156,6 +156,13 @@ export async function measureSiteEditor( options: MeasureOptions ): Promise< Mea page.waitForURL( '**/wp-admin/**', { timeout: 60_000 } ), page.click( '#wp-submit' ), ] ); + // Navigate explicitly to wp-admin to bypass plugin setup wizard redirects + // (WooCommerce, Jetpack, etc. redirect to their own onboarding pages) + await page.goto( `${ wpAdminUrl }/wp-admin/`, { + waitUntil: 'domcontentloaded', + timeout: 60_000, + } ); + await page.waitForLoadState( 'networkidle', { timeout: 30_000 } ).catch( () => {} ); await page.getByRole( 'link', { name: 'Appearance' } ).waitFor( { state: 'visible', timeout: 60_000,