From 8b36eb108a0ba90f0cfdc5529871b33879d69bdf Mon Sep 17 00:00:00 2001 From: burnedchris Date: Wed, 29 Oct 2025 19:10:25 -0700 Subject: [PATCH 01/21] Refactor benchmark projects and update dependencies. Introduce new '@consentio/benchmark' package for core benchmarking logic, replace '@cookiebench/cli' with 'cookiebench' in various benchmark configurations, and enhance package.json scripts for improved functionality. Remove deprecated CLI files and streamline project structure. --- benchmarks/baseline/package.json | 10 +- benchmarks/with-c15t-nextjs/package.json | 10 +- benchmarks/with-c15t-react/package.json | 10 +- benchmarks/with-cookie-control/package.json | 10 +- benchmarks/with-cookie-yes/package.json | 10 +- benchmarks/with-didomi/package.json | 8 +- benchmarks/with-enzuzo/package.json | 10 +- benchmarks/with-iubenda/package.json | 10 +- benchmarks/with-ketch/package.json | 8 +- benchmarks/with-onetrust/package.json | 10 +- benchmarks/with-osano/package.json | 10 +- benchmarks/with-usercentrics/package.json | 10 +- package.json | 14 +- packages/benchmark/README.md | 113 +++++ packages/benchmark/package.json | 31 ++ packages/benchmark/rslib.config.ts | 15 + packages/benchmark/src/bundle-strategy.ts | 22 + packages/benchmark/src/constants.ts | 16 + .../benchmark/src/cookie-banner-collector.ts | 229 +++++++++ packages/benchmark/src/index.ts | 24 + packages/benchmark/src/network-monitor.ts | 121 +++++ .../src/resource-timing-collector.ts | 177 +++++++ packages/benchmark/src/types.ts | 226 +++++++++ packages/benchmark/tsconfig.json | 16 + packages/cli/package.json | 35 -- .../commands/benchmark/benchmark-runner.ts | 452 ------------------ .../src/commands/benchmark/bundle-strategy.ts | 22 - .../cli/src/commands/benchmark/constants.ts | 15 - .../benchmark/cookie-banner-detector.ts | 293 ------------ packages/cli/src/commands/benchmark/index.ts | 3 - .../commands/benchmark/metrics-calculator.ts | 155 ------ .../src/commands/benchmark/network-monitor.ts | 80 ---- .../commands/benchmark/resource-collector.ts | 219 --------- packages/cli/src/commands/benchmark/types.ts | 146 ------ packages/cli/src/index.ts | 82 ---- packages/cli/src/lib/README.md | 53 -- packages/cli/src/lib/benchmark-runner.ts | 189 -------- .../lib/collectors/cookie-banner-collector.ts | 279 ----------- packages/cli/src/lib/collectors/index.ts | 14 - .../cli/src/lib/collectors/network-monitor.ts | 136 ------ .../collectors/resource-timing-collector.ts | 274 ----------- packages/cli/src/lib/metrics/index.ts | 4 - .../src/lib/metrics/performance-aggregator.ts | 286 ----------- packages/cli/src/lib/performance-enhanced.ts | 259 ---------- packages/cli/src/lib/performance.ts | 437 ----------------- packages/cli/src/lib/server.ts | 74 --- packages/cli/src/types/index.ts | 369 -------------- packages/cookiebench-cli/README.md | 213 +++++++++ packages/{cli => cookiebench-cli}/base.json | 0 packages/cookiebench-cli/package.json | 34 ++ .../{cli => cookiebench-cli}/rslib.config.ts | 0 .../cookiebench-cli/src/commands/benchmark.ts | 256 ++++++++++ .../src/commands/db.ts | 0 .../src/commands/results.ts | 22 +- packages/cookiebench-cli/src/index.ts | 82 ++++ packages/cookiebench-cli/src/types/index.ts | 39 ++ .../src/utils/index.ts | 0 .../src/utils/scoring.ts | 10 +- .../{cli => cookiebench-cli}/tsconfig.json | 0 packages/runner/README.md | 150 ++++++ packages/runner/package.json | 32 ++ packages/runner/rslib.config.ts | 15 + packages/runner/src/benchmark-runner.ts | 191 ++++++++ packages/runner/src/index.ts | 28 ++ packages/runner/src/performance-aggregator.ts | 274 +++++++++++ packages/runner/src/server.ts | 75 +++ packages/runner/src/types.ts | 272 +++++++++++ packages/runner/src/utils.ts | 56 +++ packages/runner/tsconfig.json | 16 + pnpm-lock.yaml | 142 ++++-- 70 files changed, 2899 insertions(+), 4004 deletions(-) create mode 100644 packages/benchmark/README.md create mode 100644 packages/benchmark/package.json create mode 100644 packages/benchmark/rslib.config.ts create mode 100644 packages/benchmark/src/bundle-strategy.ts create mode 100644 packages/benchmark/src/constants.ts create mode 100644 packages/benchmark/src/cookie-banner-collector.ts create mode 100644 packages/benchmark/src/index.ts create mode 100644 packages/benchmark/src/network-monitor.ts create mode 100644 packages/benchmark/src/resource-timing-collector.ts create mode 100644 packages/benchmark/src/types.ts create mode 100644 packages/benchmark/tsconfig.json delete mode 100644 packages/cli/package.json delete mode 100644 packages/cli/src/commands/benchmark/benchmark-runner.ts delete mode 100644 packages/cli/src/commands/benchmark/bundle-strategy.ts delete mode 100644 packages/cli/src/commands/benchmark/constants.ts delete mode 100644 packages/cli/src/commands/benchmark/cookie-banner-detector.ts delete mode 100644 packages/cli/src/commands/benchmark/index.ts delete mode 100644 packages/cli/src/commands/benchmark/metrics-calculator.ts delete mode 100644 packages/cli/src/commands/benchmark/network-monitor.ts delete mode 100644 packages/cli/src/commands/benchmark/resource-collector.ts delete mode 100644 packages/cli/src/commands/benchmark/types.ts delete mode 100644 packages/cli/src/index.ts delete mode 100644 packages/cli/src/lib/README.md delete mode 100644 packages/cli/src/lib/benchmark-runner.ts delete mode 100644 packages/cli/src/lib/collectors/cookie-banner-collector.ts delete mode 100644 packages/cli/src/lib/collectors/index.ts delete mode 100644 packages/cli/src/lib/collectors/network-monitor.ts delete mode 100644 packages/cli/src/lib/collectors/resource-timing-collector.ts delete mode 100644 packages/cli/src/lib/metrics/index.ts delete mode 100644 packages/cli/src/lib/metrics/performance-aggregator.ts delete mode 100644 packages/cli/src/lib/performance-enhanced.ts delete mode 100644 packages/cli/src/lib/performance.ts delete mode 100644 packages/cli/src/lib/server.ts delete mode 100644 packages/cli/src/types/index.ts create mode 100644 packages/cookiebench-cli/README.md rename packages/{cli => cookiebench-cli}/base.json (100%) create mode 100644 packages/cookiebench-cli/package.json rename packages/{cli => cookiebench-cli}/rslib.config.ts (100%) create mode 100644 packages/cookiebench-cli/src/commands/benchmark.ts rename packages/{cli => cookiebench-cli}/src/commands/db.ts (100%) rename packages/{cli => cookiebench-cli}/src/commands/results.ts (98%) create mode 100644 packages/cookiebench-cli/src/index.ts create mode 100644 packages/cookiebench-cli/src/types/index.ts rename packages/{cli => cookiebench-cli}/src/utils/index.ts (100%) rename packages/{cli => cookiebench-cli}/src/utils/scoring.ts (99%) rename packages/{cli => cookiebench-cli}/tsconfig.json (100%) create mode 100644 packages/runner/README.md create mode 100644 packages/runner/package.json create mode 100644 packages/runner/rslib.config.ts create mode 100644 packages/runner/src/benchmark-runner.ts create mode 100644 packages/runner/src/index.ts create mode 100644 packages/runner/src/performance-aggregator.ts create mode 100644 packages/runner/src/server.ts create mode 100644 packages/runner/src/types.ts create mode 100644 packages/runner/src/utils.ts create mode 100644 packages/runner/tsconfig.json diff --git a/benchmarks/baseline/package.json b/benchmarks/baseline/package.json index 3d25695..7ab3910 100644 --- a/benchmarks/baseline/package.json +++ b/benchmarks/baseline/package.json @@ -3,12 +3,12 @@ "version": "0.1.0", "private": true, "scripts": { - "dev": "next dev --port 3000 --turbopack", + "benchmark": "pnpm exec cookiebench benchmark", "build": "next build --turbopack", - "start": "next start", - "lint": "next lint", + "dev": "next dev --port 3000 --turbopack", "fmt": "pnpm biome format --write . && pnpm biome check --formatter-enabled=false --linter-enabled=false --organize-imports-enabled=true --write", - "benchmark": "pnpm exec benchmark-cli benchmark" + "lint": "next lint", + "start": "next start" }, "dependencies": { "next": "15.3.3", @@ -17,11 +17,11 @@ }, "devDependencies": { "@cookiebench/benchmark-schema": "workspace:*", - "@cookiebench/cli": "workspace:*", "@cookiebench/ts-config": "workspace:*", "@types/node": "^22.15.30", "@types/react": "^19.1.6", "@types/react-dom": "^19.1.6", + "cookiebench": "workspace:*", "typescript": "^5.8.3" } } diff --git a/benchmarks/with-c15t-nextjs/package.json b/benchmarks/with-c15t-nextjs/package.json index 297dd71..bc46b0d 100644 --- a/benchmarks/with-c15t-nextjs/package.json +++ b/benchmarks/with-c15t-nextjs/package.json @@ -3,12 +3,12 @@ "version": "0.1.0", "private": true, "scripts": { - "dev": "next dev --port 3001 --turbopack", + "benchmark": "pnpm exec cookiebench benchmark", "build": "next build --turbopack", - "start": "next start", - "lint": "next lint", + "dev": "next dev --port 3001 --turbopack", "fmt": "pnpm biome format --write . && pnpm biome check --formatter-enabled=false --linter-enabled=false --organize-imports-enabled=true --write", - "benchmark": "pnpm exec benchmark-cli benchmark" + "lint": "next lint", + "start": "next start" }, "dependencies": { "@c15t/nextjs": "1.2.2-canary-20250603153501", @@ -18,11 +18,11 @@ }, "devDependencies": { "@cookiebench/benchmark-schema": "workspace:*", - "@cookiebench/cli": "workspace:*", "@cookiebench/ts-config": "workspace:*", "@types/node": "^22.15.30", "@types/react": "^19.1.6", "@types/react-dom": "^19.1.6", + "cookiebench": "workspace:*", "typescript": "^5.8.3" } } diff --git a/benchmarks/with-c15t-react/package.json b/benchmarks/with-c15t-react/package.json index 79cab8e..f31d5b6 100644 --- a/benchmarks/with-c15t-react/package.json +++ b/benchmarks/with-c15t-react/package.json @@ -2,12 +2,12 @@ "name": "with-c15t-react", "private": true, "scripts": { - "dev": "next dev --port 3003 --turbopack", + "benchmark": "pnpm exec cookiebench benchmark", "build": "next build --turbopack", - "start": "next start --port 3003", - "lint": "next lint", + "dev": "next dev --port 3003 --turbopack", "fmt": "pnpm biome format --write . && pnpm biome check --formatter-enabled=false --linter-enabled=false --organize-imports-enabled=true --write", - "benchmark": "pnpm exec benchmark-cli benchmark" + "lint": "next lint", + "start": "next start --port 3003" }, "dependencies": { "@c15t/react": "1.2.2-canary-20250603153501", @@ -18,11 +18,11 @@ }, "devDependencies": { "@cookiebench/benchmark-schema": "workspace:*", - "@cookiebench/cli": "workspace:*", "@cookiebench/ts-config": "workspace:*", "@types/node": "^22.15.30", "@types/react": "^19.1.6", "@types/react-dom": "^19.1.6", + "cookiebench": "workspace:*", "typescript": "^5.8.3" } } diff --git a/benchmarks/with-cookie-control/package.json b/benchmarks/with-cookie-control/package.json index 94c5647..06fac9e 100644 --- a/benchmarks/with-cookie-control/package.json +++ b/benchmarks/with-cookie-control/package.json @@ -2,12 +2,12 @@ "name": "with-cookie-control", "private": true, "scripts": { - "dev": "next dev --port 3001 --turbopack", + "benchmark": "pnpm exec cookiebench benchmark", "build": "next build --turbopack", - "start": "next start", - "lint": "next lint", + "dev": "next dev --port 3001 --turbopack", "fmt": "pnpm biome format --write . && pnpm biome check --formatter-enabled=false --linter-enabled=false --organize-imports-enabled=true --write", - "benchmark": "pnpm exec benchmark-cli benchmark" + "lint": "next lint", + "start": "next start" }, "dependencies": { "next": "15.3.3", @@ -15,12 +15,12 @@ "react-dom": "^19.0.0" }, "devDependencies": { - "@cookiebench/cli": "workspace:*", "@cookiebench/benchmark-schema": "workspace:*", "@cookiebench/ts-config": "workspace:*", "@types/node": "^22.15.30", "@types/react": "^19.1.6", "@types/react-dom": "^19.1.6", + "cookiebench": "workspace:*", "typescript": "^5.8.3" } } diff --git a/benchmarks/with-cookie-yes/package.json b/benchmarks/with-cookie-yes/package.json index 3747ba8..934ec59 100644 --- a/benchmarks/with-cookie-yes/package.json +++ b/benchmarks/with-cookie-yes/package.json @@ -2,12 +2,12 @@ "name": "with-cookie-yes", "private": true, "scripts": { - "dev": "next dev --port 3001 --turbopack", + "benchmark": "pnpm exec cookiebench benchmark", "build": "next build --turbopack", - "start": "next start", - "lint": "next lint", + "dev": "next dev --port 3001 --turbopack", "fmt": "pnpm biome format --write . && pnpm biome check --formatter-enabled=false --linter-enabled=false --organize-imports-enabled=true --write", - "benchmark": "pnpm exec benchmark-cli benchmark" + "lint": "next lint", + "start": "next start" }, "dependencies": { "next": "15.3.3", @@ -15,12 +15,12 @@ "react-dom": "^19.0.0" }, "devDependencies": { - "@cookiebench/cli": "workspace:*", "@cookiebench/benchmark-schema": "workspace:*", "@cookiebench/ts-config": "workspace:*", "@types/node": "^22.15.30", "@types/react": "^19.1.6", "@types/react-dom": "^19.1.6", + "cookiebench": "workspace:*", "typescript": "^5.8.3" } } diff --git a/benchmarks/with-didomi/package.json b/benchmarks/with-didomi/package.json index 090d982..2ac50db 100644 --- a/benchmarks/with-didomi/package.json +++ b/benchmarks/with-didomi/package.json @@ -2,12 +2,12 @@ "name": "with-dodomi", "private": true, "scripts": { - "dev": "next dev --turbopack", + "benchmark": "pnpm exec cookiebench benchmark", "build": "next build --turbopack", - "start": "next start", - "lint": "next lint", + "dev": "next dev --turbopack", "fmt": "pnpm biome format --write . && pnpm biome check --formatter-enabled=false --linter-enabled=false --organize-imports-enabled=true --write", - "benchmark": "pnpm exec benchmark-cli benchmark" + "lint": "next lint", + "start": "next start" }, "dependencies": { "@didomi/react": "^1.8.8", diff --git a/benchmarks/with-enzuzo/package.json b/benchmarks/with-enzuzo/package.json index 53a87fe..6563d1f 100644 --- a/benchmarks/with-enzuzo/package.json +++ b/benchmarks/with-enzuzo/package.json @@ -2,12 +2,12 @@ "name": "with-enzuzo", "private": true, "scripts": { - "dev": "next dev --port 3001 --turbopack", + "benchmark": "pnpm exec cookiebench benchmark", "build": "next build --turbopack", - "start": "next start", - "lint": "next lint", + "dev": "next dev --port 3001 --turbopack", "fmt": "pnpm biome format --write . && pnpm biome check --formatter-enabled=false --linter-enabled=false --organize-imports-enabled=true --write", - "benchmark": "pnpm exec benchmark-cli benchmark" + "lint": "next lint", + "start": "next start" }, "dependencies": { "next": "15.3.3", @@ -15,12 +15,12 @@ "react-dom": "^19.0.0" }, "devDependencies": { - "@cookiebench/cli": "workspace:*", "@cookiebench/benchmark-schema": "workspace:*", "@cookiebench/ts-config": "workspace:*", "@types/node": "^22.15.30", "@types/react": "^19.1.6", "@types/react-dom": "^19.1.6", + "cookiebench": "workspace:*", "typescript": "^5.8.3" } } diff --git a/benchmarks/with-iubenda/package.json b/benchmarks/with-iubenda/package.json index 42406c0..e2777fb 100644 --- a/benchmarks/with-iubenda/package.json +++ b/benchmarks/with-iubenda/package.json @@ -2,12 +2,12 @@ "name": "with-iubenda", "private": true, "scripts": { - "dev": "next dev --port 3001 --turbopack", + "benchmark": "pnpm exec cookiebench benchmark", "build": "next build --turbopack", - "start": "next start", - "lint": "next lint", + "dev": "next dev --port 3001 --turbopack", "fmt": "pnpm biome format --write . && pnpm biome check --formatter-enabled=false --linter-enabled=false --organize-imports-enabled=true --write", - "benchmark": "pnpm exec benchmark-cli benchmark" + "lint": "next lint", + "start": "next start" }, "dependencies": { "next": "15.3.3", @@ -15,12 +15,12 @@ "react-dom": "^19.0.0" }, "devDependencies": { - "@cookiebench/cli": "workspace:*", "@cookiebench/benchmark-schema": "workspace:*", "@cookiebench/ts-config": "workspace:*", "@types/node": "^22.15.30", "@types/react": "^19.1.6", "@types/react-dom": "^19.1.6", + "cookiebench": "workspace:*", "typescript": "^5.8.3" } } diff --git a/benchmarks/with-ketch/package.json b/benchmarks/with-ketch/package.json index 207f482..6d3d75e 100644 --- a/benchmarks/with-ketch/package.json +++ b/benchmarks/with-ketch/package.json @@ -2,12 +2,12 @@ "name": "with-ketch", "private": true, "scripts": { - "dev": "next dev --turbopack", + "benchmark": "pnpm exec cookiebench benchmark", "build": "next build --turbopack", - "start": "next start", - "lint": "next lint", + "dev": "next dev --turbopack", "fmt": "pnpm biome format --write . && pnpm biome check --formatter-enabled=false --linter-enabled=false --organize-imports-enabled=true --write", - "benchmark": "pnpm exec benchmark-cli benchmark" + "lint": "next lint", + "start": "next start" }, "dependencies": { "next": "15.3.3", diff --git a/benchmarks/with-onetrust/package.json b/benchmarks/with-onetrust/package.json index 87ff08f..3e6ed1b 100644 --- a/benchmarks/with-onetrust/package.json +++ b/benchmarks/with-onetrust/package.json @@ -2,12 +2,12 @@ "name": "with-onetrust", "private": true, "scripts": { - "dev": "next dev --port 3006 --turbopack", + "benchmark": "pnpm exec cookiebench benchmark", "build": "next build --turbopack", - "start": "next start", - "lint": "next lint", + "dev": "next dev --port 3006 --turbopack", "fmt": "pnpm biome format --write . && pnpm biome check --formatter-enabled=false --linter-enabled=false --organize-imports-enabled=true --write", - "benchmark": "pnpm exec benchmark-cli benchmark" + "lint": "next lint", + "start": "next start" }, "dependencies": { "next": "15.3.3", @@ -16,11 +16,11 @@ }, "devDependencies": { "@cookiebench/benchmark-schema": "workspace:*", - "@cookiebench/cli": "workspace:*", "@cookiebench/ts-config": "workspace:*", "@types/node": "^22.15.30", "@types/react": "^19.1.6", "@types/react-dom": "^19.1.6", + "cookiebench": "workspace:*", "typescript": "^5.8.3" } } diff --git a/benchmarks/with-osano/package.json b/benchmarks/with-osano/package.json index 86bd27b..940620d 100644 --- a/benchmarks/with-osano/package.json +++ b/benchmarks/with-osano/package.json @@ -2,12 +2,12 @@ "name": "with-osano", "private": true, "scripts": { - "dev": "next dev --port 3006 --turbopack", + "benchmark": "pnpm exec cookiebench benchmark", "build": "next build --turbopack", - "start": "next start", - "lint": "next lint", + "dev": "next dev --port 3006 --turbopack", "fmt": "pnpm biome format --write . && pnpm biome check --formatter-enabled=false --linter-enabled=false --organize-imports-enabled=true --write", - "benchmark": "pnpm exec benchmark-cli benchmark" + "lint": "next lint", + "start": "next start" }, "dependencies": { "next": "15.3.3", @@ -16,11 +16,11 @@ }, "devDependencies": { "@cookiebench/benchmark-schema": "workspace:*", - "@cookiebench/cli": "workspace:*", "@cookiebench/ts-config": "workspace:*", "@types/node": "^22.15.30", "@types/react": "^19.1.6", "@types/react-dom": "^19.1.6", + "cookiebench": "workspace:*", "typescript": "^5.8.3" } } diff --git a/benchmarks/with-usercentrics/package.json b/benchmarks/with-usercentrics/package.json index b0b5e26..a3599ea 100644 --- a/benchmarks/with-usercentrics/package.json +++ b/benchmarks/with-usercentrics/package.json @@ -2,12 +2,12 @@ "name": "with-usercentrics", "private": true, "scripts": { - "dev": "next dev --port 3001 --turbopack", + "benchmark": "pnpm exec cookiebench benchmark", "build": "next build --turbopack", - "start": "next start", - "lint": "next lint", + "dev": "next dev --port 3001 --turbopack", "fmt": "pnpm biome format --write . && pnpm biome check --formatter-enabled=false --linter-enabled=false --organize-imports-enabled=true --write", - "benchmark": "pnpm exec benchmark-cli benchmark" + "lint": "next lint", + "start": "next start" }, "dependencies": { "next": "15.3.3", @@ -15,12 +15,12 @@ "react-dom": "^19.0.0" }, "devDependencies": { - "@cookiebench/cli": "workspace:*", "@cookiebench/benchmark-schema": "workspace:*", "@cookiebench/ts-config": "workspace:*", "@types/node": "^22.15.30", "@types/react": "^19.1.6", "@types/react-dom": "^19.1.6", + "cookiebench": "workspace:*", "typescript": "^5.8.3" } } diff --git a/package.json b/package.json index 5f84d1e..6b4d11b 100644 --- a/package.json +++ b/package.json @@ -2,21 +2,23 @@ "name": "cookiebench", "private": true, "scripts": { + "benchmark": "turbo run benchmark", "build": "turbo run build", + "check-types": "turbo run check-types", + "db": "pnpm exec cookiebench db", "dev": "turbo run dev --filter=benchmarks", - "lint": "turbo run lint", "fmt": "turbo fmt", - "check-types": "turbo run check-types", - "benchmark": "turbo run benchmark", - "results": "pnpm exec benchmark-cli results", - "db": "pnpm exec benchmark-cli db" + "lint": "turbo run lint", + "results": "pnpm exec cookiebench results" }, "devDependencies": { "@biomejs/biome": "1.9.4", "@c15t/translations": "^1.0.0", - "@cookiebench/cli": "workspace:*", + "@consentio/benchmark": "workspace:*", + "@consentio/runner": "workspace:*", "@playwright/test": "^1.42.1", "cli-table3": "^0.6.5", + "cookiebench": "workspace:*", "drizzle-kit": "^0.31.1", "p-limit": "^6.2.0", "pretty-ms": "^9.2.0", diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md new file mode 100644 index 0000000..574b95b --- /dev/null +++ b/packages/benchmark/README.md @@ -0,0 +1,113 @@ +# @consentio/benchmark + +Core benchmark measurement logic for cookie banner performance testing. + +## Overview + +This package provides the core functionality for detecting and measuring cookie banner performance impact. It includes collectors for cookie banners, network monitoring, and resource timing. + +## Features + +- **Cookie Banner Detection**: Automatically detects cookie banners using configurable selectors +- **Network Monitoring**: Tracks network requests and calculates size/timing metrics +- **Resource Collection**: Collects detailed resource timing data from the browser +- **Bundle Strategy Detection**: Identifies bundling approaches (IIFE, ESM, CJS, bundled) +- **Performance Metrics**: Measures layout shift, render time, and viewport coverage + +## Installation + +```bash +pnpm add @consentio/benchmark +``` + +## Usage + +```typescript +import { + CookieBannerCollector, + NetworkMonitor, + ResourceTimingCollector, + determineBundleStrategy, + BENCHMARK_CONSTANTS, +} from '@consentio/benchmark'; +import { chromium } from '@playwright/test'; + +// Create config +const config = { + name: 'my-app', + iterations: 5, + cookieBanner: { + selectors: ['.cookie-banner', '#cookie-consent'], + serviceHosts: ['cookiecdn.com'], + serviceName: 'CookieService', + waitForVisibility: true, + measureViewportCoverage: true, + expectedLayoutShift: true, + }, + techStack: { + bundleType: 'esm', + // ... + }, + // ... +}; + +// Initialize collectors +const cookieBannerCollector = new CookieBannerCollector(config); +const networkMonitor = new NetworkMonitor(config); +const resourceCollector = new ResourceTimingCollector(); + +// Use with Playwright +const browser = await chromium.launch(); +const page = await browser.newPage(); + +// Setup detection and monitoring +await cookieBannerCollector.setupDetection(page); +await networkMonitor.setupMonitoring(page); + +// Navigate to page +await page.goto('https://example.com'); + +// Collect metrics +const bannerData = await cookieBannerCollector.collectMetrics(page); +const resourceData = await resourceCollector.collect(page); +const networkRequests = networkMonitor.getNetworkRequests(); + +await browser.close(); +``` + +## API + +### CookieBannerCollector + +- `constructor(config: Config)`: Create a new collector +- `initializeMetrics()`: Initialize cookie banner metrics tracking +- `setupDetection(page: Page)`: Set up browser-side detection script +- `collectMetrics(page: Page)`: Collect metrics from the page + +### NetworkMonitor + +- `constructor(config: Config)`: Create a new monitor +- `setupMonitoring(page: Page)`: Set up network request interception +- `getNetworkRequests()`: Get collected network requests +- `getMetrics()`: Get network metrics +- `calculateNetworkImpact()`: Calculate network impact metrics +- `reset()`: Reset collected data + +### ResourceTimingCollector + +- `collect(page: Page)`: Collect detailed resource timing data + +### Utilities + +- `determineBundleStrategy(config: Config)`: Determine bundle strategy from config +- `BENCHMARK_CONSTANTS`: Constants for detection intervals, timeouts, etc. +- `BUNDLE_TYPES`: Bundle type constants (IIFE, ESM, CJS, BUNDLED) + +## Types + +See the [types file](./src/types.ts) for complete type definitions. + +## License + +MIT + diff --git a/packages/benchmark/package.json b/packages/benchmark/package.json new file mode 100644 index 0000000..cbfe513 --- /dev/null +++ b/packages/benchmark/package.json @@ -0,0 +1,31 @@ +{ + "name": "@consentio/benchmark", + "version": "0.0.1", + "type": "module", + "exports": { + ".": { + "import": "./dist/index.js", + "types": "./dist/index.d.ts" + } + }, + "main": "./dist/index.js", + "module": "./dist/index.js", + "types": "./dist/index.d.ts", + "scripts": { + "build": "rslib build", + "check-types": "tsc --noEmit", + "dev": "rslib build --watch", + "format": "biome format . --write", + "lint": "biome lint ." + }, + "dependencies": { + "@playwright/test": "^1.42.1", + "playwright-performance-metrics": "^1.2.2" + }, + "devDependencies": { + "@rsdoctor/rspack-plugin": "^1.1.3", + "@rslib/core": "^0.9.1", + "@types/node": "^22.15.30", + "typescript": "^5.8.3" + } +} diff --git a/packages/benchmark/rslib.config.ts b/packages/benchmark/rslib.config.ts new file mode 100644 index 0000000..8f03d65 --- /dev/null +++ b/packages/benchmark/rslib.config.ts @@ -0,0 +1,15 @@ +import { defineConfig } from '@rslib/core'; + +export default defineConfig({ + lib: [ + { + format: 'esm', + syntax: 'es2021', + dts: true, + }, + ], + output: { + target: 'node', + }, +}); + diff --git a/packages/benchmark/src/bundle-strategy.ts b/packages/benchmark/src/bundle-strategy.ts new file mode 100644 index 0000000..bd6624a --- /dev/null +++ b/packages/benchmark/src/bundle-strategy.ts @@ -0,0 +1,22 @@ +import type { Config, BundleStrategy } from './types'; +import { BUNDLE_TYPES } from './constants'; + +export function determineBundleStrategy(config: Config): BundleStrategy { + const bundleType = config.techStack?.bundleType; + + const isIIFE = + bundleType === BUNDLE_TYPES.IIFE || + (Array.isArray(bundleType) && bundleType.includes(BUNDLE_TYPES.IIFE)); + + const isBundled = + !isIIFE && + (bundleType === BUNDLE_TYPES.BUNDLED || + (Array.isArray(bundleType) && + (bundleType.includes(BUNDLE_TYPES.ESM) || + bundleType.includes(BUNDLE_TYPES.CJS))) || + bundleType === BUNDLE_TYPES.ESM || + bundleType === BUNDLE_TYPES.CJS); + + return { isBundled, isIIFE, bundleType }; +} + diff --git a/packages/benchmark/src/constants.ts b/packages/benchmark/src/constants.ts new file mode 100644 index 0000000..1b61f70 --- /dev/null +++ b/packages/benchmark/src/constants.ts @@ -0,0 +1,16 @@ +export const BENCHMARK_CONSTANTS = { + DETECTION_INTERVAL: 1000, // Wait 1 second between detection attempts + MAX_DETECTION_TIME: 15000, // Increased to 15 seconds to accommodate longer waits + INITIAL_DETECTION_DELAY: 500, // Wait 500ms before starting + TTI_BUFFER: 1000, + METRICS_TIMEOUT: 10000, + METRICS_RETRY_TIMEOUT: 5000, +} as const; + +export const BUNDLE_TYPES = { + IIFE: 'iffe', + ESM: 'esm', + CJS: 'cjs', + BUNDLED: 'bundled', +} as const; + diff --git a/packages/benchmark/src/cookie-banner-collector.ts b/packages/benchmark/src/cookie-banner-collector.ts new file mode 100644 index 0000000..57a4c60 --- /dev/null +++ b/packages/benchmark/src/cookie-banner-collector.ts @@ -0,0 +1,229 @@ +import type { Page } from '@playwright/test'; +import type { + Config, + CookieBannerMetrics, + CookieBannerData, + LayoutShiftEntry, + WindowWithCookieMetrics, +} from './types'; +import { determineBundleStrategy } from './bundle-strategy'; + +export class CookieBannerCollector { + private config: Config; + + constructor(config: Config) { + this.config = config; + } + + /** + * Initialize cookie banner metrics tracking + */ + initializeMetrics(): CookieBannerMetrics { + const { isBundled, isIIFE } = determineBundleStrategy(this.config); + + console.log( + `🔍 [BUNDLE-STRATEGY] Detected from config: ${ + isBundled ? 'Bundled' : isIIFE ? 'IIFE' : 'Unknown' + }`, + { + bundleType: this.config.techStack?.bundleType, + isBundled, + isIIFE, + } + ); + + return { + detectionStartTime: 0, + bannerRenderTime: 0, + bannerInteractiveTime: 0, + bannerScriptLoadTime: 0, + bannerLayoutShiftImpact: 0, + bannerNetworkRequests: 0, + bannerBundleSize: 0, + bannerMainThreadBlockingTime: 0, + isBundled, + isIIFE, + bannerDetected: false, + bannerSelector: null, + }; + } + + /** + * Set up cookie banner detection script in the browser + */ + async setupDetection(page: Page): Promise { + const selectors = this.config.cookieBanner?.selectors || []; + + await page.addInitScript((selectors: string[]) => { + console.log('🔍 [BROWSER] Setting up cookie banner detection...'); + + // Store initial performance baseline + (window as unknown as WindowWithCookieMetrics).__cookieBannerMetrics = { + pageLoadStart: performance.now(), + bannerDetectionStart: 0, + bannerFirstSeen: 0, + bannerInteractive: 0, + layoutShiftsBefore: 0, + layoutShiftsAfter: 0, + detected: false, + selector: null, + }; + + // Monitor for layout shifts specifically + let cumulativeLayoutShift = 0; + if ('PerformanceObserver' in window) { + const clsObserver = new PerformanceObserver((list) => { + for (const entry of list.getEntries()) { + const layoutShiftEntry = entry as LayoutShiftEntry; + if (!layoutShiftEntry.hadRecentInput) { + cumulativeLayoutShift += layoutShiftEntry.value; + ( + window as unknown as WindowWithCookieMetrics + ).__cookieBannerMetrics.layoutShiftsAfter = cumulativeLayoutShift; + } + } + }); + clsObserver.observe({ type: 'layout-shift', buffered: true }); + } + + // Cookie banner detection logic + const detectCookieBanner = () => { + ( + window as unknown as WindowWithCookieMetrics + ).__cookieBannerMetrics.bannerDetectionStart = performance.now(); + + for (const selector of selectors) { + try { + const element = document.querySelector(selector); + if (element) { + const rect = element.getBoundingClientRect(); + const isVisible = + rect.width > 0 && + rect.height > 0 && + window.getComputedStyle(element).visibility !== 'hidden' && + window.getComputedStyle(element).display !== 'none'; + + if (isVisible) { + const metrics = (window as unknown as WindowWithCookieMetrics) + .__cookieBannerMetrics; + metrics.detected = true; + metrics.selector = selector; + metrics.bannerFirstSeen = performance.now(); + metrics.layoutShiftsBefore = cumulativeLayoutShift; + + console.log('🔍 [BANNER] Cookie banner detected:', selector); + console.log( + '🔍 [BANNER] Banner render time:', + metrics.bannerFirstSeen - metrics.pageLoadStart, + 'ms' + ); + + // Check if banner is interactive + const buttons = element.querySelectorAll( + 'button, a, [role="button"], [onclick]' + ); + if (buttons.length > 0) { + // Test if buttons are actually clickable + const firstButton = buttons[0] as HTMLElement; + if (firstButton.offsetParent !== null) { + // Element is visible and clickable + metrics.bannerInteractive = performance.now(); + console.log( + '🔍 [BANNER] Banner interactive time:', + metrics.bannerInteractive - metrics.pageLoadStart, + 'ms' + ); + } + } + + return true; + } + } + } catch (error) { + console.warn( + '🔍 [BANNER] Error checking selector:', + selector, + error + ); + } + } + return false; + }; + + // Start detection after DOM is ready + if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', () => { + setTimeout(() => { + if (!detectCookieBanner()) { + // Keep checking for dynamically loaded banners + const interval = setInterval(() => { + if (detectCookieBanner()) { + clearInterval(interval); + } + }, 100); + + // Stop checking after 10 seconds + setTimeout(() => clearInterval(interval), 10000); + } + }, 100); // Small delay to allow for initial render + }); + } else { + setTimeout(() => { + if (!detectCookieBanner()) { + const interval = setInterval(() => { + if (detectCookieBanner()) { + clearInterval(interval); + } + }, 100); + + setTimeout(() => clearInterval(interval), 10000); + } + }, 100); + } + }, selectors); + } + + /** + * Collect cookie banner specific metrics from the browser + */ + async collectMetrics(page: Page): Promise { + return page.evaluate(() => { + const metrics = (window as unknown as WindowWithCookieMetrics) + .__cookieBannerMetrics; + if (!metrics) { + return null; + } + + return { + detected: metrics.detected, + selector: metrics.selector, + bannerRenderTime: metrics.bannerFirstSeen - metrics.pageLoadStart, + bannerInteractiveTime: metrics.bannerInteractive - metrics.pageLoadStart, + bannerHydrationTime: + metrics.bannerInteractive > 0 + ? metrics.bannerInteractive - metrics.bannerFirstSeen + : 0, + layoutShiftImpact: + metrics.layoutShiftsAfter - metrics.layoutShiftsBefore, + viewportCoverage: metrics.detected + ? (() => { + if (!metrics.selector) { + return 0; + } + const element = document.querySelector(metrics.selector); + if (element) { + const rect = element.getBoundingClientRect(); + return ( + ((rect.width * rect.height) / + (window.innerWidth * window.innerHeight)) * + 100 + ); + } + return 0; + })() + : 0, + }; + }); + } +} + diff --git a/packages/benchmark/src/index.ts b/packages/benchmark/src/index.ts new file mode 100644 index 0000000..ad5fc6d --- /dev/null +++ b/packages/benchmark/src/index.ts @@ -0,0 +1,24 @@ +// Collectors +export { CookieBannerCollector } from './cookie-banner-collector'; +export { NetworkMonitor } from './network-monitor'; +export { ResourceTimingCollector } from './resource-timing-collector'; + +// Utilities +export { determineBundleStrategy } from './bundle-strategy'; +export { BENCHMARK_CONSTANTS, BUNDLE_TYPES } from './constants'; + +// Types +export type { + Config, + CookieBannerConfig, + CookieBannerMetrics, + CookieBannerData, + NetworkRequest, + NetworkMetrics, + BundleStrategy, + ResourceTimingData, + CoreWebVitals, + LayoutShiftEntry, + WindowWithCookieMetrics, +} from './types'; + diff --git a/packages/benchmark/src/network-monitor.ts b/packages/benchmark/src/network-monitor.ts new file mode 100644 index 0000000..27b80ac --- /dev/null +++ b/packages/benchmark/src/network-monitor.ts @@ -0,0 +1,121 @@ +import type { Page, Route } from '@playwright/test'; +import type { Config, NetworkRequest, NetworkMetrics } from './types'; + +export class NetworkMonitor { + private config: Config; + private networkRequests: NetworkRequest[] = []; + private metrics: NetworkMetrics = { + bannerNetworkRequests: 0, + bannerBundleSize: 0, + }; + + constructor(config: Config) { + this.config = config; + } + + /** + * Set up network request monitoring + */ + async setupMonitoring(page: Page): Promise { + await page.route('**/*', async (route: Route) => { + const request = route.request(); + const url = request.url(); + + try { + const response = await route.fetch(); + const headers = response.headers(); + + // Add timing-allow-origin header for all responses + headers['timing-allow-origin'] = '*'; + + const isScript = request.resourceType() === 'script'; + const isThirdParty = !url.includes(new URL(url).hostname); + + if (isScript) { + const contentLength = response.headers()['content-length']; + const size = contentLength ? +contentLength || 0 : 0; + + this.networkRequests.push({ + url, + size: size / 1024, // Convert to KB + duration: 0, // Will be calculated later + startTime: Date.now(), + isScript, + isThirdParty, + }); + + if (isThirdParty) { + this.metrics.bannerNetworkRequests++; + this.metrics.bannerBundleSize += size / 1024; + console.log( + `🌐 [THIRD-PARTY-SCRIPT] Detected: ${url} (${(size / 1024).toFixed(2)}KB)` + ); + } + } + + await route.fulfill({ response, headers }); + } catch { + // If we can't modify the response, just continue with the original request + await route.continue(); + } + }); + } + + /** + * Get collected network requests + */ + getNetworkRequests(): NetworkRequest[] { + return this.networkRequests; + } + + /** + * Get network metrics + */ + getMetrics(): NetworkMetrics { + return this.metrics; + } + + /** + * Calculate network impact metrics + */ + calculateNetworkImpact(): { + totalImpact: number; + totalDownloadTime: number; + thirdPartyImpact: number; + scriptImpact: number; + } { + const totalImpact = this.networkRequests.reduce( + (acc, req) => acc + req.size, + 0 + ); + const totalDownloadTime = this.networkRequests.reduce( + (acc, req) => acc + req.duration, + 0 + ); + const thirdPartyImpact = this.networkRequests + .filter((req) => req.isThirdParty) + .reduce((acc, req) => acc + req.size, 0); + const scriptImpact = this.networkRequests + .filter((req) => req.isScript) + .reduce((acc, req) => acc + req.size, 0); + + return { + totalImpact, + totalDownloadTime, + thirdPartyImpact, + scriptImpact, + }; + } + + /** + * Reset collected data + */ + reset(): void { + this.networkRequests = []; + this.metrics = { + bannerNetworkRequests: 0, + bannerBundleSize: 0, + }; + } +} + diff --git a/packages/benchmark/src/resource-timing-collector.ts b/packages/benchmark/src/resource-timing-collector.ts new file mode 100644 index 0000000..377756a --- /dev/null +++ b/packages/benchmark/src/resource-timing-collector.ts @@ -0,0 +1,177 @@ +import type { Page } from '@playwright/test'; +import type { ResourceTimingData } from './types'; + +export class ResourceTimingCollector { + /** + * Collect detailed resource timing data from the browser + */ + async collect(page: Page): Promise { + console.log('🔍 [DEBUG] Collecting resource timing data...'); + + return page.evaluate(() => { + console.log('🔍 [BROWSER] Starting resource collection...'); + + const perfEntries = performance.getEntriesByType( + 'navigation' + )[0] as PerformanceNavigationTiming; + const resourceEntries = performance.getEntriesByType( + 'resource' + ) as PerformanceResourceTiming[]; + + console.log('🔍 [BROWSER] Navigation timing:', { + navigationStart: perfEntries.startTime, + domContentLoaded: + perfEntries.domContentLoadedEventEnd - perfEntries.startTime, + loadComplete: perfEntries.loadEventEnd - perfEntries.startTime, + domInteractive: perfEntries.domInteractive - perfEntries.startTime, + }); + + console.log('🔍 [BROWSER] Found', resourceEntries.length, 'resources'); + + // Categorize resources + const scriptEntries = resourceEntries.filter( + (entry) => entry.initiatorType === 'script' + ); + const styleEntries = resourceEntries.filter( + (entry) => entry.initiatorType === 'link' && entry.name.endsWith('.css') + ); + const imageEntries = resourceEntries.filter( + (entry) => entry.initiatorType === 'img' + ); + const fontEntries = resourceEntries.filter( + (entry) => entry.initiatorType === 'font' + ); + const otherEntries = resourceEntries.filter( + (entry) => + !['script', 'link', 'img', 'font'].includes(entry.initiatorType) + ); + + console.log('🔍 [BROWSER] Resource breakdown:', { + scripts: scriptEntries.length, + styles: styleEntries.length, + images: imageEntries.length, + fonts: fontEntries.length, + other: otherEntries.length, + }); + + // Calculate sizes + const calculateSize = (entries: PerformanceResourceTiming[]) => { + const total = + entries.reduce((acc, entry) => { + const size = entry.transferSize || entry.encodedBodySize || 0; + return acc + size; + }, 0) / 1024; + return total; + }; + + const navigationStart = perfEntries.startTime; + const domContentLoaded = + perfEntries.domContentLoadedEventEnd - navigationStart; + const load = perfEntries.loadEventEnd - navigationStart; + + console.log('🔍 [BROWSER] Calculated timings:', { + navigationStart, + domContentLoaded, + load, + }); + + return { + timing: { + navigationStart, + domContentLoaded, + load, + scripts: { + bundled: { + loadStart: 0, + loadEnd: scriptEntries.reduce((acc, entry) => acc + entry.duration, 0), + executeStart: 0, + executeEnd: 0, + }, + thirdParty: { + loadStart: 0, + loadEnd: scriptEntries.reduce((acc, entry) => acc + entry.duration, 0), + executeStart: 0, + executeEnd: 0, + }, + }, + }, + size: { + total: calculateSize(resourceEntries), + bundled: calculateSize( + scriptEntries.filter((e) => e.name.includes(window.location.hostname)) + ), + thirdParty: calculateSize( + scriptEntries.filter((e) => !e.name.includes(window.location.hostname)) + ), + cookieServices: 0, // Will be calculated later + scripts: { + total: calculateSize(scriptEntries), + initial: calculateSize( + scriptEntries.filter((e) => e.startTime < domContentLoaded) + ), + dynamic: calculateSize( + scriptEntries.filter((e) => e.startTime >= domContentLoaded) + ), + thirdParty: calculateSize( + scriptEntries.filter((e) => !e.name.includes(window.location.hostname)) + ), + cookieServices: 0, // Will be calculated later + }, + styles: calculateSize(styleEntries), + images: calculateSize(imageEntries), + fonts: calculateSize(fontEntries), + other: calculateSize(otherEntries), + }, + resources: { + scripts: scriptEntries.map((entry) => ({ + name: entry.name, + size: entry.transferSize ? entry.transferSize / 1024 : 0, + duration: entry.duration, + startTime: entry.startTime - navigationStart, + isThirdParty: !entry.name.includes(window.location.hostname), + isDynamic: entry.startTime >= domContentLoaded, + isCookieService: false, + dnsTime: entry.domainLookupEnd - entry.domainLookupStart, + connectionTime: entry.connectEnd - entry.connectStart, + })), + styles: styleEntries.map((entry) => ({ + name: entry.name, + size: entry.transferSize ? entry.transferSize / 1024 : 0, + duration: entry.duration, + startTime: entry.startTime - navigationStart, + isThirdParty: !entry.name.includes(window.location.hostname), + isCookieService: false, + })), + images: imageEntries.map((entry) => ({ + name: entry.name, + size: entry.transferSize ? entry.transferSize / 1024 : 0, + duration: entry.duration, + startTime: entry.startTime - navigationStart, + isThirdParty: !entry.name.includes(window.location.hostname), + isCookieService: false, + })), + fonts: fontEntries.map((entry) => ({ + name: entry.name, + size: entry.transferSize ? entry.transferSize / 1024 : 0, + duration: entry.duration, + startTime: entry.startTime - navigationStart, + isThirdParty: !entry.name.includes(window.location.hostname), + isCookieService: false, + })), + other: otherEntries.map((entry) => ({ + name: entry.name, + size: entry.transferSize ? entry.transferSize / 1024 : 0, + duration: entry.duration, + startTime: entry.startTime - navigationStart, + isThirdParty: !entry.name.includes(window.location.hostname), + isCookieService: false, + type: entry.initiatorType, + })), + }, + language: 'en', + duration: load, + }; + }); + } +} + diff --git a/packages/benchmark/src/types.ts b/packages/benchmark/src/types.ts new file mode 100644 index 0000000..a527e74 --- /dev/null +++ b/packages/benchmark/src/types.ts @@ -0,0 +1,226 @@ +import type { Page } from '@playwright/test'; + +// Config types +export interface CookieBannerConfig { + selectors: string[]; + serviceHosts: string[]; + waitForVisibility: boolean; + measureViewportCoverage: boolean; + expectedLayoutShift: boolean; + serviceName: string; +} + +export interface Config { + name: string; + url?: string; + testId?: string; + id?: string; + iterations: number; + baseline?: boolean; + custom?: (page: Page) => Promise; + remote?: { + enabled?: boolean; + url?: string; + headers?: Record; + }; + cookieBanner: CookieBannerConfig; + internationalization: { + detection: string; + stringLoading: string; + }; + techStack: { + bundler: string; + bundleType: string | string[]; + frameworks: string[]; + languages: string[]; + packageManager: string; + typescript: boolean; + }; + source: { + github: string | false; + isOpenSource: boolean | string; + license: string; + npm: string | false; + website?: string; + }; + includes: { + backend: string | string[] | false; + components: string[]; + }; + company?: { + name: string; + website: string; + avatar: string; + }; + tags?: string[]; +} + +// Performance API type definitions +export interface LayoutShiftEntry extends PerformanceEntry { + value: number; + hadRecentInput: boolean; +} + +// Cookie banner types +export interface WindowWithCookieMetrics extends Window { + __cookieBannerMetrics: { + pageLoadStart: number; + bannerDetectionStart: number; + bannerFirstSeen: number; + bannerInteractive: number; + layoutShiftsBefore: number; + layoutShiftsAfter: number; + detected: boolean; + selector: string | null; + }; +} + +export interface CookieBannerMetrics { + detectionStartTime: number; + bannerRenderTime: number; + bannerInteractiveTime: number; + bannerScriptLoadTime: number; + bannerLayoutShiftImpact: number; + bannerNetworkRequests: number; + bannerBundleSize: number; + bannerMainThreadBlockingTime: number; + isBundled: boolean; + isIIFE: boolean; + bannerDetected: boolean; + bannerSelector: string | null; +} + +export interface CookieBannerData { + detected: boolean; + selector: string | null; + bannerRenderTime: number; + bannerInteractiveTime: number; + bannerHydrationTime: number; + layoutShiftImpact: number; + viewportCoverage: number; +} + +// Network types +export interface NetworkRequest { + url: string; + size: number; + duration: number; + startTime: number; + isScript: boolean; + isThirdParty: boolean; +} + +export interface NetworkMetrics { + bannerNetworkRequests: number; + bannerBundleSize: number; +} + +// Bundle strategy types +export interface BundleStrategy { + isBundled: boolean; + isIIFE: boolean; + bundleType: string | string[] | undefined; +} + +// Resource timing types +export interface ResourceTimingData { + timing: { + navigationStart: number; + domContentLoaded: number; + load: number; + scripts: { + bundled: { + loadStart: number; + loadEnd: number; + executeStart: number; + executeEnd: number; + }; + thirdParty: { + loadStart: number; + loadEnd: number; + executeStart: number; + executeEnd: number; + }; + }; + }; + size: { + total: number; + bundled: number; + thirdParty: number; + cookieServices: number; + scripts: { + total: number; + initial: number; + dynamic: number; + thirdParty: number; + cookieServices: number; + }; + styles: number; + images: number; + fonts: number; + other: number; + }; + resources: { + scripts: Array<{ + name: string; + size: number; + duration: number; + startTime: number; + isThirdParty: boolean; + isDynamic: boolean; + isCookieService: boolean; + dnsTime: number; + connectionTime: number; + }>; + styles: Array<{ + name: string; + size: number; + duration: number; + startTime: number; + isThirdParty: boolean; + isCookieService: boolean; + }>; + images: Array<{ + name: string; + size: number; + duration: number; + startTime: number; + isThirdParty: boolean; + isCookieService: boolean; + }>; + fonts: Array<{ + name: string; + size: number; + duration: number; + startTime: number; + isThirdParty: boolean; + isCookieService: boolean; + }>; + other: Array<{ + name: string; + size: number; + duration: number; + startTime: number; + isThirdParty: boolean; + isCookieService: boolean; + type: string; + }>; + }; + language: string; + duration: number; +} + +// Core web vitals types +export interface CoreWebVitals { + paint?: { + firstPaint?: number; + firstContentfulPaint?: number; + }; + largestContentfulPaint?: number; + cumulativeLayoutShift?: number; + totalBlockingTime?: number; + domCompleteTiming?: number; + pageloadTiming?: number; + totalBytes?: number; +} + diff --git a/packages/benchmark/tsconfig.json b/packages/benchmark/tsconfig.json new file mode 100644 index 0000000..3534f4f --- /dev/null +++ b/packages/benchmark/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "ESNext", + "moduleResolution": "bundler", + "esModuleInterop": true, + "strict": true, + "skipLibCheck": true, + "declaration": true, + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"] +} + diff --git a/packages/cli/package.json b/packages/cli/package.json deleted file mode 100644 index 900fef2..0000000 --- a/packages/cli/package.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "name": "@cookiebench/cli", - "private": true, - "version": "0.0.0", - "type": "module", - "exports": "./dist/index.mjs", - "main": "./dist/index.mjs", - "module": "dist/index.mjs", - "bin": { - "benchmark-cli": "dist/index.mjs" - }, - "scripts": { - "build": "rslib build", - "check-types": "tsc --noEmit", - "dev": "rslib build --watch", - "lint": "biome lint .", - "format": "biome format . --write", - "start": "node ./dist/index.mjs" - }, - "dependencies": { - "@clack/prompts": "^1.0.0-alpha.0", - "@playwright/test": "^1.42.1", - "cli-table3": "^0.6.3", - "dotenv": "^16.5.0", - "package-manager-detector": "^1.3.0", - "picocolors": "^1.0.0" - }, - "devDependencies": { - "@rsdoctor/rspack-plugin": "^1.1.3", - "@rslib/core": "^0.9.1", - "@types/node": "^22.15.30", - "playwright-performance-metrics": "^1.2.2", - "typescript": "^5.8.3" - } -} \ No newline at end of file diff --git a/packages/cli/src/commands/benchmark/benchmark-runner.ts b/packages/cli/src/commands/benchmark/benchmark-runner.ts deleted file mode 100644 index a1f2568..0000000 --- a/packages/cli/src/commands/benchmark/benchmark-runner.ts +++ /dev/null @@ -1,452 +0,0 @@ -import { chromium, type Page } from "@playwright/test"; -import { readConfig } from "../../utils"; -import { buildAndServeNextApp, cleanupServer } from "../../lib/server"; -import { PerformanceMetricsCollector } from "playwright-performance-metrics"; -import { writeFile } from "node:fs/promises"; -import { join } from "node:path"; -import type { ServerInfo } from "../../types"; - -import type { - Config, - BenchmarkResult, - BenchmarkDetails, - CookieBannerMetrics, -} from "./types"; -import { BENCHMARK_CONSTANTS } from "./constants"; -import { determineBundleStrategy } from "./bundle-strategy"; -import { NetworkMonitor } from "./network-monitor"; -import { CookieBannerDetector } from "./cookie-banner-detector"; -import { ResourceCollector } from "./resource-collector"; -import { MetricsCalculator } from "./metrics-calculator"; -import { calculateScores, printScores } from "../../utils/scoring"; -import type { RawBenchmarkDetail } from "../results"; - -async function runBenchmark( - page: Page, - url: string, - config: Config -): Promise { - console.log(`🔍 [DEBUG] Starting cookie banner benchmark for: ${url}`); - console.log( - "🔍 [DEBUG] Cookie banner selectors:", - config.cookieBanner?.selectors || [] - ); - console.log( - "🔍 [DEBUG] Bundle type from config:", - config.techStack?.bundleType - ); - - // Set custom headers if provided for remote benchmarking - if (config.remote?.enabled && config.remote.headers) { - console.log("🔍 [DEBUG] Setting custom headers:", config.remote.headers); - await page.setExtraHTTPHeaders(config.remote.headers); - } - - // Initialize components - const collector = new PerformanceMetricsCollector(); - const networkMonitor = new NetworkMonitor(); - const bannerDetector = new CookieBannerDetector(); - const resourceCollector = new ResourceCollector(); - const metricsCalculator = new MetricsCalculator(); - - // Determine bundle strategy - const bundleStrategy = determineBundleStrategy(config); - console.log( - `🔍 [BUNDLE-STRATEGY] Detected from config: ${ - bundleStrategy.isBundled - ? "Bundled" - : bundleStrategy.isIIFE - ? "IIFE" - : "Unknown" - }`, - { - bundleType: bundleStrategy.bundleType, - isBundled: bundleStrategy.isBundled, - isIIFE: bundleStrategy.isIIFE, - } - ); - - // Initialize cookie banner metrics - const cookieBannerMetrics: CookieBannerMetrics = { - detectionStartTime: 0, - bannerRenderTime: 0, - bannerInteractiveTime: 0, - bannerScriptLoadTime: 0, - bannerLayoutShiftImpact: 0, - bannerNetworkRequests: 0, - bannerBundleSize: 0, - bannerMainThreadBlockingTime: 0, - isBundled: bundleStrategy.isBundled, - isIIFE: bundleStrategy.isIIFE, - bannerDetected: false, - bannerSelector: null, - }; - - // Enable console log capture - page.on('console', (msg) => { - const text = msg.text(); - if (text.includes('🔍')) { - console.log(`[BROWSER] ${text}`); - } - }); - - // Setup monitoring - await networkMonitor.setupRequestMonitoring(page, cookieBannerMetrics); - await bannerDetector.setupDetection(page, config); - - console.log(`🔍 [DEBUG] Navigating to: ${url}`); - await page.goto(url, { waitUntil: "networkidle" }); - - // Collect core web vitals - console.log("🔍 [DEBUG] Collecting core web vitals..."); - const coreWebVitals = await collector.collectMetrics(page, { - timeout: BENCHMARK_CONSTANTS.METRICS_TIMEOUT, - retryTimeout: BENCHMARK_CONSTANTS.METRICS_RETRY_TIMEOUT, - }); - - console.log("🔍 [DEBUG] Core web vitals collected:", { - fcp: coreWebVitals.paint?.firstContentfulPaint, - lcp: coreWebVitals.largestContentfulPaint, - cls: coreWebVitals.cumulativeLayoutShift, - tbt: coreWebVitals.totalBlockingTime, - domComplete: coreWebVitals.domCompleteTiming, - pageLoad: coreWebVitals.pageloadTiming, - totalBytes: coreWebVitals.totalBytes, - }); - - // Collect cookie banner data - const cookieBannerData = await bannerDetector.collectBannerData(page); - console.log("🔍 [DEBUG] Cookie banner metrics:", cookieBannerData); - - // Collect resource timing - console.log("🔍 [DEBUG] Collecting resource timing data..."); - const resourceMetrics = await resourceCollector.collectResourceTiming(page); - - // Calculate TTI - const tti = metricsCalculator.calculateTTI(coreWebVitals, cookieBannerData); - - // Get network impact data - const networkRequests = networkMonitor.getNetworkRequests(); - - // Merge all metrics - const finalMetrics = metricsCalculator.mergeBenchmarkMetrics( - resourceMetrics, - coreWebVitals, - cookieBannerData, - cookieBannerMetrics, - networkRequests, - config, - tti - ); - - // Log final results - metricsCalculator.logFinalResults( - finalMetrics, - cookieBannerMetrics, - bundleStrategy.bundleType - ); - - // Cleanup - await collector.cleanup(); - networkMonitor.reset(); - - return finalMetrics; -} - -async function runBenchmarks( - serverUrl: string, - config: Config -): Promise { - const browser = await chromium.launch({ - headless: true, // Keep headless mode for stability - }); - const page = await browser.newPage(); - const results: BenchmarkDetails[] = []; - - try { - for (let i = 0; i < config.iterations; i++) { - console.log( - `[Benchmark] Running iteration ${i + 1}/${config.iterations}...` - ); - const result = await runBenchmark(page, serverUrl, config); - results.push(result); - } - } finally { - await browser.close(); - } - - // Create app data for transparency scoring - const appData = { - name: config.name, - baseline: config.baseline || false, - company: config.company ? JSON.stringify(config.company) : null, - techStack: JSON.stringify(config.techStack), - source: config.source ? JSON.stringify(config.source) : null, - tags: config.tags ? JSON.stringify(config.tags) : null, - }; - - // Calculate scores - const scores = calculateScores( - { - fcp: results.reduce((acc, curr) => acc + curr.timing.firstContentfulPaint, 0) / results.length, - lcp: results.reduce((acc, curr) => acc + curr.timing.largestContentfulPaint, 0) / results.length, - cls: results.reduce((acc, curr) => acc + curr.timing.cumulativeLayoutShift, 0) / results.length, - tbt: results.reduce((acc, curr) => acc + curr.timing.mainThreadBlocking.total, 0) / results.length, - tti: results.reduce((acc, curr) => acc + curr.timing.timeToInteractive, 0) / results.length, - }, - { - totalSize: results.reduce((acc, curr) => acc + curr.size.total, 0) / results.length, - jsSize: results.reduce((acc, curr) => acc + curr.size.scripts.total, 0) / results.length, - cssSize: results.reduce((acc, curr) => acc + curr.size.styles, 0) / results.length, - imageSize: results.reduce((acc, curr) => acc + curr.size.images, 0) / results.length, - fontSize: results.reduce((acc, curr) => acc + curr.size.fonts, 0) / results.length, - otherSize: results.reduce((acc, curr) => acc + curr.size.other, 0) / results.length, - }, - { - totalRequests: results.reduce((acc, curr) => - acc + (curr.resources.scripts.length + curr.resources.styles.length + - curr.resources.images.length + curr.resources.fonts.length + - curr.resources.other.length), 0) / results.length, - thirdPartyRequests: results.reduce((acc, curr) => - acc + curr.resources.scripts.filter(s => s.isThirdParty).length, 0) / results.length, - thirdPartySize: results.reduce((acc, curr) => acc + curr.size.thirdParty, 0) / results.length, - thirdPartyDomains: 5, // Default value - }, - { - cookieBannerDetected: (() => { - // Require consistent detection across ALL iterations for true positive - const allDetected = results.every(r => r.cookieBanner.detected); - if (!allDetected) { - console.log("⚠️ [SCORING] Banner detection inconsistent or failed - marking as not detected"); - } - return allDetected; - })(), - cookieBannerTiming: (() => { - // If no banners detected across any iteration, heavily penalize - const detectionSuccess = results.some(r => r.cookieBanner.detected); - if (!detectionSuccess) { - console.log("⚠️ [SCORING] No banner detected in any iteration - applying penalty"); - return null; // This signals failed detection for scoring - } - - // Check if any results have null timing (undetected banners) - const timingValues = results.map(r => r.cookieBanner.visibilityTime); - const hasNullValues = timingValues.some(t => t === null || t === 0); - - // If we have mixed results (some detected, some not), still penalize - if (hasNullValues) { - console.log("⚠️ [SCORING] Inconsistent banner detection - applying penalty"); - return null; - } - - // Only return actual timing if all iterations successfully detected banner - const validTimings = timingValues.filter((t): t is number => t !== null && t > 0); - return validTimings.length === results.length && validTimings.length > 0 - ? validTimings.reduce((acc, curr) => acc + curr, 0) / validTimings.length - : null; - })(), - cookieBannerCoverage: (() => { - // Only calculate coverage if banner was consistently detected - const detectionSuccess = results.every(r => r.cookieBanner.detected); - if (!detectionSuccess) { - console.log("⚠️ [SCORING] Inconsistent detection - setting coverage to 0"); - return 0; // No coverage score if detection failed - } - return results.reduce((acc, curr) => acc + curr.cookieBanner.viewportCoverage, 0) / results.length / 100; - })(), - }, - { - domSize: 1500, // Default value - mainThreadBlocking: results.reduce((acc, curr) => acc + curr.timing.mainThreadBlocking.total, 0) / results.length, - layoutShifts: results.reduce((acc, curr) => acc + curr.timing.cumulativeLayoutShift, 0) / results.length, - }, - config.baseline || false, - appData - ); - - return { - name: config.name, - baseline: config.baseline || false, - techStack: config.techStack, - source: config.source, - includes: config.includes, - company: config.company, - tags: config.tags, - details: results, - average: { - firstContentfulPaint: - results.reduce( - (acc, curr) => acc + curr.timing.firstContentfulPaint, - 0 - ) / results.length, - largestContentfulPaint: - results.reduce( - (acc, curr) => acc + curr.timing.largestContentfulPaint, - 0 - ) / results.length, - timeToInteractive: - results.reduce((acc, curr) => acc + curr.timing.timeToInteractive, 0) / - results.length, - totalBlockingTime: - results.reduce( - (acc, curr) => acc + curr.timing.mainThreadBlocking.total, - 0 - ) / results.length, - speedIndex: 0, // Default value - timeToFirstByte: 0, // Default value - firstInputDelay: 0, // Default value - cumulativeLayoutShift: - results.reduce( - (acc, curr) => acc + curr.timing.cumulativeLayoutShift, - 0 - ) / results.length, - domSize: 0, // Default value - totalRequests: - results.reduce( - (acc, curr) => - acc + - (curr.resources.scripts.length + - curr.resources.styles.length + - curr.resources.images.length + - curr.resources.fonts.length + - curr.resources.other.length), - 0 - ) / results.length, - totalSize: - results.reduce((acc, curr) => acc + curr.size.total, 0) / - results.length, - jsSize: - results.reduce((acc, curr) => acc + curr.size.scripts.total, 0) / - results.length, - cssSize: - results.reduce((acc, curr) => acc + curr.size.styles, 0) / results.length, - imageSize: - results.reduce((acc, curr) => acc + curr.size.images, 0) / results.length, - fontSize: - results.reduce((acc, curr) => acc + curr.size.fonts, 0) / results.length, - otherSize: - results.reduce((acc, curr) => acc + curr.size.other, 0) / results.length, - thirdPartyRequests: - results.reduce( - (acc, curr) => acc + curr.resources.scripts.filter(s => s.isThirdParty).length, - 0 - ) / results.length, - thirdPartySize: - results.reduce((acc, curr) => acc + curr.size.thirdParty, 0) / results.length, - thirdPartyDomains: 0, // Default value - thirdPartyCookies: 0, // Default value - thirdPartyLocalStorage: 0, // Default value - thirdPartySessionStorage: 0, // Default value - thirdPartyIndexedDB: 0, // Default value - thirdPartyCache: 0, // Default value - thirdPartyServiceWorkers: 0, // Default value - thirdPartyWebWorkers: 0, // Default value - thirdPartyWebSockets: 0, // Default value - thirdPartyBeacons: 0, // Default value - thirdPartyFetch: 0, // Default value - thirdPartyXHR: 0, // Default value - thirdPartyScripts: 0, // Default value - thirdPartyStyles: 0, // Default value - thirdPartyImages: 0, // Default value - thirdPartyFonts: 0, // Default value - thirdPartyMedia: 0, // Default value - thirdPartyOther: 0, // Default value - thirdPartyTiming: { - total: 0, - blocking: 0, - dns: 0, - connect: 0, - ssl: 0, - send: 0, - wait: 0, - receive: 0, - }, - cookieBannerTiming: { - firstPaint: 0, - firstContentfulPaint: - results.reduce( - (acc, curr) => acc + curr.timing.firstContentfulPaint, - 0 - ) / results.length, - domContentLoaded: - results.reduce((acc, curr) => acc + curr.timing.domContentLoaded, 0) / - results.length, - load: - results.reduce((acc, curr) => acc + curr.timing.load, 0) / - results.length, - }, - }, - scores, - }; -} - -export async function benchmarkCommand(appPath?: string): Promise { - try { - const config = await readConfig(appPath); - if (!config) { - throw new Error("Failed to read config.json"); - } - - let serverInfo: ServerInfo | null = null; - let benchmarkUrl: string; - - // Check if remote benchmarking is enabled - if (config.remote?.enabled && config.remote.url) { - console.log(`🌐 Running remote benchmark against: ${config.remote.url}`); - benchmarkUrl = config.remote.url; - } else { - console.log("🏗️ Building and serving app locally..."); - serverInfo = await buildAndServeNextApp(appPath); - benchmarkUrl = serverInfo.url; - } - - const cwd = appPath || process.cwd(); - - try { - const result = await runBenchmarks(benchmarkUrl, config); - - // Format results for results.json - const resultsData = { - app: config.name, - techStack: config.techStack, - source: config.source, - includes: config.includes, - internationalization: config.internationalization, - company: config.company, - tags: config.tags, - results: result.details, - scores: result.scores, - metadata: { - timestamp: new Date().toISOString(), - iterations: config.iterations, - languages: config.techStack.languages, - isRemote: config.remote?.enabled || false, - url: config.remote?.enabled ? config.remote.url : undefined, - }, - }; - - // Write results to file - const outputPath = join(cwd, "results.json"); - await writeFile(outputPath, JSON.stringify(resultsData, null, 2)); - console.log(`✅ Benchmark results saved to ${outputPath}`); - - // Print scores if available - if (result.scores) { - console.log("📊 Benchmark Scores:"); - printScores(result.scores); - } - } finally { - // Only cleanup server if we started one - if (serverInfo) { - await cleanupServer(serverInfo); - } - } - } catch (error: unknown) { - if (error instanceof Error) { - console.error(`Error running benchmark: ${error.message}`); - } else { - console.error("An unknown error occurred during benchmark"); - } - process.exit(1); - } -} diff --git a/packages/cli/src/commands/benchmark/bundle-strategy.ts b/packages/cli/src/commands/benchmark/bundle-strategy.ts deleted file mode 100644 index 9da257b..0000000 --- a/packages/cli/src/commands/benchmark/bundle-strategy.ts +++ /dev/null @@ -1,22 +0,0 @@ -import type { Config } from "../../types"; -import type { BundleStrategy } from "./types"; -import { BUNDLE_TYPES } from "./constants"; - -export function determineBundleStrategy(config: Config): BundleStrategy { - const bundleType = config.techStack?.bundleType; - - const isIIFE = - bundleType === BUNDLE_TYPES.IIFE || - (Array.isArray(bundleType) && bundleType.includes(BUNDLE_TYPES.IIFE)); - - const isBundled = - !isIIFE && - (bundleType === BUNDLE_TYPES.BUNDLED || - (Array.isArray(bundleType) && - (bundleType.includes(BUNDLE_TYPES.ESM) || - bundleType.includes(BUNDLE_TYPES.CJS))) || - bundleType === BUNDLE_TYPES.ESM || - bundleType === BUNDLE_TYPES.CJS); - - return { isBundled, isIIFE, bundleType }; -} diff --git a/packages/cli/src/commands/benchmark/constants.ts b/packages/cli/src/commands/benchmark/constants.ts deleted file mode 100644 index 5cf8bc0..0000000 --- a/packages/cli/src/commands/benchmark/constants.ts +++ /dev/null @@ -1,15 +0,0 @@ -export const BENCHMARK_CONSTANTS = { - DETECTION_INTERVAL: 1000, // Wait 1 second between detection attempts - MAX_DETECTION_TIME: 15000, // Increased to 15 seconds to accommodate longer waits - INITIAL_DETECTION_DELAY: 500, // Wait 500ms before starting - TTI_BUFFER: 1000, - METRICS_TIMEOUT: 10000, - METRICS_RETRY_TIMEOUT: 5000, -} as const; - -export const BUNDLE_TYPES = { - IIFE: "iffe", - ESM: "esm", - CJS: "cjs", - BUNDLED: "bundled", -} as const; diff --git a/packages/cli/src/commands/benchmark/cookie-banner-detector.ts b/packages/cli/src/commands/benchmark/cookie-banner-detector.ts deleted file mode 100644 index bb04e2a..0000000 --- a/packages/cli/src/commands/benchmark/cookie-banner-detector.ts +++ /dev/null @@ -1,293 +0,0 @@ -import type { Page } from "@playwright/test"; -import type { Config } from "../../types"; -import type { - WindowWithCookieMetrics, - CookieBannerData, - LayoutShiftEntry, -} from "./types"; -import { BENCHMARK_CONSTANTS } from "./constants"; - -export class CookieBannerDetector { - async setupDetection(page: Page, config: Config): Promise { - await page.addInitScript( - (params: { - selectors: string[]; - constants: typeof BENCHMARK_CONSTANTS; - }) => { - const { selectors, constants } = params; - console.log("🔍 [BROWSER] Setting up cookie banner detection..."); - - - // Store initial performance baseline - (window as unknown as WindowWithCookieMetrics).__cookieBannerMetrics = { - pageLoadStart: performance.now(), - bannerDetectionStart: 0, - bannerFirstSeen: 0, - bannerInteractive: 0, - layoutShiftsBefore: 0, - layoutShiftsAfter: 0, - detected: false, - selector: null, - }; - - // Monitor for layout shifts specifically - let cumulativeLayoutShift = 0; - if ("PerformanceObserver" in window) { - const clsObserver = new PerformanceObserver((list) => { - for (const entry of list.getEntries()) { - const layoutShiftEntry = entry as LayoutShiftEntry; - if (!layoutShiftEntry.hadRecentInput) { - cumulativeLayoutShift += layoutShiftEntry.value; - ( - window as unknown as WindowWithCookieMetrics - ).__cookieBannerMetrics.layoutShiftsAfter = - cumulativeLayoutShift; - } - } - }); - clsObserver.observe({ type: "layout-shift", buffered: true }); - } - - // Cookie banner detection logic - const detectCookieBanner = (): boolean => { - ( - window as unknown as WindowWithCookieMetrics - ).__cookieBannerMetrics.bannerDetectionStart = performance.now(); - - console.log("🔍 [BANNER] Starting detection check..."); - - for (const selector of selectors) { - try { - const element = document.querySelector(selector); - console.log(`🔍 [BANNER] Checking selector "${selector}":`, element ? "found" : "not found"); - - if (element) { - // First check if element is immediately visible - let isVisible = false; - let recheckAttempts = 0; - const maxRecheckAttempts = 10; // 10 attempts * 100ms = 1 second max - - // Function to check visibility - const checkElementVisibility = () => { - const rect = element.getBoundingClientRect(); - const computedStyle = window.getComputedStyle(element); - const hasContent = rect.width > 100 && rect.height > 0; - const hasContainer = rect.width > 200 && computedStyle.display !== "none" && computedStyle.visibility !== "hidden"; - return hasContent || hasContainer; - }; - // Check visibility immediately - isVisible = checkElementVisibility(); - - // If element exists but not visible, wait and recheck up to 1 second - while (!isVisible && recheckAttempts < maxRecheckAttempts) { - const rect = element.getBoundingClientRect(); - console.log(`🔍 [BANNER] Element "${selector}" found but not visible yet (${rect.width}x${rect.height}), waiting 100ms (attempt ${recheckAttempts + 1}/${maxRecheckAttempts})`); - - // Wait 100ms synchronously (not ideal but necessary for this detection pattern) - const start = performance.now(); - while (performance.now() - start < 100) { - // Busy wait for 100ms - } - - recheckAttempts++; - isVisible = checkElementVisibility(); - } - - const rect = element.getBoundingClientRect(); - console.log(`🔍 [BANNER] Final visibility check for "${selector}":`, { - width: rect.width, - height: rect.height, - isVisible, - recheckAttempts - }); - - if (isVisible) { - const metrics = (window as unknown as WindowWithCookieMetrics) - .__cookieBannerMetrics; - metrics.detected = true; - metrics.selector = selector; - metrics.bannerFirstSeen = performance.now(); - metrics.layoutShiftsBefore = cumulativeLayoutShift; - - console.log("🔍 [BANNER] Cookie banner detected:", selector); - console.log( - "🔍 [BANNER] Banner render time:", - metrics.bannerFirstSeen - metrics.pageLoadStart, - "ms" - ); - - // Check if banner is interactive - const buttons = element.querySelectorAll( - 'button, a, [role="button"], [onclick]' - ); - if (buttons.length > 0) { - // Test if buttons are actually clickable - const firstButton = buttons[0] as HTMLElement; - if (firstButton.offsetParent !== null) { - // Element is visible and clickable - metrics.bannerInteractive = performance.now(); - console.log( - "🔍 [BANNER] Banner interactive time:", - metrics.bannerInteractive - metrics.pageLoadStart, - "ms" - ); - } - } - - return true; - } - } - } catch (error) { - console.warn( - "🔍 [BANNER] Error checking selector:", - selector, - error - ); - } - } - - console.log("🔍 [BANNER] No visible banner found in this check"); - return false; - }; - - // Enhanced detection for async-loaded banners - const startDetection = (): void => { - let detectionInterval: ReturnType; - let isDetected = false; - let attemptCount = 0; - - const runDetection = () => { - attemptCount++; - console.log(`🔍 [DETECTION] Attempt ${attemptCount} - Looking for cookie banner...`); - - if (!isDetected && detectCookieBanner()) { - isDetected = true; - console.log(`🔍 [DETECTION] Banner found on attempt ${attemptCount}!`); - if (detectionInterval) clearInterval(detectionInterval); - if (mutationObserver) mutationObserver.disconnect(); - } else if (!isDetected) { - console.log(`🔍 [DETECTION] Attempt ${attemptCount} - No banner found, will retry in ${constants.DETECTION_INTERVAL}ms`); - } - }; - - // Initial detection attempt - setTimeout(() => { - runDetection(); - - if (!isDetected) { - // Keep checking for dynamically loaded banners - detectionInterval = setInterval(runDetection, constants.DETECTION_INTERVAL); - - // Stop checking after max detection time - setTimeout(() => { - if (!isDetected) { - console.log(`🔍 [DETECTION] Giving up after ${constants.MAX_DETECTION_TIME}ms and ${attemptCount} attempts`); - } - if (detectionInterval) clearInterval(detectionInterval); - if (mutationObserver) mutationObserver.disconnect(); - }, constants.MAX_DETECTION_TIME); - } - }, constants.INITIAL_DETECTION_DELAY); - - // Enhanced: Watch for DOM changes for async-loaded content - let mutationObserver: MutationObserver | null = null; - if ('MutationObserver' in window) { - mutationObserver = new MutationObserver((mutations) => { - if (isDetected) return; - - console.log("🔍 [MUTATION] DOM mutations detected:", mutations.length); - - for (const mutation of mutations) { - if (mutation.type === 'childList' && mutation.addedNodes.length > 0) { - console.log("🔍 [MUTATION] Nodes added:", mutation.addedNodes.length); - - // Check if any added nodes might be our banner - for (const node of mutation.addedNodes) { - if (node.nodeType === Node.ELEMENT_NODE) { - const element = node as Element; - console.log("🔍 [MUTATION] Added element:", element.tagName, element.className, element.id); - - // Check if this element or its children match our selectors - for (const selector of selectors) { - if (element.matches?.(selector)) { - console.log("🔍 [MUTATION] Found matching element for selector:", selector); - setTimeout(runDetection, 50); // Small delay to ensure rendering - return; - } - if (element.querySelector?.(selector)) { - console.log("🔍 [MUTATION] Found child matching selector:", selector); - setTimeout(runDetection, 50); - return; - } - } - } - } - } - } - }); - - mutationObserver.observe(document.body, { - childList: true, - subtree: true, - attributes: false - }); - } - - }; - - if (document.readyState === "loading") { - document.addEventListener("DOMContentLoaded", startDetection); - } else { - startDetection(); - } - }, - { - selectors: config.cookieBanner?.selectors || [], - constants: BENCHMARK_CONSTANTS, - } - ); - } - - async collectBannerData(page: Page): Promise { - return page.evaluate(() => { - const metrics = (window as unknown as WindowWithCookieMetrics) - .__cookieBannerMetrics; - - if (!metrics) { - return null; - } - - return { - detected: metrics.detected, - selector: metrics.selector, - bannerRenderTime: metrics.bannerFirstSeen - metrics.pageLoadStart, - bannerInteractiveTime: - metrics.bannerInteractive - metrics.pageLoadStart, - bannerHydrationTime: - metrics.bannerInteractive > 0 - ? metrics.bannerInteractive - metrics.bannerFirstSeen - : 0, - layoutShiftImpact: - metrics.layoutShiftsAfter - metrics.layoutShiftsBefore, - viewportCoverage: metrics.detected - ? (() => { - if (!metrics.selector) { - return 0; - } - - const element = document.querySelector(metrics.selector); - if (element) { - const rect = element.getBoundingClientRect(); - return ( - ((rect.width * rect.height) / - (window.innerWidth * window.innerHeight)) * - 100 - ); - } - return 0; - })() - : 0, - }; - }); - } -} diff --git a/packages/cli/src/commands/benchmark/index.ts b/packages/cli/src/commands/benchmark/index.ts deleted file mode 100644 index 1cc5025..0000000 --- a/packages/cli/src/commands/benchmark/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export { benchmarkCommand } from "./benchmark-runner"; -export * from "./types"; -export * from "./constants"; diff --git a/packages/cli/src/commands/benchmark/metrics-calculator.ts b/packages/cli/src/commands/benchmark/metrics-calculator.ts deleted file mode 100644 index 2af5b88..0000000 --- a/packages/cli/src/commands/benchmark/metrics-calculator.ts +++ /dev/null @@ -1,155 +0,0 @@ -import type { - BenchmarkDetails, - CookieBannerData, - CookieBannerMetrics, - ResourceTimingData, - NetworkRequest, - Config, -} from "./types"; -import { BENCHMARK_CONSTANTS } from "./constants"; - -interface CoreWebVitals { - paint?: { - firstPaint?: number; - firstContentfulPaint?: number; - }; - largestContentfulPaint?: number; - cumulativeLayoutShift?: number; - totalBlockingTime?: number; - domCompleteTiming?: number; -} - -export class MetricsCalculator { - calculateTTI( - coreWebVitals: CoreWebVitals, - cookieBannerData: CookieBannerData | null - ): number { - return ( - Math.max( - coreWebVitals.paint?.firstContentfulPaint || 0, - coreWebVitals.domCompleteTiming || 0, - cookieBannerData?.bannerInteractiveTime || 0 - ) + BENCHMARK_CONSTANTS.TTI_BUFFER - ); - } - - mergeBenchmarkMetrics( - resourceMetrics: ResourceTimingData, - coreWebVitals: CoreWebVitals, - cookieBannerData: CookieBannerData | null, - cookieBannerMetrics: CookieBannerMetrics, - networkRequests: NetworkRequest[], - config: Config, - tti: number - ): BenchmarkDetails { - return { - duration: resourceMetrics.duration, - size: resourceMetrics.size, - timing: { - navigationStart: resourceMetrics.timing.navigationStart, - domContentLoaded: resourceMetrics.timing.domContentLoaded, - load: resourceMetrics.timing.load, - firstPaint: coreWebVitals.paint?.firstPaint || 0, - firstContentfulPaint: coreWebVitals.paint?.firstContentfulPaint || 0, - largestContentfulPaint: coreWebVitals.largestContentfulPaint || 0, - timeToInteractive: tti, - cumulativeLayoutShift: coreWebVitals.cumulativeLayoutShift || 0, - cookieBanner: { - renderStart: cookieBannerData?.bannerRenderTime || 0, - renderEnd: cookieBannerData?.bannerInteractiveTime || 0, - interactionStart: cookieBannerData?.bannerInteractiveTime || 0, - interactionEnd: cookieBannerData?.bannerInteractiveTime || 0, - layoutShift: cookieBannerData?.layoutShiftImpact || 0, - detected: cookieBannerData?.detected || false, - selector: cookieBannerData?.selector || null, - serviceName: config.cookieBanner?.serviceName || "unknown", - visibilityTime: cookieBannerData?.bannerRenderTime ?? null, - viewportCoverage: cookieBannerData?.viewportCoverage || 0, - }, - thirdParty: { - dnsLookupTime: 0, - connectionTime: 0, - downloadTime: networkRequests.reduce( - (acc, req) => acc + req.duration, - 0 - ), - totalImpact: networkRequests.reduce((acc, req) => acc + req.size, 0), - cookieServices: { - hosts: config.cookieBanner?.serviceHosts || [], - totalSize: cookieBannerMetrics.bannerBundleSize, - resourceCount: cookieBannerMetrics.bannerNetworkRequests, - dnsLookupTime: 0, - connectionTime: 0, - downloadTime: networkRequests.reduce( - (acc, req) => acc + req.duration, - 0 - ), - }, - }, - mainThreadBlocking: { - total: coreWebVitals.totalBlockingTime || 0, - cookieBannerEstimate: - cookieBannerMetrics.bannerMainThreadBlockingTime, - percentageFromCookies: - (coreWebVitals.totalBlockingTime || 0) > 0 - ? (cookieBannerMetrics.bannerMainThreadBlockingTime / - (coreWebVitals.totalBlockingTime || 1)) * - 100 - : 0, - }, - scripts: resourceMetrics.timing.scripts, - }, - resources: resourceMetrics.resources, - language: resourceMetrics.language, - cookieBanner: { - detected: cookieBannerData?.detected || false, - selector: cookieBannerData?.selector || null, - serviceName: config.cookieBanner?.serviceName || "unknown", - visibilityTime: cookieBannerData?.bannerRenderTime ?? null, - viewportCoverage: cookieBannerData?.viewportCoverage || 0, - }, - thirdParty: { - cookieServices: { - hosts: config.cookieBanner?.serviceHosts || [], - totalSize: cookieBannerMetrics.bannerBundleSize, - resourceCount: cookieBannerMetrics.bannerNetworkRequests, - dnsLookupTime: 0, - connectionTime: 0, - downloadTime: networkRequests.reduce( - (acc, req) => acc + req.duration, - 0 - ), - }, - totalImpact: networkRequests.reduce((acc, req) => acc + req.size, 0), - }, - }; - } - - logFinalResults( - finalMetrics: BenchmarkDetails, - cookieBannerMetrics: CookieBannerMetrics, - bundleType: string | string[] | undefined - ): void { - console.log("🔍 [DEBUG] Final cookie banner benchmark results:", { - fcp: finalMetrics.timing.firstContentfulPaint, - lcp: finalMetrics.timing.largestContentfulPaint, - cls: finalMetrics.timing.cumulativeLayoutShift, - tti: finalMetrics.timing.timeToInteractive, - tbt: finalMetrics.timing.mainThreadBlocking.total, - bannerDetected: finalMetrics.cookieBanner.detected, - bannerRenderTime: - finalMetrics.timing.cookieBanner.renderEnd - - finalMetrics.timing.cookieBanner.renderStart, - bannerLayoutShift: finalMetrics.timing.cookieBanner.layoutShift, - bannerNetworkImpact: finalMetrics.thirdParty.totalImpact, - bundleStrategy: cookieBannerMetrics.isBundled - ? "Bundled" - : cookieBannerMetrics.isIIFE - ? "IIFE" - : "Unknown", - isBundled: cookieBannerMetrics.isBundled, - isIIFE: cookieBannerMetrics.isIIFE, - configBundleType: bundleType, - }); - } -} diff --git a/packages/cli/src/commands/benchmark/network-monitor.ts b/packages/cli/src/commands/benchmark/network-monitor.ts deleted file mode 100644 index afd77ed..0000000 --- a/packages/cli/src/commands/benchmark/network-monitor.ts +++ /dev/null @@ -1,80 +0,0 @@ -import type { Page } from "@playwright/test"; -import type { NetworkRequest, CookieBannerMetrics } from "./types"; - -export class NetworkMonitor { - private networkRequests: NetworkRequest[] = []; - - async setupRequestMonitoring( - page: Page, - cookieBannerMetrics: CookieBannerMetrics - ): Promise { - await page.route("**/*", async (route) => { - const request = route.request(); - const url = request.url(); - - try { - const response = await route.fetch(); - const headers = response.headers(); - - // Add timing-allow-origin header for all responses - headers["timing-allow-origin"] = "*"; - - const isScript = request.resourceType() === "script"; - const isThirdParty = !url.includes(new URL(url).hostname); - - if (isScript) { - const contentLength = response.headers()["content-length"]; - const size = contentLength ? +contentLength || 0 : 0; - - this.networkRequests.push({ - url, - size: size / 1024, // Convert to KB - duration: 0, // Will be calculated later - startTime: Date.now(), - isScript, - isThirdParty, - }); - - if (isThirdParty) { - cookieBannerMetrics.bannerNetworkRequests++; - cookieBannerMetrics.bannerBundleSize += size / 1024; - console.log( - `🌐 [THIRD-PARTY-SCRIPT] Detected: ${url} (${( - size / 1024 - ).toFixed(2)}KB)` - ); - } - } - - await route.fulfill({ response, headers }); - } catch { - // If we can't modify the response, just continue with the original request - await route.continue(); - } - }); - } - - getNetworkRequests(): NetworkRequest[] { - return this.networkRequests; - } - - calculateNetworkImpact(): { - totalDownloadTime: number; - totalSize: number; - thirdPartyCount: number; - } { - return { - totalDownloadTime: this.networkRequests.reduce( - (acc, req) => acc + req.duration, - 0 - ), - totalSize: this.networkRequests.reduce((acc, req) => acc + req.size, 0), - thirdPartyCount: this.networkRequests.filter((req) => req.isThirdParty) - .length, - }; - } - - reset(): void { - this.networkRequests = []; - } -} diff --git a/packages/cli/src/commands/benchmark/resource-collector.ts b/packages/cli/src/commands/benchmark/resource-collector.ts deleted file mode 100644 index 3954ea6..0000000 --- a/packages/cli/src/commands/benchmark/resource-collector.ts +++ /dev/null @@ -1,219 +0,0 @@ -import type { Page } from "@playwright/test"; -import type { ResourceTimingData } from "./types"; - -export class ResourceCollector { - async collectResourceTiming(page: Page): Promise { - try { - return await page.evaluate(() => { - console.log("🔍 [BROWSER] Starting resource collection..."); - - const perfEntries = performance.getEntriesByType( - "navigation" - )[0] as PerformanceNavigationTiming; - const resourceEntries = performance.getEntriesByType( - "resource" - ) as PerformanceResourceTiming[]; - - console.log("🔍 [BROWSER] Navigation timing:", { - navigationStart: perfEntries.startTime, - domContentLoaded: - perfEntries.domContentLoadedEventEnd - perfEntries.startTime, - loadComplete: perfEntries.loadEventEnd - perfEntries.startTime, - domInteractive: perfEntries.domInteractive - perfEntries.startTime, - }); - - console.log("🔍 [BROWSER] Found", resourceEntries.length, "resources"); - - // Categorize resources - const scriptEntries = resourceEntries.filter( - (entry) => entry.initiatorType === "script" - ); - const styleEntries = resourceEntries.filter( - (entry) => entry.initiatorType === "link" && entry.name.endsWith(".css") - ); - const imageEntries = resourceEntries.filter( - (entry) => entry.initiatorType === "img" - ); - const fontEntries = resourceEntries.filter( - (entry) => entry.initiatorType === "font" - ); - const otherEntries = resourceEntries.filter( - (entry) => - !["script", "link", "img", "font"].includes(entry.initiatorType) - ); - - console.log("🔍 [BROWSER] Resource breakdown:", { - scripts: scriptEntries.length, - styles: styleEntries.length, - images: imageEntries.length, - fonts: fontEntries.length, - other: otherEntries.length, - }); - - // Calculate sizes - const calculateSize = (entries: PerformanceResourceTiming[]): number => { - const total = - entries.reduce((acc, entry) => { - const size = entry.transferSize || entry.encodedBodySize || 0; - return acc + size; - }, 0) / 1024; - return total; - }; - - const navigationStart = perfEntries.startTime; - const domContentLoaded = - perfEntries.domContentLoadedEventEnd - navigationStart; - const load = perfEntries.loadEventEnd - navigationStart; - - console.log("🔍 [BROWSER] Calculated timings:", { - navigationStart, - domContentLoaded, - load, - }); - - return { - timing: { - navigationStart, - domContentLoaded, - load, - scripts: { - bundled: { - loadStart: 0, - loadEnd: scriptEntries.reduce( - (acc, entry) => acc + entry.duration, - 0 - ), - executeStart: 0, - executeEnd: 0, - }, - thirdParty: { - loadStart: 0, - loadEnd: scriptEntries.reduce( - (acc, entry) => acc + entry.duration, - 0 - ), - executeStart: 0, - executeEnd: 0, - }, - }, - }, - size: { - total: calculateSize(resourceEntries), - bundled: calculateSize( - scriptEntries.filter((e) => - e.name.includes(window.location.hostname) - ) - ), - thirdParty: calculateSize( - scriptEntries.filter( - (e) => !e.name.includes(window.location.hostname) - ) - ), - cookieServices: 0, // Will be calculated later - scripts: { - total: calculateSize(scriptEntries), - initial: calculateSize( - scriptEntries.filter((e) => e.startTime < domContentLoaded) - ), - dynamic: calculateSize( - scriptEntries.filter((e) => e.startTime >= domContentLoaded) - ), - thirdParty: calculateSize( - scriptEntries.filter( - (e) => !e.name.includes(window.location.hostname) - ) - ), - cookieServices: 0, // Will be calculated later - }, - styles: calculateSize(styleEntries), - images: calculateSize(imageEntries), - fonts: calculateSize(fontEntries), - other: calculateSize(otherEntries), - }, - resources: { - scripts: scriptEntries.map((entry) => ({ - name: entry.name, - size: entry.transferSize ? entry.transferSize / 1024 : 0, - duration: entry.duration, - startTime: entry.startTime - navigationStart, - isThirdParty: !entry.name.includes(window.location.hostname), - isDynamic: entry.startTime >= domContentLoaded, - isCookieService: false, - dnsTime: entry.domainLookupEnd - entry.domainLookupStart, - connectionTime: entry.connectEnd - entry.connectStart, - })), - styles: styleEntries.map((entry) => ({ - name: entry.name, - size: entry.transferSize ? entry.transferSize / 1024 : 0, - duration: entry.duration, - startTime: entry.startTime - navigationStart, - isThirdParty: !entry.name.includes(window.location.hostname), - isCookieService: false, - })), - images: imageEntries.map((entry) => ({ - name: entry.name, - size: entry.transferSize ? entry.transferSize / 1024 : 0, - duration: entry.duration, - startTime: entry.startTime - navigationStart, - isThirdParty: !entry.name.includes(window.location.hostname), - isCookieService: false, - })), - fonts: fontEntries.map((entry) => ({ - name: entry.name, - size: entry.transferSize ? entry.transferSize / 1024 : 0, - duration: entry.duration, - startTime: entry.startTime - navigationStart, - isThirdParty: !entry.name.includes(window.location.hostname), - isCookieService: false, - })), - other: otherEntries.map((entry) => ({ - name: entry.name, - size: entry.transferSize ? entry.transferSize / 1024 : 0, - duration: entry.duration, - startTime: entry.startTime - navigationStart, - isThirdParty: !entry.name.includes(window.location.hostname), - isCookieService: false, - type: entry.initiatorType, - })), - }, - language: "en", - duration: load, - }; - }); - } catch (error) { - console.error("🔍 [BROWSER] Error collecting resource timing:", error); - // Return default values if resource collection fails - return { - timing: { - navigationStart: 0, - domContentLoaded: 0, - load: 0, - scripts: { - bundled: { loadStart: 0, loadEnd: 0, executeStart: 0, executeEnd: 0 }, - thirdParty: { loadStart: 0, loadEnd: 0, executeStart: 0, executeEnd: 0 }, - }, - }, - size: { - total: 0, - bundled: 0, - thirdParty: 0, - cookieServices: 0, - scripts: { total: 0, initial: 0, dynamic: 0, thirdParty: 0, cookieServices: 0 }, - styles: 0, - images: 0, - fonts: 0, - other: 0, - }, - resources: { - scripts: [], - styles: [], - images: [], - fonts: [], - other: [], - }, - language: "en", - duration: 0, - }; - } - } -} diff --git a/packages/cli/src/commands/benchmark/types.ts b/packages/cli/src/commands/benchmark/types.ts deleted file mode 100644 index ff9c2cf..0000000 --- a/packages/cli/src/commands/benchmark/types.ts +++ /dev/null @@ -1,146 +0,0 @@ -export type { - Config, - BenchmarkResult, - BenchmarkDetails, - LayoutShiftEntry, -} from "../../types"; - -export interface WindowWithCookieMetrics extends Window { - __cookieBannerMetrics: { - pageLoadStart: number; - bannerDetectionStart: number; - bannerFirstSeen: number; - bannerInteractive: number; - layoutShiftsBefore: number; - layoutShiftsAfter: number; - detected: boolean; - selector: string | null; - }; -} - -export interface NetworkRequest { - url: string; - size: number; - duration: number; - startTime: number; - isScript: boolean; - isThirdParty: boolean; -} - -export interface CookieBannerMetrics { - detectionStartTime: number; - bannerRenderTime: number; - bannerInteractiveTime: number; - bannerScriptLoadTime: number; - bannerLayoutShiftImpact: number; - bannerNetworkRequests: number; - bannerBundleSize: number; - bannerMainThreadBlockingTime: number; - isBundled: boolean; - isIIFE: boolean; - bannerDetected: boolean; - bannerSelector: string | null; -} - -export interface CookieBannerData { - detected: boolean; - selector: string | null; - bannerRenderTime: number | null; - bannerInteractiveTime: number | null; - bannerHydrationTime: number; - layoutShiftImpact: number; - viewportCoverage: number; -} - -export interface BundleStrategy { - isBundled: boolean; - isIIFE: boolean; - bundleType: string | string[] | undefined; -} - -export interface ResourceTimingData { - timing: { - navigationStart: number; - domContentLoaded: number; - load: number; - scripts: { - bundled: { - loadStart: number; - loadEnd: number; - executeStart: number; - executeEnd: number; - }; - thirdParty: { - loadStart: number; - loadEnd: number; - executeStart: number; - executeEnd: number; - }; - }; - }; - size: { - total: number; - bundled: number; - thirdParty: number; - cookieServices: number; - scripts: { - total: number; - initial: number; - dynamic: number; - thirdParty: number; - cookieServices: number; - }; - styles: number; - images: number; - fonts: number; - other: number; - }; - resources: { - scripts: Array<{ - name: string; - size: number; - duration: number; - startTime: number; - isThirdParty: boolean; - isDynamic: boolean; - isCookieService: boolean; - dnsTime: number; - connectionTime: number; - }>; - styles: Array<{ - name: string; - size: number; - duration: number; - startTime: number; - isThirdParty: boolean; - isCookieService: boolean; - }>; - images: Array<{ - name: string; - size: number; - duration: number; - startTime: number; - isThirdParty: boolean; - isCookieService: boolean; - }>; - fonts: Array<{ - name: string; - size: number; - duration: number; - startTime: number; - isThirdParty: boolean; - isCookieService: boolean; - }>; - other: Array<{ - name: string; - size: number; - duration: number; - startTime: number; - isThirdParty: boolean; - isCookieService: boolean; - type: string; - }>; - }; - language: string; - duration: number; -} diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts deleted file mode 100644 index 29b5808..0000000 --- a/packages/cli/src/index.ts +++ /dev/null @@ -1,82 +0,0 @@ -import { setTimeout } from "node:timers/promises"; -import * as p from "@clack/prompts"; -import color from "picocolors"; -import { benchmarkCommand } from "./commands/benchmark"; -import { resultsCommand } from "./commands/results"; -import { dbCommand } from "./commands/db"; - -function onCancel() { - p.cancel("Operation cancelled."); - process.exit(0); -} - -async function main() { - console.clear(); - await setTimeout(1000); - - // Check for command line arguments - const args = process.argv.slice(2); - const command = args[0]; - - // If no command specified, show the prompt - if (command) { - // Direct command execution - switch (command) { - case "benchmark": - await benchmarkCommand(); - break; - case "results": - await resultsCommand(); - break; - case "db": - await dbCommand(args[1]); - break; - default: - console.error(`Unknown command: ${command}`); - console.log("Available commands: benchmark, results, db"); - process.exit(1); - } - } else { - p.intro(`${color.bgCyan(color.black(" c15t "))}`); - - const selectedCommand = await p.select({ - message: "What would you like to do?", - options: [ - { - value: "benchmark", - label: "Run a benchmark", - hint: "Run a performance benchmark on a URL", - }, - { - value: "results", - label: "Results", - hint: "Combine and display benchmark results", - }, - { - value: "db", - label: "Database", - hint: "Manage database schema and migrations", - }, - ], - }); - - if (p.isCancel(selectedCommand)) { - return onCancel(); - } - - // biome-ignore lint/style/useDefaultSwitchClause: - switch (selectedCommand) { - case "benchmark": - await benchmarkCommand(); - break; - case "results": - await resultsCommand(); - break; - case "db": - await dbCommand(); - break; - } - } -} - -main().catch(console.error); diff --git a/packages/cli/src/lib/README.md b/packages/cli/src/lib/README.md deleted file mode 100644 index ffa3f45..0000000 --- a/packages/cli/src/lib/README.md +++ /dev/null @@ -1,53 +0,0 @@ -# Benchmark Library Structure - -This directory contains the refactored benchmark modules, organized by functionality for better maintainability and testability. - -## Architecture - -The benchmark system has been broken down into focused, single-responsibility modules: - -### Core Runner - -- **`benchmark-runner.ts`** - Main orchestrator that coordinates all collectors and aggregators - -### Collectors (`/collectors`) - -- **`cookie-banner-collector.ts`** - Detects and measures cookie banner performance impact -- **`network-monitor.ts`** - Monitors network requests and calculates size/timing metrics -- **`resource-timing-collector.ts`** - Collects detailed resource timing data from the browser - -### Metrics (`/metrics`) - -- **`performance-aggregator.ts`** - Aggregates all collected metrics into final benchmark results - -## Benefits of Refactoring - -1. **Single Responsibility**: Each module has one clear purpose -2. **Better Testability**: Individual components can be tested in isolation -3. **Improved Maintainability**: Changes to one area don't affect others -4. **Cleaner Code**: Removed ~500 lines from a single 700+ line file -5. **Type Safety**: Better TypeScript interfaces and type definitions -6. **Reusability**: Modules can be used independently or in different combinations - -## Usage - -```typescript -import { BenchmarkRunner } from "./benchmark-runner"; - -const runner = new BenchmarkRunner(config); -const results = await runner.runBenchmarks(serverUrl); -``` - -## Module Dependencies - -``` -benchmark-runner.ts -├── collectors/ -│ ├── cookie-banner-collector.ts -│ ├── network-monitor.ts -│ └── resource-timing-collector.ts -└── metrics/ - └── performance-aggregator.ts -``` - -Each collector is independent and can be used separately if needed. The performance aggregator combines all metrics into the final benchmark result format. diff --git a/packages/cli/src/lib/benchmark-runner.ts b/packages/cli/src/lib/benchmark-runner.ts deleted file mode 100644 index d311fee..0000000 --- a/packages/cli/src/lib/benchmark-runner.ts +++ /dev/null @@ -1,189 +0,0 @@ -import { chromium, type Page } from "@playwright/test"; -import { PerformanceMetricsCollector } from "playwright-performance-metrics"; -import type { Config, BenchmarkResult, BenchmarkDetails } from "../types"; -import { CookieBannerCollector } from "./collectors/cookie-banner-collector"; -import { NetworkMonitor } from "./collectors/network-monitor"; -import { ResourceTimingCollector } from "./collectors/resource-timing-collector"; -import { - PerformanceAggregator, - type CoreWebVitals, -} from "./metrics/performance-aggregator"; - -export class BenchmarkRunner { - private config: Config; - private cookieBannerCollector: CookieBannerCollector; - private networkMonitor: NetworkMonitor; - private resourceTimingCollector: ResourceTimingCollector; - private performanceAggregator: PerformanceAggregator; - - constructor(config: Config) { - this.config = config; - this.cookieBannerCollector = new CookieBannerCollector(config); - this.networkMonitor = new NetworkMonitor(config); - this.resourceTimingCollector = new ResourceTimingCollector(); - this.performanceAggregator = new PerformanceAggregator(); - } - - /** - * Run a single benchmark iteration - */ - async runSingleBenchmark(page: Page, url: string): Promise { - console.log(`🔍 [DEBUG] Starting cookie banner benchmark for: ${url}`); - console.log( - "🔍 [DEBUG] Cookie banner selectors:", - this.config.cookieBanner?.selectors || [] - ); - console.log( - "🔍 [DEBUG] Bundle type from config:", - this.config.techStack?.bundleType - ); - - // Initialize collectors - const collector = new PerformanceMetricsCollector(); - const cookieBannerMetrics = this.cookieBannerCollector.initializeMetrics(); - - // Setup monitoring and detection - await this.networkMonitor.setupMonitoring(page); - await this.cookieBannerCollector.setupDetection(page); - - // Navigate to the page - console.log(`🔍 [DEBUG] Navigating to: ${url}`); - await page.goto(url, { waitUntil: "networkidle" }); - - // Wait for the specified element - await this.waitForElement(page); - - // Wait for network to be idle - console.log("🔍 [DEBUG] Waiting for network idle..."); - await page.waitForLoadState("networkidle"); - - // Collect core web vitals - console.log("🔍 [DEBUG] Collecting core web vitals..."); - const coreWebVitals = await this.collectCoreWebVitals(collector, page); - - // Collect cookie banner specific metrics - const cookieBannerData = await this.cookieBannerCollector.collectMetrics( - page - ); - console.log("🔍 [DEBUG] Cookie banner metrics:", cookieBannerData); - - // Collect detailed resource timing data - const resourceMetrics = await this.resourceTimingCollector.collect(page); - - // Get network metrics - const networkRequests = this.networkMonitor.getNetworkRequests(); - const networkMetrics = this.networkMonitor.getMetrics(); - - // Aggregate all metrics - const finalMetrics = this.performanceAggregator.aggregateMetrics( - coreWebVitals, - cookieBannerData, - cookieBannerMetrics, - networkRequests, - networkMetrics, - resourceMetrics, - this.config - ); - - // Log results - this.performanceAggregator.logResults( - finalMetrics, - cookieBannerMetrics, - this.config - ); - - // Cleanup - await collector.cleanup(); - this.networkMonitor.reset(); - - return finalMetrics; - } - - /** - * Run multiple benchmark iterations - */ - async runBenchmarks(serverUrl: string): Promise { - const browser = await chromium.launch({ - headless: true, // Keep headless mode for stability - args: ["--remote-debugging-port=9222"], - }); - const results: BenchmarkDetails[] = []; - - try { - for (let i = 0; i < this.config.iterations; i++) { - console.log( - `[Benchmark] Running iteration ${i + 1}/${this.config.iterations}...` - ); - - const context = await browser.newContext(); - const page = await context.newPage(); - - const result = await this.runSingleBenchmark( - page, - // Add a timestamp to the URL to avoid caching - `${serverUrl}?t=${Date.now()}` - ); - results.push(result); - - await context.close(); - } - } finally { - await browser.close(); - } - - const averages = this.performanceAggregator.calculateAverages(results); - - return { - name: this.config.name, - baseline: this.config.baseline || false, - techStack: this.config.techStack, - source: this.config.source, - includes: this.config.includes, - company: this.config.company, - tags: this.config.tags, - details: results, - average: averages, - }; - } - - /** - * Wait for the specified element based on config - */ - private async waitForElement(page: Page): Promise { - if (this.config.testId) { - console.log(`🔍 [DEBUG] Waiting for testId: ${this.config.testId}`); - await page.waitForSelector(`[data-testid="${this.config.testId}"]`); - } else if (this.config.id) { - console.log(`🔍 [DEBUG] Waiting for id: ${this.config.id}`); - await page.waitForSelector(`#${this.config.id}`); - } else if (this.config.custom) { - console.log("🔍 [DEBUG] Running custom wait function"); - await this.config.custom(page); - } - } - - /** - * Collect core web vitals using playwright-performance-metrics - */ - private async collectCoreWebVitals( - collector: PerformanceMetricsCollector, - page: Page - ): Promise { - const coreWebVitals = await collector.collectMetrics(page, { - timeout: 10000, - retryTimeout: 5000, - }); - - console.log("🔍 [DEBUG] Core web vitals collected:", { - fcp: coreWebVitals.paint?.firstContentfulPaint, - lcp: coreWebVitals.largestContentfulPaint, - cls: coreWebVitals.cumulativeLayoutShift, - tbt: coreWebVitals.totalBlockingTime, - domComplete: coreWebVitals.domCompleteTiming, - pageLoad: coreWebVitals.pageloadTiming, - totalBytes: coreWebVitals.totalBytes, - }); - - return coreWebVitals; - } -} diff --git a/packages/cli/src/lib/collectors/cookie-banner-collector.ts b/packages/cli/src/lib/collectors/cookie-banner-collector.ts deleted file mode 100644 index 8f92c0b..0000000 --- a/packages/cli/src/lib/collectors/cookie-banner-collector.ts +++ /dev/null @@ -1,279 +0,0 @@ -import type { Page } from "@playwright/test"; -import type { Config, LayoutShiftEntry } from "../../types"; - -interface WindowWithCookieMetrics extends Window { - __cookieBannerMetrics: { - pageLoadStart: number; - bannerDetectionStart: number; - bannerFirstSeen: number; - bannerInteractive: number; - layoutShiftsBefore: number; - layoutShiftsAfter: number; - detected: boolean; - selector: string | null; - }; -} - -export interface CookieBannerMetrics { - detectionStartTime: number; - bannerRenderTime: number; - bannerInteractiveTime: number; - bannerScriptLoadTime: number; - bannerLayoutShiftImpact: number; - bannerNetworkRequests: number; - bannerBundleSize: number; - bannerMainThreadBlockingTime: number; - isBundled: boolean; - isIIFE: boolean; - bannerDetected: boolean; - bannerSelector: string | null; -} - -export interface CookieBannerData { - detected: boolean; - selector: string | null; - bannerRenderTime: number; - bannerInteractiveTime: number; - bannerHydrationTime: number; - layoutShiftImpact: number; - viewportCoverage: number; -} - -export class CookieBannerCollector { - private config: Config; - - constructor(config: Config) { - this.config = config; - } - - /** - * Determines bundle strategy from config - */ - getBundleStrategy(): { isBundled: boolean; isIIFE: boolean } { - const bundleType = this.config.techStack?.bundleType; - const isIIFE = - bundleType === "iffe" || - (Array.isArray(bundleType) && bundleType.includes("iffe")); - const isBundled = - !isIIFE && - (bundleType === "bundled" || - (Array.isArray(bundleType) && - (bundleType.includes("esm") || bundleType.includes("cjs"))) || - bundleType === "esm" || - bundleType === "cjs"); - - console.log( - `🔍 [BUNDLE-STRATEGY] Detected from config: ${ - isBundled ? "Bundled" : isIIFE ? "IIFE" : "Unknown" - }`, - { - bundleType, - isBundled, - isIIFE, - } - ); - - return { isBundled, isIIFE }; - } - - /** - * Initialize cookie banner metrics tracking - */ - initializeMetrics(): CookieBannerMetrics { - const { isBundled, isIIFE } = this.getBundleStrategy(); - - return { - detectionStartTime: 0, - bannerRenderTime: 0, - bannerInteractiveTime: 0, - bannerScriptLoadTime: 0, - bannerLayoutShiftImpact: 0, - bannerNetworkRequests: 0, - bannerBundleSize: 0, - bannerMainThreadBlockingTime: 0, - isBundled, - isIIFE, - bannerDetected: false, - bannerSelector: null, - }; - } - - /** - * Set up cookie banner detection script in the browser - */ - async setupDetection(page: Page): Promise { - const selectors = this.config.cookieBanner?.selectors || []; - - await page.addInitScript((selectors: string[]) => { - console.log("🔍 [BROWSER] Setting up cookie banner detection..."); - - // Store initial performance baseline - (window as unknown as WindowWithCookieMetrics).__cookieBannerMetrics = { - pageLoadStart: performance.now(), - bannerDetectionStart: 0, - bannerFirstSeen: 0, - bannerInteractive: 0, - layoutShiftsBefore: 0, - layoutShiftsAfter: 0, - detected: false, - selector: null, - }; - - // Monitor for layout shifts specifically - let cumulativeLayoutShift = 0; - if ("PerformanceObserver" in window) { - const clsObserver = new PerformanceObserver((list) => { - for (const entry of list.getEntries()) { - const layoutShiftEntry = entry as LayoutShiftEntry; - if (!layoutShiftEntry.hadRecentInput) { - cumulativeLayoutShift += layoutShiftEntry.value; - ( - window as unknown as WindowWithCookieMetrics - ).__cookieBannerMetrics.layoutShiftsAfter = cumulativeLayoutShift; - } - } - }); - clsObserver.observe({ type: "layout-shift", buffered: true }); - } - - // Cookie banner detection logic - const detectCookieBanner = () => { - ( - window as unknown as WindowWithCookieMetrics - ).__cookieBannerMetrics.bannerDetectionStart = performance.now(); - - for (const selector of selectors) { - try { - const element = document.querySelector(selector); - if (element) { - const rect = element.getBoundingClientRect(); - const isVisible = - rect.width > 0 && - rect.height > 0 && - window.getComputedStyle(element).visibility !== "hidden" && - window.getComputedStyle(element).display !== "none"; - - if (isVisible) { - const metrics = (window as unknown as WindowWithCookieMetrics) - .__cookieBannerMetrics; - metrics.detected = true; - metrics.selector = selector; - metrics.bannerFirstSeen = performance.now(); - metrics.layoutShiftsBefore = cumulativeLayoutShift; - - console.log("🔍 [BANNER] Cookie banner detected:", selector); - console.log( - "🔍 [BANNER] Banner render time:", - metrics.bannerFirstSeen - metrics.pageLoadStart, - "ms" - ); - - // Check if banner is interactive - const buttons = element.querySelectorAll( - 'button, a, [role="button"], [onclick]' - ); - if (buttons.length > 0) { - // Test if buttons are actually clickable - const firstButton = buttons[0] as HTMLElement; - if (firstButton.offsetParent !== null) { - // Element is visible and clickable - metrics.bannerInteractive = performance.now(); - console.log( - "🔍 [BANNER] Banner interactive time:", - metrics.bannerInteractive - metrics.pageLoadStart, - "ms" - ); - } - } - - return true; - } - } - } catch (error) { - console.warn( - "🔍 [BANNER] Error checking selector:", - selector, - error - ); - } - } - return false; - }; - - // Start detection after DOM is ready - if (document.readyState === "loading") { - document.addEventListener("DOMContentLoaded", () => { - setTimeout(() => { - if (!detectCookieBanner()) { - // Keep checking for dynamically loaded banners - const interval = setInterval(() => { - if (detectCookieBanner()) { - clearInterval(interval); - } - }, 100); - - // Stop checking after 10 seconds - setTimeout(() => clearInterval(interval), 10000); - } - }, 100); // Small delay to allow for initial render - }); - } else { - setTimeout(() => { - if (!detectCookieBanner()) { - const interval = setInterval(() => { - if (detectCookieBanner()) { - clearInterval(interval); - } - }, 100); - - setTimeout(() => clearInterval(interval), 10000); - } - }, 100); - } - }, selectors); - } - - /** - * Collect cookie banner specific metrics from the browser - */ - async collectMetrics(page: Page): Promise { - return page.evaluate(() => { - const metrics = (window as unknown as WindowWithCookieMetrics) - .__cookieBannerMetrics; - if (!metrics) { - return null; - } - - return { - detected: metrics.detected, - selector: metrics.selector, - bannerRenderTime: metrics.bannerFirstSeen - metrics.pageLoadStart, - bannerInteractiveTime: - metrics.bannerInteractive - metrics.pageLoadStart, - bannerHydrationTime: - metrics.bannerInteractive > 0 - ? metrics.bannerInteractive - metrics.bannerFirstSeen - : 0, - layoutShiftImpact: - metrics.layoutShiftsAfter - metrics.layoutShiftsBefore, - viewportCoverage: metrics.detected - ? (() => { - if (!metrics.selector) { - return 0; - } - const element = document.querySelector(metrics.selector); - if (element) { - const rect = element.getBoundingClientRect(); - return ( - ((rect.width * rect.height) / - (window.innerWidth * window.innerHeight)) * - 100 - ); - } - return 0; - })() - : 0, - }; - }); - } -} diff --git a/packages/cli/src/lib/collectors/index.ts b/packages/cli/src/lib/collectors/index.ts deleted file mode 100644 index 0eb6d64..0000000 --- a/packages/cli/src/lib/collectors/index.ts +++ /dev/null @@ -1,14 +0,0 @@ -export { - CookieBannerCollector, - type CookieBannerMetrics, - type CookieBannerData, -} from "./cookie-banner-collector"; -export { - NetworkMonitor, - type NetworkRequest, - type NetworkMetrics, -} from "./network-monitor"; -export { - ResourceTimingCollector, - type ResourceTimingData, -} from "./resource-timing-collector"; diff --git a/packages/cli/src/lib/collectors/network-monitor.ts b/packages/cli/src/lib/collectors/network-monitor.ts deleted file mode 100644 index 8be8c34..0000000 --- a/packages/cli/src/lib/collectors/network-monitor.ts +++ /dev/null @@ -1,136 +0,0 @@ -import type { Page, Route } from "@playwright/test"; -import type { Config } from "../../types"; - -export interface NetworkRequest { - url: string; - size: number; - duration: number; - startTime: number; - isScript: boolean; - isThirdParty: boolean; -} - -export interface NetworkMetrics { - bannerNetworkRequests: number; - bannerBundleSize: number; -} - -export class NetworkMonitor { - private config: Config; - private networkRequests: NetworkRequest[] = []; - private metrics: NetworkMetrics = { - bannerNetworkRequests: 0, - bannerBundleSize: 0, - }; - - constructor(config: Config) { - this.config = config; - } - - /** - * Set up network request monitoring - */ - async setupMonitoring(page: Page): Promise { - await page.route("**/*", async (route: Route) => { - const request = route.request(); - const url = request.url(); - - try { - const response = await route.fetch(); - const headers = response.headers(); - - // Add timing-allow-origin header for all responses - headers["timing-allow-origin"] = "*"; - - const isScript = request.resourceType() === "script"; - const isThirdParty = !url.includes(new URL(url).hostname); - - if (isScript) { - const contentLength = response.headers()["content-length"]; - const size = contentLength ? +contentLength || 0 : 0; - - this.networkRequests.push({ - url, - size: size / 1024, // Convert to KB - duration: 0, // Will be calculated later - startTime: Date.now(), - isScript, - isThirdParty, - }); - - if (isThirdParty) { - this.metrics.bannerNetworkRequests++; - this.metrics.bannerBundleSize += size / 1024; - console.log( - `🌐 [THIRD-PARTY-SCRIPT] Detected: ${url} (${( - size / 1024 - ).toFixed(2)}KB)` - ); - } - } - - await route.fulfill({ response, headers }); - } catch { - // If we can't modify the response, just continue with the original request - await route.continue(); - } - }); - } - - /** - * Get collected network requests - */ - getNetworkRequests(): NetworkRequest[] { - return this.networkRequests; - } - - /** - * Get network metrics - */ - getMetrics(): NetworkMetrics { - return this.metrics; - } - - /** - * Calculate network impact metrics - */ - calculateNetworkImpact(): { - totalImpact: number; - totalDownloadTime: number; - thirdPartyImpact: number; - scriptImpact: number; - } { - const totalImpact = this.networkRequests.reduce( - (acc, req) => acc + req.size, - 0 - ); - const totalDownloadTime = this.networkRequests.reduce( - (acc, req) => acc + req.duration, - 0 - ); - const thirdPartyImpact = this.networkRequests - .filter((req) => req.isThirdParty) - .reduce((acc, req) => acc + req.size, 0); - const scriptImpact = this.networkRequests - .filter((req) => req.isScript) - .reduce((acc, req) => acc + req.size, 0); - - return { - totalImpact, - totalDownloadTime, - thirdPartyImpact, - scriptImpact, - }; - } - - /** - * Reset collected data - */ - reset(): void { - this.networkRequests = []; - this.metrics = { - bannerNetworkRequests: 0, - bannerBundleSize: 0, - }; - } -} diff --git a/packages/cli/src/lib/collectors/resource-timing-collector.ts b/packages/cli/src/lib/collectors/resource-timing-collector.ts deleted file mode 100644 index 26b09f3..0000000 --- a/packages/cli/src/lib/collectors/resource-timing-collector.ts +++ /dev/null @@ -1,274 +0,0 @@ -import type { Page } from "@playwright/test"; - -export interface ResourceTimingData { - timing: { - navigationStart: number; - domContentLoaded: number; - load: number; - scripts: { - bundled: { - loadStart: number; - loadEnd: number; - executeStart: number; - executeEnd: number; - }; - thirdParty: { - loadStart: number; - loadEnd: number; - executeStart: number; - executeEnd: number; - }; - }; - }; - size: { - total: number; - bundled: number; - thirdParty: number; - cookieServices: number; - scripts: { - total: number; - initial: number; - dynamic: number; - thirdParty: number; - cookieServices: number; - }; - styles: number; - images: number; - fonts: number; - other: number; - }; - resources: { - scripts: Array<{ - name: string; - size: number; - duration: number; - startTime: number; - isThirdParty: boolean; - isDynamic: boolean; - isCookieService: boolean; - dnsTime: number; - connectionTime: number; - }>; - styles: Array<{ - name: string; - size: number; - duration: number; - startTime: number; - isThirdParty: boolean; - isCookieService: boolean; - }>; - images: Array<{ - name: string; - size: number; - duration: number; - startTime: number; - isThirdParty: boolean; - isCookieService: boolean; - }>; - fonts: Array<{ - name: string; - size: number; - duration: number; - startTime: number; - isThirdParty: boolean; - isCookieService: boolean; - }>; - other: Array<{ - name: string; - size: number; - duration: number; - startTime: number; - isThirdParty: boolean; - isCookieService: boolean; - type: string; - }>; - }; - language: string; - duration: number; -} - -export class ResourceTimingCollector { - /** - * Collect detailed resource timing data from the browser - */ - async collect(page: Page): Promise { - console.log("🔍 [DEBUG] Collecting resource timing data..."); - - return page.evaluate(() => { - console.log("🔍 [BROWSER] Starting resource collection..."); - - const perfEntries = performance.getEntriesByType( - "navigation" - )[0] as PerformanceNavigationTiming; - const resourceEntries = performance.getEntriesByType( - "resource" - ) as PerformanceResourceTiming[]; - - console.log("🔍 [BROWSER] Navigation timing:", { - navigationStart: perfEntries.startTime, - domContentLoaded: - perfEntries.domContentLoadedEventEnd - perfEntries.startTime, - loadComplete: perfEntries.loadEventEnd - perfEntries.startTime, - domInteractive: perfEntries.domInteractive - perfEntries.startTime, - }); - - console.log("🔍 [BROWSER] Found", resourceEntries.length, "resources"); - - // Categorize resources - const scriptEntries = resourceEntries.filter( - (entry) => entry.initiatorType === "script" - ); - const styleEntries = resourceEntries.filter( - (entry) => entry.initiatorType === "link" && entry.name.endsWith(".css") - ); - const imageEntries = resourceEntries.filter( - (entry) => entry.initiatorType === "img" - ); - const fontEntries = resourceEntries.filter( - (entry) => entry.initiatorType === "font" - ); - const otherEntries = resourceEntries.filter( - (entry) => - !["script", "link", "img", "font"].includes(entry.initiatorType) - ); - - console.log("🔍 [BROWSER] Resource breakdown:", { - scripts: scriptEntries.length, - styles: styleEntries.length, - images: imageEntries.length, - fonts: fontEntries.length, - other: otherEntries.length, - }); - - // Calculate sizes - const calculateSize = (entries: PerformanceResourceTiming[]) => { - const total = - entries.reduce((acc, entry) => { - const size = entry.transferSize || entry.encodedBodySize || 0; - return acc + size; - }, 0) / 1024; - return total; - }; - - const navigationStart = perfEntries.startTime; - const domContentLoaded = - perfEntries.domContentLoadedEventEnd - navigationStart; - const load = perfEntries.loadEventEnd - navigationStart; - - console.log("🔍 [BROWSER] Calculated timings:", { - navigationStart, - domContentLoaded, - load, - }); - - return { - timing: { - navigationStart, - domContentLoaded, - load, - scripts: { - bundled: { - loadStart: 0, - loadEnd: scriptEntries.reduce( - (acc, entry) => acc + entry.duration, - 0 - ), - executeStart: 0, - executeEnd: 0, - }, - thirdParty: { - loadStart: 0, - loadEnd: scriptEntries.reduce( - (acc, entry) => acc + entry.duration, - 0 - ), - executeStart: 0, - executeEnd: 0, - }, - }, - }, - size: { - total: calculateSize(resourceEntries), - bundled: calculateSize( - scriptEntries.filter((e) => - e.name.includes(window.location.hostname) - ) - ), - thirdParty: calculateSize( - scriptEntries.filter( - (e) => !e.name.includes(window.location.hostname) - ) - ), - cookieServices: 0, // Will be calculated later - scripts: { - total: calculateSize(scriptEntries), - initial: calculateSize( - scriptEntries.filter((e) => e.startTime < domContentLoaded) - ), - dynamic: calculateSize( - scriptEntries.filter((e) => e.startTime >= domContentLoaded) - ), - thirdParty: calculateSize( - scriptEntries.filter( - (e) => !e.name.includes(window.location.hostname) - ) - ), - cookieServices: 0, // Will be calculated later - }, - styles: calculateSize(styleEntries), - images: calculateSize(imageEntries), - fonts: calculateSize(fontEntries), - other: calculateSize(otherEntries), - }, - resources: { - scripts: scriptEntries.map((entry) => ({ - name: entry.name, - size: entry.transferSize ? entry.transferSize / 1024 : 0, - duration: entry.duration, - startTime: entry.startTime - navigationStart, - isThirdParty: !entry.name.includes(window.location.hostname), - isDynamic: entry.startTime >= domContentLoaded, - isCookieService: false, - dnsTime: entry.domainLookupEnd - entry.domainLookupStart, - connectionTime: entry.connectEnd - entry.connectStart, - })), - styles: styleEntries.map((entry) => ({ - name: entry.name, - size: entry.transferSize ? entry.transferSize / 1024 : 0, - duration: entry.duration, - startTime: entry.startTime - navigationStart, - isThirdParty: !entry.name.includes(window.location.hostname), - isCookieService: false, - })), - images: imageEntries.map((entry) => ({ - name: entry.name, - size: entry.transferSize ? entry.transferSize / 1024 : 0, - duration: entry.duration, - startTime: entry.startTime - navigationStart, - isThirdParty: !entry.name.includes(window.location.hostname), - isCookieService: false, - })), - fonts: fontEntries.map((entry) => ({ - name: entry.name, - size: entry.transferSize ? entry.transferSize / 1024 : 0, - duration: entry.duration, - startTime: entry.startTime - navigationStart, - isThirdParty: !entry.name.includes(window.location.hostname), - isCookieService: false, - })), - other: otherEntries.map((entry) => ({ - name: entry.name, - size: entry.transferSize ? entry.transferSize / 1024 : 0, - duration: entry.duration, - startTime: entry.startTime - navigationStart, - isThirdParty: !entry.name.includes(window.location.hostname), - isCookieService: false, - type: entry.initiatorType, - })), - }, - language: "en", - duration: load, - }; - }); - } -} diff --git a/packages/cli/src/lib/metrics/index.ts b/packages/cli/src/lib/metrics/index.ts deleted file mode 100644 index a6b3319..0000000 --- a/packages/cli/src/lib/metrics/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export { - PerformanceAggregator, - type CoreWebVitals, -} from "./performance-aggregator"; diff --git a/packages/cli/src/lib/metrics/performance-aggregator.ts b/packages/cli/src/lib/metrics/performance-aggregator.ts deleted file mode 100644 index 38b6a81..0000000 --- a/packages/cli/src/lib/metrics/performance-aggregator.ts +++ /dev/null @@ -1,286 +0,0 @@ -import type { BenchmarkDetails, BenchmarkResult, Config } from "../../types"; -import type { - CookieBannerData, - CookieBannerMetrics, -} from "../collectors/cookie-banner-collector"; -import type { - NetworkRequest, - NetworkMetrics, -} from "../collectors/network-monitor"; -import type { ResourceTimingData } from "../collectors/resource-timing-collector"; - -export interface CoreWebVitals { - paint?: { - firstPaint?: number; - firstContentfulPaint?: number; - }; - largestContentfulPaint?: number; - cumulativeLayoutShift?: number; - totalBlockingTime?: number; - domCompleteTiming?: number; - pageloadTiming?: number; - totalBytes?: number; -} - -export class PerformanceAggregator { - /** - * Calculate Time to Interactive based on core web vitals and cookie banner interaction - */ - calculateTTI( - coreWebVitals: CoreWebVitals, - cookieBannerData: CookieBannerData | null - ): number { - return ( - Math.max( - coreWebVitals.paint?.firstContentfulPaint || 0, - coreWebVitals.domCompleteTiming || 0, - cookieBannerData?.bannerInteractiveTime || 0 - ) + 1000 - ); // Add buffer for true interactivity - } - - /** - * Merge all collected metrics into final benchmark details - */ - aggregateMetrics( - coreWebVitals: CoreWebVitals, - cookieBannerData: CookieBannerData | null, - cookieBannerMetrics: CookieBannerMetrics, - networkRequests: NetworkRequest[], - networkMetrics: NetworkMetrics, - resourceMetrics: ResourceTimingData, - config: Config - ): BenchmarkDetails { - const tti = this.calculateTTI(coreWebVitals, cookieBannerData); - - const networkImpact = this.calculateNetworkImpact(networkRequests); - - return { - duration: resourceMetrics.duration, - size: resourceMetrics.size, - timing: { - navigationStart: resourceMetrics.timing.navigationStart, - domContentLoaded: resourceMetrics.timing.domContentLoaded, - load: resourceMetrics.timing.load, - firstPaint: coreWebVitals.paint?.firstPaint || 0, - firstContentfulPaint: coreWebVitals.paint?.firstContentfulPaint || 0, - largestContentfulPaint: coreWebVitals.largestContentfulPaint || 0, - timeToInteractive: tti, - cumulativeLayoutShift: coreWebVitals.cumulativeLayoutShift || 0, - cookieBanner: { - renderStart: cookieBannerData?.bannerRenderTime || 0, - renderEnd: cookieBannerData?.bannerInteractiveTime || 0, - interactionStart: cookieBannerData?.bannerInteractiveTime || 0, - interactionEnd: cookieBannerData?.bannerInteractiveTime || 0, - layoutShift: cookieBannerData?.layoutShiftImpact || 0, - detected: cookieBannerData?.detected || false, - selector: cookieBannerData?.selector || null, - serviceName: config.cookieBanner?.serviceName || "unknown", - visibilityTime: cookieBannerData?.bannerInteractiveTime || 0, - viewportCoverage: cookieBannerData?.viewportCoverage || 0, - }, - thirdParty: { - dnsLookupTime: 0, - connectionTime: 0, - downloadTime: networkImpact.totalDownloadTime, - totalImpact: networkImpact.totalImpact, - cookieServices: { - hosts: config.cookieBanner?.serviceHosts || [], - totalSize: networkMetrics.bannerBundleSize, - resourceCount: networkMetrics.bannerNetworkRequests, - dnsLookupTime: 0, - connectionTime: 0, - downloadTime: networkImpact.totalDownloadTime, - }, - }, - mainThreadBlocking: { - total: coreWebVitals.totalBlockingTime || 0, - cookieBannerEstimate: - cookieBannerMetrics.bannerMainThreadBlockingTime, - percentageFromCookies: - (coreWebVitals.totalBlockingTime || 0) > 0 - ? (cookieBannerMetrics.bannerMainThreadBlockingTime / - (coreWebVitals.totalBlockingTime || 1)) * - 100 - : 0, - }, - scripts: resourceMetrics.timing.scripts, - }, - resources: resourceMetrics.resources, - language: resourceMetrics.language, - cookieBanner: { - detected: cookieBannerData?.detected || false, - selector: cookieBannerData?.selector || null, - serviceName: config.cookieBanner?.serviceName || "unknown", - visibilityTime: cookieBannerData?.bannerInteractiveTime || 0, - viewportCoverage: cookieBannerData?.viewportCoverage || 0, - }, - thirdParty: { - cookieServices: { - hosts: config.cookieBanner?.serviceHosts || [], - totalSize: networkMetrics.bannerBundleSize, - resourceCount: networkMetrics.bannerNetworkRequests, - dnsLookupTime: 0, - connectionTime: 0, - downloadTime: networkImpact.totalDownloadTime, - }, - totalImpact: networkImpact.totalImpact, - }, - }; - } - - /** - * Calculate network impact metrics - */ - private calculateNetworkImpact(networkRequests: NetworkRequest[]): { - totalImpact: number; - totalDownloadTime: number; - } { - const totalImpact = networkRequests.reduce((acc, req) => acc + req.size, 0); - const totalDownloadTime = networkRequests.reduce( - (acc, req) => acc + req.duration, - 0 - ); - - return { totalImpact, totalDownloadTime }; - } - - /** - * Calculate average metrics from multiple benchmark results - */ - calculateAverages(results: BenchmarkDetails[]): BenchmarkResult["average"] { - if (results.length === 0) { - throw new Error("Cannot calculate averages from empty results array"); - } - - return { - firstContentfulPaint: - results.reduce( - (acc, curr) => acc + curr.timing.firstContentfulPaint, - 0 - ) / results.length, - largestContentfulPaint: - results.reduce( - (acc, curr) => acc + curr.timing.largestContentfulPaint, - 0 - ) / results.length, - timeToInteractive: - results.reduce((acc, curr) => acc + curr.timing.timeToInteractive, 0) / - results.length, - totalBlockingTime: - results.reduce( - (acc, curr) => acc + curr.timing.mainThreadBlocking.total, - 0 - ) / results.length, - speedIndex: 0, // Default value - timeToFirstByte: 0, // Default value - firstInputDelay: 0, // Default value - cumulativeLayoutShift: - results.reduce( - (acc, curr) => acc + curr.timing.cumulativeLayoutShift, - 0 - ) / results.length, - domSize: 0, // Default value - totalRequests: - results.reduce( - (acc, curr) => - acc + - (curr.resources.scripts.length + - curr.resources.styles.length + - curr.resources.images.length + - curr.resources.fonts.length + - curr.resources.other.length), - 0 - ) / results.length, - totalSize: - results.reduce((acc, curr) => acc + curr.size.total, 0) / - results.length, - jsSize: - results.reduce((acc, curr) => acc + curr.size.scripts.total, 0) / - results.length, - cssSize: - results.reduce((acc, curr) => acc + curr.size.styles, 0) / results.length, - imageSize: - results.reduce((acc, curr) => acc + curr.size.images, 0) / results.length, - fontSize: - results.reduce((acc, curr) => acc + curr.size.fonts, 0) / results.length, - otherSize: - results.reduce((acc, curr) => acc + curr.size.other, 0) / results.length, - thirdPartyRequests: 0, // Default value - thirdPartySize: 0, // Default value - thirdPartyDomains: 0, // Default value - thirdPartyCookies: 0, // Default value - thirdPartyLocalStorage: 0, // Default value - thirdPartySessionStorage: 0, // Default value - thirdPartyIndexedDB: 0, // Default value - thirdPartyCache: 0, // Default value - thirdPartyServiceWorkers: 0, // Default value - thirdPartyWebWorkers: 0, // Default value - thirdPartyWebSockets: 0, // Default value - thirdPartyBeacons: 0, // Default value - thirdPartyFetch: 0, // Default value - thirdPartyXHR: 0, // Default value - thirdPartyScripts: 0, // Default value - thirdPartyStyles: 0, // Default value - thirdPartyImages: 0, // Default value - thirdPartyFonts: 0, // Default value - thirdPartyMedia: 0, // Default value - thirdPartyOther: 0, // Default value - thirdPartyTiming: { - total: 0, - blocking: 0, - dns: 0, - connect: 0, - ssl: 0, - send: 0, - wait: 0, - receive: 0, - }, - cookieBannerTiming: { - firstPaint: 0, - firstContentfulPaint: - results.reduce( - (acc, curr) => acc + curr.timing.firstContentfulPaint, - 0 - ) / results.length, - domContentLoaded: - results.reduce((acc, curr) => acc + curr.timing.domContentLoaded, 0) / - results.length, - load: - results.reduce((acc, curr) => acc + curr.timing.load, 0) / - results.length, - }, - }; - } - - /** - * Log comprehensive benchmark results - */ - logResults( - finalMetrics: BenchmarkDetails, - cookieBannerMetrics: CookieBannerMetrics, - config: Config - ): void { - console.log("🔍 [DEBUG] Final cookie banner benchmark results:", { - fcp: finalMetrics.timing.firstContentfulPaint, - lcp: finalMetrics.timing.largestContentfulPaint, - cls: finalMetrics.timing.cumulativeLayoutShift, - tti: finalMetrics.timing.timeToInteractive, - tbt: finalMetrics.timing.mainThreadBlocking.total, - bannerDetected: finalMetrics.cookieBanner.detected, - bannerRenderTime: - finalMetrics.timing.cookieBanner.renderEnd - - finalMetrics.timing.cookieBanner.renderStart, - bannerLayoutShift: finalMetrics.timing.cookieBanner.layoutShift, - bannerNetworkImpact: finalMetrics.thirdParty.totalImpact, - bundleStrategy: cookieBannerMetrics.isBundled - ? "Bundled" - : cookieBannerMetrics.isIIFE - ? "IIFE" - : "Unknown", - isBundled: cookieBannerMetrics.isBundled, - isIIFE: cookieBannerMetrics.isIIFE, - configBundleType: config.techStack?.bundleType, - }); - } -} diff --git a/packages/cli/src/lib/performance-enhanced.ts b/packages/cli/src/lib/performance-enhanced.ts deleted file mode 100644 index e2e9e71..0000000 --- a/packages/cli/src/lib/performance-enhanced.ts +++ /dev/null @@ -1,259 +0,0 @@ -import type { Page } from "@playwright/test"; -import { - PerformanceMetricsCollector, - DefaultNetworkPresets, -} from "playwright-performance-metrics"; -import type { CookieBannerMetrics } from "./performance"; - -export interface EnhancedPerformanceOptions { - networkCondition?: keyof typeof DefaultNetworkPresets | "none"; - timeout?: number; - initialDelay?: number; - retryTimeout?: number; -} - -export class EnhancedPerformanceCollector { - private collector: PerformanceMetricsCollector; - - constructor() { - this.collector = new PerformanceMetricsCollector(); - } - - async initialize( - page: Page, - options: EnhancedPerformanceOptions = {} - ): Promise { - console.log( - "🔍 [ENHANCED-PERF] Initializing enhanced performance collector..." - ); - - // Set up network conditions if specified - if (options.networkCondition && options.networkCondition !== "none") { - const networkPreset = DefaultNetworkPresets[options.networkCondition]; - if (networkPreset) { - console.log( - `🔍 [ENHANCED-PERF] Applying network preset: ${options.networkCondition}` - ); - await this.collector.initialize(page, networkPreset); - } else { - console.warn( - `🔍 [ENHANCED-PERF] Unknown network preset: ${options.networkCondition}` - ); - } - } - } - - async collectMetrics( - page: Page, - cookieBannerSelectors: string[] = [], - options: EnhancedPerformanceOptions = {} - ): Promise { - console.log("🔍 [ENHANCED-PERF] Collecting performance metrics..."); - - try { - // Use the robust playwright-performance-metrics library - const metrics = await this.collector.collectMetrics(page, { - timeout: options.timeout || 10000, - retryTimeout: options.retryTimeout || 5000, - ...options, - }); - - console.log("🔍 [ENHANCED-PERF] Raw metrics collected:", { - fcp: metrics.paint?.firstContentfulPaint, - lcp: metrics.largestContentfulPaint, - cls: metrics.cumulativeLayoutShift, - tbt: metrics.totalBlockingTime, - domComplete: metrics.domCompleteTiming, - pageLoad: metrics.pageloadTiming, - totalBytes: metrics.totalBytes, - }); - - // Detect cookie banners using our existing logic - const bannerMetrics = await this.detectCookieBanner( - page, - cookieBannerSelectors - ); - - // Convert to our CookieBannerMetrics format - const enhancedMetrics: CookieBannerMetrics = { - // Core Web Vitals from the library - fcp: metrics.paint?.firstContentfulPaint || 0, - lcp: metrics.largestContentfulPaint || 0, - cls: metrics.cumulativeLayoutShift || 0, - tti: this.calculateTTI(metrics), - tbt: metrics.totalBlockingTime || 0, - - // Cookie Banner Specific - bannerDetected: bannerMetrics.detected, - bannerSelector: bannerMetrics.selector, - bannerFirstPaint: bannerMetrics.firstPaint, - bannerLargestContentfulPaint: bannerMetrics.lcp, - bannerTimeToInteractive: bannerMetrics.timeToInteractive, - bannerHydrationTime: bannerMetrics.hydrationTime, - bannerLayoutShift: bannerMetrics.layoutShift, - bannerMainThreadBlocking: bannerMetrics.mainThreadBlocking, - bannerNetworkImpact: bannerMetrics.networkImpact, - bannerVisibilityTime: bannerMetrics.visibilityTime, - bannerViewportCoverage: bannerMetrics.viewportCoverage, - }; - - console.log("🔍 [ENHANCED-PERF] Final enhanced metrics:", { - fcp: enhancedMetrics.fcp, - lcp: enhancedMetrics.lcp, - cls: enhancedMetrics.cls, - tti: enhancedMetrics.tti, - tbt: enhancedMetrics.tbt, - bannerDetected: enhancedMetrics.bannerDetected, - }); - - return enhancedMetrics; - } catch (error) { - console.error("🔍 [ENHANCED-PERF] Error collecting metrics:", error); - // Fallback to basic metrics - return this.getDefaultMetrics(); - } - } - - private async detectCookieBanner( - page: Page, - selectors: string[] - ): Promise<{ - detected: boolean; - selector: string | null; - firstPaint: number; - lcp: number; - timeToInteractive: number; - hydrationTime: number; - layoutShift: number; - mainThreadBlocking: number; - networkImpact: number; - visibilityTime: number; - viewportCoverage: number; - }> { - if (selectors.length === 0) { - return { - detected: false, - selector: null, - firstPaint: 0, - lcp: 0, - timeToInteractive: 0, - hydrationTime: 0, - layoutShift: 0, - mainThreadBlocking: 0, - networkImpact: 0, - visibilityTime: 0, - viewportCoverage: 0, - }; - } - - return page.evaluate((selectors: string[]) => { - for (const selector of selectors) { - try { - const element = document.querySelector(selector); - if (element) { - const rect = element.getBoundingClientRect(); - const isVisible = - rect.width > 0 && - rect.height > 0 && - window.getComputedStyle(element).visibility !== "hidden" && - window.getComputedStyle(element).display !== "none"; - - if (isVisible) { - const viewportCoverage = - ((rect.width * rect.height) / - (window.innerWidth * window.innerHeight)) * - 100; - const now = performance.now(); - - // Check if banner is interactive - const buttons = element.querySelectorAll( - 'button, a, [role="button"], [onclick]' - ); - const timeToInteractive = buttons.length > 0 ? now : 0; - - return { - detected: true, - selector: selector, - firstPaint: now, - lcp: now, - timeToInteractive: timeToInteractive, - hydrationTime: - timeToInteractive > 0 ? timeToInteractive - now : 0, - layoutShift: 0, // Would need more complex measurement - mainThreadBlocking: 0, // Would need more complex measurement - networkImpact: 0, // Would need more complex measurement - visibilityTime: 0, // Would need time tracking - viewportCoverage: viewportCoverage, - }; - } - } - } catch (error) { - console.warn("🔍 [BANNER] Error checking selector:", selector, error); - } - } - - return { - detected: false, - selector: null, - firstPaint: 0, - lcp: 0, - timeToInteractive: 0, - hydrationTime: 0, - layoutShift: 0, - mainThreadBlocking: 0, - networkImpact: 0, - visibilityTime: 0, - viewportCoverage: 0, - }; - }, selectors); - } - - private calculateTTI(metrics: { - paint?: { firstContentfulPaint?: number }; - domCompleteTiming?: number | null; - }): number { - // TTI calculation based on the available metrics - // This is a simplified approach - the library might provide better TTI measurement in the future - const baseTime = Math.max( - metrics.paint?.firstContentfulPaint || 0, - metrics.domCompleteTiming || 0 - ); - - // Add some buffer time for interactivity (simplified approach) - return baseTime + 2000; // Add 2 seconds as a buffer - } - - private getDefaultMetrics(): CookieBannerMetrics { - return { - fcp: 0, - lcp: 0, - cls: 0, - tti: 0, - tbt: 0, - bannerDetected: false, - bannerSelector: null, - bannerFirstPaint: 0, - bannerLargestContentfulPaint: 0, - bannerTimeToInteractive: 0, - bannerHydrationTime: 0, - bannerLayoutShift: 0, - bannerMainThreadBlocking: 0, - bannerNetworkImpact: 0, - bannerVisibilityTime: 0, - bannerViewportCoverage: 0, - }; - } - - async cleanup(): Promise { - await this.collector.cleanup(); - } -} - -// Network presets for easy access -export const NetworkPresets = { - REGULAR_4G: "REGULAR_4G" as const, - SLOW_3G: "SLOW_3G" as const, - FAST_3G: "FAST_3G" as const, - FAST_WIFI: "FAST_WIFI" as const, - NONE: "none" as const, -}; diff --git a/packages/cli/src/lib/performance.ts b/packages/cli/src/lib/performance.ts deleted file mode 100644 index a9953bb..0000000 --- a/packages/cli/src/lib/performance.ts +++ /dev/null @@ -1,437 +0,0 @@ -import type { Page } from "@playwright/test"; -import type { LayoutShiftEntry } from "../types"; - -interface WebVitalsMetrics { - fcp: number; - lcp: number; - cls: number; - tti: number; - tbt: number; - bannerDetected: boolean; - bannerSelector: string | null; - bannerVisibilityTime: number; - bannerViewportCoverage: number; - bannerFirstPaint: number; - bannerLargestContentfulPaint: number; - bannerTimeToInteractive: number; - bannerHydrationTime: number; - bannerLayoutShift: number; - bannerMainThreadBlocking: number; - bannerNetworkImpact: number; - measurementComplete: boolean; -} - -interface WindowWithWebVitals extends Window { - __webVitalsMetrics: WebVitalsMetrics; -} - -export interface CookieBannerMetrics { - // Core Web Vitals - fcp: number; // First Contentful Paint - lcp: number; // Largest Contentful Paint - cls: number; // Cumulative Layout Shift - tti: number; // Time to Interactive - tbt: number; // Total Blocking Time - - // Cookie Banner Specific Timings - bannerFirstPaint: number; // When banner first appears - bannerLargestContentfulPaint: number; // When banner's main content renders - bannerTimeToInteractive: number; // When banner buttons become clickable - bannerHydrationTime: number; // Time from first paint to interactive (hydration) - - // Cookie Banner Impact Measurements - bannerLayoutShift: number; // Layout shifts caused by banner - bannerMainThreadBlocking: number; // Main thread blocking from banner scripts - bannerNetworkImpact: number; // Network requests caused by banner - - // Cookie Banner Detection - bannerDetected: boolean; - bannerSelector: string | null; - bannerVisibilityTime: number; // How long banner was visible - bannerViewportCoverage: number; // Percentage of viewport covered -} - -export async function collectCookieBannerMetrics( - page: Page, - cookieBannerSelectors: string[], - serviceHosts: string[] -): Promise { - console.log( - "🔍 [PERFORMANCE] Setting up Web Vitals measurement before navigation..." - ); - - // Set up performance measurement BEFORE navigation - await page.addInitScript((selectors: string[]) => { - console.log( - "🔍 [BROWSER] Setting up performance observers before page load..." - ); - - // Initialize global storage for metrics - const webVitalsMetrics = { - fcp: 0, - lcp: 0, - cls: 0, - tti: 0, - tbt: 0, - bannerDetected: false, - bannerSelector: null as string | null, - bannerVisibilityTime: 0, - bannerViewportCoverage: 0, - bannerFirstPaint: 0, - bannerLargestContentfulPaint: 0, - bannerTimeToInteractive: 0, - bannerHydrationTime: 0, - bannerLayoutShift: 0, - bannerMainThreadBlocking: 0, - bannerNetworkImpact: 0, - measurementComplete: false, - }; - - (window as unknown as WindowWithWebVitals).__webVitalsMetrics = - webVitalsMetrics; - - // Set up observers for paint timing - try { - // First Contentful Paint - const fcpObserver = new PerformanceObserver((list) => { - for (const entry of list.getEntries()) { - if (entry.name === "first-contentful-paint") { - webVitalsMetrics.fcp = entry.startTime; - console.log("🔍 [BROWSER] FCP captured:", webVitalsMetrics.fcp); - fcpObserver.disconnect(); - } - } - }); - fcpObserver.observe({ type: "paint", buffered: true }); - - // Largest Contentful Paint - const lcpObserver = new PerformanceObserver((list) => { - const entries = list.getEntries(); - const lastEntry = entries.at(-1); - if (lastEntry) { - webVitalsMetrics.lcp = lastEntry.startTime; - console.log("🔍 [BROWSER] LCP captured:", webVitalsMetrics.lcp); - } - }); - lcpObserver.observe({ type: "largest-contentful-paint", buffered: true }); - - // Cumulative Layout Shift - let clsValue = 0; - const clsObserver = new PerformanceObserver((list) => { - for (const entry of list.getEntries()) { - const layoutShiftEntry = entry as LayoutShiftEntry; - if (!layoutShiftEntry.hadRecentInput) { - clsValue += layoutShiftEntry.value; - } - } - webVitalsMetrics.cls = clsValue; - if (clsValue > 0) { - console.log("🔍 [BROWSER] CLS updated:", webVitalsMetrics.cls); - } - }); - clsObserver.observe({ type: "layout-shift", buffered: true }); - - // Total Blocking Time - let tbtValue = 0; - const longTaskObserver = new PerformanceObserver((list) => { - for (const entry of list.getEntries()) { - if (entry.duration > 50) { - tbtValue += entry.duration - 50; - } - } - webVitalsMetrics.tbt = tbtValue; - if (tbtValue > 0) { - console.log("🔍 [BROWSER] TBT updated:", webVitalsMetrics.tbt); - } - }); - longTaskObserver.observe({ type: "longtask", buffered: true }); - - console.log("🔍 [BROWSER] Performance observers set up successfully"); - } catch (error) { - console.warn( - "🔍 [BROWSER] Error setting up performance observers:", - error - ); - } - - // Cookie banner detection setup - const detectCookieBanner = () => { - for (const selector of selectors) { - try { - const element = document.querySelector(selector); - if (element) { - const rect = element.getBoundingClientRect(); - const isVisible = - rect.width > 0 && - rect.height > 0 && - window.getComputedStyle(element).visibility !== "hidden" && - window.getComputedStyle(element).display !== "none"; - - if (isVisible) { - webVitalsMetrics.bannerDetected = true; - webVitalsMetrics.bannerSelector = selector; - webVitalsMetrics.bannerFirstPaint = performance.now(); - webVitalsMetrics.bannerViewportCoverage = - ((rect.width * rect.height) / - (window.innerWidth * window.innerHeight)) * - 100; - - console.log("🔍 [BANNER] Cookie banner detected:", selector); - - // Check if banner is interactive - const buttons = element.querySelectorAll( - 'button, a, [role="button"], [onclick]' - ); - if (buttons.length > 0) { - webVitalsMetrics.bannerTimeToInteractive = performance.now(); - webVitalsMetrics.bannerHydrationTime = - webVitalsMetrics.bannerTimeToInteractive - - webVitalsMetrics.bannerFirstPaint; - } - - return true; - } - } - } catch (error) { - console.warn("🔍 [BANNER] Error checking selector:", selector, error); - } - } - return false; - }; - - // Set up banner detection after DOM is ready - if (document.readyState === "loading") { - document.addEventListener("DOMContentLoaded", () => { - // Check immediately and then periodically - if (!detectCookieBanner()) { - const interval = setInterval(() => { - if (detectCookieBanner()) { - clearInterval(interval); - } - }, 100); - - // Stop checking after 10 seconds - setTimeout(() => clearInterval(interval), 10000); - } - }); - } else { - // DOM is already ready - if (!detectCookieBanner()) { - const interval = setInterval(() => { - if (detectCookieBanner()) { - clearInterval(interval); - } - }, 100); - - // Stop checking after 10 seconds - setTimeout(() => clearInterval(interval), 10000); - } - } - }, cookieBannerSelectors); - - return { - fcp: 0, - lcp: 0, - cls: 0, - tti: 0, - tbt: 0, - bannerDetected: false, - bannerSelector: null, - bannerVisibilityTime: 0, - bannerViewportCoverage: 0, - bannerFirstPaint: 0, - bannerLargestContentfulPaint: 0, - bannerTimeToInteractive: 0, - bannerHydrationTime: 0, - bannerLayoutShift: 0, - bannerMainThreadBlocking: 0, - bannerNetworkImpact: 0, - }; -} - -export async function collectWebVitalsAfterLoad( - page: Page -): Promise { - console.log("🔍 [PERFORMANCE] Collecting final Web Vitals..."); - - // Wait for page to be ready - await page.waitForLoadState("networkidle"); - - // Get final metrics from the browser - const webVitals = await page.evaluate(() => { - console.log("🔍 [BROWSER] Collecting final Web Vitals..."); - - const storedMetrics = (window as unknown as WindowWithWebVitals) - .__webVitalsMetrics; - if (!storedMetrics) { - console.error("🔍 [BROWSER] No stored metrics found!"); - return null; - } - - // Get additional paint entries that might have been missed - const paintEntries = performance.getEntriesByType("paint"); - console.log("🔍 [BROWSER] Paint entries found:", paintEntries.length); - for (const entry of paintEntries) { - console.log( - `🔍 [BROWSER] Paint entry: ${entry.name} = ${entry.startTime}ms` - ); - if (entry.name === "first-contentful-paint" && storedMetrics.fcp === 0) { - storedMetrics.fcp = entry.startTime; - console.log("🔍 [BROWSER] FCP from fallback:", storedMetrics.fcp); - } - } - - // Get LCP entries that might have been missed - const lcpEntries = performance.getEntriesByType("largest-contentful-paint"); - console.log("🔍 [BROWSER] LCP entries found:", lcpEntries.length); - if (lcpEntries.length > 0 && storedMetrics.lcp === 0) { - const lastLCP = lcpEntries[lcpEntries.length - 1]; - storedMetrics.lcp = lastLCP.startTime; - console.log("🔍 [BROWSER] LCP from fallback:", storedMetrics.lcp); - } - - console.log("🔍 [BROWSER] Final stored metrics:", { - fcp: storedMetrics.fcp, - lcp: storedMetrics.lcp, - cls: storedMetrics.cls, - tbt: storedMetrics.tbt, - bannerDetected: storedMetrics.bannerDetected, - }); - - return storedMetrics; - }); - - if (!webVitals) { - console.warn( - "🔍 [PERFORMANCE] No web vitals data available, returning defaults" - ); - return { - fcp: 0, - lcp: 0, - cls: 0, - tti: 0, - tbt: 0, - bannerDetected: false, - bannerSelector: null, - bannerVisibilityTime: 0, - bannerViewportCoverage: 0, - bannerFirstPaint: 0, - bannerLargestContentfulPaint: 0, - bannerTimeToInteractive: 0, - bannerHydrationTime: 0, - bannerLayoutShift: 0, - bannerMainThreadBlocking: 0, - bannerNetworkImpact: 0, - }; - } - - // Calculate TTI (simplified approach) - const navigationTiming = await page.evaluate(() => { - const navigation = performance.getEntriesByType( - "navigation" - )[0] as PerformanceNavigationTiming; - return { - domContentLoaded: - navigation.domContentLoadedEventEnd - navigation.fetchStart, - loadComplete: navigation.loadEventEnd - navigation.fetchStart, - }; - }); - - // Calculate TTI as a simple estimation - const tti = - Math.max( - webVitals.fcp || 0, - navigationTiming.domContentLoaded, - webVitals.bannerTimeToInteractive || 0 - ) + 5000; // Add 5 seconds for page to become truly interactive - - const finalMetrics: CookieBannerMetrics = { - fcp: webVitals.fcp, - lcp: webVitals.lcp, - cls: webVitals.cls, - tti: tti, - tbt: webVitals.tbt, - bannerDetected: webVitals.bannerDetected, - bannerSelector: webVitals.bannerSelector, - bannerVisibilityTime: webVitals.bannerVisibilityTime, - bannerViewportCoverage: webVitals.bannerViewportCoverage, - bannerFirstPaint: webVitals.bannerFirstPaint, - bannerLargestContentfulPaint: - webVitals.bannerLargestContentfulPaint || webVitals.bannerFirstPaint, - bannerTimeToInteractive: webVitals.bannerTimeToInteractive, - bannerHydrationTime: webVitals.bannerHydrationTime, - bannerLayoutShift: webVitals.bannerLayoutShift, - bannerMainThreadBlocking: webVitals.bannerMainThreadBlocking, - bannerNetworkImpact: webVitals.bannerNetworkImpact, - }; - - console.log("🔍 [PERFORMANCE] Final metrics collected:", { - fcp: finalMetrics.fcp, - lcp: finalMetrics.lcp, - cls: finalMetrics.cls, - tti: finalMetrics.tti, - tbt: finalMetrics.tbt, - bannerDetected: finalMetrics.bannerDetected, - }); - - return finalMetrics; -} - -// Legacy function for backwards compatibility -export async function collectPerformanceMetrics( - page: Page, - cookieBannerSelectors: string[] = [] -): Promise<{ - fcp: number; - lcp: number; - cls: number; - tbt: number; - tti: number; -}> { - const metrics = await collectCookieBannerMetrics( - page, - cookieBannerSelectors, - [] - ); - return { - fcp: metrics.fcp, - lcp: metrics.lcp, - cls: metrics.cls, - tbt: metrics.tbt, - tti: metrics.tti, - }; -} - -export async function collectResourceTiming(page: Page): Promise<{ - scriptLoadTime: number; - totalSize: number; - scriptSize: number; - resourceCount: number; - scriptCount: number; -}> { - return page.evaluate(() => { - const resources = performance.getEntriesByType( - "resource" - ) as PerformanceResourceTiming[]; - const scripts = resources.filter((r) => r.initiatorType === "script"); - - const totalSize = resources.reduce( - (sum, r) => sum + (r.transferSize || 0), - 0 - ); - const scriptSize = scripts.reduce( - (sum, r) => sum + (r.transferSize || 0), - 0 - ); - const scriptLoadTime = - scripts.reduce((sum, r) => sum + r.duration, 0) / (scripts.length || 1); - - return { - scriptLoadTime, - totalSize, - scriptSize, - resourceCount: resources.length, - scriptCount: scripts.length, - }; - }); -} diff --git a/packages/cli/src/lib/server.ts b/packages/cli/src/lib/server.ts deleted file mode 100644 index 7a9d4ed..0000000 --- a/packages/cli/src/lib/server.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { spawn } from "node:child_process"; -import { getPackageManager } from "../utils"; -import type { ServerInfo } from "../types"; - -export async function buildAndServeNextApp( - appPath?: string -): Promise { - const pm = await getPackageManager(); - const cwd = appPath || process.cwd(); - - // Build the app - console.log("[Build] Building Next.js app..."); - const buildProcess = spawn(pm.command, [...pm.args, "build"], { - cwd, - stdio: "inherit", - }); - - await new Promise((resolve, reject) => { - buildProcess.on("close", (code) => { - if (code === 0) { - resolve(); - } else { - reject(new Error(`Build failed with code ${code}`)); - } - }); - }); - - // Start the server - console.log("[Build] Starting Next.js server..."); - const port = Math.floor(Math.random() * (9000 - 3000 + 1)) + 3000; - console.log("command", [ - ...pm.args, - "start", - "--", - "--port", - port.toString(), - ]); - const serverProcess = spawn( - pm.command, - [...pm.args, "start", "--", "--port", port.toString()], - { - cwd, - stdio: ["inherit", "pipe", "inherit"], - } - ); - - // Wait for server to be ready - const url = `http://localhost:${port}`; - let retries = 0; - const maxRetries = 30; - - while (retries < maxRetries) { - try { - const response = await fetch(url); - if (response.ok) { - console.log("[Build] Server is ready!"); - return { serverProcess, url }; - } - } catch { - // Ignore error and retry - } - - await new Promise((resolve) => setTimeout(resolve, 1000)); - retries++; - } - - throw new Error("Server failed to start"); -} - -export function cleanupServer(serverInfo: ServerInfo): void { - if (serverInfo.serverProcess) { - serverInfo.serverProcess.kill(); - } -} diff --git a/packages/cli/src/types/index.ts b/packages/cli/src/types/index.ts deleted file mode 100644 index 26d1967..0000000 --- a/packages/cli/src/types/index.ts +++ /dev/null @@ -1,369 +0,0 @@ -import type { Page } from "@playwright/test"; -import type { ChildProcess } from "node:child_process"; - -export interface Config { - name: string; - url?: string; - testId?: string; - id?: string; - iterations: number; - baseline?: boolean; - custom?: (page: Page) => Promise; - remote?: { - enabled?: boolean; - url?: string; - headers?: Record; - }; - cookieBanner: CookieBannerConfig; - internationalization: { - detection: string; - stringLoading: string; - }; - techStack: { - bundler: string; - bundleType: string | string[]; - frameworks: string[]; - languages: string[]; - packageManager: string; - typescript: boolean; - }; - source: { - github: string | false; - isOpenSource: boolean | string; - license: string; - npm: string | false; - website?: string; - }; - includes: { - backend: string | string[] | false; - components: string[]; - }; - company?: { - name: string; - website: string; - avatar: string; - }; - tags?: string[]; -} - -export interface BenchmarkDetails { - duration: number; - size: { - total: number; - bundled: number; - thirdParty: number; - scripts: { - total: number; - initial: number; - dynamic: number; - }; - styles: number; - images: number; - fonts: number; - other: number; - }; - timing: { - navigationStart: number; - domContentLoaded: number; - load: number; - firstPaint: number; - firstContentfulPaint: number; - largestContentfulPaint: number; - timeToInteractive: number; - cumulativeLayoutShift: number; - cookieBanner: { - renderStart: number; - renderEnd: number; - interactionStart: number; - interactionEnd: number; - layoutShift: number; - detected: boolean; - selector: string | null; - serviceName: string; - visibilityTime: number | null; - viewportCoverage: number; - }; - thirdParty: { - dnsLookupTime: number; - connectionTime: number; - downloadTime: number; - totalImpact: number; - cookieServices: { - hosts: string[]; - totalSize: number; - resourceCount: number; - dnsLookupTime: number; - connectionTime: number; - downloadTime: number; - }; - }; - mainThreadBlocking: { - total: number; - cookieBannerEstimate: number; - percentageFromCookies: number; - }; - scripts: { - bundled: { - loadStart: number; - loadEnd: number; - executeStart: number; - executeEnd: number; - }; - thirdParty: { - loadStart: number; - loadEnd: number; - executeStart: number; - executeEnd: number; - }; - }; - }; - language: string; - resources: { - scripts: Array<{ - name: string; - size: number; - duration: number; - startTime: number; - isThirdParty: boolean; - isDynamic: boolean; - }>; - styles: Array<{ - name: string; - size: number; - duration: number; - startTime: number; - isThirdParty: boolean; - }>; - images: Array<{ - name: string; - size: number; - duration: number; - startTime: number; - isThirdParty: boolean; - }>; - fonts: Array<{ - name: string; - size: number; - duration: number; - startTime: number; - isThirdParty: boolean; - }>; - other: Array<{ - name: string; - size: number; - duration: number; - startTime: number; - isThirdParty: boolean; - type: string; - }>; - }; - cookieBanner: EnhancedCookieBannerTiming; - thirdParty: ThirdPartyMetrics; -} - -export interface BenchmarkResult { - name: string; - baseline: boolean; - techStack: Config["techStack"]; - source: Config["source"]; - includes: Config["includes"]; - company?: Config["company"]; - tags?: string[]; - details: BenchmarkDetails[]; - average: { - firstContentfulPaint: number; - largestContentfulPaint: number; - timeToInteractive: number; - totalBlockingTime: number; - speedIndex: number; - timeToFirstByte: number; - firstInputDelay: number; - cumulativeLayoutShift: number; - domSize: number; - totalRequests: number; - totalSize: number; - jsSize: number; - cssSize: number; - imageSize: number; - fontSize: number; - otherSize: number; - thirdPartyRequests: number; - thirdPartySize: number; - thirdPartyDomains: number; - thirdPartyCookies: number; - thirdPartyLocalStorage: number; - thirdPartySessionStorage: number; - thirdPartyIndexedDB: number; - thirdPartyCache: number; - thirdPartyServiceWorkers: number; - thirdPartyWebWorkers: number; - thirdPartyWebSockets: number; - thirdPartyBeacons: number; - thirdPartyFetch: number; - thirdPartyXHR: number; - thirdPartyScripts: number; - thirdPartyStyles: number; - thirdPartyImages: number; - thirdPartyFonts: number; - thirdPartyMedia: number; - thirdPartyOther: number; - thirdPartyTiming: { - total: number; - blocking: number; - dns: number; - connect: number; - ssl: number; - send: number; - wait: number; - receive: number; - }; - cookieBannerTiming: { - firstPaint: number; - firstContentfulPaint: number; - domContentLoaded: number; - load: number; - }; - }; - scores?: { - totalScore: number; - grade: "Excellent" | "Good" | "Fair" | "Poor" | "Critical"; - categoryScores: { - performance: number; - bundleStrategy: number; - networkImpact: number; - transparency: number; - userExperience: number; - }; - categories: Array<{ - name: string; - score: number; - maxScore: number; - weight: number; - details: Array<{ - name: string; - score: number; - maxScore: number; - weight: number; - status: "excellent" | "good" | "fair" | "poor"; - reason: string; - }>; - status: "excellent" | "good" | "fair" | "poor"; - reason: string; - }>; - insights: string[]; - recommendations: string[]; - }; -} - -export interface ServerInfo { - serverProcess: ChildProcess; - url: string; -} - -// Performance API type definitions -export interface LayoutShiftEntry extends PerformanceEntry { - value: number; - hadRecentInput: boolean; -} - -export interface ResourceTimingEntry extends PerformanceEntry { - initiatorType: string; - transferSize: number; -} - -export interface LargestContentfulPaintEntry extends PerformanceEntry { - startTime: number; -} - -export interface FirstContentfulPaintEntry extends PerformanceEntry { - startTime: number; -} - -export interface LongTaskEntry extends PerformanceEntry { - duration: number; -} - -export interface PerformanceMetrics { - fcp: number; - lcp: number; - cls: number; - tbt: number; - tti: number; -} - -interface CookieBannerConfig { - selectors: string[]; - serviceHosts: string[]; - waitForVisibility: boolean; - measureViewportCoverage: boolean; - expectedLayoutShift: boolean; - serviceName: string; -} - -interface EnhancedCookieBannerTiming { - detected: boolean; - selector: string | null; - serviceName: string; - visibilityTime: number | null; - viewportCoverage: number; -} - -interface ThirdPartyMetrics { - cookieServices: { - hosts: string[]; - totalSize: number; - resourceCount: number; - dnsLookupTime: number; - connectionTime: number; - downloadTime: number; - }; - totalImpact: number; -} - -declare global { - interface Window { - CLS?: number; - TBT?: number; - TTI?: number; - __scriptTiming?: { - bundled: { - executeStart?: number; - executeEnd?: number; - }; - thirdParty: { - executeStart?: number; - executeEnd?: number; - }; - }; - } -} - -export interface BenchmarkScores { - totalScore: number; - grade: 'Excellent' | 'Good' | 'Fair' | 'Poor' | 'Critical'; - categoryScores: { - performance: number; - bundleStrategy: number; - networkImpact: number; - transparency: number; - userExperience: number; - }; - categories: Array<{ - name: string; - score: number; - maxScore: number; - weight: number; - details: Array<{ - name: string; - score: number; - maxScore: number; - weight: number; - status: 'excellent' | 'good' | 'fair' | 'poor'; - reason: string; - }>; - status: 'excellent' | 'good' | 'fair' | 'poor'; - reason: string; - }>; - insights: string[]; - recommendations: string[]; -} diff --git a/packages/cookiebench-cli/README.md b/packages/cookiebench-cli/README.md new file mode 100644 index 0000000..01037cd --- /dev/null +++ b/packages/cookiebench-cli/README.md @@ -0,0 +1,213 @@ +# cookiebench + +Command-line interface for cookie banner performance benchmarking. + +## Overview + +This CLI tool provides an interactive interface for running cookie banner benchmarks, viewing results, and managing the benchmark database. + +## Installation + +```bash +pnpm add -g cookiebench +``` + +Or use in a workspace: + +```bash +pnpm add cookiebench +``` + +## Usage + +### Interactive Mode + +Run without arguments for interactive prompts: + +```bash +cookiebench +``` + +### Direct Commands + +Run specific commands directly: + +```bash +# Run a benchmark +cookiebench benchmark + +# View and aggregate results +cookiebench results + +# Manage database +cookiebench db push +``` + +## Commands + +### benchmark + +Run performance benchmarks on configured applications. + +```bash +cookiebench benchmark [appPath] +``` + +The command will: +1. Read `config.json` from the current directory or specified path +2. Build and serve the Next.js app (or use remote URL if configured) +3. Run benchmarks for the specified number of iterations +4. Calculate performance scores +5. Save results to `results.json` + +### results + +Aggregate and display benchmark results from multiple apps. + +```bash +cookiebench results +``` + +Features: +- Aggregates results from all `results.json` files in benchmark directories +- Displays comparison table with metrics and deltas from baseline +- Calculates scores for each app +- Saves results to database (if configured) + +### db + +Manage the benchmark database. + +```bash +# Push database schema +cookiebench db push + +# View database status +cookiebench db +``` + +## Configuration + +Create a `config.json` file in your project: + +```json +{ + "name": "my-app-with-cookieyes", + "iterations": 5, + "baseline": false, + "cookieBanner": { + "serviceName": "CookieYes", + "selectors": [ + "#cookieyes-banner", + ".cky-consent-container" + ], + "serviceHosts": [ + "cdn-cookieyes.com" + ], + "waitForVisibility": true, + "measureViewportCoverage": true, + "expectedLayoutShift": true + }, + "techStack": { + "frameworks": ["react", "nextjs"], + "languages": ["typescript"], + "bundler": "webpack", + "bundleType": "esm", + "packageManager": "pnpm", + "typescript": true + }, + "source": { + "license": "MIT", + "isOpenSource": true, + "github": "https://github.com/org/repo", + "npm": false + }, + "includes": { + "backend": false, + "components": ["button", "banner"] + }, + "company": { + "name": "Company Name", + "website": "https://company.com", + "avatar": "https://company.com/logo.png" + }, + "tags": ["cookie-banner", "consent-management"] +} +``` + +### Remote Benchmarking + +To benchmark a remote URL instead of building locally: + +```json +{ + "name": "production-app", + "remote": { + "enabled": true, + "url": "https://production.example.com", + "headers": { + "Authorization": "Bearer token" + } + } +} +``` + +## Environment Variables + +```bash +# Database configuration (optional) +DATABASE_URL=libsql://your-turso-db.turso.io +DATABASE_AUTH_TOKEN=your-auth-token + +# API endpoint for saving results (optional) +API_URL=http://localhost:3000 +``` + +## Output + +### results.json + +Each benchmark creates a `results.json` file with: +- Raw performance metrics for each iteration +- Calculated scores (performance, bundle strategy, network impact, transparency, UX) +- Metadata (timestamp, iterations, configuration) + +### Console Output + +The CLI displays: +- Progress indicators for each iteration +- Performance metrics (FCP, LCP, CLS, TTI, TBT) +- Cookie banner detection results +- Comprehensive scores with grades (Excellent/Good/Fair/Poor/Critical) +- Insights and recommendations + +## Scoring + +The tool calculates scores across five categories: + +1. **Performance** (30%): FCP, LCP, CLS, TTI, TBT +2. **Bundle Strategy** (25%): Bundle size, loading approach, execution time +3. **Network Impact** (20%): Third-party requests, cookie service overhead +4. **Transparency** (15%): Open source status, documentation, licensing +5. **User Experience** (10%): Banner timing, coverage, layout shifts + +## Development + +```bash +# Build the CLI +pnpm build + +# Run in development +pnpm dev + +# Type checking +pnpm check-types + +# Linting +pnpm lint +``` + +## License + +MIT + diff --git a/packages/cli/base.json b/packages/cookiebench-cli/base.json similarity index 100% rename from packages/cli/base.json rename to packages/cookiebench-cli/base.json diff --git a/packages/cookiebench-cli/package.json b/packages/cookiebench-cli/package.json new file mode 100644 index 0000000..5fda68c --- /dev/null +++ b/packages/cookiebench-cli/package.json @@ -0,0 +1,34 @@ +{ + "name": "cookiebench", + "version": "0.0.1", + "private": true, + "type": "module", + "exports": "./dist/index.mjs", + "main": "./dist/index.mjs", + "module": "dist/index.mjs", + "bin": { + "cookiebench": "dist/index.mjs" + }, + "scripts": { + "build": "rslib build", + "check-types": "tsc --noEmit", + "dev": "rslib build --watch", + "format": "biome format . --write", + "lint": "biome lint .", + "start": "node ./dist/index.mjs" + }, + "dependencies": { + "@clack/prompts": "^1.0.0-alpha.0", + "@consentio/benchmark": "workspace:*", + "@consentio/runner": "workspace:*", + "cli-table3": "^0.6.3", + "dotenv": "^16.5.0", + "picocolors": "^1.0.0" + }, + "devDependencies": { + "@rsdoctor/rspack-plugin": "^1.1.3", + "@rslib/core": "^0.9.1", + "@types/node": "^22.15.30", + "typescript": "^5.8.3" + } +} diff --git a/packages/cli/rslib.config.ts b/packages/cookiebench-cli/rslib.config.ts similarity index 100% rename from packages/cli/rslib.config.ts rename to packages/cookiebench-cli/rslib.config.ts diff --git a/packages/cookiebench-cli/src/commands/benchmark.ts b/packages/cookiebench-cli/src/commands/benchmark.ts new file mode 100644 index 0000000..2812995 --- /dev/null +++ b/packages/cookiebench-cli/src/commands/benchmark.ts @@ -0,0 +1,256 @@ +import { writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { + BenchmarkRunner, + buildAndServeNextApp, + cleanupServer, + readConfig, + type ServerInfo, +} from '@consentio/runner'; +import { calculateScores, printScores } from '../utils/scoring'; + +export async function benchmarkCommand(appPath?: string): Promise { + try { + const config = readConfig(appPath ? join(appPath, 'config.json') : undefined); + if (!config) { + throw new Error('Failed to read config.json'); + } + + let serverInfo: ServerInfo | null = null; + let benchmarkUrl: string; + + // Check if remote benchmarking is enabled + if (config.remote?.enabled && config.remote.url) { + console.log(`🌐 Running remote benchmark against: ${config.remote.url}`); + benchmarkUrl = config.remote.url; + } else { + console.log('🏗️ Building and serving app locally...'); + serverInfo = await buildAndServeNextApp(appPath); + benchmarkUrl = serverInfo.url; + } + + const cwd = appPath || process.cwd(); + + try { + // Create benchmark runner and run benchmarks + const runner = new BenchmarkRunner(config); + const result = await runner.runBenchmarks(benchmarkUrl); + + // Create app data for transparency scoring + const appData = { + name: config.name, + baseline: config.baseline || false, + company: config.company ? JSON.stringify(config.company) : null, + techStack: JSON.stringify(config.techStack), + source: config.source ? JSON.stringify(config.source) : null, + tags: config.tags ? JSON.stringify(config.tags) : null, + }; + + // Calculate scores + const scores = calculateScores( + { + fcp: + result.details.reduce( + (acc, curr) => acc + curr.timing.firstContentfulPaint, + 0 + ) / result.details.length, + lcp: + result.details.reduce( + (acc, curr) => acc + curr.timing.largestContentfulPaint, + 0 + ) / result.details.length, + cls: + result.details.reduce( + (acc, curr) => acc + curr.timing.cumulativeLayoutShift, + 0 + ) / result.details.length, + tbt: + result.details.reduce( + (acc, curr) => acc + curr.timing.mainThreadBlocking.total, + 0 + ) / result.details.length, + tti: + result.details.reduce( + (acc, curr) => acc + curr.timing.timeToInteractive, + 0 + ) / result.details.length, + }, + { + totalSize: + result.details.reduce((acc, curr) => acc + curr.size.total, 0) / + result.details.length, + jsSize: + result.details.reduce( + (acc, curr) => acc + curr.size.scripts.total, + 0 + ) / result.details.length, + cssSize: + result.details.reduce((acc, curr) => acc + curr.size.styles, 0) / + result.details.length, + imageSize: + result.details.reduce((acc, curr) => acc + curr.size.images, 0) / + result.details.length, + fontSize: + result.details.reduce((acc, curr) => acc + curr.size.fonts, 0) / + result.details.length, + otherSize: + result.details.reduce((acc, curr) => acc + curr.size.other, 0) / + result.details.length, + }, + { + totalRequests: + result.details.reduce( + (acc, curr) => + acc + + (curr.resources.scripts.length + + curr.resources.styles.length + + curr.resources.images.length + + curr.resources.fonts.length + + curr.resources.other.length), + 0 + ) / result.details.length, + thirdPartyRequests: + result.details.reduce( + (acc, curr) => + acc + + curr.resources.scripts.filter((s) => s.isThirdParty).length, + 0 + ) / result.details.length, + thirdPartySize: + result.details.reduce((acc, curr) => acc + curr.size.thirdParty, 0) / + result.details.length, + thirdPartyDomains: 5, // Default value + }, + { + cookieBannerDetected: (() => { + // Require consistent detection across ALL iterations for true positive + const allDetected = result.details.every( + (r) => r.cookieBanner.detected + ); + if (!allDetected) { + console.log( + '⚠️ [SCORING] Banner detection inconsistent or failed - marking as not detected' + ); + } + return allDetected; + })(), + cookieBannerTiming: (() => { + // If no banners detected across any iteration, heavily penalize + const detectionSuccess = result.details.some( + (r) => r.cookieBanner.detected + ); + if (!detectionSuccess) { + console.log( + '⚠️ [SCORING] No banner detected in any iteration - applying penalty' + ); + return null; // This signals failed detection for scoring + } + + // Check if any results have null timing (undetected banners) + const timingValues = result.details.map( + (r) => r.cookieBanner.visibilityTime + ); + const hasNullValues = timingValues.some((t) => t === null || t === 0); + + // If we have mixed results (some detected, some not), still penalize + if (hasNullValues) { + console.log( + '⚠️ [SCORING] Inconsistent banner detection - applying penalty' + ); + return null; + } + + // Only return actual timing if all iterations successfully detected banner + const validTimings = timingValues.filter( + (t): t is number => t !== null && t > 0 + ); + return validTimings.length === result.details.length && + validTimings.length > 0 + ? validTimings.reduce((acc, curr) => acc + curr, 0) / + validTimings.length + : null; + })(), + cookieBannerCoverage: (() => { + // Only calculate coverage if banner was consistently detected + const detectionSuccess = result.details.every( + (r) => r.cookieBanner.detected + ); + if (!detectionSuccess) { + console.log( + '⚠️ [SCORING] Inconsistent detection - setting coverage to 0' + ); + return 0; // No coverage score if detection failed + } + return ( + result.details.reduce( + (acc, curr) => acc + curr.cookieBanner.viewportCoverage, + 0 + ) / + result.details.length / + 100 + ); + })(), + }, + { + domSize: 1500, // Default value + mainThreadBlocking: + result.details.reduce( + (acc, curr) => acc + curr.timing.mainThreadBlocking.total, + 0 + ) / result.details.length, + layoutShifts: + result.details.reduce( + (acc, curr) => acc + curr.timing.cumulativeLayoutShift, + 0 + ) / result.details.length, + }, + config.baseline || false, + appData + ); + + // Format results for results.json + const resultsData = { + app: config.name, + techStack: config.techStack, + source: config.source, + includes: config.includes, + internationalization: config.internationalization, + company: config.company, + tags: config.tags, + results: result.details, + scores, + metadata: { + timestamp: new Date().toISOString(), + iterations: config.iterations, + languages: config.techStack.languages, + isRemote: config.remote?.enabled || false, + url: config.remote?.enabled ? config.remote.url : undefined, + }, + }; + + // Write results to file + const outputPath = join(cwd, 'results.json'); + await writeFile(outputPath, JSON.stringify(resultsData, null, 2)); + console.log(`✅ Benchmark results saved to ${outputPath}`); + + // Print scores if available + if (scores) { + console.log('📊 Benchmark Scores:'); + printScores(scores); + } + } finally { + // Only cleanup server if we started one + if (serverInfo) { + cleanupServer(serverInfo); + } + } + } catch (error: unknown) { + if (error instanceof Error) { + console.error(`Error running benchmark: ${error.message}`); + } else { + console.error('An unknown error occurred during benchmark'); + } + process.exit(1); + } +} + diff --git a/packages/cli/src/commands/db.ts b/packages/cookiebench-cli/src/commands/db.ts similarity index 100% rename from packages/cli/src/commands/db.ts rename to packages/cookiebench-cli/src/commands/db.ts diff --git a/packages/cli/src/commands/results.ts b/packages/cookiebench-cli/src/commands/results.ts similarity index 98% rename from packages/cli/src/commands/results.ts rename to packages/cookiebench-cli/src/commands/results.ts index 39054bc..c2882e2 100644 --- a/packages/cli/src/commands/results.ts +++ b/packages/cookiebench-cli/src/commands/results.ts @@ -1,14 +1,14 @@ -import { setTimeout } from "node:timers/promises"; -import * as p from "@clack/prompts"; -import color from "picocolors"; -import { readFile, readdir } from "node:fs/promises"; -import { join } from "node:path"; -import Table from "cli-table3"; -import prettyMilliseconds from "pretty-ms"; -import { config } from "dotenv"; -import { calculateScores, printScores } from "../utils/scoring"; -import type { BenchmarkScores } from "../types"; -import type { Config } from "../types"; +import { setTimeout } from 'node:timers/promises'; +import * as p from '@clack/prompts'; +import color from 'picocolors'; +import { readFile, readdir } from 'node:fs/promises'; +import { join } from 'node:path'; +import Table from 'cli-table3'; +import prettyMilliseconds from 'pretty-ms'; +import { config } from 'dotenv'; +import { calculateScores, printScores } from '../utils/scoring'; +import type { BenchmarkScores } from '../types'; +import type { Config } from '@consentio/runner'; // Load environment variables from .env files config({ path: ".env" }); diff --git a/packages/cookiebench-cli/src/index.ts b/packages/cookiebench-cli/src/index.ts new file mode 100644 index 0000000..f571566 --- /dev/null +++ b/packages/cookiebench-cli/src/index.ts @@ -0,0 +1,82 @@ +import { setTimeout } from 'node:timers/promises'; +import * as p from '@clack/prompts'; +import color from 'picocolors'; +import { benchmarkCommand } from './commands/benchmark'; +import { resultsCommand } from './commands/results'; +import { dbCommand } from './commands/db'; + +function onCancel() { + p.cancel('Operation cancelled.'); + process.exit(0); +} + +async function main() { + console.clear(); + await setTimeout(1000); + + // Check for command line arguments + const args = process.argv.slice(2); + const command = args[0]; + + // If no command specified, show the prompt + if (command) { + // Direct command execution + switch (command) { + case 'benchmark': + await benchmarkCommand(); + break; + case 'results': + await resultsCommand(); + break; + case 'db': + await dbCommand(args[1]); + break; + default: + console.error(`Unknown command: ${command}`); + console.log('Available commands: benchmark, results, db'); + process.exit(1); + } + } else { + p.intro(`${color.bgCyan(color.black(' cookiebench '))}`); + + const selectedCommand = await p.select({ + message: 'What would you like to do?', + options: [ + { + value: 'benchmark', + label: 'Run a benchmark', + hint: 'Run a performance benchmark on a URL', + }, + { + value: 'results', + label: 'Results', + hint: 'Combine and display benchmark results', + }, + { + value: 'db', + label: 'Database', + hint: 'Manage database schema and migrations', + }, + ], + }); + + if (p.isCancel(selectedCommand)) { + return onCancel(); + } + + // biome-ignore lint/style/useDefaultSwitchClause: + switch (selectedCommand) { + case 'benchmark': + await benchmarkCommand(); + break; + case 'results': + await resultsCommand(); + break; + case 'db': + await dbCommand(); + break; + } + } +} + +main().catch(console.error); diff --git a/packages/cookiebench-cli/src/types/index.ts b/packages/cookiebench-cli/src/types/index.ts new file mode 100644 index 0000000..e568eca --- /dev/null +++ b/packages/cookiebench-cli/src/types/index.ts @@ -0,0 +1,39 @@ +// Re-export types from runner package +export type { + Config, + BenchmarkResult, + BenchmarkDetails, + ServerInfo, +} from '@consentio/runner'; + +// CLI-specific scoring types +export interface BenchmarkScores { + totalScore: number; + grade: 'Excellent' | 'Good' | 'Fair' | 'Poor' | 'Critical'; + categoryScores: { + performance: number; + bundleStrategy: number; + networkImpact: number; + transparency: number; + userExperience: number; + }; + categories: Array<{ + name: string; + score: number; + maxScore: number; + weight: number; + details: Array<{ + name: string; + score: number; + maxScore: number; + weight: number; + status: 'excellent' | 'good' | 'fair' | 'poor'; + reason: string; + }>; + status: 'excellent' | 'good' | 'fair' | 'poor'; + reason: string; + }>; + insights: string[]; + recommendations: string[]; +} + diff --git a/packages/cli/src/utils/index.ts b/packages/cookiebench-cli/src/utils/index.ts similarity index 100% rename from packages/cli/src/utils/index.ts rename to packages/cookiebench-cli/src/utils/index.ts diff --git a/packages/cli/src/utils/scoring.ts b/packages/cookiebench-cli/src/utils/scoring.ts similarity index 99% rename from packages/cli/src/utils/scoring.ts rename to packages/cookiebench-cli/src/utils/scoring.ts index 9b4552c..c5c68a9 100644 --- a/packages/cli/src/utils/scoring.ts +++ b/packages/cookiebench-cli/src/utils/scoring.ts @@ -1,8 +1,8 @@ -import type { RawBenchmarkDetail } from "../commands/results"; -import type { Config } from "../types"; -import Table from "cli-table3"; -import color from "picocolors"; -import { determineBundleStrategy } from "../commands/benchmark/bundle-strategy"; +import type { RawBenchmarkDetail } from '../commands/results'; +import type { Config } from '@consentio/runner'; +import Table from 'cli-table3'; +import color from 'picocolors'; +import { determineBundleStrategy } from '@consentio/benchmark'; import type { BenchmarkScores } from '../types'; // Type definitions for better type safety diff --git a/packages/cli/tsconfig.json b/packages/cookiebench-cli/tsconfig.json similarity index 100% rename from packages/cli/tsconfig.json rename to packages/cookiebench-cli/tsconfig.json diff --git a/packages/runner/README.md b/packages/runner/README.md new file mode 100644 index 0000000..bfcd144 --- /dev/null +++ b/packages/runner/README.md @@ -0,0 +1,150 @@ +# @consentio/runner + +Benchmark orchestration for running cookie banner performance tests. + +## Overview + +This package orchestrates benchmark execution, managing browser instances, running iterations, aggregating results, and serving Next.js applications for testing. + +## Features + +- **Benchmark Orchestration**: Run multiple benchmark iterations with automated browser management +- **Next.js Server Management**: Build and serve Next.js apps for local testing +- **Performance Aggregation**: Calculate averages and aggregate metrics across iterations +- **Config Loading**: Load and validate benchmark configurations +- **Remote & Local Testing**: Support for both remote URLs and local development + +## Installation + +```bash +pnpm add @consentio/runner @consentio/benchmark +``` + +## Usage + +### Basic Usage + +```typescript +import { BenchmarkRunner, readConfig } from '@consentio/runner'; + +// Load config +const config = readConfig('./config.json'); + +// Create runner +const runner = new BenchmarkRunner(config); + +// Run benchmarks +const results = await runner.runBenchmarks('http://localhost:3000'); + +console.log('Benchmark complete:', results); +``` + +### With Server Management + +```typescript +import { + BenchmarkRunner, + buildAndServeNextApp, + cleanupServer, + readConfig, +} from '@consentio/runner'; + +const config = readConfig(); +const serverInfo = await buildAndServeNextApp('./my-next-app'); + +try { + const runner = new BenchmarkRunner(config); + const results = await runner.runBenchmarks(serverInfo.url); + + console.log('Results:', results); +} finally { + cleanupServer(serverInfo); +} +``` + +### Remote Benchmarking + +```typescript +import { BenchmarkRunner } from '@consentio/runner'; + +const config = { + name: 'production-test', + iterations: 5, + remote: { + enabled: true, + url: 'https://production.example.com', + headers: { + 'Authorization': 'Bearer token', + }, + }, + // ... other config +}; + +const runner = new BenchmarkRunner(config); +const results = await runner.runBenchmarks(config.remote.url); +``` + +## API + +### BenchmarkRunner + +- `constructor(config: Config)`: Create a new benchmark runner +- `runBenchmarks(serverUrl: string)`: Run multiple benchmark iterations +- `runSingleBenchmark(page: Page, url: string)`: Run a single benchmark iteration + +### Server Management + +- `buildAndServeNextApp(appPath?: string)`: Build and serve a Next.js app +- `cleanupServer(serverInfo: ServerInfo)`: Stop the server process + +### Utilities + +- `readConfig(configPath?: string)`: Read and parse config.json +- `formatTime(ms: number)`: Format milliseconds to human-readable string +- `getPackageManager()`: Detect package manager (npm/yarn/pnpm) + +### PerformanceAggregator + +- `calculateTTI(coreWebVitals, cookieBannerData)`: Calculate Time to Interactive +- `aggregateMetrics(...)`: Merge all collected metrics into final benchmark details +- `calculateAverages(results)`: Calculate average metrics from multiple runs +- `logResults(...)`: Log comprehensive benchmark results + +## Configuration + +```json +{ + "name": "my-app", + "iterations": 5, + "baseline": false, + "remote": { + "enabled": false, + "url": "https://example.com" + }, + "cookieBanner": { + "selectors": [".cookie-banner"], + "serviceHosts": ["cookiecdn.com"], + "serviceName": "CookieService", + "waitForVisibility": true, + "measureViewportCoverage": true, + "expectedLayoutShift": true + }, + "techStack": { + "bundler": "webpack", + "bundleType": "esm", + "frameworks": ["react", "nextjs"], + "languages": ["typescript"], + "packageManager": "pnpm", + "typescript": true + } +} +``` + +## Types + +See the [types file](./src/types.ts) for complete type definitions. + +## License + +MIT + diff --git a/packages/runner/package.json b/packages/runner/package.json new file mode 100644 index 0000000..bc64fcf --- /dev/null +++ b/packages/runner/package.json @@ -0,0 +1,32 @@ +{ + "name": "@consentio/runner", + "version": "0.0.1", + "type": "module", + "exports": { + ".": { + "import": "./dist/index.js", + "types": "./dist/index.d.ts" + } + }, + "main": "./dist/index.js", + "module": "./dist/index.js", + "types": "./dist/index.d.ts", + "scripts": { + "build": "rslib build", + "check-types": "tsc --noEmit", + "dev": "rslib build --watch", + "format": "biome format . --write", + "lint": "biome lint ." + }, + "dependencies": { + "@consentio/benchmark": "workspace:*", + "@playwright/test": "^1.42.1", + "playwright-performance-metrics": "^1.2.2" + }, + "devDependencies": { + "@rsdoctor/rspack-plugin": "^1.1.3", + "@rslib/core": "^0.9.1", + "@types/node": "^22.15.30", + "typescript": "^5.8.3" + } +} diff --git a/packages/runner/rslib.config.ts b/packages/runner/rslib.config.ts new file mode 100644 index 0000000..8f03d65 --- /dev/null +++ b/packages/runner/rslib.config.ts @@ -0,0 +1,15 @@ +import { defineConfig } from '@rslib/core'; + +export default defineConfig({ + lib: [ + { + format: 'esm', + syntax: 'es2021', + dts: true, + }, + ], + output: { + target: 'node', + }, +}); + diff --git a/packages/runner/src/benchmark-runner.ts b/packages/runner/src/benchmark-runner.ts new file mode 100644 index 0000000..9f8ce4e --- /dev/null +++ b/packages/runner/src/benchmark-runner.ts @@ -0,0 +1,191 @@ +import { chromium, type Page } from '@playwright/test'; +import { PerformanceMetricsCollector } from 'playwright-performance-metrics'; +import type { Config } from '@consentio/benchmark'; +import { + CookieBannerCollector, + NetworkMonitor, + ResourceTimingCollector, + BENCHMARK_CONSTANTS, +} from '@consentio/benchmark'; +import type { BenchmarkResult, BenchmarkDetails, CoreWebVitals } from './types'; +import { PerformanceAggregator } from './performance-aggregator'; + +export class BenchmarkRunner { + private config: Config; + private cookieBannerCollector: CookieBannerCollector; + private networkMonitor: NetworkMonitor; + private resourceTimingCollector: ResourceTimingCollector; + private performanceAggregator: PerformanceAggregator; + + constructor(config: Config) { + this.config = config; + this.cookieBannerCollector = new CookieBannerCollector(config); + this.networkMonitor = new NetworkMonitor(config); + this.resourceTimingCollector = new ResourceTimingCollector(); + this.performanceAggregator = new PerformanceAggregator(); + } + + /** + * Run a single benchmark iteration + */ + async runSingleBenchmark(page: Page, url: string): Promise { + console.log(`🔍 [DEBUG] Starting cookie banner benchmark for: ${url}`); + console.log( + '🔍 [DEBUG] Cookie banner selectors:', + this.config.cookieBanner?.selectors || [] + ); + console.log( + '🔍 [DEBUG] Bundle type from config:', + this.config.techStack?.bundleType + ); + + // Initialize collectors + const collector = new PerformanceMetricsCollector(); + const cookieBannerMetrics = this.cookieBannerCollector.initializeMetrics(); + + // Setup monitoring and detection + await this.networkMonitor.setupMonitoring(page); + await this.cookieBannerCollector.setupDetection(page); + + // Navigate to the page + console.log(`🔍 [DEBUG] Navigating to: ${url}`); + await page.goto(url, { waitUntil: 'networkidle' }); + + // Wait for the specified element + await this.waitForElement(page); + + // Wait for network to be idle + console.log('🔍 [DEBUG] Waiting for network idle...'); + await page.waitForLoadState('networkidle'); + + // Collect core web vitals + console.log('🔍 [DEBUG] Collecting core web vitals...'); + const coreWebVitals = await this.collectCoreWebVitals(collector, page); + + // Collect cookie banner specific metrics + const cookieBannerData = await this.cookieBannerCollector.collectMetrics( + page + ); + console.log('🔍 [DEBUG] Cookie banner metrics:', cookieBannerData); + + // Collect detailed resource timing data + const resourceMetrics = await this.resourceTimingCollector.collect(page); + + // Get network metrics + const networkRequests = this.networkMonitor.getNetworkRequests(); + const networkMetrics = this.networkMonitor.getMetrics(); + + // Aggregate all metrics + const finalMetrics = this.performanceAggregator.aggregateMetrics( + coreWebVitals, + cookieBannerData, + cookieBannerMetrics, + networkRequests, + networkMetrics, + resourceMetrics, + this.config + ); + + // Log results + this.performanceAggregator.logResults( + finalMetrics, + cookieBannerMetrics, + this.config + ); + + // Cleanup + await collector.cleanup(); + this.networkMonitor.reset(); + + return finalMetrics; + } + + /** + * Run multiple benchmark iterations + */ + async runBenchmarks(serverUrl: string): Promise { + const browser = await chromium.launch({ + headless: true, // Keep headless mode for stability + args: ['--remote-debugging-port=9222'], + }); + const results: BenchmarkDetails[] = []; + + try { + for (let i = 0; i < this.config.iterations; i++) { + console.log( + `[Benchmark] Running iteration ${i + 1}/${this.config.iterations}...` + ); + + const context = await browser.newContext(); + const page = await context.newPage(); + + const result = await this.runSingleBenchmark( + page, + // Add a timestamp to the URL to avoid caching + `${serverUrl}?t=${Date.now()}` + ); + results.push(result); + + await context.close(); + } + } finally { + await browser.close(); + } + + const averages = this.performanceAggregator.calculateAverages(results); + + return { + name: this.config.name, + baseline: this.config.baseline || false, + techStack: this.config.techStack, + source: this.config.source, + includes: this.config.includes, + company: this.config.company, + tags: this.config.tags, + details: results, + average: averages, + }; + } + + /** + * Wait for the specified element based on config + */ + private async waitForElement(page: Page): Promise { + if (this.config.testId) { + console.log(`🔍 [DEBUG] Waiting for testId: ${this.config.testId}`); + await page.waitForSelector(`[data-testid="${this.config.testId}"]`); + } else if (this.config.id) { + console.log(`🔍 [DEBUG] Waiting for id: ${this.config.id}`); + await page.waitForSelector(`#${this.config.id}`); + } else if (this.config.custom) { + console.log('🔍 [DEBUG] Running custom wait function'); + await this.config.custom(page); + } + } + + /** + * Collect core web vitals using playwright-performance-metrics + */ + private async collectCoreWebVitals( + collector: PerformanceMetricsCollector, + page: Page + ): Promise { + const coreWebVitals = await collector.collectMetrics(page, { + timeout: BENCHMARK_CONSTANTS.METRICS_TIMEOUT, + retryTimeout: BENCHMARK_CONSTANTS.METRICS_RETRY_TIMEOUT, + }); + + console.log('🔍 [DEBUG] Core web vitals collected:', { + fcp: coreWebVitals.paint?.firstContentfulPaint, + lcp: coreWebVitals.largestContentfulPaint, + cls: coreWebVitals.cumulativeLayoutShift, + tbt: coreWebVitals.totalBlockingTime, + domComplete: coreWebVitals.domCompleteTiming, + pageLoad: coreWebVitals.pageloadTiming, + totalBytes: coreWebVitals.totalBytes, + }); + + return coreWebVitals; + } +} + diff --git a/packages/runner/src/index.ts b/packages/runner/src/index.ts new file mode 100644 index 0000000..7814079 --- /dev/null +++ b/packages/runner/src/index.ts @@ -0,0 +1,28 @@ +// Main runner +export { BenchmarkRunner } from './benchmark-runner'; + +// Server management +export { buildAndServeNextApp, cleanupServer } from './server'; + +// Utilities +export { readConfig, formatTime, getPackageManager } from './utils'; + +// Performance aggregation +export { PerformanceAggregator } from './performance-aggregator'; + +// Types +export type { + BenchmarkResult, + BenchmarkDetails, + ServerInfo, + Config, + CookieBannerConfig, + CookieBannerMetrics, + CookieBannerData, + NetworkRequest, + NetworkMetrics, + BundleStrategy, + ResourceTimingData, + CoreWebVitals, +} from './types'; + diff --git a/packages/runner/src/performance-aggregator.ts b/packages/runner/src/performance-aggregator.ts new file mode 100644 index 0000000..d28d84a --- /dev/null +++ b/packages/runner/src/performance-aggregator.ts @@ -0,0 +1,274 @@ +import type { + Config, + CoreWebVitals, + CookieBannerData, + CookieBannerMetrics, + NetworkRequest, + NetworkMetrics, + ResourceTimingData, +} from '@consentio/benchmark'; +import type { BenchmarkDetails, BenchmarkResult } from './types'; + +export class PerformanceAggregator { + /** + * Calculate Time to Interactive based on core web vitals and cookie banner interaction + */ + calculateTTI( + coreWebVitals: CoreWebVitals, + cookieBannerData: CookieBannerData | null + ): number { + return ( + Math.max( + coreWebVitals.paint?.firstContentfulPaint || 0, + coreWebVitals.domCompleteTiming || 0, + cookieBannerData?.bannerInteractiveTime || 0 + ) + 1000 + ); // Add buffer for true interactivity + } + + /** + * Merge all collected metrics into final benchmark details + */ + aggregateMetrics( + coreWebVitals: CoreWebVitals, + cookieBannerData: CookieBannerData | null, + cookieBannerMetrics: CookieBannerMetrics, + networkRequests: NetworkRequest[], + networkMetrics: NetworkMetrics, + resourceMetrics: ResourceTimingData, + config: Config + ): BenchmarkDetails { + const tti = this.calculateTTI(coreWebVitals, cookieBannerData); + + const networkImpact = this.calculateNetworkImpact(networkRequests); + + return { + duration: resourceMetrics.duration, + size: resourceMetrics.size, + timing: { + navigationStart: resourceMetrics.timing.navigationStart, + domContentLoaded: resourceMetrics.timing.domContentLoaded, + load: resourceMetrics.timing.load, + firstPaint: coreWebVitals.paint?.firstPaint || 0, + firstContentfulPaint: coreWebVitals.paint?.firstContentfulPaint || 0, + largestContentfulPaint: coreWebVitals.largestContentfulPaint || 0, + timeToInteractive: tti, + cumulativeLayoutShift: coreWebVitals.cumulativeLayoutShift || 0, + cookieBanner: { + renderStart: cookieBannerData?.bannerRenderTime || 0, + renderEnd: cookieBannerData?.bannerInteractiveTime || 0, + interactionStart: cookieBannerData?.bannerInteractiveTime || 0, + interactionEnd: cookieBannerData?.bannerInteractiveTime || 0, + layoutShift: cookieBannerData?.layoutShiftImpact || 0, + detected: cookieBannerData?.detected || false, + selector: cookieBannerData?.selector || null, + serviceName: config.cookieBanner?.serviceName || 'unknown', + visibilityTime: cookieBannerData?.bannerInteractiveTime || 0, + viewportCoverage: cookieBannerData?.viewportCoverage || 0, + }, + thirdParty: { + dnsLookupTime: 0, + connectionTime: 0, + downloadTime: networkImpact.totalDownloadTime, + totalImpact: networkImpact.totalImpact, + cookieServices: { + hosts: config.cookieBanner?.serviceHosts || [], + totalSize: networkMetrics.bannerBundleSize, + resourceCount: networkMetrics.bannerNetworkRequests, + dnsLookupTime: 0, + connectionTime: 0, + downloadTime: networkImpact.totalDownloadTime, + }, + }, + mainThreadBlocking: { + total: coreWebVitals.totalBlockingTime || 0, + cookieBannerEstimate: + cookieBannerMetrics.bannerMainThreadBlockingTime, + percentageFromCookies: + (coreWebVitals.totalBlockingTime || 0) > 0 + ? (cookieBannerMetrics.bannerMainThreadBlockingTime / + (coreWebVitals.totalBlockingTime || 1)) * + 100 + : 0, + }, + scripts: resourceMetrics.timing.scripts, + }, + resources: resourceMetrics.resources, + language: resourceMetrics.language, + cookieBanner: { + detected: cookieBannerData?.detected || false, + selector: cookieBannerData?.selector || null, + serviceName: config.cookieBanner?.serviceName || 'unknown', + visibilityTime: cookieBannerData?.bannerInteractiveTime || 0, + viewportCoverage: cookieBannerData?.viewportCoverage || 0, + }, + thirdParty: { + cookieServices: { + hosts: config.cookieBanner?.serviceHosts || [], + totalSize: networkMetrics.bannerBundleSize, + resourceCount: networkMetrics.bannerNetworkRequests, + dnsLookupTime: 0, + connectionTime: 0, + downloadTime: networkImpact.totalDownloadTime, + }, + totalImpact: networkImpact.totalImpact, + }, + }; + } + + /** + * Calculate network impact metrics + */ + private calculateNetworkImpact(networkRequests: NetworkRequest[]): { + totalImpact: number; + totalDownloadTime: number; + } { + const totalImpact = networkRequests.reduce((acc, req) => acc + req.size, 0); + const totalDownloadTime = networkRequests.reduce( + (acc, req) => acc + req.duration, + 0 + ); + + return { totalImpact, totalDownloadTime }; + } + + /** + * Calculate average metrics from multiple benchmark results + */ + calculateAverages(results: BenchmarkDetails[]): BenchmarkResult['average'] { + if (results.length === 0) { + throw new Error('Cannot calculate averages from empty results array'); + } + + return { + firstContentfulPaint: + results.reduce( + (acc, curr) => acc + curr.timing.firstContentfulPaint, + 0 + ) / results.length, + largestContentfulPaint: + results.reduce( + (acc, curr) => acc + curr.timing.largestContentfulPaint, + 0 + ) / results.length, + timeToInteractive: + results.reduce((acc, curr) => acc + curr.timing.timeToInteractive, 0) / + results.length, + totalBlockingTime: + results.reduce( + (acc, curr) => acc + curr.timing.mainThreadBlocking.total, + 0 + ) / results.length, + speedIndex: 0, // Default value + timeToFirstByte: 0, // Default value + firstInputDelay: 0, // Default value + cumulativeLayoutShift: + results.reduce( + (acc, curr) => acc + curr.timing.cumulativeLayoutShift, + 0 + ) / results.length, + domSize: 0, // Default value + totalRequests: + results.reduce( + (acc, curr) => + acc + + (curr.resources.scripts.length + + curr.resources.styles.length + + curr.resources.images.length + + curr.resources.fonts.length + + curr.resources.other.length), + 0 + ) / results.length, + totalSize: + results.reduce((acc, curr) => acc + curr.size.total, 0) / + results.length, + jsSize: + results.reduce((acc, curr) => acc + curr.size.scripts.total, 0) / + results.length, + cssSize: + results.reduce((acc, curr) => acc + curr.size.styles, 0) / results.length, + imageSize: + results.reduce((acc, curr) => acc + curr.size.images, 0) / results.length, + fontSize: + results.reduce((acc, curr) => acc + curr.size.fonts, 0) / results.length, + otherSize: + results.reduce((acc, curr) => acc + curr.size.other, 0) / results.length, + thirdPartyRequests: 0, // Default value + thirdPartySize: 0, // Default value + thirdPartyDomains: 0, // Default value + thirdPartyCookies: 0, // Default value + thirdPartyLocalStorage: 0, // Default value + thirdPartySessionStorage: 0, // Default value + thirdPartyIndexedDB: 0, // Default value + thirdPartyCache: 0, // Default value + thirdPartyServiceWorkers: 0, // Default value + thirdPartyWebWorkers: 0, // Default value + thirdPartyWebSockets: 0, // Default value + thirdPartyBeacons: 0, // Default value + thirdPartyFetch: 0, // Default value + thirdPartyXHR: 0, // Default value + thirdPartyScripts: 0, // Default value + thirdPartyStyles: 0, // Default value + thirdPartyImages: 0, // Default value + thirdPartyFonts: 0, // Default value + thirdPartyMedia: 0, // Default value + thirdPartyOther: 0, // Default value + thirdPartyTiming: { + total: 0, + blocking: 0, + dns: 0, + connect: 0, + ssl: 0, + send: 0, + wait: 0, + receive: 0, + }, + cookieBannerTiming: { + firstPaint: 0, + firstContentfulPaint: + results.reduce( + (acc, curr) => acc + curr.timing.firstContentfulPaint, + 0 + ) / results.length, + domContentLoaded: + results.reduce((acc, curr) => acc + curr.timing.domContentLoaded, 0) / + results.length, + load: + results.reduce((acc, curr) => acc + curr.timing.load, 0) / + results.length, + }, + }; + } + + /** + * Log comprehensive benchmark results + */ + logResults( + finalMetrics: BenchmarkDetails, + cookieBannerMetrics: CookieBannerMetrics, + config: Config + ): void { + console.log('🔍 [DEBUG] Final cookie banner benchmark results:', { + fcp: finalMetrics.timing.firstContentfulPaint, + lcp: finalMetrics.timing.largestContentfulPaint, + cls: finalMetrics.timing.cumulativeLayoutShift, + tti: finalMetrics.timing.timeToInteractive, + tbt: finalMetrics.timing.mainThreadBlocking.total, + bannerDetected: finalMetrics.cookieBanner.detected, + bannerRenderTime: + finalMetrics.timing.cookieBanner.renderEnd - + finalMetrics.timing.cookieBanner.renderStart, + bannerLayoutShift: finalMetrics.timing.cookieBanner.layoutShift, + bannerNetworkImpact: finalMetrics.thirdParty.totalImpact, + bundleStrategy: cookieBannerMetrics.isBundled + ? 'Bundled' + : cookieBannerMetrics.isIIFE + ? 'IIFE' + : 'Unknown', + isBundled: cookieBannerMetrics.isBundled, + isIIFE: cookieBannerMetrics.isIIFE, + configBundleType: config.techStack?.bundleType, + }); + } +} + diff --git a/packages/runner/src/server.ts b/packages/runner/src/server.ts new file mode 100644 index 0000000..a391c9c --- /dev/null +++ b/packages/runner/src/server.ts @@ -0,0 +1,75 @@ +import { spawn } from 'node:child_process'; +import { getPackageManager } from './utils'; +import type { ServerInfo } from './types'; + +export async function buildAndServeNextApp( + appPath?: string +): Promise { + const pm = await getPackageManager(); + const cwd = appPath || process.cwd(); + + // Build the app + console.log('[Build] Building Next.js app...'); + const buildProcess = spawn(pm.command, [...pm.args, 'build'], { + cwd, + stdio: 'inherit', + }); + + await new Promise((resolve, reject) => { + buildProcess.on('close', (code) => { + if (code === 0) { + resolve(); + } else { + reject(new Error(`Build failed with code ${code}`)); + } + }); + }); + + // Start the server + console.log('[Build] Starting Next.js server...'); + const port = Math.floor(Math.random() * (9000 - 3000 + 1)) + 3000; + console.log('command', [ + ...pm.args, + 'start', + '--', + '--port', + port.toString(), + ]); + const serverProcess = spawn( + pm.command, + [...pm.args, 'start', '--', '--port', port.toString()], + { + cwd, + stdio: ['inherit', 'pipe', 'inherit'], + } + ); + + // Wait for server to be ready + const url = `http://localhost:${port}`; + let retries = 0; + const maxRetries = 30; + + while (retries < maxRetries) { + try { + const response = await fetch(url); + if (response.ok) { + console.log('[Build] Server is ready!'); + return { serverProcess, url }; + } + } catch { + // Ignore error and retry + } + + await new Promise((resolve) => setTimeout(resolve, 1000)); + retries++; + } + + throw new Error('Server failed to start'); +} + +export function cleanupServer(serverInfo: ServerInfo): void { + if (serverInfo.serverProcess) { + serverInfo.serverProcess.kill(); + } +} + diff --git a/packages/runner/src/types.ts b/packages/runner/src/types.ts new file mode 100644 index 0000000..09fb357 --- /dev/null +++ b/packages/runner/src/types.ts @@ -0,0 +1,272 @@ +import type { Page } from '@playwright/test'; +import type { ChildProcess } from 'node:child_process'; + +// Re-export common types from benchmark package +export type { + Config, + CookieBannerConfig, + CookieBannerMetrics, + CookieBannerData, + NetworkRequest, + NetworkMetrics, + BundleStrategy, + ResourceTimingData, + CoreWebVitals, +} from '@consentio/benchmark'; + +// Server types +export interface ServerInfo { + serverProcess: ChildProcess; + url: string; +} + +// Benchmark result types +export interface BenchmarkDetails { + duration: number; + size: { + total: number; + bundled: number; + thirdParty: number; + scripts: { + total: number; + initial: number; + dynamic: number; + }; + styles: number; + images: number; + fonts: number; + other: number; + }; + timing: { + navigationStart: number; + domContentLoaded: number; + load: number; + firstPaint: number; + firstContentfulPaint: number; + largestContentfulPaint: number; + timeToInteractive: number; + cumulativeLayoutShift: number; + cookieBanner: { + renderStart: number; + renderEnd: number; + interactionStart: number; + interactionEnd: number; + layoutShift: number; + detected: boolean; + selector: string | null; + serviceName: string; + visibilityTime: number | null; + viewportCoverage: number; + }; + thirdParty: { + dnsLookupTime: number; + connectionTime: number; + downloadTime: number; + totalImpact: number; + cookieServices: { + hosts: string[]; + totalSize: number; + resourceCount: number; + dnsLookupTime: number; + connectionTime: number; + downloadTime: number; + }; + }; + mainThreadBlocking: { + total: number; + cookieBannerEstimate: number; + percentageFromCookies: number; + }; + scripts: { + bundled: { + loadStart: number; + loadEnd: number; + executeStart: number; + executeEnd: number; + }; + thirdParty: { + loadStart: number; + loadEnd: number; + executeStart: number; + executeEnd: number; + }; + }; + }; + language: string; + resources: { + scripts: Array<{ + name: string; + size: number; + duration: number; + startTime: number; + isThirdParty: boolean; + isDynamic: boolean; + }>; + styles: Array<{ + name: string; + size: number; + duration: number; + startTime: number; + isThirdParty: boolean; + }>; + images: Array<{ + name: string; + size: number; + duration: number; + startTime: number; + isThirdParty: boolean; + }>; + fonts: Array<{ + name: string; + size: number; + duration: number; + startTime: number; + isThirdParty: boolean; + }>; + other: Array<{ + name: string; + size: number; + duration: number; + startTime: number; + isThirdParty: boolean; + type: string; + }>; + }; + cookieBanner: EnhancedCookieBannerTiming; + thirdParty: ThirdPartyMetrics; +} + +export interface BenchmarkResult { + name: string; + baseline: boolean; + techStack: { + bundler: string; + bundleType: string | string[]; + frameworks: string[]; + languages: string[]; + packageManager: string; + typescript: boolean; + }; + source: { + github: string | false; + isOpenSource: boolean | string; + license: string; + npm: string | false; + website?: string; + }; + includes: { + backend: string | string[] | false; + components: string[]; + }; + company?: { + name: string; + website: string; + avatar: string; + }; + tags?: string[]; + details: BenchmarkDetails[]; + average: { + firstContentfulPaint: number; + largestContentfulPaint: number; + timeToInteractive: number; + totalBlockingTime: number; + speedIndex: number; + timeToFirstByte: number; + firstInputDelay: number; + cumulativeLayoutShift: number; + domSize: number; + totalRequests: number; + totalSize: number; + jsSize: number; + cssSize: number; + imageSize: number; + fontSize: number; + otherSize: number; + thirdPartyRequests: number; + thirdPartySize: number; + thirdPartyDomains: number; + thirdPartyCookies: number; + thirdPartyLocalStorage: number; + thirdPartySessionStorage: number; + thirdPartyIndexedDB: number; + thirdPartyCache: number; + thirdPartyServiceWorkers: number; + thirdPartyWebWorkers: number; + thirdPartyWebSockets: number; + thirdPartyBeacons: number; + thirdPartyFetch: number; + thirdPartyXHR: number; + thirdPartyScripts: number; + thirdPartyStyles: number; + thirdPartyImages: number; + thirdPartyFonts: number; + thirdPartyMedia: number; + thirdPartyOther: number; + thirdPartyTiming: { + total: number; + blocking: number; + dns: number; + connect: number; + ssl: number; + send: number; + wait: number; + receive: number; + }; + cookieBannerTiming: { + firstPaint: number; + firstContentfulPaint: number; + domContentLoaded: number; + load: number; + }; + }; + scores?: { + totalScore: number; + grade: 'Excellent' | 'Good' | 'Fair' | 'Poor' | 'Critical'; + categoryScores: { + performance: number; + bundleStrategy: number; + networkImpact: number; + transparency: number; + userExperience: number; + }; + categories: Array<{ + name: string; + score: number; + maxScore: number; + weight: number; + details: Array<{ + name: string; + score: number; + maxScore: number; + weight: number; + status: 'excellent' | 'good' | 'fair' | 'poor'; + reason: string; + }>; + status: 'excellent' | 'good' | 'fair' | 'poor'; + reason: string; + }>; + insights: string[]; + recommendations: string[]; + }; +} + +interface EnhancedCookieBannerTiming { + detected: boolean; + selector: string | null; + serviceName: string; + visibilityTime: number | null; + viewportCoverage: number; +} + +interface ThirdPartyMetrics { + cookieServices: { + hosts: string[]; + totalSize: number; + resourceCount: number; + dnsLookupTime: number; + connectionTime: number; + downloadTime: number; + }; + totalImpact: number; +} + diff --git a/packages/runner/src/utils.ts b/packages/runner/src/utils.ts new file mode 100644 index 0000000..afb120e --- /dev/null +++ b/packages/runner/src/utils.ts @@ -0,0 +1,56 @@ +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import type { Config } from '@consentio/benchmark'; + +export function readConfig(configPath?: string): Config | null { + try { + const path = configPath || join(process.cwd(), 'config.json'); + const configContent = readFileSync(path, 'utf-8'); + return JSON.parse(configContent) as Config; + } catch (error) { + console.error(`Failed to read config.json:`, error); + return null; + } +} + +export function formatTime(ms: number): string { + if (ms < 1000) { + return `${ms.toFixed(0)}ms`; + } + return `${(ms / 1000).toFixed(2)}s`; +} + +export async function getPackageManager(): Promise<{ + command: string; + args: string[]; +}> { + try { + const { execSync } = await import('node:child_process'); + const output = execSync('npm -v', { encoding: 'utf-8' }); + if (output) { + return { command: 'npm', args: ['run'] }; + } + } catch { + try { + const { execSync } = await import('node:child_process'); + const output = execSync('yarn -v', { encoding: 'utf-8' }); + if (output) { + return { command: 'yarn', args: [] }; + } + } catch { + try { + const { execSync } = await import('node:child_process'); + const output = execSync('pnpm -v', { encoding: 'utf-8' }); + if (output) { + return { command: 'pnpm', args: [] }; + } + } catch { + // Default to npm if no package manager is found + return { command: 'npm', args: ['run'] }; + } + } + } + // Default to npm if no package manager is found + return { command: 'npm', args: ['run'] }; +} + diff --git a/packages/runner/tsconfig.json b/packages/runner/tsconfig.json new file mode 100644 index 0000000..3534f4f --- /dev/null +++ b/packages/runner/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "ESNext", + "moduleResolution": "bundler", + "esModuleInterop": true, + "strict": true, + "skipLibCheck": true, + "declaration": true, + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"] +} + diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index afcb82d..642676d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -14,9 +14,15 @@ importers: '@c15t/translations': specifier: ^1.0.0 version: 1.0.0 - '@cookiebench/cli': + '@consentio/benchmark': specifier: workspace:* - version: link:packages/cli + version: link:packages/benchmark + 'cookiebench': + specifier: workspace:* + version: link:packages/cookiebench-cli + '@consentio/runner': + specifier: workspace:* + version: link:packages/runner '@playwright/test': specifier: ^1.42.1 version: 1.52.0 @@ -51,12 +57,12 @@ importers: specifier: ^19.0.0 version: 19.1.0(react@19.1.0) devDependencies: + 'cookiebench': + specifier: workspace:* + version: link:../../packages/cookiebench-cli '@cookiebench/benchmark-schema': specifier: workspace:* version: link:../../packages/benchmark-schema - '@cookiebench/cli': - specifier: workspace:* - version: link:../../packages/cli '@cookiebench/ts-config': specifier: workspace:* version: link:../../packages/typescript-config @@ -77,10 +83,10 @@ importers: dependencies: '@c15t/nextjs': specifier: 1.2.2-canary-20250603153501 - version: 1.2.2-canary-20250603153501(@libsql/client@0.15.8)(@playwright/test@1.52.0)(@types/better-sqlite3@7.6.13)(@types/react-dom@19.1.6(@types/react@19.1.6))(@types/react@19.1.6)(better-sqlite3@11.10.0)(next@15.3.3(@opentelemetry/api@1.8.0)(@playwright/test@1.52.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(use-sync-external-store@1.5.0(react@19.1.0))(ws@8.18.2) + version: 1.2.2-canary-20250603153501(@playwright/test@1.52.0)(@types/react-dom@19.1.6(@types/react@19.1.6))(@types/react@19.1.6)(next@15.3.3(@opentelemetry/api@1.9.0)(@playwright/test@1.52.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react-dom@19.1.0(react@19.1.0))(react@19.1.0) next: specifier: 15.3.3 - version: 15.3.3(@opentelemetry/api@1.8.0)(@playwright/test@1.52.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0) + version: 15.3.3(@opentelemetry/api@1.9.0)(@playwright/test@1.52.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0) react: specifier: ^19.0.0 version: 19.1.0 @@ -88,12 +94,12 @@ importers: specifier: ^19.0.0 version: 19.1.0(react@19.1.0) devDependencies: + 'cookiebench': + specifier: workspace:* + version: link:../../packages/cookiebench-cli '@cookiebench/benchmark-schema': specifier: workspace:* version: link:../../packages/benchmark-schema - '@cookiebench/cli': - specifier: workspace:* - version: link:../../packages/cli '@cookiebench/ts-config': specifier: workspace:* version: link:../../packages/typescript-config @@ -128,12 +134,12 @@ importers: specifier: ^19.0.0 version: 19.1.0(react@19.1.0) devDependencies: + 'cookiebench': + specifier: workspace:* + version: link:../../packages/cookiebench-cli '@cookiebench/benchmark-schema': specifier: workspace:* version: link:../../packages/benchmark-schema - '@cookiebench/cli': - specifier: workspace:* - version: link:../../packages/cli '@cookiebench/ts-config': specifier: workspace:* version: link:../../packages/typescript-config @@ -162,12 +168,12 @@ importers: specifier: ^19.0.0 version: 19.1.0(react@19.1.0) devDependencies: + 'cookiebench': + specifier: workspace:* + version: link:../../packages/cookiebench-cli '@cookiebench/benchmark-schema': specifier: workspace:* version: link:../../packages/benchmark-schema - '@cookiebench/cli': - specifier: workspace:* - version: link:../../packages/cli '@cookiebench/ts-config': specifier: workspace:* version: link:../../packages/typescript-config @@ -196,12 +202,12 @@ importers: specifier: ^19.0.0 version: 19.1.0(react@19.1.0) devDependencies: + 'cookiebench': + specifier: workspace:* + version: link:../../packages/cookiebench-cli '@cookiebench/benchmark-schema': specifier: workspace:* version: link:../../packages/benchmark-schema - '@cookiebench/cli': - specifier: workspace:* - version: link:../../packages/cli '@cookiebench/ts-config': specifier: workspace:* version: link:../../packages/typescript-config @@ -264,12 +270,12 @@ importers: specifier: ^19.0.0 version: 19.1.0(react@19.1.0) devDependencies: + 'cookiebench': + specifier: workspace:* + version: link:../../packages/cookiebench-cli '@cookiebench/benchmark-schema': specifier: workspace:* version: link:../../packages/benchmark-schema - '@cookiebench/cli': - specifier: workspace:* - version: link:../../packages/cli '@cookiebench/ts-config': specifier: workspace:* version: link:../../packages/typescript-config @@ -298,12 +304,12 @@ importers: specifier: ^19.0.0 version: 19.1.0(react@19.1.0) devDependencies: + 'cookiebench': + specifier: workspace:* + version: link:../../packages/cookiebench-cli '@cookiebench/benchmark-schema': specifier: workspace:* version: link:../../packages/benchmark-schema - '@cookiebench/cli': - specifier: workspace:* - version: link:../../packages/cli '@cookiebench/ts-config': specifier: workspace:* version: link:../../packages/typescript-config @@ -363,12 +369,12 @@ importers: specifier: ^19.0.0 version: 19.1.0(react@19.1.0) devDependencies: + 'cookiebench': + specifier: workspace:* + version: link:../../packages/cookiebench-cli '@cookiebench/benchmark-schema': specifier: workspace:* version: link:../../packages/benchmark-schema - '@cookiebench/cli': - specifier: workspace:* - version: link:../../packages/cli '@cookiebench/ts-config': specifier: workspace:* version: link:../../packages/typescript-config @@ -397,12 +403,12 @@ importers: specifier: ^19.0.0 version: 19.1.0(react@19.1.0) devDependencies: + 'cookiebench': + specifier: workspace:* + version: link:../../packages/cookiebench-cli '@cookiebench/benchmark-schema': specifier: workspace:* version: link:../../packages/benchmark-schema - '@cookiebench/cli': - specifier: workspace:* - version: link:../../packages/cli '@cookiebench/ts-config': specifier: workspace:* version: link:../../packages/typescript-config @@ -431,12 +437,12 @@ importers: specifier: ^19.0.0 version: 19.1.0(react@19.1.0) devDependencies: + 'cookiebench': + specifier: workspace:* + version: link:../../packages/cookiebench-cli '@cookiebench/benchmark-schema': specifier: workspace:* version: link:../../packages/benchmark-schema - '@cookiebench/cli': - specifier: workspace:* - version: link:../../packages/cli '@cookiebench/ts-config': specifier: workspace:* version: link:../../packages/typescript-config @@ -453,25 +459,47 @@ importers: specifier: ^5.8.3 version: 5.8.3 + packages/benchmark: + dependencies: + '@playwright/test': + specifier: ^1.42.1 + version: 1.52.0 + playwright-performance-metrics: + specifier: ^1.2.2 + version: 1.2.2(@playwright/test@1.52.0) + devDependencies: + '@rsdoctor/rspack-plugin': + specifier: ^1.1.3 + version: 1.1.3(@rsbuild/core@1.3.22)(@rspack/core@1.3.15(@swc/helpers@0.5.17)) + '@rslib/core': + specifier: ^0.9.1 + version: 0.9.1(typescript@5.8.3) + '@types/node': + specifier: ^22.15.30 + version: 22.15.30 + typescript: + specifier: ^5.8.3 + version: 5.8.3 + packages/benchmark-schema: {} - packages/cli: + packages/cookiebench-cli: dependencies: '@clack/prompts': specifier: ^1.0.0-alpha.0 version: 1.0.0-alpha.0 - '@playwright/test': - specifier: ^1.42.1 - version: 1.52.0 + '@consentio/benchmark': + specifier: workspace:* + version: link:../benchmark + '@consentio/runner': + specifier: workspace:* + version: link:../runner cli-table3: specifier: ^0.6.3 version: 0.6.5 dotenv: specifier: ^16.5.0 version: 16.5.0 - package-manager-detector: - specifier: ^1.3.0 - version: 1.3.0 picocolors: specifier: ^1.0.0 version: 1.1.1 @@ -485,9 +513,31 @@ importers: '@types/node': specifier: ^22.15.30 version: 22.15.30 + typescript: + specifier: ^5.8.3 + version: 5.8.3 + + packages/runner: + dependencies: + '@consentio/benchmark': + specifier: workspace:* + version: link:../benchmark + '@playwright/test': + specifier: ^1.42.1 + version: 1.52.0 playwright-performance-metrics: specifier: ^1.2.2 version: 1.2.2(@playwright/test@1.52.0) + devDependencies: + '@rsdoctor/rspack-plugin': + specifier: ^1.1.3 + version: 1.1.3(@rsbuild/core@1.3.22)(@rspack/core@1.3.15(@swc/helpers@0.5.17)) + '@rslib/core': + specifier: ^0.9.1 + version: 0.9.1(typescript@5.8.3) + '@types/node': + specifier: ^22.15.30 + version: 22.15.30 typescript: specifier: ^5.8.3 version: 5.8.3 @@ -2083,6 +2133,7 @@ packages: acorn@8.14.1: resolution: {integrity: sha512-OvQ/2pUDKmgfCg++xsTX1wGxfTaszcHVcTctW4UJB4hibJx2HXxxO5UmVgyjMa+ZDsiaf5wWLXYpRWMmBI0QHg==} engines: {node: '>=0.4.0'} + hasBin: true ansi-regex@5.0.1: resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} @@ -2889,9 +2940,6 @@ packages: resolution: {integrity: sha512-kuUqqHNUqoIWp/c467RI4X6mmyuojY5jGutNU0wVTmEOOfcuwLqyMVoAi9MKi2Ak+5i9+nhmrK4ufZE8069kHA==} engines: {node: '>=18'} - package-manager-detector@1.3.0: - resolution: {integrity: sha512-ZsEbbZORsyHuO00lY1kV3/t72yp6Ysay6Pd17ZAlNGuGwmWDLCJxFpRs0IzfXfj1o4icJOkUEioexFHzyPurSQ==} - parse-ms@4.0.0: resolution: {integrity: sha512-TXfryirbmq34y8QBwgqCVLi+8oA3oWx2eAnSn62ITyEhEYaWRlVZ2DvMM9eZbMs/RfxPu/PK/aBLyGj4IrqMHw==} engines: {node: '>=18'} @@ -3525,11 +3573,11 @@ snapshots: - supports-color - ws - '@c15t/nextjs@1.2.2-canary-20250603153501(@libsql/client@0.15.8)(@playwright/test@1.52.0)(@types/better-sqlite3@7.6.13)(@types/react-dom@19.1.6(@types/react@19.1.6))(@types/react@19.1.6)(better-sqlite3@11.10.0)(next@15.3.3(@opentelemetry/api@1.8.0)(@playwright/test@1.52.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(use-sync-external-store@1.5.0(react@19.1.0))(ws@8.18.2)': + '@c15t/nextjs@1.2.2-canary-20250603153501(@playwright/test@1.52.0)(@types/react-dom@19.1.6(@types/react@19.1.6))(@types/react@19.1.6)(next@15.3.3(@opentelemetry/api@1.9.0)(@playwright/test@1.52.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react-dom@19.1.0(react@19.1.0))(react@19.1.0)': dependencies: '@c15t/react': 1.2.2-canary-20250603153501(@libsql/client@0.15.8)(@playwright/test@1.52.0)(@types/better-sqlite3@7.6.13)(@types/react-dom@19.1.6(@types/react@19.1.6))(@types/react@19.1.6)(better-sqlite3@11.10.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(use-sync-external-store@1.5.0(react@19.1.0))(ws@8.18.2) '@c15t/translations': 1.2.2-canary-20250514203718 - next: 15.3.3(@opentelemetry/api@1.8.0)(@playwright/test@1.52.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0) + next: 15.3.3(@opentelemetry/api@1.9.0)(@playwright/test@1.52.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0) react: 19.1.0 react-dom: 19.1.0(react@19.1.0) transitivePeerDependencies: @@ -5858,8 +5906,6 @@ snapshots: dependencies: yocto-queue: 1.2.1 - package-manager-detector@1.3.0: {} - parse-ms@4.0.0: {} parseurl@1.3.3: {} From fa3e4a06c7a35d57ab5af2346243b691dbb8701b Mon Sep 17 00:00:00 2001 From: burnedchris Date: Wed, 29 Oct 2025 20:28:45 -0700 Subject: [PATCH 02/21] Enhance cookiebench CLI with new features and improved logging. Introduce interactive multi-select mode for benchmark execution, add scores command to view existing results, and integrate Perfume.js for enhanced performance metrics. Update logger utility for better CLI output and refactor commands to utilize the new logging system. Update package dependencies and documentation accordingly. --- .gitignore | 96 +++++- packages/benchmark/package.json | 2 + .../benchmark/src/cookie-banner-collector.ts | 19 +- packages/benchmark/src/index.ts | 3 + packages/benchmark/src/network-monitor.ts | 19 +- packages/benchmark/src/perfume-collector.ts | 150 ++++++++ .../src/resource-timing-collector.ts | 8 +- packages/benchmark/src/types.ts | 52 +++ packages/cookiebench-cli/README.md | 30 ++ packages/cookiebench-cli/package.json | 6 +- .../cookiebench-cli/src/commands/benchmark.ts | 189 +++++++++-- packages/cookiebench-cli/src/commands/db.ts | 115 +++---- .../cookiebench-cli/src/commands/results.ts | 126 +++---- .../cookiebench-cli/src/commands/scores.ts | 321 ++++++++++++++++++ .../cookiebench-cli/src/components/intro.ts | 93 +++++ packages/cookiebench-cli/src/index.ts | 50 ++- packages/cookiebench-cli/src/utils/index.ts | 5 +- packages/cookiebench-cli/src/utils/logger.ts | 152 +++++++++ packages/cookiebench-cli/src/utils/scoring.ts | 112 +++++- packages/runner/package.json | 1 + packages/runner/src/benchmark-runner.ts | 93 +++-- packages/runner/src/performance-aggregator.ts | 45 ++- packages/runner/src/server.ts | 10 +- packages/runner/src/types.ts | 22 ++ pnpm-lock.yaml | 135 ++++++-- 25 files changed, 1587 insertions(+), 267 deletions(-) create mode 100644 packages/benchmark/src/perfume-collector.ts create mode 100644 packages/cookiebench-cli/src/commands/scores.ts create mode 100644 packages/cookiebench-cli/src/components/intro.ts create mode 100644 packages/cookiebench-cli/src/utils/logger.ts diff --git a/.gitignore b/.gitignore index 3a70dd7..6fc4ccb 100644 --- a/.gitignore +++ b/.gitignore @@ -42,4 +42,98 @@ yarn-error.log* test-results -playwright-report \ No newline at end of file +playwright-report +c15t/cli/.gitignore +c15t/cli/CHANGELOG.md +c15t/cli/knip.json +c15t/cli/package.json +c15t/cli/readme.json +c15t/cli/README.md +c15t/cli/rslib.config.ts +c15t/cli/tsconfig.json +c15t/cli/vitest.config.ts +c15t/cli/src/index.ts +c15t/cli/src/actions/show-help-menu.ts +c15t/cli/src/commands/generate/index.ts +c15t/cli/src/commands/generate/options/c15t-mode.ts +c15t/cli/src/commands/generate/options/custom-mode.ts +c15t/cli/src/commands/generate/options/offline-mode.ts +c15t/cli/src/commands/generate/options/self-hosted-mode.ts +c15t/cli/src/commands/generate/options/types.ts +c15t/cli/src/commands/generate/options/shared/scripts.ts +c15t/cli/src/commands/generate/options/utils/dependencies.ts +c15t/cli/src/commands/generate/options/utils/generate-files.ts +c15t/cli/src/commands/generate/options/utils/shared-frontend.ts +c15t/cli/src/commands/generate/templates/backend.ts +c15t/cli/src/commands/generate/templates/config.ts +c15t/cli/src/commands/generate/templates/env.ts +c15t/cli/src/commands/generate/templates/index.ts +c15t/cli/src/commands/generate/templates/layout.ts +c15t/cli/src/commands/generate/templates/next-config.ts +c15t/cli/src/commands/generate/templates/next/index.ts +c15t/cli/src/commands/generate/templates/next/app/components.ts +c15t/cli/src/commands/generate/templates/next/app/layout.ts +c15t/cli/src/commands/generate/templates/next/pages/components.ts +c15t/cli/src/commands/generate/templates/next/pages/layout.ts +c15t/cli/src/commands/generate/templates/react/components.ts +c15t/cli/src/commands/generate/templates/shared/options.ts +c15t/cli/src/commands/self-host/index.ts +c15t/cli/src/commands/self-host/migrate/ensure-backend-config.test.ts +c15t/cli/src/commands/self-host/migrate/ensure-backend-config.ts +c15t/cli/src/commands/self-host/migrate/index.test.ts +c15t/cli/src/commands/self-host/migrate/index.ts +c15t/cli/src/commands/self-host/migrate/migrator-result.test.ts +c15t/cli/src/commands/self-host/migrate/migrator-result.ts +c15t/cli/src/commands/self-host/migrate/orm-result.test.ts +c15t/cli/src/commands/self-host/migrate/orm-result.ts +c15t/cli/src/commands/self-host/migrate/read-config.test.ts +c15t/cli/src/commands/self-host/migrate/read-config.ts +c15t/cli/src/components/intro.ts +c15t/cli/src/context/creator.ts +c15t/cli/src/context/error-handlers.ts +c15t/cli/src/context/file-system.ts +c15t/cli/src/context/framework-detection.ts +c15t/cli/src/context/package-manager-detection.ts +c15t/cli/src/context/parser.ts +c15t/cli/src/context/types.ts +c15t/cli/src/context/user-interaction.ts +c15t/cli/src/utils/capitalize-first-letter.ts +c15t/cli/src/utils/logger.ts +c15t/cli/src/utils/telemetry.ts +c15t/cli/test/command-integration.test.ts +c15t/cli/test/__snapshots__/drizzle/mysql.txt +c15t/cli/test/__snapshots__/drizzle/pg.txt +c15t/cli/test/__snapshots__/drizzle/sqlite.txt +c15t/cli/test/__snapshots__/kysely/d1.sql +c15t/cli/test/__snapshots__/kysely/mysql.sql +c15t/cli/test/__snapshots__/kysely/postgresql.sql +c15t/cli/test/__snapshots__/kysely/sqlite.sql +c15t/cli/test/__snapshots__/prisma/mysql.prisma +c15t/cli/test/__snapshots__/prisma/pg.prisma +c15t/cli/test/__snapshots__/prisma/sqlite.prisma +c15t/cli/test/utils/telemetry.test.ts +c15t/cli/test/utils/utils.test.ts +c15t/logger/.npmignore +c15t/logger/CHANGELOG.md +c15t/logger/knip.json +c15t/logger/package.json +c15t/logger/readme.json +c15t/logger/README.md +c15t/logger/rslib.config.ts +c15t/logger/tsconfig.json +c15t/logger/vitest.config.ts +c15t/logger/src/index.ts +c15t/logger/src/__tests__/core/integration.test.ts +c15t/logger/src/__tests__/core/levels.test.ts +c15t/logger/src/__tests__/core/logger.test.ts +c15t/logger/src/__tests__/core/types.test.ts +c15t/logger/src/__tests__/formatting/formatter.test.ts +c15t/logger/src/__tests__/utils/result.test.ts +c15t/logger/src/core/index.ts +c15t/logger/src/core/levels.ts +c15t/logger/src/core/logger.ts +c15t/logger/src/core/types.ts +c15t/logger/src/formatting/formatter.ts +c15t/logger/src/formatting/index.ts +c15t/logger/src/utils/index.ts +c15t/logger/src/utils/result.ts diff --git a/packages/benchmark/package.json b/packages/benchmark/package.json index cbfe513..dff508d 100644 --- a/packages/benchmark/package.json +++ b/packages/benchmark/package.json @@ -19,7 +19,9 @@ "lint": "biome lint ." }, "dependencies": { + "@c15t/logger": "1.0.0", "@playwright/test": "^1.42.1", + "perfume.js": "^9.4.0", "playwright-performance-metrics": "^1.2.2" }, "devDependencies": { diff --git a/packages/benchmark/src/cookie-banner-collector.ts b/packages/benchmark/src/cookie-banner-collector.ts index 57a4c60..74a04e2 100644 --- a/packages/benchmark/src/cookie-banner-collector.ts +++ b/packages/benchmark/src/cookie-banner-collector.ts @@ -6,13 +6,16 @@ import type { LayoutShiftEntry, WindowWithCookieMetrics, } from './types'; +import type { Logger } from '@c15t/logger'; import { determineBundleStrategy } from './bundle-strategy'; export class CookieBannerCollector { private config: Config; + private logger: Logger; - constructor(config: Config) { + constructor(config: Config, logger: Logger) { this.config = config; + this.logger = logger; } /** @@ -21,8 +24,8 @@ export class CookieBannerCollector { initializeMetrics(): CookieBannerMetrics { const { isBundled, isIIFE } = determineBundleStrategy(this.config); - console.log( - `🔍 [BUNDLE-STRATEGY] Detected from config: ${ + this.logger.debug( + `Bundle strategy detected from config: ${ isBundled ? 'Bundled' : isIIFE ? 'IIFE' : 'Unknown' }`, { @@ -197,8 +200,14 @@ export class CookieBannerCollector { return { detected: metrics.detected, selector: metrics.selector, - bannerRenderTime: metrics.bannerFirstSeen - metrics.pageLoadStart, - bannerInteractiveTime: metrics.bannerInteractive - metrics.pageLoadStart, + bannerRenderTime: + metrics.detected && metrics.bannerFirstSeen > 0 + ? metrics.bannerFirstSeen - metrics.pageLoadStart + : 0, + bannerInteractiveTime: + metrics.detected && metrics.bannerInteractive > 0 + ? metrics.bannerInteractive - metrics.pageLoadStart + : 0, bannerHydrationTime: metrics.bannerInteractive > 0 ? metrics.bannerInteractive - metrics.bannerFirstSeen diff --git a/packages/benchmark/src/index.ts b/packages/benchmark/src/index.ts index ad5fc6d..80b2e69 100644 --- a/packages/benchmark/src/index.ts +++ b/packages/benchmark/src/index.ts @@ -2,6 +2,7 @@ export { CookieBannerCollector } from './cookie-banner-collector'; export { NetworkMonitor } from './network-monitor'; export { ResourceTimingCollector } from './resource-timing-collector'; +export { PerfumeCollector } from './perfume-collector'; // Utilities export { determineBundleStrategy } from './bundle-strategy'; @@ -20,5 +21,7 @@ export type { CoreWebVitals, LayoutShiftEntry, WindowWithCookieMetrics, + PerfumeMetrics, + WindowWithPerfumeMetrics, } from './types'; diff --git a/packages/benchmark/src/network-monitor.ts b/packages/benchmark/src/network-monitor.ts index 27b80ac..1afbae8 100644 --- a/packages/benchmark/src/network-monitor.ts +++ b/packages/benchmark/src/network-monitor.ts @@ -1,16 +1,19 @@ import type { Page, Route } from '@playwright/test'; import type { Config, NetworkRequest, NetworkMetrics } from './types'; +import type { Logger } from '@c15t/logger'; export class NetworkMonitor { private config: Config; + private logger: Logger; private networkRequests: NetworkRequest[] = []; private metrics: NetworkMetrics = { bannerNetworkRequests: 0, bannerBundleSize: 0, }; - constructor(config: Config) { + constructor(config: Config, logger: Logger) { this.config = config; + this.logger = logger; } /** @@ -44,13 +47,13 @@ export class NetworkMonitor { isThirdParty, }); - if (isThirdParty) { - this.metrics.bannerNetworkRequests++; - this.metrics.bannerBundleSize += size / 1024; - console.log( - `🌐 [THIRD-PARTY-SCRIPT] Detected: ${url} (${(size / 1024).toFixed(2)}KB)` - ); - } + if (isThirdParty) { + this.metrics.bannerNetworkRequests++; + this.metrics.bannerBundleSize += size / 1024; + this.logger.debug( + `Third-party script detected: ${url} (${(size / 1024).toFixed(2)}KB)` + ); + } } await route.fulfill({ response, headers }); diff --git a/packages/benchmark/src/perfume-collector.ts b/packages/benchmark/src/perfume-collector.ts new file mode 100644 index 0000000..1358132 --- /dev/null +++ b/packages/benchmark/src/perfume-collector.ts @@ -0,0 +1,150 @@ +import type { Page } from '@playwright/test'; +import type { PerfumeMetrics, WindowWithPerfumeMetrics } from './types'; +import type { Logger } from '@c15t/logger'; + +export class PerfumeCollector { + private logger: Logger; + + constructor(logger: Logger) { + this.logger = logger; + } + /** + * Setup Perfume.js in the browser to collect performance metrics + */ + async setupPerfume(page: Page): Promise { + await page.addInitScript(() => { + // Initialize storage object + const win = window as WindowWithPerfumeMetrics; + win.__perfumeMetrics = {}; + + // Load Perfume.js from CDN + const script = document.createElement('script'); + script.src = 'https://unpkg.com/perfume.js@9.4.0/dist/perfume.umd.min.js'; + script.onload = () => { + console.log('🔍 [PERFUME] Perfume.js loaded from CDN'); + + // Initialize Perfume with analytics tracker + // @ts-ignore - Perfume is loaded from CDN + new window.Perfume({ + analyticsTracker: ({ + metricName, + data, + rating, + attribution, + navigatorInformation, + }: { + metricName: string; + data: number; + rating: string; + attribution?: unknown; + navigatorInformation?: { + deviceMemory?: number; + hardwareConcurrency?: number; + isLowEndDevice?: boolean; + isLowEndExperience?: boolean; + serviceWorkerStatus?: string; + }; + }) => { + const win = window as WindowWithPerfumeMetrics; + const metrics = win.__perfumeMetrics; + + // Store metric with all available data + if (metrics) { + metrics[metricName] = { + value: data, + rating, + attribution, + navigatorInformation, + }; + } + + console.log( + `🔍 [PERFUME] ${metricName}:`, + data, + `(${rating})`, + attribution + ); + }, + }); + }; + + // Handle script load errors + script.onerror = () => { + console.error('🔍 [PERFUME] Failed to load Perfume.js from CDN'); + }; + + document.head.appendChild(script); + }); + } + + /** + * Collect all metrics from Perfume.js + */ + async collectMetrics(page: Page): Promise { + try { + // Wait a bit for metrics to be collected + await page.waitForTimeout(1000); + + const rawMetrics = await page.evaluate(() => { + const win = window as WindowWithPerfumeMetrics; + const perfumeData = win.__perfumeMetrics; + return perfumeData || {}; + }); + + this.logger.debug('Raw Perfume metrics:', rawMetrics); + + // Get navigation timing separately + const navigationTiming = await page.evaluate(() => { + const timing = performance.timing; + const navigationStart = timing.navigationStart; + + return { + timeToFirstByte: timing.responseStart - navigationStart, + domInteractive: timing.domInteractive - navigationStart, + domContentLoadedEventStart: + timing.domContentLoadedEventStart - navigationStart, + domContentLoadedEventEnd: + timing.domContentLoadedEventEnd - navigationStart, + domComplete: timing.domComplete - navigationStart, + loadEventStart: timing.loadEventStart - navigationStart, + loadEventEnd: timing.loadEventEnd - navigationStart, + }; + }); + + // Get network information + const networkInformation = await page.evaluate(() => { + // @ts-ignore - navigator.connection is experimental + const connection = navigator.connection || navigator.mozConnection; + if (connection) { + return { + effectiveType: connection.effectiveType || 'unknown', + downlink: connection.downlink || 0, + rtt: connection.rtt || 0, + saveData: connection.saveData || false, + }; + } + return undefined; + }); + + // Convert raw metrics to PerfumeMetrics format + const metrics: PerfumeMetrics = { + firstPaint: rawMetrics.FP?.value || 0, + firstContentfulPaint: rawMetrics.FCP?.value || 0, + largestContentfulPaint: rawMetrics.LCP?.value || 0, + cumulativeLayoutShift: rawMetrics.CLS?.value || 0, + totalBlockingTime: rawMetrics.TBT?.value || 0, + firstInputDelay: rawMetrics.FID?.value || null, + interactionToNextPaint: rawMetrics.INP?.value || null, + timeToFirstByte: rawMetrics.TTFB?.value || navigationTiming.timeToFirstByte, + navigationTiming, + networkInformation, + }; + + return metrics; + } catch (error) { + this.logger.error('Failed to collect Perfume metrics:', error); + return null; + } + } +} + diff --git a/packages/benchmark/src/resource-timing-collector.ts b/packages/benchmark/src/resource-timing-collector.ts index 377756a..a05db7b 100644 --- a/packages/benchmark/src/resource-timing-collector.ts +++ b/packages/benchmark/src/resource-timing-collector.ts @@ -1,12 +1,18 @@ import type { Page } from '@playwright/test'; import type { ResourceTimingData } from './types'; +import type { Logger } from '@c15t/logger'; export class ResourceTimingCollector { + private logger: Logger; + + constructor(logger: Logger) { + this.logger = logger; + } /** * Collect detailed resource timing data from the browser */ async collect(page: Page): Promise { - console.log('🔍 [DEBUG] Collecting resource timing data...'); + this.logger.debug('Collecting resource timing data...'); return page.evaluate(() => { console.log('🔍 [BROWSER] Starting resource collection...'); diff --git a/packages/benchmark/src/types.ts b/packages/benchmark/src/types.ts index a527e74..617e46f 100644 --- a/packages/benchmark/src/types.ts +++ b/packages/benchmark/src/types.ts @@ -224,3 +224,55 @@ export interface CoreWebVitals { totalBytes?: number; } +// Perfume.js metrics types +export interface PerfumeMetrics { + // Core Web Vitals (replacing playwright-performance-metrics) + firstPaint: number; + firstContentfulPaint: number; + largestContentfulPaint: number; + cumulativeLayoutShift: number; + totalBlockingTime: number; + + // Enhanced metrics (new) + firstInputDelay: number | null; + interactionToNextPaint: number | null; + timeToFirstByte: number; + + // Detailed navigation timing + navigationTiming: { + timeToFirstByte: number; + domInteractive: number; + domContentLoadedEventStart: number; + domContentLoadedEventEnd: number; + domComplete: number; + loadEventStart: number; + loadEventEnd: number; + }; + + // Network information (optional) + networkInformation?: { + effectiveType: string; + downlink: number; + rtt: number; + saveData: boolean; + }; +} + +export interface WindowWithPerfumeMetrics extends Window { + __perfumeMetrics?: Record< + string, + { + value: number; + rating: string; + attribution?: unknown; + navigatorInformation?: { + deviceMemory?: number; + hardwareConcurrency?: number; + isLowEndDevice?: boolean; + isLowEndExperience?: boolean; + serviceWorkerStatus?: string; + }; + } + >; +} + diff --git a/packages/cookiebench-cli/README.md b/packages/cookiebench-cli/README.md index 01037cd..c79ac90 100644 --- a/packages/cookiebench-cli/README.md +++ b/packages/cookiebench-cli/README.md @@ -39,6 +39,9 @@ cookiebench benchmark # View and aggregate results cookiebench results +# View scores from existing results +cookiebench scores + # Manage database cookiebench db push ``` @@ -74,6 +77,33 @@ Features: - Calculates scores for each app - Saves results to database (if configured) +### scores + +View calculated scores from existing benchmark results without re-running benchmarks. + +```bash +# Interactive: choose which app to view +cookiebench scores + +# View scores for a specific app +cookiebench scores with-c15t-nextjs + +# View scores for all apps +cookiebench scores __all__ +``` + +Features: +- Reads existing `results.json` files from benchmark directories +- Uses pre-calculated scores if available, or calculates them on-demand +- Displays detailed score breakdowns by category +- Shows insights and recommendations +- Much faster than re-running full benchmarks + +Perfect for: +- Reviewing benchmark results later +- Comparing scores across different runs +- Generating reports without re-benchmarking + ### db Manage the benchmark database. diff --git a/packages/cookiebench-cli/package.json b/packages/cookiebench-cli/package.json index 5fda68c..df4a9fa 100644 --- a/packages/cookiebench-cli/package.json +++ b/packages/cookiebench-cli/package.json @@ -18,12 +18,16 @@ "start": "node ./dist/index.mjs" }, "dependencies": { + "@c15t/logger": "^1.0.0", "@clack/prompts": "^1.0.0-alpha.0", "@consentio/benchmark": "workspace:*", "@consentio/runner": "workspace:*", + "@types/figlet": "^1.7.0", "cli-table3": "^0.6.3", "dotenv": "^16.5.0", - "picocolors": "^1.0.0" + "figlet": "^1.9.3", + "picocolors": "^1.0.0", + "pretty-ms": "^9.2.0" }, "devDependencies": { "@rsdoctor/rspack-plugin": "^1.1.3", diff --git a/packages/cookiebench-cli/src/commands/benchmark.ts b/packages/cookiebench-cli/src/commands/benchmark.ts index 2812995..d2e33b0 100644 --- a/packages/cookiebench-cli/src/commands/benchmark.ts +++ b/packages/cookiebench-cli/src/commands/benchmark.ts @@ -1,5 +1,8 @@ -import { writeFile } from 'node:fs/promises'; +import { writeFile, readdir } from 'node:fs/promises'; import { join } from 'node:path'; +import { setTimeout } from 'node:timers/promises'; +import * as p from '@clack/prompts'; +import color from 'picocolors'; import { BenchmarkRunner, buildAndServeNextApp, @@ -7,25 +10,52 @@ import { readConfig, type ServerInfo, } from '@consentio/runner'; -import { calculateScores, printScores } from '../utils/scoring'; +import { calculateScores, printScores, type CliLogger } from '../utils'; + +/** + * Find all benchmark directories + */ +async function findBenchmarkDirs(logger: CliLogger): Promise { + const benchmarksDir = 'benchmarks'; + try { + const entries = await readdir(benchmarksDir, { withFileTypes: true }); + const dirs = entries + .filter((entry) => entry.isDirectory() && !entry.name.startsWith('.')) + .map((entry) => entry.name); + return dirs; + } catch (error) { + logger.debug('Failed to read benchmarks directory:', error); + return []; + } +} + +/** + * Run a single benchmark for a specific app + */ +async function runSingleBenchmark( + logger: CliLogger, + appPath: string, + showScores = true +): Promise { + const configPath = appPath ? join(appPath, 'config.json') : undefined; + const config = readConfig(configPath); + if (!config) { + logger.error(`Failed to read config.json for ${appPath || 'current directory'}`); + return false; + } -export async function benchmarkCommand(appPath?: string): Promise { try { - const config = readConfig(appPath ? join(appPath, 'config.json') : undefined); - if (!config) { - throw new Error('Failed to read config.json'); - } let serverInfo: ServerInfo | null = null; let benchmarkUrl: string; // Check if remote benchmarking is enabled if (config.remote?.enabled && config.remote.url) { - console.log(`🌐 Running remote benchmark against: ${config.remote.url}`); + logger.info(`🌐 Running remote benchmark against: ${config.remote.url}`); benchmarkUrl = config.remote.url; } else { - console.log('🏗️ Building and serving app locally...'); - serverInfo = await buildAndServeNextApp(appPath); + logger.info('🏗️ Building and serving app locally...'); + serverInfo = await buildAndServeNextApp(logger, appPath); benchmarkUrl = serverInfo.url; } @@ -33,7 +63,7 @@ export async function benchmarkCommand(appPath?: string): Promise { try { // Create benchmark runner and run benchmarks - const runner = new BenchmarkRunner(config); + const runner = new BenchmarkRunner(config, logger); const result = await runner.runBenchmarks(benchmarkUrl); // Create app data for transparency scoring @@ -128,7 +158,7 @@ export async function benchmarkCommand(appPath?: string): Promise { (r) => r.cookieBanner.detected ); if (!allDetected) { - console.log( + logger.warn( '⚠️ [SCORING] Banner detection inconsistent or failed - marking as not detected' ); } @@ -140,7 +170,7 @@ export async function benchmarkCommand(appPath?: string): Promise { (r) => r.cookieBanner.detected ); if (!detectionSuccess) { - console.log( + logger.warn( '⚠️ [SCORING] No banner detected in any iteration - applying penalty' ); return null; // This signals failed detection for scoring @@ -154,7 +184,7 @@ export async function benchmarkCommand(appPath?: string): Promise { // If we have mixed results (some detected, some not), still penalize if (hasNullValues) { - console.log( + logger.warn( '⚠️ [SCORING] Inconsistent banner detection - applying penalty' ); return null; @@ -176,7 +206,7 @@ export async function benchmarkCommand(appPath?: string): Promise { (r) => r.cookieBanner.detected ); if (!detectionSuccess) { - console.log( + logger.warn( '⚠️ [SCORING] Inconsistent detection - setting coverage to 0' ); return 0; // No coverage score if detection failed @@ -231,13 +261,15 @@ export async function benchmarkCommand(appPath?: string): Promise { // Write results to file const outputPath = join(cwd, 'results.json'); await writeFile(outputPath, JSON.stringify(resultsData, null, 2)); - console.log(`✅ Benchmark results saved to ${outputPath}`); + logger.success(`Benchmark results saved to ${outputPath}`); - // Print scores if available - if (scores) { - console.log('📊 Benchmark Scores:'); + // Print scores if requested + if (showScores && scores) { + logger.info('📊 Benchmark Scores:'); printScores(scores); } + + return true; } finally { // Only cleanup server if we started one if (serverInfo) { @@ -246,11 +278,126 @@ export async function benchmarkCommand(appPath?: string): Promise { } } catch (error: unknown) { if (error instanceof Error) { - console.error(`Error running benchmark: ${error.message}`); + logger.error(`Error running benchmark: ${error.message}`); } else { - console.error('An unknown error occurred during benchmark'); + logger.error('An unknown error occurred during benchmark'); + } + return false; + } +} + +/** + * Main benchmark command with multi-select support + */ +export async function benchmarkCommand( + logger: CliLogger, + appPath?: string +): Promise { + // If a specific app path is provided, run that benchmark directly + if (appPath) { + const success = await runSingleBenchmark(logger, appPath, true); + if (!success) { + process.exit(1); } + return; + } + + // Otherwise, show multi-select for available benchmarks + logger.clear(); + await setTimeout(500); + + p.intro(`${color.bgMagenta(color.white(' benchmark '))}`); + + // Find available benchmarks + const availableBenchmarks = await findBenchmarkDirs(logger); + + if (availableBenchmarks.length === 0) { + logger.error('No benchmarks found in the benchmarks/ directory'); + logger.info( + 'Create benchmark directories with config.json files to get started' + ); process.exit(1); } + + logger.info( + `Found ${availableBenchmarks.length} benchmark(s): ${color.cyan(availableBenchmarks.join(', '))}` + ); + + // Ask user to select benchmarks + const selectedBenchmarks = await p.multiselect({ + message: 'Select benchmarks to run (use space to toggle):', + options: availableBenchmarks.map((name) => ({ + value: name, + label: name, + hint: join('benchmarks', name), + })), + required: true, + }); + + if (p.isCancel(selectedBenchmarks)) { + p.cancel('Operation cancelled'); + return; + } + + if (!Array.isArray(selectedBenchmarks) || selectedBenchmarks.length === 0) { + logger.warn('No benchmarks selected'); + return; + } + + // Ask if user wants to see scores after each benchmark + const showScores = await p.confirm({ + message: 'Show scores after each benchmark?', + initialValue: true, + }); + + if (p.isCancel(showScores)) { + p.cancel('Operation cancelled'); + return; + } + + // Run selected benchmarks sequentially + const results: Array<{ name: string; success: boolean }> = []; + + for (let i = 0; i < selectedBenchmarks.length; i++) { + const benchmarkName = selectedBenchmarks[i]; + const benchmarkPath = join('benchmarks', benchmarkName); + + logger.info( + `\n${color.bold(color.cyan(`[${i + 1}/${selectedBenchmarks.length}]`))} Running benchmark: ${color.bold(benchmarkName)}` + ); + + const success = await runSingleBenchmark( + logger, + benchmarkPath, + showScores === true + ); + + results.push({ name: benchmarkName, success }); + + if (!success) { + logger.error( + `Failed to complete benchmark for ${benchmarkName}, continuing...` + ); + } + + // Add spacing between benchmarks + if (i < selectedBenchmarks.length - 1) { + logger.message('\n' + '─'.repeat(80) + '\n'); + } + } + + // Summary + logger.message('\n'); + p.outro( + `${color.bold('Summary:')} ${results.filter((r) => r.success).length}/${results.length} benchmarks completed successfully` + ); + + // Show failed benchmarks if any + const failed = results.filter((r) => !r.success); + if (failed.length > 0) { + logger.warn( + `Failed benchmarks: ${failed.map((r) => r.name).join(', ')}` + ); + } } diff --git a/packages/cookiebench-cli/src/commands/db.ts b/packages/cookiebench-cli/src/commands/db.ts index 4e861ee..6644308 100644 --- a/packages/cookiebench-cli/src/commands/db.ts +++ b/packages/cookiebench-cli/src/commands/db.ts @@ -4,45 +4,46 @@ import { join, dirname } from 'node:path'; import { existsSync } from 'node:fs'; import * as p from '@clack/prompts'; import color from 'picocolors'; +import type { CliLogger } from '../utils'; const DB_PACKAGE_PATH = join(process.cwd(), 'packages', 'db'); const DRIZZLE_CONFIG_PATH = join(DB_PACKAGE_PATH, 'drizzle.config.ts'); -function ensureDbPackage() { +function ensureDbPackage(logger: CliLogger) { if (!existsSync(DB_PACKAGE_PATH)) { - p.log.error('Database package not found. Make sure you are running this from the project root.'); + logger.error('Database package not found. Make sure you are running this from the project root.'); process.exit(1); } if (!existsSync(DRIZZLE_CONFIG_PATH)) { - p.log.error('Drizzle config not found. Make sure drizzle.config.ts exists in packages/db/'); + logger.error('Drizzle config not found. Make sure drizzle.config.ts exists in packages/db/'); process.exit(1); } } -function runDrizzleCommand(command: string): void { +function runDrizzleCommand(logger: CliLogger, command: string): void { try { - p.log.step(`Running: ${color.cyan(`drizzle-kit ${command}`)}`); + logger.step(`Running: ${color.cyan(`drizzle-kit ${command}`)}`); execSync(`cd ${DB_PACKAGE_PATH} && pnpm drizzle-kit ${command}`, { stdio: 'inherit', encoding: 'utf-8' }); } catch (error) { - p.log.error(`Failed to run drizzle-kit ${command}`); + logger.error(`Failed to run drizzle-kit ${command}`); if (error instanceof Error) { - p.log.error(error.message); + logger.error(error.message); } process.exit(1); } } -export async function dbCommand(subcommand?: string) { - console.clear(); +export async function dbCommand(logger: CliLogger, subcommand?: string) { + logger.clear(); await setTimeout(1000); p.intro(`${color.bgBlue(color.white(' database '))} ${color.dim('v0.1.0')}`); - ensureDbPackage(); + ensureDbPackage(logger); let selectedCommand = subcommand; @@ -88,31 +89,31 @@ export async function dbCommand(subcommand?: string) { switch (selectedCommand) { case 'push': - await pushCommand(); + await pushCommand(logger); break; case 'generate': - await generateCommand(); + await generateCommand(logger); break; case 'migrate': - await migrateCommand(); + await migrateCommand(logger); break; case 'studio': - await studioCommand(); + await studioCommand(logger); break; case 'status': - await statusCommand(); + await statusCommand(logger); break; default: - p.log.error(`Unknown subcommand: ${selectedCommand}`); - p.log.info('Available commands: push, generate, migrate, studio, status'); + logger.error(`Unknown subcommand: ${selectedCommand}`); + logger.info('Available commands: push, generate, migrate, studio, status'); process.exit(1); } } -async function pushCommand() { - p.log.step('Pushing schema changes to database...'); - p.log.info('This will apply schema changes directly to your database.'); - p.log.warn('This is recommended for development only!'); +async function pushCommand(logger: CliLogger) { + logger.step('Pushing schema changes to database...'); + logger.info('This will apply schema changes directly to your database.'); + logger.warn('This is recommended for development only!'); const confirm = await p.confirm({ message: 'Are you sure you want to push schema changes?', @@ -124,24 +125,24 @@ async function pushCommand() { return; } - runDrizzleCommand('push'); - p.log.success('Schema pushed successfully!'); - p.outro('Database is now up to date with your schema.'); + runDrizzleCommand(logger, 'push'); + logger.success('Schema pushed successfully!'); + logger.outro('Database is now up to date with your schema.'); } -async function generateCommand() { - p.log.step('Generating migration files...'); - p.log.info('This will create SQL migration files based on schema changes.'); +async function generateCommand(logger: CliLogger) { + logger.step('Generating migration files...'); + logger.info('This will create SQL migration files based on schema changes.'); - runDrizzleCommand('generate'); - p.log.success('Migration files generated!'); - p.log.info('Review the generated files in packages/db/drizzle/ before applying them.'); - p.outro(`Run ${color.cyan('cli db migrate')} to apply the migrations.`); + runDrizzleCommand(logger, 'generate'); + logger.success('Migration files generated!'); + logger.info('Review the generated files in packages/db/drizzle/ before applying them.'); + logger.outro(`Run ${color.cyan('cli db migrate')} to apply the migrations.`); } -async function migrateCommand() { - p.log.step('Running migrations...'); - p.log.info('This will apply pending migration files to your database.'); +async function migrateCommand(logger: CliLogger) { + logger.step('Running migrations...'); + logger.info('This will apply pending migration files to your database.'); const confirm = await p.confirm({ message: 'Are you sure you want to run migrations?', @@ -154,28 +155,28 @@ async function migrateCommand() { } try { - runDrizzleCommand('migrate'); - p.log.success('Migrations completed successfully!'); - p.outro('Database is now up to date.'); + runDrizzleCommand(logger, 'migrate'); + logger.success('Migrations completed successfully!'); + logger.outro('Database is now up to date.'); } catch (error) { - p.log.error('Migration failed!'); + logger.error('Migration failed!'); if (error instanceof Error) { - p.log.error(error.message); + logger.error(error.message); } process.exit(1); } } -async function studioCommand() { - p.log.step('Opening Drizzle Studio...'); - p.log.info('This will start a web interface to browse and edit your database.'); - p.log.info('Press Ctrl+C to stop the studio when you\'re done.'); +async function studioCommand(logger: CliLogger) { + logger.step('Opening Drizzle Studio...'); + logger.info('This will start a web interface to browse and edit your database.'); + logger.info('Press Ctrl+C to stop the studio when you\'re done.'); try { - runDrizzleCommand('studio'); + runDrizzleCommand(logger, 'studio'); } catch (error) { // Studio command might be interrupted by Ctrl+C, which is normal - p.log.info('Studio closed.'); + logger.info('Studio closed.'); } } @@ -196,24 +197,24 @@ function findProjectRoot(): string { return process.cwd(); } -async function statusCommand() { - p.log.step('Checking migration status...'); +async function statusCommand(logger: CliLogger) { + logger.step('Checking migration status...'); try { // Check if database exists at project root const projectRoot = findProjectRoot(); const dbPath = join(projectRoot, 'benchmarks.db'); if (!existsSync(dbPath)) { - p.log.warn('Database file does not exist yet.'); - p.log.info(`Run ${color.cyan('cli db push')} or ${color.cyan('cli db migrate')} to create it.`); + logger.warn('Database file does not exist yet.'); + logger.info(`Run ${color.cyan('cli db push')} or ${color.cyan('cli db migrate')} to create it.`); return; } // Check migrations folder const migrationsPath = join(DB_PACKAGE_PATH, 'drizzle'); if (!existsSync(migrationsPath)) { - p.log.warn('No migrations found.'); - p.log.info(`Run ${color.cyan('cli db generate')} to create migration files.`); + logger.warn('No migrations found.'); + logger.info(`Run ${color.cyan('cli db generate')} to create migration files.`); return; } @@ -226,19 +227,19 @@ async function statusCommand() { .sort(); if (migrations.length === 0) { - p.log.info('No migration files found.'); + logger.info('No migration files found.'); } else { - p.log.info(`Found ${migrations.length} migration(s):`); + logger.info(`Found ${migrations.length} migration(s):`); for (const migration of migrations) { - p.log.info(` - ${migration}`); + logger.info(` - ${migration}`); } } - p.log.success('Status check complete.'); + logger.success('Status check complete.'); } catch (error) { - p.log.error('Failed to check status.'); + logger.error('Failed to check status.'); if (error instanceof Error) { - p.log.error(error.message); + logger.error(error.message); } } } \ No newline at end of file diff --git a/packages/cookiebench-cli/src/commands/results.ts b/packages/cookiebench-cli/src/commands/results.ts index c2882e2..0e953d4 100644 --- a/packages/cookiebench-cli/src/commands/results.ts +++ b/packages/cookiebench-cli/src/commands/results.ts @@ -6,7 +6,7 @@ import { join } from 'node:path'; import Table from 'cli-table3'; import prettyMilliseconds from 'pretty-ms'; import { config } from 'dotenv'; -import { calculateScores, printScores } from '../utils/scoring'; +import { calculateScores, printScores, type CliLogger } from '../utils'; import type { BenchmarkScores } from '../types'; import type { Config } from '@consentio/runner'; @@ -16,12 +16,12 @@ config({ path: ".env.local" }); config({ path: "www/.env.local" }); // Also check www directory // Function to save benchmark result via oRPC endpoint -async function saveBenchmarkResult(result: BenchmarkResult): Promise { +async function saveBenchmarkResult(logger: CliLogger, result: BenchmarkResult): Promise { const apiUrl = process.env.API_URL || "http://localhost:3000"; const endpoint = `${apiUrl}/api/orpc/benchmarks/save`; try { - p.log.info(`Attempting to save ${result.name} to ${endpoint}`); + logger.info(`Attempting to save ${result.name} to ${endpoint}`); const response = await fetch(endpoint, { method: "POST", @@ -37,17 +37,17 @@ async function saveBenchmarkResult(result: BenchmarkResult): Promise { } const responseData = await response.json(); - p.log.success( + logger.success( `Saved benchmark result for ${result.name} (App ID: ${responseData.appId})` ); } catch (error) { if (error instanceof Error) { - p.log.error(`Failed to save benchmark result for ${result.name}: ${error.message}`); + logger.error(`Failed to save benchmark result for ${result.name}: ${error.message}`); if (error.message.includes('fetch failed')) { - p.log.error(`Connection failed. Is the server running on ${apiUrl}?`); + logger.error(`Connection failed. Is the server running on ${apiUrl}?`); } } else { - p.log.error(`Failed to save benchmark result for ${result.name}: Unknown error`); + logger.error(`Failed to save benchmark result for ${result.name}: Unknown error`); } throw error; } @@ -143,6 +143,25 @@ export interface RawBenchmarkDetail { largestContentfulPaint: number; timeToInteractive: number; cumulativeLayoutShift: number; + // NEW: Perfume.js enhanced metrics + timeToFirstByte?: number; + firstInputDelay?: number | null; + interactionToNextPaint?: number | null; + navigationTiming?: { + timeToFirstByte: number; + domInteractive: number; + domContentLoadedEventStart: number; + domContentLoadedEventEnd: number; + domComplete: number; + loadEventStart: number; + loadEventEnd: number; + }; + networkInformation?: { + effectiveType: string; + downlink: number; + rtt: number; + saveData: boolean; + }; cookieBanner: { renderStart: number; renderEnd: number; @@ -290,7 +309,7 @@ async function findResultsFiles(dir: string): Promise { return files; } -async function loadConfigForApp(appName: string): Promise { +async function loadConfigForApp(logger: CliLogger, appName: string): Promise { const configPath = join("benchmarks", appName, "config.json"); try { @@ -331,7 +350,7 @@ async function loadConfigForApp(appName: string): Promise { }, }; } catch (error) { - p.log.warn( + logger.warn( `Could not load config for ${appName}: ${ error instanceof Error ? error.message : "Unknown error" }` @@ -392,13 +411,13 @@ function safeGet(obj: unknown, path: string, defaultValue: T): T { } } -async function aggregateResults(resultsDir: string) { +async function aggregateResults(logger: CliLogger, resultsDir: string) { const resultsFiles = await findResultsFiles(resultsDir); const results: Record = {}; - p.log.info(`Found ${resultsFiles.length} results files:`); + logger.info(`Found ${resultsFiles.length} results files:`); for (const file of resultsFiles) { - p.log.info(` - ${file}`); + logger.info(` - ${file}`); } for (const file of resultsFiles) { @@ -407,41 +426,41 @@ async function aggregateResults(resultsDir: string) { const data: BenchmarkOutput = JSON.parse(content); if (!data.app || !data.results) { - p.log.warn( + logger.warn( `Skipping invalid results file: ${file} (missing app or results)` ); continue; } // Log the actual app name from the file - p.log.info(`Processing ${file} with app name: "${data.app}"`); + logger.info(`Processing ${file} with app name: "${data.app}"`); if (results[data.app]) { - p.log.warn( + logger.warn( `Duplicate app name "${data.app}" found in ${file}. Previous results will be overwritten.` ); } results[data.app] = data.results; - p.log.success( + logger.success( `Loaded results for ${data.app} (${data.results.length} iterations)` ); } catch (error) { - p.log.error( + logger.error( `Failed to process ${file}: ${ error instanceof Error ? error.message : "Unknown error" }` ); if (error instanceof Error && error.stack) { - p.log.error(`Stack trace: ${error.stack}`); + logger.error(`Stack trace: ${error.stack}`); } } } // Log final results summary - p.log.info("Final results summary:"); + logger.info("Final results summary:"); for (const [app, appResults] of Object.entries(results)) { - p.log.info(` - ${app}: ${appResults.length} iterations`); + logger.info(` - ${app}: ${appResults.length} iterations`); } return results; @@ -670,21 +689,8 @@ function printResults(results: Record) { } // Print the table to console + // Table output is user-facing, so we use console.log directly console.log(table.toString()); - - // Log a summary to console - p.log.info("Summary:"); - for (const r of sorted.slice(0, 5)) { - // Show top 5 results - const deltaColor = r.timeDelta > 0 ? color.red : color.green; - const deltaStr = - r.app === "baseline" - ? "" - : ` (${deltaColor(r.timeDelta > 0 ? "+" : "")}${deltaColor( - r.timeDelta.toFixed(1) - )}${deltaColor("%")})`; - p.log.info(` ${r.app}: ${formatTime(r.avgTime)}${deltaStr}`); - } } // Function to transform BenchmarkScores to match oRPC contract @@ -718,8 +724,8 @@ function mapStatusToContract(status: 'excellent' | 'good' | 'fair' | 'poor'): 'e return status; } -export async function resultsCommand() { - console.clear(); +export async function resultsCommand(logger: CliLogger) { + logger.clear(); await setTimeout(1000); p.intro(`${color.bgCyan(color.black(" results "))}`); @@ -734,34 +740,34 @@ export async function resultsCommand() { databaseUrl?.startsWith("libsql://") || databaseUrl?.startsWith("wss://") ) { - p.log.info( + logger.info( `🌐 Using Turso remote database: ${color.cyan( `${databaseUrl.split("@")[0]}@***` )}` ); if (!authToken) { - p.log.warn("⚠️ No auth token found. Database operations may fail."); + logger.warn("⚠️ No auth token found. Database operations may fail."); } } else if (databaseUrl?.startsWith("file:")) { - p.log.info(`📁 Using file database: ${color.cyan(databaseUrl)}`); + logger.info(`📁 Using file database: ${color.cyan(databaseUrl)}`); } else if (process.env.VERCEL || process.env.NODE_ENV === "production") { - p.log.warn("⚠️ Using in-memory database. Data will not persist!"); + logger.warn("⚠️ Using in-memory database. Data will not persist!"); } else { - p.log.info( + logger.info( `📁 Using local SQLite database: ${color.cyan("benchmarks.db")}` ); } const resultsDir = "benchmarks"; - p.log.step("Aggregating results..."); - const results = await aggregateResults(resultsDir); + logger.step("Aggregating results..."); + const results = await aggregateResults(logger, resultsDir); if (Object.keys(results).length === 0) { - p.log.error("No benchmark results found!"); + logger.error("No benchmark results found!"); return; } - p.log.info( + logger.info( `Found results for ${Object.keys(results).length} apps: ${Object.keys( results ).join(", ")}` @@ -769,7 +775,7 @@ export async function resultsCommand() { const appConfigs: Record = {}; for (const appName of Object.keys(results)) { - appConfigs[appName] = await loadConfigForApp(appName); + appConfigs[appName] = await loadConfigForApp(logger, appName); } // Calculate scores for each app @@ -794,6 +800,9 @@ export async function resultsCommand() { cls: appResults.reduce((a, b) => a + b.timing.cumulativeLayoutShift, 0) / appResults.length, tbt: appResults.reduce((a, b) => a + b.timing.mainThreadBlocking.total, 0) / appResults.length, tti: appResults.reduce((a, b) => a + b.timing.timeToInteractive, 0) / appResults.length, + // NEW: Add Perfume.js metrics + timeToFirstByte: appResults.reduce((a, b) => a + (b.timing.timeToFirstByte || 0), 0) / appResults.length, + interactionToNextPaint: appResults[0]?.timing.interactionToNextPaint || null, }, { totalSize: appResults.reduce((a, b) => a + b.size.total, 0) / appResults.length, @@ -824,19 +833,20 @@ export async function resultsCommand() { layoutShifts: appResults.reduce((a, b) => a + b.timing.cumulativeLayoutShift, 0) / appResults.length, }, appName === "baseline", - appData + appData, + appResults[0]?.timing.networkInformation ); } // Print scores - console.log("\n📊 Benchmark Scores:"); + logger.info("\n📊 Benchmark Scores:"); for (const [appName, appScores] of Object.entries(scores)) { - console.log(`\n${appName}:`); + logger.info(`\n${appName}:`); printScores(appScores); } // Save results to database - p.log.step("Saving results to database..."); + logger.step("Saving results to database..."); let savedCount = 0; let errorCount = 0; @@ -904,11 +914,11 @@ export async function resultsCommand() { scores: transformScoresToContract(scores[appName]), }; - await saveBenchmarkResult(benchmarkResult); - p.log.success(`Saved results for ${appName}`); + await saveBenchmarkResult(logger, benchmarkResult); + logger.success(`Saved results for ${appName}`); savedCount++; } catch (error) { - p.log.error( + logger.error( `Failed to save results for ${appName}: ${ error instanceof Error ? error.message : "Unknown error" }` @@ -922,28 +932,28 @@ export async function resultsCommand() { // Summary of database operations if (savedCount > 0) { - p.log.success(`Successfully saved ${savedCount} app(s) to database.`); + logger.success(`Successfully saved ${savedCount} app(s) to database.`); } if (errorCount > 0) { - p.log.warn(`Failed to save ${errorCount} app(s) to database.`); + logger.warn(`Failed to save ${errorCount} app(s) to database.`); } if ( databaseUrl?.startsWith("libsql://") || databaseUrl?.startsWith("wss://") ) { - p.log.info( + logger.info( `Results have been saved to Turso database: ${color.cyan( `${databaseUrl.split("@")[0]}@***` )}` ); } else { - p.log.info( + logger.info( `Results have been saved to local database: ${color.cyan( "benchmarks.db" )} (in project root)` ); } - p.outro("Results aggregated and saved to database successfully!"); + logger.outro("Results aggregated and saved to database successfully!"); } diff --git a/packages/cookiebench-cli/src/commands/scores.ts b/packages/cookiebench-cli/src/commands/scores.ts new file mode 100644 index 0000000..b7bfe02 --- /dev/null +++ b/packages/cookiebench-cli/src/commands/scores.ts @@ -0,0 +1,321 @@ +import { setTimeout } from 'node:timers/promises'; +import * as p from '@clack/prompts'; +import color from 'picocolors'; +import { readFile, readdir } from 'node:fs/promises'; +import { join } from 'node:path'; +import { calculateScores, printScores, type CliLogger } from '../utils'; +import type { BenchmarkScores } from '../types'; +import type { Config } from '@consentio/runner'; +import type { RawBenchmarkDetail } from './results'; + +interface BenchmarkOutput { + app: string; + results: RawBenchmarkDetail[]; + scores?: BenchmarkScores; + metadata?: { + timestamp: string; + iterations: number; + languages?: string[]; + }; +} + +async function findResultsFiles(dir: string): Promise { + const files: string[] = []; + try { + const entries = await readdir(dir, { withFileTypes: true }); + + for (const entry of entries) { + const fullPath = join(dir, entry.name); + if (entry.isDirectory()) { + files.push(...(await findResultsFiles(fullPath))); + } else if (entry.name === 'results.json') { + files.push(fullPath); + } + } + } catch (error) { + // Directory doesn't exist or can't be read + } + + return files; +} + +async function loadConfigForApp( + logger: CliLogger, + appName: string +): Promise { + const configPath = join('benchmarks', appName, 'config.json'); + + try { + const configContent = await readFile(configPath, 'utf-8'); + const config = JSON.parse(configContent); + + return { + name: config.name || appName, + iterations: config.iterations || 0, + techStack: config.techStack || { + languages: [], + frameworks: [], + bundler: 'unknown', + bundleType: 'unknown', + packageManager: 'unknown', + typescript: false, + }, + source: config.source || { + license: 'unknown', + isOpenSource: false, + github: false, + npm: false, + }, + includes: config.includes || { backend: [], components: [] }, + company: config.company || undefined, + tags: config.tags || [], + cookieBanner: config.cookieBanner || { + serviceName: 'Unknown', + selectors: [], + serviceHosts: [], + waitForVisibility: false, + measureViewportCoverage: false, + expectedLayoutShift: false, + }, + internationalization: config.internationalization || { + detection: 'none', + stringLoading: 'bundled', + }, + }; + } catch (error) { + logger.debug(`Could not load config for ${appName}:`, error); + return null; + } +} + +export async function scoresCommand(logger: CliLogger, appName?: string) { + logger.clear(); + await setTimeout(500); + + p.intro(`${color.bgCyan(color.black(' scores '))}`); + + const resultsDir = 'benchmarks'; + const resultsFiles = await findResultsFiles(resultsDir); + + if (resultsFiles.length === 0) { + logger.error('No benchmark results found!'); + logger.info( + `Run ${color.cyan('cookiebench benchmark')} first to generate results.` + ); + return; + } + + logger.debug(`Found ${resultsFiles.length} results files`); + + // Load all results + const allResults: Record = {}; + for (const file of resultsFiles) { + try { + const content = await readFile(file, 'utf-8'); + const data: BenchmarkOutput = JSON.parse(content); + + if (data.app && data.results) { + allResults[data.app] = data; + } + } catch (error) { + logger.debug(`Failed to load ${file}:`, error); + } + } + + if (Object.keys(allResults).length === 0) { + logger.error('No valid benchmark results found!'); + return; + } + + // If specific app requested, show only that one + if (appName) { + const result = allResults[appName]; + if (!result) { + logger.error(`No results found for app: ${appName}`); + logger.info( + `Available apps: ${Object.keys(allResults).join(', ')}` + ); + return; + } + + await displayAppScores(logger, appName, result); + return; + } + + // Otherwise, show interactive selection + const appOptions = Object.keys(allResults).map((name) => ({ + value: name, + label: name, + hint: `${allResults[name].results.length} iterations`, + })); + + appOptions.push({ + value: '__all__', + label: 'Show all apps', + hint: 'Display scores for all benchmarks', + }); + + const selectedApp = await p.select({ + message: 'Which benchmark scores would you like to view?', + options: appOptions, + }); + + if (p.isCancel(selectedApp)) { + logger.info('Operation cancelled'); + return; + } + + if (selectedApp === '__all__') { + // Show all apps + for (const [name, result] of Object.entries(allResults)) { + await displayAppScores(logger, name, result); + logger.message(''); // Add spacing between apps + } + } else { + await displayAppScores(logger, selectedApp as string, allResults[selectedApp as string]); + } + + logger.outro('Done!'); +} + +async function displayAppScores( + logger: CliLogger, + appName: string, + result: BenchmarkOutput +) { + logger.info(`\n${color.bold(color.cyan(`📊 ${appName}`))}`); + + // Show metadata if available + if (result.metadata) { + logger.debug(`Iterations: ${result.metadata.iterations}`); + logger.debug(`Timestamp: ${result.metadata.timestamp}`); + } + + // If scores are already calculated and stored, use them + if (result.scores) { + logger.debug('Using pre-calculated scores from results file'); + printScores(result.scores); + return; + } + + // Otherwise, calculate scores from raw results + logger.debug('Calculating scores from raw benchmark data'); + + const appResults = result.results; + const config = await loadConfigForApp(logger, appName); + + if (!config) { + logger.warn(`Could not load config for ${appName}, using default values`); + } + + // Create app data for transparency scoring + const appData = { + name: appName, + baseline: appName === 'baseline', + company: config?.company ? JSON.stringify(config.company) : null, + techStack: config?.techStack ? JSON.stringify(config.techStack) : '{}', + source: config?.source ? JSON.stringify(config.source) : null, + tags: config?.tags ? JSON.stringify(config.tags) : null, + }; + + const scores = calculateScores( + { + fcp: + appResults.reduce((a, b) => a + b.timing.firstContentfulPaint, 0) / + appResults.length, + lcp: + appResults.reduce( + (a, b) => a + b.timing.largestContentfulPaint, + 0 + ) / appResults.length, + cls: + appResults.reduce((a, b) => a + b.timing.cumulativeLayoutShift, 0) / + appResults.length, + tbt: + appResults.reduce( + (a, b) => a + b.timing.mainThreadBlocking.total, + 0 + ) / appResults.length, + tti: + appResults.reduce((a, b) => a + b.timing.timeToInteractive, 0) / + appResults.length, + timeToFirstByte: + appResults.reduce((a, b) => a + (b.timing.timeToFirstByte || 0), 0) / + appResults.length, + interactionToNextPaint: + appResults[0]?.timing.interactionToNextPaint || null, + }, + { + totalSize: + appResults.reduce((a, b) => a + b.size.total, 0) / appResults.length, + jsSize: + appResults.reduce((a, b) => a + b.size.scripts.total, 0) / + appResults.length, + cssSize: + appResults.reduce((a, b) => a + b.size.styles, 0) / appResults.length, + imageSize: + appResults.reduce((a, b) => a + b.size.images, 0) / appResults.length, + fontSize: + appResults.reduce((a, b) => a + b.size.fonts, 0) / appResults.length, + otherSize: + appResults.reduce((a, b) => a + b.size.other, 0) / appResults.length, + }, + { + totalRequests: + appResults.reduce( + (a, b) => + a + + b.resources.scripts.length + + b.resources.styles.length + + b.resources.images.length + + b.resources.fonts.length + + b.resources.other.length, + 0 + ) / appResults.length, + thirdPartyRequests: + appResults.reduce( + (a, b) => + a + b.resources.scripts.filter((s) => s.isThirdParty).length, + 0 + ) / appResults.length, + thirdPartySize: + appResults.reduce((a, b) => a + b.size.thirdParty, 0) / + appResults.length, + thirdPartyDomains: 5, // Default value + }, + { + cookieBannerDetected: appResults.some( + (r) => r.timing.cookieBanner.detected + ), + cookieBannerTiming: + appResults.reduce( + (a, b) => a + b.timing.cookieBanner.visibilityTime, + 0 + ) / appResults.length, + cookieBannerCoverage: + appResults.reduce( + (a, b) => a + b.timing.cookieBanner.viewportCoverage, + 0 + ) / + appResults.length / + 100, + }, + { + domSize: 1500, // Default value + mainThreadBlocking: + appResults.reduce( + (a, b) => a + b.timing.mainThreadBlocking.total, + 0 + ) / appResults.length, + layoutShifts: + appResults.reduce((a, b) => a + b.timing.cumulativeLayoutShift, 0) / + appResults.length, + }, + appName === 'baseline', + appData, + appResults[0]?.timing.networkInformation + ); + + printScores(scores); +} + diff --git a/packages/cookiebench-cli/src/components/intro.ts b/packages/cookiebench-cli/src/components/intro.ts new file mode 100644 index 0000000..e0ecb79 --- /dev/null +++ b/packages/cookiebench-cli/src/components/intro.ts @@ -0,0 +1,93 @@ +import figlet from 'figlet'; +import color from 'picocolors'; +import type { CliLogger } from '../utils'; + +/** + * Displays the CLI introduction sequence with figlet art + * @param logger - The CLI logger instance + * @param version - The CLI version string + */ +export async function displayIntro( + logger: CliLogger, + version?: string +): Promise { + // Generate and display Figlet text (async) + let figletText = 'cookiebench'; // Default + try { + figletText = await new Promise((resolve) => { + figlet.text( + 'cookiebench', + { + font: 'Slant', + horizontalLayout: 'default', + verticalLayout: 'default', + width: 80, + whitespaceBreak: true, + }, + (err, data) => { + if (err) { + logger.debug('Failed to generate figlet text'); + resolve('cookiebench'); + } else { + resolve(data || 'cookiebench'); + } + } + ); + }); + } catch (error) { + logger.debug('Error generating figlet text', error); + } + + // Display the figlet text with cyan/teal gradient + const customColor = { + cyan10: (text: string) => `\x1b[38;2;10;80;90m${text}\x1b[0m`, + cyan20: (text: string) => `\x1b[38;2;15;100;110m${text}\x1b[0m`, + cyan30: (text: string) => `\x1b[38;2;20;120;130m${text}\x1b[0m`, + cyan40: (text: string) => `\x1b[38;2;25;150;170m${text}\x1b[0m`, + cyan50: (text: string) => `\x1b[38;2;30;170;190m${text}\x1b[0m`, + cyan75: (text: string) => `\x1b[38;2;34;211;230m${text}\x1b[0m`, + cyan90: (text: string) => `\x1b[38;2;45;225;245m${text}\x1b[0m`, + cyan100: (text: string) => `\x1b[38;2;65;235;255m${text}\x1b[0m`, + }; + + const lines = figletText.split('\n'); + const coloredLines = lines.map((line, index) => { + // Calculate the position in the gradient based on line index + const position = index / (lines.length - 1); + + if (position < 0.1) { + return customColor.cyan10(line); + } + if (position < 0.2) { + return customColor.cyan20(line); + } + if (position < 0.3) { + return customColor.cyan30(line); + } + if (position < 0.4) { + return customColor.cyan40(line); + } + if (position < 0.5) { + return customColor.cyan50(line); + } + if (position < 0.65) { + return customColor.cyan75(line); + } + if (position < 0.8) { + return customColor.cyan90(line); + } + return customColor.cyan100(line); + }); + + // Join all colored lines and send as a single message + logger.message(coloredLines.join('\n')); + + // Display version if provided + if (version) { + logger.message(color.dim(`v${version}`)); + } + + // Spacing before next step + logger.message(''); +} + diff --git a/packages/cookiebench-cli/src/index.ts b/packages/cookiebench-cli/src/index.ts index f571566..099e635 100644 --- a/packages/cookiebench-cli/src/index.ts +++ b/packages/cookiebench-cli/src/index.ts @@ -4,6 +4,13 @@ import color from 'picocolors'; import { benchmarkCommand } from './commands/benchmark'; import { resultsCommand } from './commands/results'; import { dbCommand } from './commands/db'; +import { scoresCommand } from './commands/scores'; +import { createCliLogger, type CliLogger } from './utils/logger'; +import { displayIntro } from './components/intro'; + +// Get log level from env or default to info +const logLevel = (process.env.LOG_LEVEL as 'error' | 'warn' | 'info' | 'debug') || 'info'; +const logger: CliLogger = createCliLogger(logLevel); function onCancel() { p.cancel('Operation cancelled.'); @@ -11,34 +18,40 @@ function onCancel() { } async function main() { - console.clear(); - await setTimeout(1000); + logger.clear(); + await setTimeout(500); // Check for command line arguments const args = process.argv.slice(2); const command = args[0]; + // Show intro for interactive mode + if (!command) { + await displayIntro(logger); + } + // If no command specified, show the prompt if (command) { // Direct command execution switch (command) { case 'benchmark': - await benchmarkCommand(); + await benchmarkCommand(logger); break; case 'results': - await resultsCommand(); + await resultsCommand(logger); + break; + case 'scores': + await scoresCommand(logger, args[1]); break; case 'db': - await dbCommand(args[1]); + await dbCommand(logger, args[1]); break; default: - console.error(`Unknown command: ${command}`); - console.log('Available commands: benchmark, results, db'); + logger.error(`Unknown command: ${command}`); + logger.info('Available commands: benchmark, results, scores, db'); process.exit(1); } } else { - p.intro(`${color.bgCyan(color.black(' cookiebench '))}`); - const selectedCommand = await p.select({ message: 'What would you like to do?', options: [ @@ -52,6 +65,11 @@ async function main() { label: 'Results', hint: 'Combine and display benchmark results', }, + { + value: 'scores', + label: 'View scores', + hint: 'View scores from existing benchmark results', + }, { value: 'db', label: 'Database', @@ -67,16 +85,22 @@ async function main() { // biome-ignore lint/style/useDefaultSwitchClause: switch (selectedCommand) { case 'benchmark': - await benchmarkCommand(); + await benchmarkCommand(logger); break; case 'results': - await resultsCommand(); + await resultsCommand(logger); + break; + case 'scores': + await scoresCommand(logger); break; case 'db': - await dbCommand(); + await dbCommand(logger); break; } } } -main().catch(console.error); +main().catch((error) => { + logger.error('Fatal error:', error); + process.exit(1); +}); diff --git a/packages/cookiebench-cli/src/utils/index.ts b/packages/cookiebench-cli/src/utils/index.ts index 372ea35..d3abec4 100644 --- a/packages/cookiebench-cli/src/utils/index.ts +++ b/packages/cookiebench-cli/src/utils/index.ts @@ -2,13 +2,16 @@ import { readFileSync } from "node:fs"; import { join } from "node:path"; import type { Config } from "../types"; +export * from './logger'; +export * from './scoring'; + export function readConfig(configName = "config"): Config | null { try { const configPath = join(process.cwd(), "config.json"); const configContent = readFileSync(configPath, "utf-8"); return JSON.parse(configContent) as Config; } catch (error) { - console.error(`Failed to read ${configName}/config.json:`, error); + // Error will be logged by caller return null; } } diff --git a/packages/cookiebench-cli/src/utils/logger.ts b/packages/cookiebench-cli/src/utils/logger.ts new file mode 100644 index 0000000..f0ccf1b --- /dev/null +++ b/packages/cookiebench-cli/src/utils/logger.ts @@ -0,0 +1,152 @@ +import { createLogger, type Logger } from '@c15t/logger'; +import * as p from '@clack/prompts'; +import color from 'picocolors'; + +// Define standard log levels +export type LogLevel = 'error' | 'warn' | 'info' | 'debug'; +export const validLogLevels: LogLevel[] = ['error', 'warn', 'info', 'debug']; +export type CliLogger = Logger & CliExtensions; + +// Define CLI-specific extension levels with their method signatures +export interface CliExtensions { + message: (message: string, ...args: unknown[]) => void; + note: (message: string, title?: string) => void; + outro: (message: string) => void; + step: (message: string) => void; + clear: () => void; +} + +const formatArgs = (args: unknown[]): string => { + if (args.length === 0) { + return ''; + } + return `\n${args.map((arg) => ` - ${JSON.stringify(arg, null, 2)}`).join('\n')}`; +}; + +/** + * Formats a log message with appropriate styling based on log level + * + * @param logLevel - The log level to format for + * @param message - The message to format + * @param args - Additional arguments to format + * @returns The formatted message string + */ +export const formatLogMessage = ( + logLevel: LogLevel | string, + message: unknown, + args: unknown[] = [] +): string => { + const messageStr = typeof message === 'string' ? message : String(message); + const formattedArgs = formatArgs(args); + + switch (logLevel) { + case 'error': { + return `${color.bgRed(color.black(' error '))} ${messageStr}${formattedArgs}`; + } + case 'warn': { + return `${color.bgYellow(color.black(' warning '))} ${messageStr}${formattedArgs}`; + } + case 'info': { + return `${color.bgCyan(color.black(' info '))} ${messageStr}${formattedArgs}`; + } + case 'debug': { + return `${color.bgBlack(color.white(' debug '))} ${messageStr}${formattedArgs}`; + } + case 'success': { + return `${color.bgGreen(color.white(' success '))} ${messageStr}${formattedArgs}`; + } + default: { + // Handle unexpected levels + const levelStr = logLevel as string; + return `[${levelStr.toUpperCase()}] ${messageStr}${formattedArgs}`; + } + } +}; + +/** + * Logs a message with the appropriate clack prompt styling + * Can be used before logger initialization + * + * @param logLevel - The log level to use + * @param message - The message to log + * @param args - Additional arguments to include + */ +export const logMessage = ( + logLevel: LogLevel | 'success' | string, + message: unknown, + ...args: unknown[] +): void => { + const formattedMessage = formatLogMessage(logLevel, message, args); + + switch (logLevel) { + case 'error': + p.log.error(formattedMessage); + break; + case 'warn': + p.log.warn(formattedMessage); + break; + case 'info': + case 'debug': + p.log.info(formattedMessage); + break; + case 'success': + p.log.success(formattedMessage); + break; + default: + p.log.message(formattedMessage); + } +}; + +// This function creates a logger instance based on the provided level +// It includes the custom log handler for clack integration. +export const createCliLogger = (level: LogLevel = 'info'): CliLogger => { + // Create the base logger with standard levels + const baseLogger = createLogger({ + level, + appName: 'cookiebench', + log: ( + logLevel: LogLevel | 'success', + message: string, + ...args: unknown[] + ) => { + // Level filtering is primarily handled by the createLogger factory's level setting. + // This function now just focuses on routing output. + logMessage(logLevel, message, ...args); + }, + }); + + // Extend the logger with CLI-specific methods + const extendedLogger = baseLogger as CliLogger; + + // Add message method (plain text without prefix) + extendedLogger.message = (message: string) => { + p.log.message(message); + }; + + // Add note method (creates a note box) + extendedLogger.note = (message: string, title?: string) => { + const messageStr = typeof message === 'string' ? message : String(message); + p.note(messageStr, title); + }; + + // Add step method + extendedLogger.step = (message: string) => { + p.log.step(message); + }; + + // Add outro method (uses plain message) + extendedLogger.outro = (message: string) => { + p.outro(message); + }; + + // Add clear method + extendedLogger.clear = () => { + console.clear(); + }; + + return extendedLogger; +}; + +// Export a default logger instance +export const logger = createCliLogger(); + diff --git a/packages/cookiebench-cli/src/utils/scoring.ts b/packages/cookiebench-cli/src/utils/scoring.ts index c5c68a9..e5fc379 100644 --- a/packages/cookiebench-cli/src/utils/scoring.ts +++ b/packages/cookiebench-cli/src/utils/scoring.ts @@ -30,6 +30,15 @@ interface MetricsData { scriptLoadTime?: number; isBundled?: boolean; isIIFE?: boolean; + // NEW: Perfume.js enhanced metrics + timeToFirstByte?: number; + interactionToNextPaint?: number | null; + networkInformation?: { + effectiveType: string; + downlink: number; + rtt: number; + saveData: boolean; + }; } interface ResourceData { @@ -256,25 +265,25 @@ function calculatePerformanceScore(metrics: MetricsData): { const tti = Number.isFinite(metrics.tti) ? metrics.tti : 0; const tbt = Number.isFinite(metrics.tbt) ? metrics.tbt : 0; - // FCP Score (20 points) - More sensitive for fast sites - const fcpScore = fcp <= 50 ? 20 : fcp <= 100 ? 18 : fcp <= 200 ? 15 : fcp <= 500 ? 10 : 5; + // FCP Score (15 points) - More sensitive for fast sites + const fcpScore = fcp <= 50 ? 15 : fcp <= 100 ? 13 : fcp <= 200 ? 10 : fcp <= 500 ? 7 : 3; totalScore += fcpScore; details.push({ metric: 'First Contentful Paint', value: formatTime(fcp), score: fcpScore, - maxScore: 20, + maxScore: 15, reason: fcp <= 50 ? 'Excellent' : fcp <= 100 ? 'Very Good' : fcp <= 200 ? 'Good' : fcp <= 500 ? 'Fair' : 'Poor', }); - // LCP Score (25 points) - More sensitive for banner rendering - const lcpScore = lcp <= 100 ? 25 : lcp <= 300 ? 20 : lcp <= 500 ? 15 : lcp <= 1000 ? 10 : 5; + // LCP Score (20 points) - More sensitive for banner rendering + const lcpScore = lcp <= 100 ? 20 : lcp <= 300 ? 16 : lcp <= 500 ? 12 : lcp <= 1000 ? 8 : 4; totalScore += lcpScore; details.push({ metric: 'Largest Contentful Paint', value: formatTime(lcp), score: lcpScore, - maxScore: 25, + maxScore: 20, reason: lcp <= 100 ? 'Excellent' : lcp <= 300 ? 'Very Good' : lcp <= 500 ? 'Good' : lcp <= 1000 ? 'Fair' : 'Poor', }); @@ -289,28 +298,52 @@ function calculatePerformanceScore(metrics: MetricsData): { reason: cls <= 0.01 ? 'Excellent' : cls <= 0.05 ? 'Very Good' : cls <= 0.1 ? 'Good' : cls <= 0.25 ? 'Fair' : 'Poor', }); - // TTI Score (20 points) - Cookie banners should be interactive quickly - const ttiScore = tti <= 1000 ? 20 : tti <= 1500 ? 15 : tti <= 2000 ? 10 : tti <= 3000 ? 5 : 0; + // TTI Score (15 points) - Cookie banners should be interactive quickly + const ttiScore = tti <= 1000 ? 15 : tti <= 1500 ? 12 : tti <= 2000 ? 8 : tti <= 3000 ? 4 : 0; totalScore += ttiScore; details.push({ metric: 'Time to Interactive', value: formatTime(tti), score: ttiScore, - maxScore: 20, + maxScore: 15, reason: tti <= 1000 ? 'Excellent' : tti <= 1500 ? 'Very Good' : tti <= 2000 ? 'Good' : tti <= 3000 ? 'Fair' : 'Poor', }); - // TBT Score (15 points) - Main thread blocking - const tbtScore = tbt <= 50 ? 15 : tbt <= 200 ? 10 : tbt <= 500 ? 5 : 0; + // TBT Score (10 points) - Main thread blocking + const tbtScore = tbt <= 50 ? 10 : tbt <= 200 ? 7 : tbt <= 500 ? 3 : 0; totalScore += tbtScore; details.push({ metric: 'Total Blocking Time', value: formatTime(tbt), score: tbtScore, - maxScore: 15, + maxScore: 10, reason: tbt <= 50 ? 'Excellent' : tbt <= 200 ? 'Good' : tbt <= 500 ? 'Fair' : 'Poor', }); + // TTFB Score (10 points) - Server response time + const ttfb = Number.isFinite(metrics.timeToFirstByte) && metrics.timeToFirstByte !== undefined ? metrics.timeToFirstByte : 0; + const ttfbScore = ttfb <= 100 ? 10 : ttfb <= 200 ? 8 : ttfb <= 400 ? 5 : ttfb <= 600 ? 3 : 0; + totalScore += ttfbScore; + details.push({ + metric: 'Time to First Byte', + value: formatTime(ttfb), + score: ttfbScore, + maxScore: 10, + reason: ttfb <= 100 ? 'Excellent' : ttfb <= 200 ? 'Good' : ttfb <= 400 ? 'Fair' : ttfb <= 600 ? 'Poor' : 'Very Poor', + }); + + // INP Score (10 points) - Interaction responsiveness (replaces/complements FID) + const inp = Number.isFinite(metrics.interactionToNextPaint) && metrics.interactionToNextPaint !== undefined && metrics.interactionToNextPaint !== null ? metrics.interactionToNextPaint : null; + const inpScore = inp === null ? 5 : inp <= 200 ? 10 : inp <= 500 ? 7 : inp <= 1000 ? 4 : 0; + totalScore += inpScore; + details.push({ + metric: 'Interaction to Next Paint', + value: inp === null ? 'N/A' : formatTime(inp), + score: inpScore, + maxScore: 10, + reason: inp === null ? 'No interactions detected' : inp <= 200 ? 'Excellent' : inp <= 500 ? 'Good' : inp <= 1000 ? 'Fair' : 'Poor', + }); + return { score: totalScore, maxScore, details }; } @@ -699,6 +732,30 @@ function generateInsights( ); } + // Network quality context (from Perfume.js) + if (metrics.networkInformation) { + const netInfo = metrics.networkInformation; + insights.push( + `Tested on ${netInfo.effectiveType} connection (${netInfo.downlink} Mbps, ${netInfo.rtt}ms RTT).` + ); + } + + // TTFB insights + if (metrics.timeToFirstByte) { + if (metrics.timeToFirstByte <= 100) { + insights.push('Excellent server response time (TTFB) ensures fast initial loading.'); + } else if (metrics.timeToFirstByte > 600) { + insights.push('Server response time (TTFB) is slow - consider CDN or server optimization.'); + } + } + + // INP insights (if interactions detected) + if (metrics.interactionToNextPaint !== null && metrics.interactionToNextPaint !== undefined) { + if (metrics.interactionToNextPaint > 500) { + insights.push('Interaction responsiveness (INP) needs improvement for better user experience.'); + } + } + return insights; } @@ -734,6 +791,18 @@ function generateRecommendations( } } + // TTFB and INP recommendations + if (metrics.timeToFirstByte && metrics.timeToFirstByte > 200) { + recommendations.push( + 'Improve Time to First Byte (TTFB) through server optimization, CDN usage, or caching strategies.' + ); + } + if (metrics.interactionToNextPaint && metrics.interactionToNextPaint > 200) { + recommendations.push( + 'Optimize Interaction to Next Paint (INP) by reducing JavaScript execution time and improving event handler performance.' + ); + } + // Bundle strategy recommendations if (categoryScores.bundleStrategy < 70) { if (!metrics.isBundled) { @@ -797,6 +866,8 @@ export function calculateScores( cls: number; tbt: number; tti: number; + timeToFirstByte?: number; + interactionToNextPaint?: number | null; }, bundleMetrics: { totalSize: number; @@ -823,7 +894,13 @@ export function calculateScores( layoutShifts: number; }, isBaseline = false, - appData?: AppData + appData?: AppData, + networkInformation?: { + effectiveType: string; + downlink: number; + rtt: number; + saveData: boolean; + } ): BenchmarkScores { if (isBaseline) { return { @@ -958,6 +1035,10 @@ export function calculateScores( scriptLoadTime: 0, // TODO: Calculate from timing data isBundled: networkMetrics.thirdPartyRequests === 0, isIIFE: networkMetrics.thirdPartyRequests > 0, + // NEW: Add Perfume.js metrics + timeToFirstByte: metrics.timeToFirstByte || 0, + interactionToNextPaint: metrics.interactionToNextPaint, + networkInformation: networkInformation, }; // Create mock resource data @@ -1127,6 +1208,7 @@ export function calculateScores( } // Function to print scores in a table format +// Note: This outputs directly to console as it's user-facing display output, not logging export function printScores(scores: BenchmarkScores): void { // Create a table for overall scores const overallTable = new Table({ @@ -1150,6 +1232,7 @@ export function printScores(scores: BenchmarkScores): void { ]); } + // User-facing output - always display console.log('\nOverall Scores:'); console.log(overallTable.toString()); @@ -1171,11 +1254,13 @@ export function printScores(scores: BenchmarkScores): void { } } + // User-facing output - always display console.log('\nDetailed Scores:'); console.log(detailsTable.toString()); // Print insights if (scores.insights.length > 0) { + // User-facing output - always display console.log('\nInsights:'); for (const insight of scores.insights) { console.log(`• ${insight}`); @@ -1184,6 +1269,7 @@ export function printScores(scores: BenchmarkScores): void { // Print recommendations if (scores.recommendations.length > 0) { + // User-facing output - always display console.log('\nRecommendations:'); for (const recommendation of scores.recommendations) { console.log(`• ${recommendation}`); diff --git a/packages/runner/package.json b/packages/runner/package.json index bc64fcf..a55d27b 100644 --- a/packages/runner/package.json +++ b/packages/runner/package.json @@ -19,6 +19,7 @@ "lint": "biome lint ." }, "dependencies": { + "@c15t/logger": "1.0.0", "@consentio/benchmark": "workspace:*", "@playwright/test": "^1.42.1", "playwright-performance-metrics": "^1.2.2" diff --git a/packages/runner/src/benchmark-runner.ts b/packages/runner/src/benchmark-runner.ts index 9f8ce4e..5babe8d 100644 --- a/packages/runner/src/benchmark-runner.ts +++ b/packages/runner/src/benchmark-runner.ts @@ -1,41 +1,47 @@ import { chromium, type Page } from '@playwright/test'; import { PerformanceMetricsCollector } from 'playwright-performance-metrics'; -import type { Config } from '@consentio/benchmark'; +import type { Config, PerfumeMetrics } from '@consentio/benchmark'; import { CookieBannerCollector, NetworkMonitor, ResourceTimingCollector, + PerfumeCollector, BENCHMARK_CONSTANTS, } from '@consentio/benchmark'; +import type { Logger } from '@c15t/logger'; import type { BenchmarkResult, BenchmarkDetails, CoreWebVitals } from './types'; import { PerformanceAggregator } from './performance-aggregator'; export class BenchmarkRunner { private config: Config; + private logger: Logger; private cookieBannerCollector: CookieBannerCollector; private networkMonitor: NetworkMonitor; private resourceTimingCollector: ResourceTimingCollector; + private perfumeCollector: PerfumeCollector; private performanceAggregator: PerformanceAggregator; - constructor(config: Config) { + constructor(config: Config, logger: Logger) { this.config = config; - this.cookieBannerCollector = new CookieBannerCollector(config); - this.networkMonitor = new NetworkMonitor(config); - this.resourceTimingCollector = new ResourceTimingCollector(); - this.performanceAggregator = new PerformanceAggregator(); + this.logger = logger; + this.cookieBannerCollector = new CookieBannerCollector(config, logger); + this.networkMonitor = new NetworkMonitor(config, logger); + this.resourceTimingCollector = new ResourceTimingCollector(logger); + this.perfumeCollector = new PerfumeCollector(logger); + this.performanceAggregator = new PerformanceAggregator(logger); } /** * Run a single benchmark iteration */ async runSingleBenchmark(page: Page, url: string): Promise { - console.log(`🔍 [DEBUG] Starting cookie banner benchmark for: ${url}`); - console.log( - '🔍 [DEBUG] Cookie banner selectors:', + this.logger.debug(`Starting cookie banner benchmark for: ${url}`); + this.logger.debug( + 'Cookie banner selectors:', this.config.cookieBanner?.selectors || [] ); - console.log( - '🔍 [DEBUG] Bundle type from config:', + this.logger.debug( + 'Bundle type from config:', this.config.techStack?.bundleType ); @@ -46,27 +52,43 @@ export class BenchmarkRunner { // Setup monitoring and detection await this.networkMonitor.setupMonitoring(page); await this.cookieBannerCollector.setupDetection(page); + await this.perfumeCollector.setupPerfume(page); // Navigate to the page - console.log(`🔍 [DEBUG] Navigating to: ${url}`); + this.logger.debug(`Navigating to: ${url}`); await page.goto(url, { waitUntil: 'networkidle' }); // Wait for the specified element await this.waitForElement(page); // Wait for network to be idle - console.log('🔍 [DEBUG] Waiting for network idle...'); + this.logger.debug('Waiting for network idle...'); await page.waitForLoadState('networkidle'); - // Collect core web vitals - console.log('🔍 [DEBUG] Collecting core web vitals...'); - const coreWebVitals = await this.collectCoreWebVitals(collector, page); + // Collect core web vitals from playwright-performance-metrics (primary source) + this.logger.debug('Collecting core web vitals...'); + const coreWebVitals = await collector.collectMetrics(page, { + timeout: BENCHMARK_CONSTANTS.METRICS_TIMEOUT, + retryTimeout: BENCHMARK_CONSTANTS.METRICS_RETRY_TIMEOUT, + }); + + this.logger.debug('Core web vitals collected:', { + fcp: coreWebVitals.paint?.firstContentfulPaint, + lcp: coreWebVitals.largestContentfulPaint, + cls: coreWebVitals.cumulativeLayoutShift, + tbt: coreWebVitals.totalBlockingTime, + }); + + // Collect Perfume.js metrics (supplementary - TTFB, navigation timing, network info) + this.logger.debug('Collecting Perfume.js supplementary metrics...'); + const perfumeMetrics = await this.perfumeCollector.collectMetrics(page); + this.logger.debug('Perfume.js metrics:', perfumeMetrics); // Collect cookie banner specific metrics const cookieBannerData = await this.cookieBannerCollector.collectMetrics( page ); - console.log('🔍 [DEBUG] Cookie banner metrics:', cookieBannerData); + this.logger.debug('Cookie banner metrics:', cookieBannerData); // Collect detailed resource timing data const resourceMetrics = await this.resourceTimingCollector.collect(page); @@ -83,7 +105,8 @@ export class BenchmarkRunner { networkRequests, networkMetrics, resourceMetrics, - this.config + this.config, + perfumeMetrics ); // Log results @@ -112,8 +135,8 @@ export class BenchmarkRunner { try { for (let i = 0; i < this.config.iterations; i++) { - console.log( - `[Benchmark] Running iteration ${i + 1}/${this.config.iterations}...` + this.logger.info( + `Running iteration ${i + 1}/${this.config.iterations}...` ); const context = await browser.newContext(); @@ -152,40 +175,16 @@ export class BenchmarkRunner { */ private async waitForElement(page: Page): Promise { if (this.config.testId) { - console.log(`🔍 [DEBUG] Waiting for testId: ${this.config.testId}`); + this.logger.debug(`Waiting for testId: ${this.config.testId}`); await page.waitForSelector(`[data-testid="${this.config.testId}"]`); } else if (this.config.id) { - console.log(`🔍 [DEBUG] Waiting for id: ${this.config.id}`); + this.logger.debug(`Waiting for id: ${this.config.id}`); await page.waitForSelector(`#${this.config.id}`); } else if (this.config.custom) { - console.log('🔍 [DEBUG] Running custom wait function'); + this.logger.debug('Running custom wait function'); await this.config.custom(page); } } - /** - * Collect core web vitals using playwright-performance-metrics - */ - private async collectCoreWebVitals( - collector: PerformanceMetricsCollector, - page: Page - ): Promise { - const coreWebVitals = await collector.collectMetrics(page, { - timeout: BENCHMARK_CONSTANTS.METRICS_TIMEOUT, - retryTimeout: BENCHMARK_CONSTANTS.METRICS_RETRY_TIMEOUT, - }); - - console.log('🔍 [DEBUG] Core web vitals collected:', { - fcp: coreWebVitals.paint?.firstContentfulPaint, - lcp: coreWebVitals.largestContentfulPaint, - cls: coreWebVitals.cumulativeLayoutShift, - tbt: coreWebVitals.totalBlockingTime, - domComplete: coreWebVitals.domCompleteTiming, - pageLoad: coreWebVitals.pageloadTiming, - totalBytes: coreWebVitals.totalBytes, - }); - - return coreWebVitals; - } } diff --git a/packages/runner/src/performance-aggregator.ts b/packages/runner/src/performance-aggregator.ts index d28d84a..b1f1a33 100644 --- a/packages/runner/src/performance-aggregator.ts +++ b/packages/runner/src/performance-aggregator.ts @@ -6,10 +6,17 @@ import type { NetworkRequest, NetworkMetrics, ResourceTimingData, + PerfumeMetrics, } from '@consentio/benchmark'; +import type { Logger } from '@c15t/logger'; import type { BenchmarkDetails, BenchmarkResult } from './types'; export class PerformanceAggregator { + private logger: Logger; + + constructor(logger: Logger) { + this.logger = logger; + } /** * Calculate Time to Interactive based on core web vitals and cookie banner interaction */ @@ -36,7 +43,8 @@ export class PerformanceAggregator { networkRequests: NetworkRequest[], networkMetrics: NetworkMetrics, resourceMetrics: ResourceTimingData, - config: Config + config: Config, + perfumeMetrics: PerfumeMetrics | null ): BenchmarkDetails { const tti = this.calculateTTI(coreWebVitals, cookieBannerData); @@ -54,6 +62,22 @@ export class PerformanceAggregator { largestContentfulPaint: coreWebVitals.largestContentfulPaint || 0, timeToInteractive: tti, cumulativeLayoutShift: coreWebVitals.cumulativeLayoutShift || 0, + // Enhanced metrics from Perfume.js + timeToFirstByte: perfumeMetrics?.timeToFirstByte || 0, + firstInputDelay: perfumeMetrics?.firstInputDelay || null, + interactionToNextPaint: perfumeMetrics?.interactionToNextPaint || null, + // Detailed navigation timing from Perfume.js + navigationTiming: perfumeMetrics?.navigationTiming || { + timeToFirstByte: 0, + domInteractive: 0, + domContentLoadedEventStart: 0, + domContentLoadedEventEnd: 0, + domComplete: 0, + loadEventStart: 0, + loadEventEnd: 0, + }, + // Network information from Perfume.js + networkInformation: perfumeMetrics?.networkInformation || undefined, cookieBanner: { renderStart: cookieBannerData?.bannerRenderTime || 0, renderEnd: cookieBannerData?.bannerInteractiveTime || 0, @@ -160,8 +184,21 @@ export class PerformanceAggregator { 0 ) / results.length, speedIndex: 0, // Default value - timeToFirstByte: 0, // Default value - firstInputDelay: 0, // Default value + timeToFirstByte: + results.reduce( + (acc, curr) => acc + (curr.timing.timeToFirstByte || 0), + 0 + ) / results.length, + firstInputDelay: + results.reduce( + (acc, curr) => acc + (curr.timing.firstInputDelay || 0), + 0 + ) / results.length, + interactionToNextPaint: + results.reduce( + (acc, curr) => acc + (curr.timing.interactionToNextPaint || 0), + 0 + ) / results.length, cumulativeLayoutShift: results.reduce( (acc, curr) => acc + curr.timing.cumulativeLayoutShift, @@ -248,7 +285,7 @@ export class PerformanceAggregator { cookieBannerMetrics: CookieBannerMetrics, config: Config ): void { - console.log('🔍 [DEBUG] Final cookie banner benchmark results:', { + this.logger.debug('Final cookie banner benchmark results:', { fcp: finalMetrics.timing.firstContentfulPaint, lcp: finalMetrics.timing.largestContentfulPaint, cls: finalMetrics.timing.cumulativeLayoutShift, diff --git a/packages/runner/src/server.ts b/packages/runner/src/server.ts index a391c9c..70b5437 100644 --- a/packages/runner/src/server.ts +++ b/packages/runner/src/server.ts @@ -1,15 +1,17 @@ import { spawn } from 'node:child_process'; +import type { Logger } from '@c15t/logger'; import { getPackageManager } from './utils'; import type { ServerInfo } from './types'; export async function buildAndServeNextApp( + logger: Logger, appPath?: string ): Promise { const pm = await getPackageManager(); const cwd = appPath || process.cwd(); // Build the app - console.log('[Build] Building Next.js app...'); + logger.info('Building Next.js app...'); const buildProcess = spawn(pm.command, [...pm.args, 'build'], { cwd, stdio: 'inherit', @@ -26,9 +28,9 @@ export async function buildAndServeNextApp( }); // Start the server - console.log('[Build] Starting Next.js server...'); + logger.info('Starting Next.js server...'); const port = Math.floor(Math.random() * (9000 - 3000 + 1)) + 3000; - console.log('command', [ + logger.debug('Server command:', [ ...pm.args, 'start', '--', @@ -53,7 +55,7 @@ export async function buildAndServeNextApp( try { const response = await fetch(url); if (response.ok) { - console.log('[Build] Server is ready!'); + logger.success('Server is ready!'); return { serverProcess, url }; } } catch { diff --git a/packages/runner/src/types.ts b/packages/runner/src/types.ts index 09fb357..d67183e 100644 --- a/packages/runner/src/types.ts +++ b/packages/runner/src/types.ts @@ -1,5 +1,6 @@ import type { Page } from '@playwright/test'; import type { ChildProcess } from 'node:child_process'; +import type { PerfumeMetrics } from '@consentio/benchmark'; // Re-export common types from benchmark package export type { @@ -12,6 +13,7 @@ export type { BundleStrategy, ResourceTimingData, CoreWebVitals, + PerfumeMetrics, } from '@consentio/benchmark'; // Server types @@ -46,6 +48,25 @@ export interface BenchmarkDetails { largestContentfulPaint: number; timeToInteractive: number; cumulativeLayoutShift: number; + // Enhanced metrics from Perfume.js + timeToFirstByte: number; + firstInputDelay: number | null; + interactionToNextPaint: number | null; + navigationTiming: { + timeToFirstByte: number; + domInteractive: number; + domContentLoadedEventStart: number; + domContentLoadedEventEnd: number; + domComplete: number; + loadEventStart: number; + loadEventEnd: number; + }; + networkInformation?: { + effectiveType: string; + downlink: number; + rtt: number; + saveData: boolean; + }; cookieBanner: { renderStart: number; renderEnd: number; @@ -173,6 +194,7 @@ export interface BenchmarkResult { speedIndex: number; timeToFirstByte: number; firstInputDelay: number; + interactionToNextPaint: number; cumulativeLayoutShift: number; domSize: number; totalRequests: number; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 642676d..323f3c5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -17,9 +17,6 @@ importers: '@consentio/benchmark': specifier: workspace:* version: link:packages/benchmark - 'cookiebench': - specifier: workspace:* - version: link:packages/cookiebench-cli '@consentio/runner': specifier: workspace:* version: link:packages/runner @@ -29,6 +26,9 @@ importers: cli-table3: specifier: ^0.6.5 version: 0.6.5 + cookiebench: + specifier: workspace:* + version: link:packages/cookiebench-cli drizzle-kit: specifier: ^0.31.1 version: 0.31.1 @@ -57,9 +57,6 @@ importers: specifier: ^19.0.0 version: 19.1.0(react@19.1.0) devDependencies: - 'cookiebench': - specifier: workspace:* - version: link:../../packages/cookiebench-cli '@cookiebench/benchmark-schema': specifier: workspace:* version: link:../../packages/benchmark-schema @@ -75,6 +72,9 @@ importers: '@types/react-dom': specifier: ^19.1.6 version: 19.1.6(@types/react@19.1.6) + cookiebench: + specifier: workspace:* + version: link:../../packages/cookiebench-cli typescript: specifier: ^5.8.3 version: 5.8.3 @@ -94,9 +94,6 @@ importers: specifier: ^19.0.0 version: 19.1.0(react@19.1.0) devDependencies: - 'cookiebench': - specifier: workspace:* - version: link:../../packages/cookiebench-cli '@cookiebench/benchmark-schema': specifier: workspace:* version: link:../../packages/benchmark-schema @@ -112,6 +109,9 @@ importers: '@types/react-dom': specifier: ^19.1.6 version: 19.1.6(@types/react@19.1.6) + cookiebench: + specifier: workspace:* + version: link:../../packages/cookiebench-cli typescript: specifier: ^5.8.3 version: 5.8.3 @@ -134,9 +134,6 @@ importers: specifier: ^19.0.0 version: 19.1.0(react@19.1.0) devDependencies: - 'cookiebench': - specifier: workspace:* - version: link:../../packages/cookiebench-cli '@cookiebench/benchmark-schema': specifier: workspace:* version: link:../../packages/benchmark-schema @@ -152,6 +149,9 @@ importers: '@types/react-dom': specifier: ^19.1.6 version: 19.1.6(@types/react@19.1.6) + cookiebench: + specifier: workspace:* + version: link:../../packages/cookiebench-cli typescript: specifier: ^5.8.3 version: 5.8.3 @@ -168,9 +168,6 @@ importers: specifier: ^19.0.0 version: 19.1.0(react@19.1.0) devDependencies: - 'cookiebench': - specifier: workspace:* - version: link:../../packages/cookiebench-cli '@cookiebench/benchmark-schema': specifier: workspace:* version: link:../../packages/benchmark-schema @@ -186,6 +183,9 @@ importers: '@types/react-dom': specifier: ^19.1.6 version: 19.1.6(@types/react@19.1.6) + cookiebench: + specifier: workspace:* + version: link:../../packages/cookiebench-cli typescript: specifier: ^5.8.3 version: 5.8.3 @@ -202,9 +202,6 @@ importers: specifier: ^19.0.0 version: 19.1.0(react@19.1.0) devDependencies: - 'cookiebench': - specifier: workspace:* - version: link:../../packages/cookiebench-cli '@cookiebench/benchmark-schema': specifier: workspace:* version: link:../../packages/benchmark-schema @@ -220,6 +217,9 @@ importers: '@types/react-dom': specifier: ^19.1.6 version: 19.1.6(@types/react@19.1.6) + cookiebench: + specifier: workspace:* + version: link:../../packages/cookiebench-cli typescript: specifier: ^5.8.3 version: 5.8.3 @@ -270,9 +270,6 @@ importers: specifier: ^19.0.0 version: 19.1.0(react@19.1.0) devDependencies: - 'cookiebench': - specifier: workspace:* - version: link:../../packages/cookiebench-cli '@cookiebench/benchmark-schema': specifier: workspace:* version: link:../../packages/benchmark-schema @@ -288,6 +285,9 @@ importers: '@types/react-dom': specifier: ^19.1.6 version: 19.1.6(@types/react@19.1.6) + cookiebench: + specifier: workspace:* + version: link:../../packages/cookiebench-cli typescript: specifier: ^5.8.3 version: 5.8.3 @@ -304,9 +304,6 @@ importers: specifier: ^19.0.0 version: 19.1.0(react@19.1.0) devDependencies: - 'cookiebench': - specifier: workspace:* - version: link:../../packages/cookiebench-cli '@cookiebench/benchmark-schema': specifier: workspace:* version: link:../../packages/benchmark-schema @@ -322,6 +319,9 @@ importers: '@types/react-dom': specifier: ^19.1.6 version: 19.1.6(@types/react@19.1.6) + cookiebench: + specifier: workspace:* + version: link:../../packages/cookiebench-cli typescript: specifier: ^5.8.3 version: 5.8.3 @@ -369,9 +369,6 @@ importers: specifier: ^19.0.0 version: 19.1.0(react@19.1.0) devDependencies: - 'cookiebench': - specifier: workspace:* - version: link:../../packages/cookiebench-cli '@cookiebench/benchmark-schema': specifier: workspace:* version: link:../../packages/benchmark-schema @@ -387,6 +384,9 @@ importers: '@types/react-dom': specifier: ^19.1.6 version: 19.1.6(@types/react@19.1.6) + cookiebench: + specifier: workspace:* + version: link:../../packages/cookiebench-cli typescript: specifier: ^5.8.3 version: 5.8.3 @@ -403,9 +403,6 @@ importers: specifier: ^19.0.0 version: 19.1.0(react@19.1.0) devDependencies: - 'cookiebench': - specifier: workspace:* - version: link:../../packages/cookiebench-cli '@cookiebench/benchmark-schema': specifier: workspace:* version: link:../../packages/benchmark-schema @@ -421,6 +418,9 @@ importers: '@types/react-dom': specifier: ^19.1.6 version: 19.1.6(@types/react@19.1.6) + cookiebench: + specifier: workspace:* + version: link:../../packages/cookiebench-cli typescript: specifier: ^5.8.3 version: 5.8.3 @@ -437,9 +437,6 @@ importers: specifier: ^19.0.0 version: 19.1.0(react@19.1.0) devDependencies: - 'cookiebench': - specifier: workspace:* - version: link:../../packages/cookiebench-cli '@cookiebench/benchmark-schema': specifier: workspace:* version: link:../../packages/benchmark-schema @@ -455,15 +452,24 @@ importers: '@types/react-dom': specifier: ^19.1.6 version: 19.1.6(@types/react@19.1.6) + cookiebench: + specifier: workspace:* + version: link:../../packages/cookiebench-cli typescript: specifier: ^5.8.3 version: 5.8.3 packages/benchmark: dependencies: + '@c15t/logger': + specifier: 1.0.0 + version: 1.0.0 '@playwright/test': specifier: ^1.42.1 version: 1.52.0 + perfume.js: + specifier: ^9.4.0 + version: 9.4.0 playwright-performance-metrics: specifier: ^1.2.2 version: 1.2.2(@playwright/test@1.52.0) @@ -485,6 +491,9 @@ importers: packages/cookiebench-cli: dependencies: + '@c15t/logger': + specifier: ^1.0.0 + version: 1.0.0 '@clack/prompts': specifier: ^1.0.0-alpha.0 version: 1.0.0-alpha.0 @@ -494,15 +503,24 @@ importers: '@consentio/runner': specifier: workspace:* version: link:../runner + '@types/figlet': + specifier: ^1.7.0 + version: 1.7.0 cli-table3: specifier: ^0.6.3 version: 0.6.5 dotenv: specifier: ^16.5.0 version: 16.5.0 + figlet: + specifier: ^1.9.3 + version: 1.9.3 picocolors: specifier: ^1.0.0 version: 1.1.1 + pretty-ms: + specifier: ^9.2.0 + version: 9.2.0 devDependencies: '@rsdoctor/rspack-plugin': specifier: ^1.1.3 @@ -519,6 +537,9 @@ importers: packages/runner: dependencies: + '@c15t/logger': + specifier: 1.0.0 + version: 1.0.0 '@consentio/benchmark': specifier: workspace:* version: link:../benchmark @@ -668,6 +689,9 @@ packages: '@c15t/backend@1.2.2-canary-20250602152741': resolution: {integrity: sha512-LlL5GkopE4pPQmL2LZeYZttrPTrApKamcJt3H9gLJyRf54McG4slgNb9UwLi1M4NPneZmdhpTt5YiWfkqYogHA==} + '@c15t/logger@1.0.0': + resolution: {integrity: sha512-z2RrUnvO5bbEg/qd9iD/TqdBi04vfvALf2uovDVLJiQTNfOVUb7FM/GkPH5mugiDwErlrtVhAW8hW/2VmBXbUA==} + '@c15t/nextjs@1.2.2-canary-20250603153501': resolution: {integrity: sha512-2KjPTqyO8xzfoKQ4fax/HsoYGn9jVvBtS3dTWLJcHOYPuyCfY1e5KgqM7WvQfZqS8YhYGpIeC5THRpC2nzPXww==} peerDependencies: @@ -2091,6 +2115,9 @@ packages: '@types/estree@1.0.5': resolution: {integrity: sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw==} + '@types/figlet@1.7.0': + resolution: {integrity: sha512-KwrT7p/8Eo3Op/HBSIwGXOsTZKYiM9NpWRBJ5sVjWP/SmlS+oxxRvJht/FNAtliJvja44N3ul1yATgohnVBV0Q==} + '@types/fs-extra@11.0.4': resolution: {integrity: sha512-yTbItCNreRooED33qjunPthRcSjERP1r4MqCZc7wv0u2sUkzTFp45tgUfS5+r7FrZPdmCCNflLhVSP/o+SemsQ==} @@ -2215,6 +2242,10 @@ packages: caniuse-lite@1.0.30001721: resolution: {integrity: sha512-cOuvmUVtKrtEaoKiO0rSc29jcjwMwX5tOHDy4MgVFEWiUXj4uBMJkwI8MDySkgXidpMiHUcviogAvFi4pA2hDQ==} + chalk@5.6.2: + resolution: {integrity: sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==} + engines: {node: ^12.17.0 || ^14.13 || >=16.0.0} + chownr@1.1.4: resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==} @@ -2254,6 +2285,10 @@ packages: resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} engines: {node: '>= 0.8'} + commander@14.0.2: + resolution: {integrity: sha512-TywoWNNRbhoD0BXs1P3ZEScW8W5iKrnbithIl0YH+uCmBd0QpPOA8yc82DS3BIE5Ma6FnBVUsJ7wVUDz4dvOWQ==} + engines: {node: '>=20'} + commander@7.2.0: resolution: {integrity: sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==} engines: {node: '>= 10'} @@ -2587,6 +2622,11 @@ packages: resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==} engines: {node: ^12.20 || >= 14.13} + figlet@1.9.3: + resolution: {integrity: sha512-majPgOpVtrZN1iyNGbsUP6bOtZ6eaJgg5HHh0vFvm5DJhh8dc+FJpOC4GABvMZ/A7XHAJUuJujhgUY/2jPWgMA==} + engines: {node: '>= 17.0.0'} + hasBin: true + file-uri-to-path@1.0.0: resolution: {integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==} @@ -2954,6 +2994,10 @@ packages: path-parse@1.0.7: resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==} + perfume.js@9.4.0: + resolution: {integrity: sha512-YYxGBYm2OcDx68GhzX/N3h4RrtViAz7Whgk7dA6j1bC9NxBGIG8c+rs+K3ql/dW4KReszN8ptAC1ghUGdWNsMQ==} + engines: {node: '>=10.0.0'} + picocolors@1.1.1: resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} @@ -3343,6 +3387,9 @@ packages: resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==} engines: {node: '>= 8'} + web-vitals@3.5.2: + resolution: {integrity: sha512-c0rhqNcHXRkY/ogGDJQxZ9Im9D19hDihbzSQJrsioex+KnFgmMzBiy57Z1EjkhX/+OjyBpclDCzz2ITtjokFmg==} + webpack-bundle-analyzer@4.10.2: resolution: {integrity: sha512-vJptkMm9pk5si4Bv922ZbKLV8UTT4zib4FPgXMhgzUny0bfDDkLXAVQs3ly3fS4/TN9ROFtb0NFrm04UXFE/Vw==} engines: {node: '>= 10.13.0'} @@ -3573,6 +3620,12 @@ snapshots: - supports-color - ws + '@c15t/logger@1.0.0': + dependencies: + chalk: 5.6.2 + neverthrow: 8.2.0 + picocolors: 1.1.1 + '@c15t/nextjs@1.2.2-canary-20250603153501(@playwright/test@1.52.0)(@types/react-dom@19.1.6(@types/react@19.1.6))(@types/react@19.1.6)(next@15.3.3(@opentelemetry/api@1.9.0)(@playwright/test@1.52.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0))(react-dom@19.1.0(react@19.1.0))(react@19.1.0)': dependencies: '@c15t/react': 1.2.2-canary-20250603153501(@libsql/client@0.15.8)(@playwright/test@1.52.0)(@types/better-sqlite3@7.6.13)(@types/react-dom@19.1.6(@types/react@19.1.6))(@types/react@19.1.6)(better-sqlite3@11.10.0)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(use-sync-external-store@1.5.0(react@19.1.0))(ws@8.18.2) @@ -5025,6 +5078,8 @@ snapshots: '@types/estree@1.0.5': {} + '@types/figlet@1.7.0': {} + '@types/fs-extra@11.0.4': dependencies: '@types/jsonfile': 6.1.4 @@ -5222,6 +5277,8 @@ snapshots: caniuse-lite@1.0.30001721: {} + chalk@5.6.2: {} + chownr@1.1.4: optional: true @@ -5265,6 +5322,8 @@ snapshots: dependencies: delayed-stream: 1.0.0 + commander@14.0.2: {} + commander@7.2.0: {} connect@3.7.0: @@ -5530,6 +5589,10 @@ snapshots: web-streams-polyfill: 3.3.3 optional: true + figlet@1.9.3: + dependencies: + commander: 14.0.2 + file-uri-to-path@1.0.0: optional: true @@ -5914,6 +5977,10 @@ snapshots: path-parse@1.0.7: {} + perfume.js@9.4.0: + dependencies: + web-vitals: 3.5.2 + picocolors@1.1.1: {} picomatch@4.0.2: {} @@ -6370,6 +6437,8 @@ snapshots: web-streams-polyfill@3.3.3: optional: true + web-vitals@3.5.2: {} + webpack-bundle-analyzer@4.10.2: dependencies: '@discoveryjs/json-ext': 0.5.7 From c9d323210b5ce6143603ba724460b22a9982bb73 Mon Sep 17 00:00:00 2001 From: burnedchris Date: Wed, 29 Oct 2025 21:13:39 -0700 Subject: [PATCH 03/21] Enhance cookiebench CLI with new save command for syncing benchmark results to the database, and implement interactive multi-select mode for benchmark execution and results viewing. Introduce admin access checks for restricted commands, improve logging, and update documentation to reflect new features and usage instructions. --- packages/cookiebench-cli/README.md | 190 +++- .../cookiebench-cli/src/commands/benchmark.ts | 87 +- packages/cookiebench-cli/src/commands/db.ts | 8 +- .../cookiebench-cli/src/commands/results.ts | 987 +++++++++--------- packages/cookiebench-cli/src/commands/save.ts | 618 +++++++++++ packages/cookiebench-cli/src/index.ts | 82 +- packages/cookiebench-cli/src/utils/auth.ts | 12 + packages/cookiebench-cli/src/utils/index.ts | 1 + 8 files changed, 1430 insertions(+), 555 deletions(-) create mode 100644 packages/cookiebench-cli/src/commands/save.ts create mode 100644 packages/cookiebench-cli/src/utils/auth.ts diff --git a/packages/cookiebench-cli/README.md b/packages/cookiebench-cli/README.md index c79ac90..01b5bc2 100644 --- a/packages/cookiebench-cli/README.md +++ b/packages/cookiebench-cli/README.md @@ -36,13 +36,16 @@ Run specific commands directly: # Run a benchmark cookiebench benchmark -# View and aggregate results +# View detailed benchmark results cookiebench results -# View scores from existing results +# View scores from existing results (available via CLI only) cookiebench scores -# Manage database +# Sync results to database (admin only) +cookiebench save + +# Manage database (admin only) cookiebench db push ``` @@ -53,33 +56,129 @@ cookiebench db push Run performance benchmarks on configured applications. ```bash -cookiebench benchmark [appPath] +# Interactive mode - select multiple benchmarks to run +cookiebench benchmark + +# Run a specific benchmark +cookiebench benchmark benchmarks/with-c15t-nextjs ``` -The command will: -1. Read `config.json` from the current directory or specified path +**Interactive Multi-Select Mode:** + +When run without arguments, the command will: +1. Scan the `benchmarks/` directory for available benchmarks +2. Present a multi-select interface (use space to toggle selections) +3. Show iteration counts from each config file +4. Ask for iterations override (or press Enter to use config values) +5. Ask if you want to show the results panel after completion +6. Run selected benchmarks sequentially +7. Show a summary of completed benchmarks +8. Display the full results panel with all metrics (if enabled) + +**Single Benchmark Mode:** + +When a specific path is provided: +1. Read `config.json` from the specified path 2. Build and serve the Next.js app (or use remote URL if configured) 3. Run benchmarks for the specified number of iterations 4. Calculate performance scores 5. Save results to `results.json` +6. Display scores + +**Features:** +- ✅ Multi-select with space bar toggle +- ✅ View default iterations from config files +- ✅ Override iterations for all benchmarks or use individual config values +- ✅ Sequential execution with progress indicators +- ✅ Automatic results panel display after completion (toggle on/off) +- ✅ Comprehensive metrics: scores, insights, Core Web Vitals, resource breakdown, network waterfall +- ✅ Error handling - continues to next benchmark on failure +- ✅ Summary report at the end + +**Example:** +``` +? Select benchmarks to run: + ◼ baseline + ◼ with-c15t-nextjs + ◼ with-cookieyes + +● info Config iterations: baseline: 5, with-c15t-nextjs: 3, with-cookieyes: 5 + +? Number of iterations (press Enter to use config values): +› Default: 5 + + [Just press Enter to use config values, or type a number to override all] + +? Show results panel after completion? › Yes +``` ### results -Aggregate and display benchmark results from multiple apps. +View comprehensive benchmark results with detailed metrics and analysis. ```bash +# Interactive mode - select which benchmarks to view (all selected by default) cookiebench results + +# View results for a specific app +cookiebench results with-c15t-nextjs + +# View all results +cookiebench results __all__ ``` -Features: +**Interactive Multi-Select Mode:** + +When run without arguments, you can: +1. Select which benchmarks to view using space bar (all selected by default) +2. Press Enter to view detailed results for selected benchmarks +3. View baseline comparisons and deltas + +**Features:** + +Displays a detailed panel for each selected benchmark with: + +1. **🎯 Overall Score** - Color-coded score (0-100) with grade (Excellent/Good/Fair/Poor/Critical) + - 🟢 Green (90+) = Excellent + - 🟡 Yellow (70-89) = Good/Fair + - 🔴 Red (<70) = Poor/Critical + +2. **💡 Key Insights** - Auto-generated bullet points highlighting strengths and areas for improvement + +3. **🍪 Cookie Banner Impact** - Banner visibility, viewport coverage, network impact, bundle strategy + +4. **⚡ Core Web Vitals** - FCP, LCP, TTI, CLS with performance ratings + +5. **📦 Resource Breakdown** - Detailed size analysis by type (JS, CSS, Images, Fonts, Other) + +6. **📊 Performance Impact Summary** - Loading strategy, render performance, network overhead, layout stability + +7. **🌐 Network Chart** - ASCII waterfall visualization showing resource loading timeline with color-coded bars + +8. **📋 Resource Details** - Sortable table with resource names, types, sources, sizes, and durations + +**Notes:** +- Interactive multi-select UI - choose which benchmarks to view +- All benchmarks selected by default for quick viewing - Aggregates results from all `results.json` files in benchmark directories -- Displays comparison table with metrics and deltas from baseline -- Calculates scores for each app -- Saves results to database (if configured) +- Calculates scores on-demand from raw metrics +- Baseline comparison with delta values +- No database writes (read-only) +- Automatically shown after running benchmarks (if enabled) + +**Example:** +``` +? Select benchmarks to view (use space to toggle, all selected by default): + ◼ baseline (benchmarks/baseline) + ◼ with-c15t-nextjs (benchmarks/with-c15t-nextjs) + ◼ with-cookieyes (benchmarks/with-cookieyes) + +● info Viewing results for: baseline, with-c15t-nextjs, with-cookieyes +``` ### scores -View calculated scores from existing benchmark results without re-running benchmarks. +**Available via direct CLI only** - View calculated scores from existing benchmark results. ```bash # Interactive: choose which app to view @@ -99,14 +198,40 @@ Features: - Shows insights and recommendations - Much faster than re-running full benchmarks -Perfect for: -- Reviewing benchmark results later -- Comparing scores across different runs -- Generating reports without re-benchmarking +**Note:** This command is not shown in the interactive menu. Use the `results` command instead for a comprehensive view with all metrics, or access `scores` directly via CLI for score-only output. + +### save + +**🔒 Admin Only** - Sync benchmark results to database. + +```bash +# Interactive: choose which benchmarks to save +cookiebench save + +# Save a specific benchmark +cookiebench save with-c15t-nextjs + +# Save all benchmarks (can also select in interactive mode) +cookiebench save __all__ +``` + +**Requirements:** +- `CONSENT_ADMIN=true` environment variable must be set +- `API_URL` for the database endpoint +- `DATABASE_URL` and `DATABASE_AUTH_TOKEN` for database access + +Features: +- Multi-select interface for choosing which benchmarks to sync +- Sends results to API endpoint (oRPC) +- Persists to Turso/SQLite database +- Confirmation prompt before syncing +- Shows success/failure count + +**Note:** This command is hidden from the menu and unavailable unless you have admin access. Contact the Consent.io team for access credentials if needed. ### db -Manage the benchmark database. +**🔒 Admin Only** - Manage the benchmark database. ```bash # Push database schema @@ -185,14 +310,41 @@ To benchmark a remote URL instead of building locally: ## Environment Variables ```bash -# Database configuration (optional) +# Logging level (optional, default: info) +LOG_LEVEL=debug # error | warn | info | debug +# Note: Use LOG_LEVEL=debug to see detailed processing information + +# Admin access (required for save and db commands) +CONSENT_ADMIN=true + +# Database configuration (required for save command) DATABASE_URL=libsql://your-turso-db.turso.io DATABASE_AUTH_TOKEN=your-auth-token -# API endpoint for saving results (optional) +# API endpoint for saving results (required for save command) API_URL=http://localhost:3000 ``` +### Admin Commands + +Some commands require admin access and are only available when `CONSENT_ADMIN=true` is set: + +- `save` - Sync benchmark results to database (admin only) +- `db` - Manage database schema and migrations (admin only) + +**For Consent.io team members:** +```bash +# Enable admin mode +export CONSENT_ADMIN=true + +# Now admin commands are available +cookiebench save +cookiebench db push +``` + +**For public users:** +The `benchmark`, `results`, and `scores` commands work without admin access. Admin commands will be hidden from the menu and show an error if attempted. + ## Output ### results.json diff --git a/packages/cookiebench-cli/src/commands/benchmark.ts b/packages/cookiebench-cli/src/commands/benchmark.ts index d2e33b0..3241d51 100644 --- a/packages/cookiebench-cli/src/commands/benchmark.ts +++ b/packages/cookiebench-cli/src/commands/benchmark.ts @@ -35,7 +35,8 @@ async function findBenchmarkDirs(logger: CliLogger): Promise { async function runSingleBenchmark( logger: CliLogger, appPath: string, - showScores = true + showScores = true, + iterationsOverride?: number ): Promise { const configPath = appPath ? join(appPath, 'config.json') : undefined; const config = readConfig(configPath); @@ -44,6 +45,11 @@ async function runSingleBenchmark( return false; } + // Override iterations if provided + if (iterationsOverride !== undefined && iterationsOverride > 0) { + config.iterations = iterationsOverride; + } + try { let serverInfo: ServerInfo | null = null; @@ -344,13 +350,71 @@ export async function benchmarkCommand( return; } - // Ask if user wants to see scores after each benchmark - const showScores = await p.confirm({ - message: 'Show scores after each benchmark?', + // Load configs to get default iterations + const benchmarkConfigs = new Map(); + for (const benchmarkName of selectedBenchmarks) { + const benchmarkPath = join('benchmarks', benchmarkName); + const configPath = join(benchmarkPath, 'config.json'); + const config = readConfig(configPath); + if (config) { + benchmarkConfigs.set(benchmarkName, config.iterations); + } + } + + // Find the most common iteration count or first one + const defaultIterations = + benchmarkConfigs.size > 0 + ? Array.from(benchmarkConfigs.values())[0] + : 5; + + // Show iteration counts for selected benchmarks + const iterationsList = Array.from(selectedBenchmarks) + .map((name) => { + const iterations = benchmarkConfigs.get(name) || '?'; + return `${name}: ${iterations}`; + }) + .join(', '); + + logger.info(`Config iterations: ${color.dim(iterationsList)}`); + + // Ask for iterations override + const iterationsInput = await p.text({ + message: 'Number of iterations (press Enter to use config values):', + placeholder: `Default: ${defaultIterations}`, + defaultValue: '', + validate: (value) => { + if (value === '') return; // Empty is valid (use defaults) + const num = Number.parseInt(value, 10); + if (Number.isNaN(num) || num < 1) { + return 'Please enter a valid number greater than 0'; + } + }, + }); + + if (p.isCancel(iterationsInput)) { + p.cancel('Operation cancelled'); + return; + } + + // Parse iterations - if empty string, use undefined to let each benchmark use its config + const iterationsOverride = + iterationsInput === '' ? undefined : Number.parseInt(iterationsInput, 10); + + if (iterationsOverride !== undefined) { + logger.info( + `Using ${color.bold(color.cyan(String(iterationsOverride)))} iterations for all benchmarks` + ); + } else { + logger.info('Using iteration counts from each benchmark config'); + } + + // Ask if user wants to see results panel after completion + const showResults = await p.confirm({ + message: 'Show results panel after completion?', initialValue: true, }); - if (p.isCancel(showScores)) { + if (p.isCancel(showResults)) { p.cancel('Operation cancelled'); return; } @@ -369,7 +433,8 @@ export async function benchmarkCommand( const success = await runSingleBenchmark( logger, benchmarkPath, - showScores === true + false, // Don't show inline scores anymore + iterationsOverride ); results.push({ name: benchmarkName, success }); @@ -399,5 +464,15 @@ export async function benchmarkCommand( `Failed benchmarks: ${failed.map((r) => r.name).join(', ')}` ); } + + // Show results panel if requested + if (showResults === true && results.some((r) => r.success)) { + logger.message('\n' + '═'.repeat(80) + '\n'); + logger.info('Loading results panel...\n'); + + // Dynamically import and run the results command + const { resultsCommand } = await import('./results.js'); + await resultsCommand(logger); + } } diff --git a/packages/cookiebench-cli/src/commands/db.ts b/packages/cookiebench-cli/src/commands/db.ts index 6644308..1323a26 100644 --- a/packages/cookiebench-cli/src/commands/db.ts +++ b/packages/cookiebench-cli/src/commands/db.ts @@ -4,7 +4,7 @@ import { join, dirname } from 'node:path'; import { existsSync } from 'node:fs'; import * as p from '@clack/prompts'; import color from 'picocolors'; -import type { CliLogger } from '../utils'; +import { isAdminUser, type CliLogger } from '../utils'; const DB_PACKAGE_PATH = join(process.cwd(), 'packages', 'db'); const DRIZZLE_CONFIG_PATH = join(DB_PACKAGE_PATH, 'drizzle.config.ts'); @@ -38,6 +38,12 @@ function runDrizzleCommand(logger: CliLogger, command: string): void { } export async function dbCommand(logger: CliLogger, subcommand?: string) { + // Double-check admin access (safeguard) + if (!isAdminUser()) { + logger.error('This command requires admin access'); + process.exit(1); + } + logger.clear(); await setTimeout(1000); diff --git a/packages/cookiebench-cli/src/commands/results.ts b/packages/cookiebench-cli/src/commands/results.ts index 0e953d4..cb926a1 100644 --- a/packages/cookiebench-cli/src/commands/results.ts +++ b/packages/cookiebench-cli/src/commands/results.ts @@ -5,115 +5,10 @@ import { readFile, readdir } from 'node:fs/promises'; import { join } from 'node:path'; import Table from 'cli-table3'; import prettyMilliseconds from 'pretty-ms'; -import { config } from 'dotenv'; -import { calculateScores, printScores, type CliLogger } from '../utils'; +import { calculateScores, isAdminUser, type CliLogger } from '../utils'; import type { BenchmarkScores } from '../types'; import type { Config } from '@consentio/runner'; -// Load environment variables from .env files -config({ path: ".env" }); -config({ path: ".env.local" }); -config({ path: "www/.env.local" }); // Also check www directory - -// Function to save benchmark result via oRPC endpoint -async function saveBenchmarkResult(logger: CliLogger, result: BenchmarkResult): Promise { - const apiUrl = process.env.API_URL || "http://localhost:3000"; - const endpoint = `${apiUrl}/api/orpc/benchmarks/save`; - - try { - logger.info(`Attempting to save ${result.name} to ${endpoint}`); - - const response = await fetch(endpoint, { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify(result), - }); - - if (!response.ok) { - const errorText = await response.text(); - throw new Error(`HTTP error! status: ${response.status}, body: ${errorText}`); - } - - const responseData = await response.json(); - logger.success( - `Saved benchmark result for ${result.name} (App ID: ${responseData.appId})` - ); - } catch (error) { - if (error instanceof Error) { - logger.error(`Failed to save benchmark result for ${result.name}: ${error.message}`); - if (error.message.includes('fetch failed')) { - logger.error(`Connection failed. Is the server running on ${apiUrl}?`); - } - } else { - logger.error(`Failed to save benchmark result for ${result.name}: Unknown error`); - } - throw error; - } -} - -// Benchmark result type (matching the oRPC contract) -interface BenchmarkResult { - name: string; - baseline: boolean; - cookieBannerConfig: unknown; - techStack: unknown; - internationalization: unknown; - source: unknown; - includes: string[]; - company?: unknown; - tags: string[]; - details: unknown[]; - average: { - fcp: number; - lcp: number; - cls: number; - tbt: number; - tti: number; - scriptLoadTime: number; - totalSize: number; - scriptSize: number; - resourceCount: number; - scriptCount: number; - time: number; - thirdPartySize: number; - cookieServiceSize: number; - bannerVisibilityTime: number; - viewportCoverage: number; - thirdPartyImpact: number; - mainThreadBlocking: number; - cookieBannerBlocking: number; - }; - scores?: { - totalScore: number; - grade: 'Excellent' | 'Good' | 'Fair' | 'Poor' | 'Critical'; - categoryScores: { - performance: number; - bundleStrategy: number; - networkImpact: number; - transparency: number; - userExperience: number; - }; - categories: Array<{ - name: string; - score: number; - maxScore: number; - weight: number; - details: Array<{ - metric: string; - value: string | number; - score: number; - maxScore: number; - reason: string; - }>; - status: 'excellent' | 'good' | 'fair' | 'poor'; - }>; - insights: string[]; - recommendations: string[]; - }; -} - // Raw benchmark data structure from JSON files export interface RawBenchmarkDetail { duration: number; @@ -350,7 +245,7 @@ async function loadConfigForApp(logger: CliLogger, appName: string): Promise = {}; - logger.info(`Found ${resultsFiles.length} results files:`); + logger.debug(`Found ${resultsFiles.length} results files:`); for (const file of resultsFiles) { - logger.info(` - ${file}`); + logger.debug(` - ${file}`); } for (const file of resultsFiles) { @@ -432,8 +327,7 @@ async function aggregateResults(logger: CliLogger, resultsDir: string) { continue; } - // Log the actual app name from the file - logger.info(`Processing ${file} with app name: "${data.app}"`); + logger.debug(`Processing ${file} with app name: "${data.app}"`); if (results[data.app]) { logger.warn( @@ -442,7 +336,7 @@ async function aggregateResults(logger: CliLogger, resultsDir: string) { } results[data.app] = data.results; - logger.success( + logger.debug( `Loaded results for ${data.app} (${data.results.length} iterations)` ); } catch (error) { @@ -452,15 +346,14 @@ async function aggregateResults(logger: CliLogger, resultsDir: string) { }` ); if (error instanceof Error && error.stack) { - logger.error(`Stack trace: ${error.stack}`); + logger.debug(`Stack trace: ${error.stack}`); } } } - // Log final results summary - logger.info("Final results summary:"); + logger.debug("Final results summary:"); for (const [app, appResults] of Object.entries(results)) { - logger.info(` - ${app}: ${appResults.length} iterations`); + logger.debug(` - ${app}: ${appResults.length} iterations`); } return results; @@ -474,292 +367,437 @@ function formatTime(ms: number): string { }); } -function printResults(results: Record) { - // Calculate baseline averages - const baseline = results.baseline; - const baselineAvgTime = baseline - ? baseline.reduce((a, b) => a + b.duration, 0) / baseline.length - : 1; - const baselineAvgSize = baseline - ? baseline.reduce((a, b) => a + b.size.total, 0) / baseline.length - : 1; - const baselineAvgFCP = baseline - ? baseline.reduce((a, b) => a + b.timing.firstContentfulPaint, 0) / - baseline.length - : 1; - const baselineAvgLCP = baseline - ? baseline.reduce((a, b) => a + b.timing.largestContentfulPaint, 0) / - baseline.length - : 1; - const baselineAvgCLS = baseline - ? baseline.reduce((a, b) => a + b.timing.cumulativeLayoutShift, 0) / - baseline.length - : 1; - const baselineAvgTTI = baseline - ? baseline.reduce((a, b) => a + b.timing.timeToInteractive, 0) / - baseline.length - : 1; - const baselineAvgBannerRender = baseline - ? baseline.reduce( - (a, b) => - a + b.timing.cookieBanner.visibilityTime, - 0 - ) / baseline.length - : 1; - const baselineAvgScriptLoad = baseline - ? baseline.reduce( - (a, b) => - a + - (b.timing.scripts.thirdParty.loadEnd - - b.timing.scripts.thirdParty.loadStart), - 0 - ) / baseline.length - : 1; - - // Prepare and sort results by avg time - const sorted = Object.entries(results) - .map(([app, arr]) => { - const avgTime = arr.reduce((a, b) => a + b.duration, 0) / arr.length; - const avgSize = arr.reduce((a, b) => a + b.size.total, 0) / arr.length; +function formatBytes(bytes: number): string { + if (bytes === 0) return '0bytes'; + if (bytes < 1024) return `${bytes.toFixed(0)}bytes`; + return `${(bytes / 1024).toFixed(0)}KB`; +} + +function getPerformanceRating(metric: string, value: number): string { + const ratings: Record = { + fcp: { good: 1800, poor: 3000 }, + lcp: { good: 2500, poor: 4000 }, + cls: { good: 0.1, poor: 0.25 }, + tti: { good: 3800, poor: 7300 }, + tbt: { good: 200, poor: 600 }, + }; + + const thresholds = ratings[metric]; + if (!thresholds) return 'N/A'; + + if (value <= thresholds.good) { + return color.green('Good'); + } else if (value <= thresholds.poor) { + return color.yellow('Fair'); + } else { + return color.red('Poor'); + } +} + +function printDetailedResults( + appName: string, + results: RawBenchmarkDetail[], + scores: BenchmarkScores, + baseline?: RawBenchmarkDetail[] +) { + console.log('\n' + color.bold(color.cyan(`━━━ ${appName.toUpperCase()} ━━━`))); + + // ━━━ Score Display ━━━ + const score = Math.round(scores.totalScore); + let scoreColor = color.green; + let scoreBgColor = color.bgGreen; + + if (score < 70) { + scoreColor = color.red; + scoreBgColor = color.bgRed; + } else if (score < 90) { + scoreColor = color.yellow; + scoreBgColor = color.bgYellow; + } + + console.log('\n' + color.bold('🎯 Overall Score')); + console.log( + scoreColor(` ${score}/100`) + + ' ' + + scoreBgColor(color.black(` ${scores.grade} `)) + ); + + // ━━━ Key Insights ━━━ + if (scores.insights && scores.insights.length > 0) { + console.log('\n' + color.bold('💡 Key Insights')); + for (const insight of scores.insights) { + console.log(color.blue(' •') + ' ' + color.dim(insight)); + } + } + + // Calculate averages + const avgBannerVisibility = + results.reduce((a, b) => a + b.timing.cookieBanner.visibilityTime, 0) / + results.length; + const avgViewportCoverage = + results.reduce((a, b) => a + b.timing.cookieBanner.viewportCoverage, 0) / + results.length; + const avgNetworkImpact = + results.reduce((a, b) => a + b.size.thirdParty, 0) / results.length; + const bannerDetected = results.some((r) => r.timing.cookieBanner.detected); + const isBundled = results[0]?.size.thirdParty === 0; + const avgFCP = - arr.reduce((a, b) => a + b.timing.firstContentfulPaint, 0) / arr.length; + results.reduce((a, b) => a + b.timing.firstContentfulPaint, 0) / + results.length; const avgLCP = - arr.reduce((a, b) => a + b.timing.largestContentfulPaint, 0) / - arr.length; - const avgCLS = - arr.reduce((a, b) => a + b.timing.cumulativeLayoutShift, 0) / - arr.length; + results.reduce((a, b) => a + b.timing.largestContentfulPaint, 0) / + results.length; const avgTTI = - arr.reduce((a, b) => a + b.timing.timeToInteractive, 0) / arr.length; - const avgBannerRender = - arr.reduce( - (a, b) => - a + b.timing.cookieBanner.visibilityTime, - 0 - ) / arr.length; - const avgScriptLoad = - arr.reduce( - (a, b) => - a + - (b.timing.scripts.thirdParty.loadEnd - - b.timing.scripts.thirdParty.loadStart), - 0 - ) / arr.length; - - // Calculate average sizes for each resource type - const avgScriptsTotal = - arr.reduce((a, b) => a + b.size.scripts.total, 0) / arr.length; - const avgScriptsInitial = - arr.reduce((a, b) => a + b.size.scripts.initial, 0) / arr.length; - const avgScriptsDynamic = - arr.reduce((a, b) => a + b.size.scripts.dynamic, 0) / arr.length; - const avgStyles = arr.reduce((a, b) => a + b.size.styles, 0) / arr.length; - const avgImages = arr.reduce((a, b) => a + b.size.images, 0) / arr.length; - const avgFonts = arr.reduce((a, b) => a + b.size.fonts, 0) / arr.length; - const avgOther = arr.reduce((a, b) => a + b.size.other, 0) / arr.length; - - return { - app, - avgTime, - avgSize, - avgFCP, - avgLCP, - avgCLS, - avgTTI, - avgBannerRender, - avgScriptLoad, - avgScriptsTotal, - avgScriptsInitial, - avgScriptsDynamic, - avgStyles, - avgImages, - avgFonts, - avgOther, - timeDelta: ((avgTime - baselineAvgTime) / baselineAvgTime) * 100, - sizeDelta: ((avgSize - baselineAvgSize) / baselineAvgSize) * 100, - fcpDelta: ((avgFCP - baselineAvgFCP) / baselineAvgFCP) * 100, - lcpDelta: ((avgLCP - baselineAvgLCP) / baselineAvgLCP) * 100, - clsDelta: ((avgCLS - baselineAvgCLS) / baselineAvgCLS) * 100, - ttiDelta: ((avgTTI - baselineAvgTTI) / baselineAvgTTI) * 100, - bannerRenderDelta: - ((avgBannerRender - baselineAvgBannerRender) / - baselineAvgBannerRender) * - 100, - scriptLoadDelta: - ((avgScriptLoad - baselineAvgScriptLoad) / baselineAvgScriptLoad) * - 100, - }; - }) - .sort((a, b) => a.avgTime - b.avgTime); - - // Setup cli-table3 - const table = new Table({ - head: [ - "App", - "Total Time", - "ΔTime", - "FCP", - "ΔFCP", - "LCP", - "ΔLCP", - "CLS", - "ΔCLS", - "TTI", - "ΔTTI", - "Banner", - "ΔBanner", - "Script", - "ΔScript", - "Total Size", - "ΔSize", - "Scripts", - "Styles", - "Images", - "Fonts", - "Other", + results.reduce((a, b) => a + b.timing.timeToInteractive, 0) / + results.length; + const avgCLS = + results.reduce((a, b) => a + b.timing.cumulativeLayoutShift, 0) / + results.length; + const avgTBT = + results.reduce((a, b) => a + b.timing.mainThreadBlocking.total, 0) / + results.length; + + const totalSize = results.reduce((a, b) => a + b.size.total, 0) / results.length; + const jsSize = + results.reduce((a, b) => a + b.size.scripts.total, 0) / results.length; + const cssSize = results.reduce((a, b) => a + b.size.styles, 0) / results.length; + const imageSize = + results.reduce((a, b) => a + b.size.images, 0) / results.length; + const fontSize = results.reduce((a, b) => a + b.size.fonts, 0) / results.length; + const otherSize = + results.reduce((a, b) => a + b.size.other, 0) / results.length; + + const jsFiles = + results.reduce((a, b) => a + b.resources.scripts.length, 0) / results.length; + const cssFiles = + results.reduce((a, b) => a + b.resources.styles.length, 0) / results.length; + const imageFiles = + results.reduce((a, b) => a + b.resources.images.length, 0) / results.length; + const fontFiles = + results.reduce((a, b) => a + b.resources.fonts.length, 0) / results.length; + const otherFiles = + results.reduce((a, b) => a + b.resources.other.length, 0) / results.length; + + // Calculate deltas if baseline exists + let bannerDelta = ''; + if (baseline && appName !== 'baseline') { + const baselineAvgBanner = + baseline.reduce((a, b) => a + b.timing.cookieBanner.visibilityTime, 0) / + baseline.length; + const delta = avgBannerVisibility - baselineAvgBanner; + bannerDelta = ` ${delta > 0 ? '+' : ''}${formatTime(delta)}`; + } + + // ━━━ Cookie Banner Impact ━━━ + console.log('\n' + color.bold('🍪 Cookie Banner Impact')); + const bannerTable = new Table({ + chars: { mid: '', 'left-mid': '', 'mid-mid': '', 'right-mid': '' }, + style: { 'padding-left': 2, 'padding-right': 2, border: ['grey'] }, + }); + + bannerTable.push( + [ + { content: 'Banner Visibility', colSpan: 1 }, + { content: 'Viewport Coverage', colSpan: 1 }, + { content: 'Network Impact', colSpan: 1 }, + { content: 'Bundle Strategy', colSpan: 1 }, ], - colWidths: [ - 15, 10, 8, 10, 8, 10, 8, 8, 8, 10, 8, 10, 8, 10, 8, 10, 8, 10, 10, 10, 10, - 10, + [ + `${color.bold(formatTime(avgBannerVisibility))}\n${color.dim(bannerDelta || 'baseline')}`, + `${color.bold(avgViewportCoverage.toFixed(1) + '%')}\n${color.dim('Screen real estate')}`, + `${color.bold(formatBytes(avgNetworkImpact * 1024))}\n${color.dim(isBundled ? 'Bundled (no network)' : 'External requests')}`, + `${color.bold(isBundled ? 'Bundled' : 'External')}\n${color.dim(isBundled ? 'Included in main bundle' : 'Loaded from CDN')}`, + ] + ); + + console.log(bannerTable.toString()); + + // ━━━ Core Web Vitals ━━━ + console.log('\n' + color.bold('⚡ Core Web Vitals')); + const vitalsTable = new Table({ + chars: { mid: '', 'left-mid': '', 'mid-mid': '', 'right-mid': '' }, + style: { 'padding-left': 2, 'padding-right': 2, border: ['grey'] }, + }); + + vitalsTable.push( + [ + { content: 'First Contentful Paint', colSpan: 1 }, + { content: 'Largest Contentful Paint', colSpan: 1 }, + { content: 'Time to Interactive', colSpan: 1 }, + { content: 'Cumulative Layout Shift', colSpan: 1 }, ], - style: { head: ["cyan"], border: ["grey"] }, + [ + `${color.bold(formatTime(avgFCP))}\n${getPerformanceRating('fcp', avgFCP)}`, + `${color.bold(formatTime(avgLCP))}\n${getPerformanceRating('lcp', avgLCP)}`, + `${color.bold(formatTime(avgTTI))}\n${getPerformanceRating('tti', avgTTI)}`, + `${color.bold(avgCLS.toFixed(3))}\n${getPerformanceRating('cls', avgCLS)}`, + ] + ); + + console.log(vitalsTable.toString()); + + // ━━━ Resource Breakdown ━━━ + console.log('\n' + color.bold('📦 Resource Breakdown')); + + const totalFiles = jsFiles + cssFiles + imageFiles + fontFiles + otherFiles; + const jsPercentage = totalSize > 0 ? (jsSize / totalSize) * 100 : 0; + const cssPercentage = totalSize > 0 ? (cssSize / totalSize) * 100 : 0; + const imagePercentage = totalSize > 0 ? (imageSize / totalSize) * 100 : 0; + const fontPercentage = totalSize > 0 ? (fontSize / totalSize) * 100 : 0; + const otherPercentage = totalSize > 0 ? (otherSize / totalSize) * 100 : 0; + + const resourceTable = new Table({ + chars: { mid: '', 'left-mid': '', 'mid-mid': '', 'right-mid': '' }, + style: { 'padding-left': 2, 'padding-right': 2, border: ['grey'] }, + }); + + resourceTable.push( + [ + { content: 'Type', colSpan: 1 }, + { content: 'Size', colSpan: 1 }, + { content: 'Files', colSpan: 1 }, + { content: '% of Total', colSpan: 1 }, + ], + [ + color.cyan('JavaScript'), + formatBytes(jsSize * 1024), + Math.round(jsFiles).toString(), + `${jsPercentage.toFixed(1)}%`, + ], + [ + color.cyan('CSS'), + formatBytes(cssSize * 1024), + Math.round(cssFiles).toString(), + `${cssPercentage.toFixed(1)}%`, + ], + [ + color.cyan('Images'), + formatBytes(imageSize * 1024), + Math.round(imageFiles).toString(), + `${imagePercentage.toFixed(1)}%`, + ], + [ + color.cyan('Fonts'), + formatBytes(fontSize * 1024), + Math.round(fontFiles).toString(), + `${fontPercentage.toFixed(1)}%`, + ], + [ + color.cyan('Other'), + formatBytes(otherSize * 1024), + Math.round(otherFiles).toString(), + `${otherPercentage.toFixed(1)}%`, + ], + [ + color.bold('Total'), + color.bold(formatBytes(totalSize * 1024)), + color.bold(Math.round(totalFiles).toString()), + color.bold('100%'), + ] + ); + + console.log(resourceTable.toString()); + + // ━━━ Performance Impact Summary ━━━ + console.log('\n' + color.bold('📊 Performance Impact Summary')); + const summaryTable = new Table({ + chars: { mid: '', 'left-mid': '', 'mid-mid': '', 'right-mid': '' }, + style: { 'padding-left': 2, 'padding-right': 2, border: ['grey'] }, }); - // Add rows - for (const r of sorted) { - const timeDeltaStr = - r.app === "baseline" - ? "-" - : `${r.timeDelta > 0 ? "+" : ""}${r.timeDelta.toFixed(1)}%`; - const sizeDeltaStr = - r.app === "baseline" - ? "-" - : `${r.sizeDelta > 0 ? "+" : ""}${r.sizeDelta.toFixed(1)}%`; - const fcpDeltaStr = - r.app === "baseline" - ? "-" - : `${r.fcpDelta > 0 ? "+" : ""}${r.fcpDelta.toFixed(1)}%`; - const lcpDeltaStr = - r.app === "baseline" - ? "-" - : `${r.lcpDelta > 0 ? "+" : ""}${r.lcpDelta.toFixed(1)}%`; - const clsDeltaStr = - r.app === "baseline" - ? "-" - : `${r.clsDelta > 0 ? "+" : ""}${r.clsDelta.toFixed(1)}%`; - const ttiDeltaStr = - r.app === "baseline" - ? "-" - : `${r.ttiDelta > 0 ? "+" : ""}${r.ttiDelta.toFixed(1)}%`; - const bannerDeltaStr = - r.app === "baseline" - ? "-" - : `${r.bannerRenderDelta > 0 ? "+" : ""}${r.bannerRenderDelta.toFixed( - 1 - )}%`; - const scriptDeltaStr = - r.app === "baseline" - ? "-" - : `${r.scriptLoadDelta > 0 ? "+" : ""}${r.scriptLoadDelta.toFixed(1)}%`; - - table.push([ - r.app, - formatTime(r.avgTime), - timeDeltaStr, - formatTime(r.avgFCP), - fcpDeltaStr, - formatTime(r.avgLCP), - lcpDeltaStr, - r.avgCLS.toFixed(3), - clsDeltaStr, - formatTime(r.avgTTI), - ttiDeltaStr, - formatTime(r.avgBannerRender), - bannerDeltaStr, - formatTime(r.avgScriptLoad), - scriptDeltaStr, - `${r.avgSize.toFixed(2)}KB`, - sizeDeltaStr, - `${r.avgScriptsTotal.toFixed(2)}KB`, - `${r.avgStyles.toFixed(2)}KB`, - `${r.avgImages.toFixed(2)}KB`, - `${r.avgFonts.toFixed(2)}KB`, - `${r.avgOther.toFixed(2)}KB`, - ]); + const layoutStability = avgCLS === 0 ? 'Perfect' : avgCLS < 0.1 ? 'Good' : avgCLS < 0.25 ? 'Fair' : 'Poor'; + + summaryTable.push( + ['Loading Strategy', color.bold(isBundled ? 'Bundled' : 'External')], + ['Render Performance', color.bold(formatTime(avgBannerVisibility))], + ['Network Overhead', color.bold(formatBytes(avgNetworkImpact * 1024))], + ['Main Thread Impact', color.bold(formatTime(avgTBT))], + ['Layout Stability', color.bold(layoutStability)], + ['User Disruption', color.bold(`${avgViewportCoverage.toFixed(1)}%`)], + ); + + console.log(summaryTable.toString()); + + // ━━━ Network Chart (Waterfall) ━━━ + console.log('\n' + color.bold('🌐 Network Chart')); + + // Get first iteration's resources for waterfall + const firstResult = results[0]; + if (firstResult && firstResult.resources) { + const allResources = [ + ...firstResult.resources.scripts.map(r => ({ ...r, type: 'script' })), + ...firstResult.resources.styles.map(r => ({ ...r, type: 'style' })), + ...firstResult.resources.images.map(r => ({ ...r, type: 'image' })), + ...firstResult.resources.fonts.map(r => ({ ...r, type: 'font' })), + ...firstResult.resources.other.map(r => ({ ...r, type: 'other' })), + ].sort((a, b) => a.startTime - b.startTime); + + // Take top 10 resources for waterfall + const topResources = allResources.slice(0, 10); + + if (topResources.length > 0) { + const maxEndTime = Math.max(...topResources.map(r => r.startTime + r.duration)); + const chartWidth = 60; // Width of the waterfall bars + + const waterfallTable = new Table({ + chars: { mid: '', 'left-mid': '', 'mid-mid': '', 'right-mid': '' }, + colWidths: [25, chartWidth + 10], + style: { 'padding-left': 1, 'padding-right': 1, border: ['grey'] }, + wordWrap: true, + }); + + waterfallTable.push([ + color.dim('Resource'), + color.dim('Timeline (0ms ───────────────────────────► ' + formatTime(maxEndTime) + ')'), + ]); + + for (const resource of topResources) { + const fileName = resource.name.split('/').pop() || resource.name; + const shortName = fileName.length > 23 ? fileName.substring(0, 20) + '...' : fileName; + + const startPos = Math.floor((resource.startTime / maxEndTime) * chartWidth); + const barLength = Math.max(1, Math.floor((resource.duration / maxEndTime) * chartWidth)); + + const emptyBefore = ' '.repeat(startPos); + const bar = '█'.repeat(barLength); + const durationLabel = resource.duration > maxEndTime * 0.1 ? formatTime(resource.duration) : ''; + + let barColor = color.blue; + if (resource.isThirdParty) barColor = color.yellow; + if (resource.isCookieService) barColor = color.red; + + waterfallTable.push([ + color.dim(shortName), + emptyBefore + barColor(bar) + ' ' + color.dim(durationLabel), + ]); + } + + console.log(waterfallTable.toString()); + } } - // Print the table to console - // Table output is user-facing, so we use console.log directly - console.log(table.toString()); -} + // ━━━ Resource Details ━━━ + console.log('\n' + color.bold('📋 Resource Details')); + + // Aggregate resource data across all results + const aggregatedResources: Array<{ + name: string; + type: string; + source: string; + size: number; + duration: number; + tags: string[]; + }> = []; + + // Use first result for resource list (assuming resources are consistent) + const sampleResult = results[0]; + if (sampleResult && sampleResult.resources) { + const allSampleResources = [ + ...sampleResult.resources.scripts.map(r => ({ ...r, type: 'JavaScript' })), + ...sampleResult.resources.styles.map(r => ({ ...r, type: 'CSS' })), + ...sampleResult.resources.images.map(r => ({ ...r, type: 'Image' })), + ...sampleResult.resources.fonts.map(r => ({ ...r, type: 'Font' })), + ...sampleResult.resources.other.map(r => ({ ...r, type: 'Other' })), + ]; + + // Calculate averages for each resource + for (const sampleResource of allSampleResources) { + const resourceName = sampleResource.name; + + // Find this resource in all results and average the values + const avgSize = results.reduce((sum, result) => { + const allResources = [ + ...result.resources.scripts, + ...result.resources.styles, + ...result.resources.images, + ...result.resources.fonts, + ...result.resources.other, + ]; + const found = allResources.find(r => r.name === resourceName); + return sum + (found ? found.size : 0); + }, 0) / results.length; + + const avgDuration = results.reduce((sum, result) => { + const allResources = [ + ...result.resources.scripts, + ...result.resources.styles, + ...result.resources.images, + ...result.resources.fonts, + ...result.resources.other, + ]; + const found = allResources.find(r => r.name === resourceName); + return sum + (found ? found.duration : 0); + }, 0) / results.length; + + const source = sampleResource.isThirdParty + ? (sampleResource.isCookieService ? 'Cookie Service' : 'Third-Party') + : 'Bundled'; + + const tags = []; + if (!sampleResource.isThirdParty) tags.push('bundled'); + if (sampleResource.isThirdParty) tags.push('third-party'); + if (sampleResource.isCookieService) tags.push('cookie-service'); + if ('isDynamic' in sampleResource && sampleResource.isDynamic) tags.push('dynamic'); + + // Add core/other categorization for bundled scripts + if (!sampleResource.isThirdParty && sampleResource.type === 'JavaScript') { + tags.push('core'); + } -// Function to transform BenchmarkScores to match oRPC contract -function transformScoresToContract(scores: BenchmarkScores): BenchmarkResult['scores'] { - return { - totalScore: scores.totalScore, - grade: scores.grade, - categoryScores: scores.categoryScores, - categories: scores.categories.map(category => ({ - name: category.name, - score: category.score, - maxScore: category.maxScore, - weight: category.weight, - details: category.details.map(detail => ({ - metric: detail.name, - value: detail.score, - score: detail.score, - maxScore: detail.maxScore, - reason: detail.reason, - })), - status: mapStatusToContract(category.status), - })), - insights: scores.insights, - recommendations: scores.recommendations, - }; -} + aggregatedResources.push({ + name: resourceName, + type: sampleResource.type, + source, + size: avgSize, + duration: avgDuration, + tags, + }); + } + } -// Function to map status values from CLI format to contract format -function mapStatusToContract(status: 'excellent' | 'good' | 'fair' | 'poor'): 'excellent' | 'good' | 'fair' | 'poor' { - // Now that both CLI and contract use the same format, just return as is - return status; + // Sort by size (descending) and take top 10 + const topResources = aggregatedResources + .sort((a, b) => b.size - a.size) + .slice(0, 10); + + if (topResources.length > 0) { + const detailsTable = new Table({ + head: ['Resource Name', 'Type', 'Source', 'Size', 'Duration', 'Tags'], + colWidths: [25, 12, 15, 10, 10, 20], + style: { head: ['cyan'], border: ['grey'] }, + wordWrap: true, + }); + + for (const resource of topResources) { + const fileName = resource.name.split('/').pop() || resource.name; + const shortName = fileName.length > 23 ? fileName.substring(0, 20) + '...' : fileName; + + let sourceColor = color.green; + if (resource.source === 'Third-Party') sourceColor = color.yellow; + if (resource.source === 'Cookie Service') sourceColor = color.red; + + detailsTable.push([ + shortName, + resource.type, + sourceColor(resource.source), + formatBytes(resource.size * 1024), + color.blue(formatTime(resource.duration)), + resource.tags.join(', '), + ]); + } + + console.log(detailsTable.toString()); + } } -export async function resultsCommand(logger: CliLogger) { +export async function resultsCommand(logger: CliLogger, appName?: string) { logger.clear(); await setTimeout(1000); - p.intro(`${color.bgCyan(color.black(" results "))}`); - - // Check database configuration - const databaseUrl = - process.env.DATABASE_URL || process.env.TURSO_DATABASE_URL; - const authToken = - process.env.DATABASE_AUTH_TOKEN || process.env.TURSO_AUTH_TOKEN; - - if ( - databaseUrl?.startsWith("libsql://") || - databaseUrl?.startsWith("wss://") - ) { - logger.info( - `🌐 Using Turso remote database: ${color.cyan( - `${databaseUrl.split("@")[0]}@***` - )}` - ); - if (!authToken) { - logger.warn("⚠️ No auth token found. Database operations may fail."); - } - } else if (databaseUrl?.startsWith("file:")) { - logger.info(`📁 Using file database: ${color.cyan(databaseUrl)}`); - } else if (process.env.VERCEL || process.env.NODE_ENV === "production") { - logger.warn("⚠️ Using in-memory database. Data will not persist!"); - } else { - logger.info( - `📁 Using local SQLite database: ${color.cyan("benchmarks.db")}` - ); - } + p.intro(`${color.bgCyan(color.black(" results "))} ${color.dim('Compare benchmarks')}`); const resultsDir = "benchmarks"; - logger.step("Aggregating results..."); const results = await aggregateResults(logger, resultsDir); if (Object.keys(results).length === 0) { @@ -767,12 +805,61 @@ export async function resultsCommand(logger: CliLogger) { return; } - logger.info( + logger.debug( `Found results for ${Object.keys(results).length} apps: ${Object.keys( results ).join(", ")}` ); + // If a specific app is requested, filter to that + let selectedApps: string[]; + + if (appName && appName !== '__all__') { + // Direct command with specific app + if (!results[appName]) { + logger.error(`No results found for "${appName}"`); + logger.info(`Available apps: ${Object.keys(results).join(', ')}`); + return; + } + selectedApps = [appName]; + } else if (appName === '__all__') { + // Show all results + selectedApps = Object.keys(results); + } else { + // Interactive mode - let user select which apps to view + const availableApps = Object.keys(results).sort((a, b) => { + if (a === 'baseline') return -1; + if (b === 'baseline') return 1; + return a.localeCompare(b); + }); + + const selected = await p.multiselect({ + message: 'Select benchmarks to view (use space to toggle, all selected by default):', + options: availableApps.map((name) => ({ + value: name, + label: name, + hint: `benchmarks/${name}`, + })), + initialValues: availableApps, // All selected by default + required: true, + }); + + if (p.isCancel(selected)) { + p.cancel('Operation cancelled'); + return; + } + + if (!Array.isArray(selected) || selected.length === 0) { + logger.warn('No benchmarks selected'); + return; + } + + selectedApps = selected; + } + + logger.debug(`Viewing results for: ${selectedApps.join(', ')}`); + + // Load configs for each app const appConfigs: Record = {}; for (const appName of Object.keys(results)) { appConfigs[appName] = await loadConfigForApp(logger, appName); @@ -800,7 +887,6 @@ export async function resultsCommand(logger: CliLogger) { cls: appResults.reduce((a, b) => a + b.timing.cumulativeLayoutShift, 0) / appResults.length, tbt: appResults.reduce((a, b) => a + b.timing.mainThreadBlocking.total, 0) / appResults.length, tti: appResults.reduce((a, b) => a + b.timing.timeToInteractive, 0) / appResults.length, - // NEW: Add Perfume.js metrics timeToFirstByte: appResults.reduce((a, b) => a + (b.timing.timeToFirstByte || 0), 0) / appResults.length, interactionToNextPaint: appResults[0]?.timing.interactionToNextPaint || null, }, @@ -820,7 +906,7 @@ export async function resultsCommand(logger: CliLogger) { thirdPartyRequests: appResults.reduce((a, b) => a + b.resources.scripts.filter(s => s.isThirdParty).length, 0) / appResults.length, thirdPartySize: appResults.reduce((a, b) => a + b.size.thirdParty, 0) / appResults.length, - thirdPartyDomains: 5, // Default value + thirdPartyDomains: 5, }, { cookieBannerDetected: appResults.some(r => r.timing.cookieBanner.detected), @@ -828,7 +914,7 @@ export async function resultsCommand(logger: CliLogger) { cookieBannerCoverage: appResults.reduce((a, b) => a + b.timing.cookieBanner.viewportCoverage, 0) / appResults.length / 100, }, { - domSize: 1500, // Default value + domSize: 1500, mainThreadBlocking: appResults.reduce((a, b) => a + b.timing.mainThreadBlocking.total, 0) / appResults.length, layoutShifts: appResults.reduce((a, b) => a + b.timing.cumulativeLayoutShift, 0) / appResults.length, }, @@ -838,122 +924,21 @@ export async function resultsCommand(logger: CliLogger) { ); } - // Print scores - logger.info("\n📊 Benchmark Scores:"); - for (const [appName, appScores] of Object.entries(scores)) { - logger.info(`\n${appName}:`); - printScores(appScores); - } - - // Save results to database - logger.step("Saving results to database..."); - let savedCount = 0; - let errorCount = 0; - - for (const [appName, appResults] of Object.entries(results)) { - try { - // Load config data for this app - const config = appConfigs[appName]; - - // Convert raw benchmark data to BenchmarkResult format - const benchmarkResult: BenchmarkResult = { - name: appName, - baseline: appName === "baseline", - cookieBannerConfig: { - selectors: config.cookieBanner.selectors, - serviceHosts: config.cookieBanner.serviceHosts, - waitForVisibility: config.cookieBanner.waitForVisibility, - measureViewportCoverage: config.cookieBanner.measureViewportCoverage, - expectedLayoutShift: config.cookieBanner.expectedLayoutShift, - serviceName: config.cookieBanner.serviceName, - }, - techStack: config.techStack, - internationalization: config.internationalization, - source: config.source, - includes: Object.values(config.includes || {}) - .flat() - .filter((v): v is string => typeof v === "string"), - company: config.company, - tags: config.tags || [], - details: appResults, - average: { - fcp: appResults.reduce((a, b) => a + b.timing.firstContentfulPaint, 0) / - appResults.length, - lcp: appResults.reduce((a, b) => a + b.timing.largestContentfulPaint, 0) / - appResults.length, - cls: appResults.reduce((a, b) => a + b.timing.cumulativeLayoutShift, 0) / - appResults.length, - tbt: appResults.reduce((a, b) => a + b.timing.mainThreadBlocking.total, 0) / - appResults.length, - tti: appResults.reduce((a, b) => a + b.timing.timeToInteractive, 0) / - appResults.length, - scriptLoadTime: 0, - totalSize: appResults.reduce((a, b) => a + b.size.total, 0) / - appResults.length, - scriptSize: 0, - resourceCount: appResults.reduce((a, b) => a + b.resources.scripts.length, 0) / - appResults.length, - scriptCount: appResults.reduce((a, b) => a + b.resources.scripts.length, 0) / - appResults.length, - time: appResults.reduce((a, b) => a + b.duration, 0) / appResults.length, - thirdPartySize: appResults.reduce((a, b) => a + b.size.thirdParty, 0) / - appResults.length, - cookieServiceSize: appResults.reduce((a, b) => a + b.size.cookieServices, 0) / - appResults.length, - bannerVisibilityTime: appResults.reduce((a, b) => a + b.timing.cookieBanner.visibilityTime, 0) / - appResults.length, - viewportCoverage: appResults.reduce((a, b) => a + b.timing.cookieBanner.viewportCoverage, 0) / - appResults.length, - thirdPartyImpact: appResults.reduce((a, b) => a + b.timing.thirdParty.totalImpact, 0) / - appResults.length, - mainThreadBlocking: appResults.reduce((a, b) => a + b.timing.mainThreadBlocking.total, 0) / - appResults.length, - cookieBannerBlocking: appResults.reduce((a, b) => a + b.timing.mainThreadBlocking.cookieBannerEstimate, 0) / - appResults.length, - }, - scores: transformScoresToContract(scores[appName]), - }; - - await saveBenchmarkResult(logger, benchmarkResult); - logger.success(`Saved results for ${appName}`); - savedCount++; - } catch (error) { - logger.error( - `Failed to save results for ${appName}: ${ - error instanceof Error ? error.message : "Unknown error" - }` - ); - errorCount++; - } - } - - // Print results table - printResults(results); + // Print detailed results for selected apps only + const baselineResults = results.baseline; + const sortedApps = selectedApps.sort((a, b) => { + if (a === 'baseline') return -1; + if (b === 'baseline') return 1; + return a.localeCompare(b); + }); - // Summary of database operations - if (savedCount > 0) { - logger.success(`Successfully saved ${savedCount} app(s) to database.`); - } - if (errorCount > 0) { - logger.warn(`Failed to save ${errorCount} app(s) to database.`); + for (const appName of sortedApps) { + printDetailedResults(appName, results[appName], scores[appName], baselineResults); } - if ( - databaseUrl?.startsWith("libsql://") || - databaseUrl?.startsWith("wss://") - ) { - logger.info( - `Results have been saved to Turso database: ${color.cyan( - `${databaseUrl.split("@")[0]}@***` - )}` - ); + if (isAdminUser()) { + logger.outro(`\nDisplayed ${selectedApps.length} of ${Object.keys(results).length} benchmark(s) - Use ${color.cyan('cookiebench save')} to sync to database`); } else { - logger.info( - `Results have been saved to local database: ${color.cyan( - "benchmarks.db" - )} (in project root)` - ); + logger.outro(`\nDisplayed ${selectedApps.length} of ${Object.keys(results).length} benchmark(s)`); } - - logger.outro("Results aggregated and saved to database successfully!"); } diff --git a/packages/cookiebench-cli/src/commands/save.ts b/packages/cookiebench-cli/src/commands/save.ts new file mode 100644 index 0000000..392aaa1 --- /dev/null +++ b/packages/cookiebench-cli/src/commands/save.ts @@ -0,0 +1,618 @@ +import { setTimeout } from 'node:timers/promises'; +import * as p from '@clack/prompts'; +import color from 'picocolors'; +import { readFile, readdir } from 'node:fs/promises'; +import { join } from 'node:path'; +import { config } from 'dotenv'; +import { calculateScores, type CliLogger, isAdminUser } from '../utils'; +import type { BenchmarkScores } from '../types'; +import type { Config } from '@consentio/runner'; +import type { RawBenchmarkDetail } from './results'; + +// Load environment variables from .env files +config({ path: '.env' }); +config({ path: '.env.local' }); +config({ path: 'www/.env.local' }); + +interface BenchmarkOutput { + app: string; + results: RawBenchmarkDetail[]; + scores?: BenchmarkScores; + metadata?: { + timestamp: string; + iterations: number; + languages?: string[]; + }; +} + +// Benchmark result type (matching the oRPC contract) +interface BenchmarkResult { + name: string; + baseline: boolean; + cookieBannerConfig: unknown; + techStack: unknown; + internationalization: unknown; + source: unknown; + includes: string[]; + company?: unknown; + tags: string[]; + details: unknown[]; + average: { + fcp: number; + lcp: number; + cls: number; + tbt: number; + tti: number; + scriptLoadTime: number; + totalSize: number; + scriptSize: number; + resourceCount: number; + scriptCount: number; + time: number; + thirdPartySize: number; + cookieServiceSize: number; + bannerVisibilityTime: number; + viewportCoverage: number; + thirdPartyImpact: number; + mainThreadBlocking: number; + cookieBannerBlocking: number; + }; + scores?: { + totalScore: number; + grade: 'Excellent' | 'Good' | 'Fair' | 'Poor' | 'Critical'; + categoryScores: { + performance: number; + bundleStrategy: number; + networkImpact: number; + transparency: number; + userExperience: number; + }; + categories: Array<{ + name: string; + score: number; + maxScore: number; + weight: number; + details: Array<{ + metric: string; + value: string | number; + score: number; + maxScore: number; + reason: string; + }>; + status: 'excellent' | 'good' | 'fair' | 'poor'; + }>; + insights: string[]; + recommendations: string[]; + }; +} + +async function saveBenchmarkResult( + logger: CliLogger, + result: BenchmarkResult +): Promise { + const apiUrl = process.env.API_URL || 'http://localhost:3000'; + const endpoint = `${apiUrl}/api/orpc/benchmarks/save`; + + try { + logger.debug(`Attempting to save ${result.name} to ${endpoint}`); + + const response = await fetch(endpoint, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(result), + }); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error( + `HTTP error! status: ${response.status}, body: ${errorText}` + ); + } + + const responseData = await response.json(); + logger.success( + `Saved ${result.name} (App ID: ${responseData.appId})` + ); + } catch (error) { + if (error instanceof Error) { + logger.error( + `Failed to save ${result.name}: ${error.message}` + ); + if (error.message.includes('fetch failed')) { + logger.error( + `Connection failed. Is the server running on ${apiUrl}?` + ); + } + } else { + logger.error( + `Failed to save ${result.name}: Unknown error` + ); + } + throw error; + } +} + +async function findResultsFiles(dir: string): Promise { + const files: string[] = []; + try { + const entries = await readdir(dir, { withFileTypes: true }); + + for (const entry of entries) { + const fullPath = join(dir, entry.name); + if (entry.isDirectory()) { + files.push(...(await findResultsFiles(fullPath))); + } else if (entry.name === 'results.json') { + files.push(fullPath); + } + } + } catch (error) { + // Silently fail if directory doesn't exist + } + + return files; +} + +async function loadConfigForApp( + logger: CliLogger, + appName: string +): Promise { + const configPath = join('benchmarks', appName, 'config.json'); + + try { + const configContent = await readFile(configPath, 'utf-8'); + const config = JSON.parse(configContent); + + return { + name: config.name || appName, + iterations: config.iterations || 0, + techStack: config.techStack || { + languages: [], + frameworks: [], + bundler: 'unknown', + bundleType: 'unknown', + packageManager: 'unknown', + typescript: false, + }, + source: config.source || { + license: 'unknown', + isOpenSource: false, + github: false, + npm: false, + }, + includes: config.includes || { backend: [], components: [] }, + company: config.company || undefined, + tags: config.tags || [], + cookieBanner: config.cookieBanner || { + serviceName: 'Unknown', + selectors: [], + serviceHosts: [], + waitForVisibility: false, + measureViewportCoverage: false, + expectedLayoutShift: false, + }, + internationalization: config.internationalization || { + detection: 'none', + stringLoading: 'bundled', + }, + }; + } catch (error) { + logger.debug(`Could not load config for ${appName}:`, error); + return null; + } +} + +function transformScoresToContract( + scores: BenchmarkScores +): BenchmarkResult['scores'] { + return { + totalScore: scores.totalScore, + grade: scores.grade, + categoryScores: scores.categoryScores, + categories: scores.categories.map((category) => ({ + name: category.name, + score: category.score, + maxScore: category.maxScore, + weight: category.weight, + details: category.details.map((detail) => ({ + metric: detail.name, + value: detail.score, + score: detail.score, + maxScore: detail.maxScore, + reason: detail.reason, + })), + status: category.status, + })), + insights: scores.insights, + recommendations: scores.recommendations, + }; +} + +export async function saveCommand( + logger: CliLogger, + appName?: string +): Promise { + // Double-check admin access (safeguard) + if (!isAdminUser()) { + logger.error('This command requires admin access'); + process.exit(1); + } + + logger.clear(); + await setTimeout(500); + + p.intro(`${color.bgBlue(color.white(' save '))} ${color.dim('Sync results to database')}`); + + // Check database configuration + const databaseUrl = + process.env.DATABASE_URL || process.env.TURSO_DATABASE_URL; + const authToken = + process.env.DATABASE_AUTH_TOKEN || process.env.TURSO_AUTH_TOKEN; + const apiUrl = process.env.API_URL || 'http://localhost:3000'; + + logger.info(`API endpoint: ${color.cyan(apiUrl)}`); + + if ( + databaseUrl?.startsWith('libsql://') || + databaseUrl?.startsWith('wss://') + ) { + logger.info( + `Database: ${color.cyan(`Turso (${databaseUrl.split('@')[0]}@***)`)}` + ); + if (!authToken) { + logger.warn('⚠️ No auth token found. Database operations may fail.'); + } + } else if (databaseUrl?.startsWith('file:')) { + logger.info(`Database: ${color.cyan(`Local (${databaseUrl})`)}`); + } else { + logger.info(`Database: ${color.cyan('Local SQLite (benchmarks.db)')}`); + } + + const resultsDir = 'benchmarks'; + const resultsFiles = await findResultsFiles(resultsDir); + + if (resultsFiles.length === 0) { + logger.error('No benchmark results found!'); + logger.info( + `Run ${color.cyan('cookiebench benchmark')} first to generate results.` + ); + return; + } + + logger.info(`Found ${resultsFiles.length} results file(s)`); + + // Load all results + const allResults: Record = {}; + for (const file of resultsFiles) { + try { + const content = await readFile(file, 'utf-8'); + const data: BenchmarkOutput = JSON.parse(content); + + if (data.app && data.results) { + allResults[data.app] = data; + } + } catch (error) { + logger.debug(`Failed to load ${file}:`, error); + } + } + + if (Object.keys(allResults).length === 0) { + logger.error('No valid benchmark results found!'); + return; + } + + // If specific app requested, save only that one + if (appName) { + const result = allResults[appName]; + if (!result) { + logger.error(`No results found for app: ${appName}`); + logger.info( + `Available apps: ${Object.keys(allResults).join(', ')}` + ); + return; + } + + await saveAppToDatabase(logger, appName, result); + logger.outro('Done!'); + return; + } + + // Otherwise, show interactive selection + const appOptions = Object.keys(allResults).map((name) => ({ + value: name, + label: name, + hint: `${allResults[name].results.length} iterations`, + })); + + appOptions.push({ + value: '__all__', + label: 'Save all apps', + hint: 'Sync all benchmark results to database', + }); + + const selectedApps = await p.multiselect({ + message: 'Select benchmarks to save to database:', + options: appOptions, + required: true, + }); + + if (p.isCancel(selectedApps)) { + p.cancel('Operation cancelled'); + return; + } + + if (!Array.isArray(selectedApps) || selectedApps.length === 0) { + logger.warn('No benchmarks selected'); + return; + } + + // Confirm before saving + const appsToSave = + selectedApps.includes('__all__') + ? Object.keys(allResults) + : (selectedApps as string[]); + + const confirm = await p.confirm({ + message: `Save ${appsToSave.length} benchmark(s) to ${apiUrl}?`, + initialValue: true, + }); + + if (p.isCancel(confirm) || !confirm) { + p.cancel('Operation cancelled'); + return; + } + + // Save selected apps + let savedCount = 0; + let errorCount = 0; + + for (const name of appsToSave) { + try { + await saveAppToDatabase(logger, name, allResults[name]); + savedCount++; + } catch (error) { + errorCount++; + } + } + + // Summary + logger.message(''); + if (savedCount > 0) { + logger.success(`Successfully saved ${savedCount} app(s)`); + } + if (errorCount > 0) { + logger.warn(`Failed to save ${errorCount} app(s)`); + } + + logger.outro( + `Saved ${savedCount}/${appsToSave.length} benchmarks to database` + ); +} + +async function saveAppToDatabase( + logger: CliLogger, + appName: string, + result: BenchmarkOutput +): Promise { + const config = await loadConfigForApp(logger, appName); + const appResults = result.results; + + // Calculate scores if not already in results + let scores = result.scores; + if (!scores) { + const appData = { + name: appName, + baseline: appName === 'baseline', + company: config?.company ? JSON.stringify(config.company) : null, + techStack: config?.techStack + ? JSON.stringify(config.techStack) + : '{}', + source: config?.source ? JSON.stringify(config.source) : null, + tags: config?.tags ? JSON.stringify(config.tags) : null, + }; + + scores = calculateScores( + { + fcp: + appResults.reduce( + (a, b) => a + b.timing.firstContentfulPaint, + 0 + ) / appResults.length, + lcp: + appResults.reduce( + (a, b) => a + b.timing.largestContentfulPaint, + 0 + ) / appResults.length, + cls: + appResults.reduce( + (a, b) => a + b.timing.cumulativeLayoutShift, + 0 + ) / appResults.length, + tbt: + appResults.reduce( + (a, b) => a + b.timing.mainThreadBlocking.total, + 0 + ) / appResults.length, + tti: + appResults.reduce((a, b) => a + b.timing.timeToInteractive, 0) / + appResults.length, + timeToFirstByte: + appResults.reduce( + (a, b) => a + (b.timing.timeToFirstByte || 0), + 0 + ) / appResults.length, + interactionToNextPaint: + appResults[0]?.timing.interactionToNextPaint || null, + }, + { + totalSize: + appResults.reduce((a, b) => a + b.size.total, 0) / + appResults.length, + jsSize: + appResults.reduce((a, b) => a + b.size.scripts.total, 0) / + appResults.length, + cssSize: + appResults.reduce((a, b) => a + b.size.styles, 0) / + appResults.length, + imageSize: + appResults.reduce((a, b) => a + b.size.images, 0) / + appResults.length, + fontSize: + appResults.reduce((a, b) => a + b.size.fonts, 0) / + appResults.length, + otherSize: + appResults.reduce((a, b) => a + b.size.other, 0) / + appResults.length, + }, + { + totalRequests: + appResults.reduce( + (a, b) => + a + + b.resources.scripts.length + + b.resources.styles.length + + b.resources.images.length + + b.resources.fonts.length + + b.resources.other.length, + 0 + ) / appResults.length, + thirdPartyRequests: + appResults.reduce( + (a, b) => + a + + b.resources.scripts.filter((s) => s.isThirdParty).length, + 0 + ) / appResults.length, + thirdPartySize: + appResults.reduce((a, b) => a + b.size.thirdParty, 0) / + appResults.length, + thirdPartyDomains: 5, + }, + { + cookieBannerDetected: appResults.some( + (r) => r.timing.cookieBanner.detected + ), + cookieBannerTiming: + appResults.reduce( + (a, b) => a + b.timing.cookieBanner.visibilityTime, + 0 + ) / appResults.length, + cookieBannerCoverage: + appResults.reduce( + (a, b) => a + b.timing.cookieBanner.viewportCoverage, + 0 + ) / + appResults.length / + 100, + }, + { + domSize: 1500, + mainThreadBlocking: + appResults.reduce( + (a, b) => a + b.timing.mainThreadBlocking.total, + 0 + ) / appResults.length, + layoutShifts: + appResults.reduce( + (a, b) => a + b.timing.cumulativeLayoutShift, + 0 + ) / appResults.length, + }, + appName === 'baseline', + appData, + appResults[0]?.timing.networkInformation + ); + } + + // Convert to API format + const benchmarkResult: BenchmarkResult = { + name: appName, + baseline: appName === 'baseline', + cookieBannerConfig: config?.cookieBanner || {}, + techStack: config?.techStack || {}, + internationalization: config?.internationalization || {}, + source: config?.source || {}, + includes: config?.includes + ? Object.values(config.includes) + .flat() + .filter((v): v is string => typeof v === 'string') + : [], + company: config?.company, + tags: config?.tags || [], + details: appResults, + average: { + fcp: + appResults.reduce( + (a, b) => a + b.timing.firstContentfulPaint, + 0 + ) / appResults.length, + lcp: + appResults.reduce( + (a, b) => a + b.timing.largestContentfulPaint, + 0 + ) / appResults.length, + cls: + appResults.reduce( + (a, b) => a + b.timing.cumulativeLayoutShift, + 0 + ) / appResults.length, + tbt: + appResults.reduce( + (a, b) => a + b.timing.mainThreadBlocking.total, + 0 + ) / appResults.length, + tti: + appResults.reduce((a, b) => a + b.timing.timeToInteractive, 0) / + appResults.length, + scriptLoadTime: 0, + totalSize: + appResults.reduce((a, b) => a + b.size.total, 0) / + appResults.length, + scriptSize: 0, + resourceCount: + appResults.reduce((a, b) => a + b.resources.scripts.length, 0) / + appResults.length, + scriptCount: + appResults.reduce((a, b) => a + b.resources.scripts.length, 0) / + appResults.length, + time: + appResults.reduce((a, b) => a + b.duration, 0) / appResults.length, + thirdPartySize: + appResults.reduce((a, b) => a + b.size.thirdParty, 0) / + appResults.length, + cookieServiceSize: + appResults.reduce((a, b) => a + b.size.cookieServices, 0) / + appResults.length, + bannerVisibilityTime: + appResults.reduce( + (a, b) => a + b.timing.cookieBanner.visibilityTime, + 0 + ) / appResults.length, + viewportCoverage: + appResults.reduce( + (a, b) => a + b.timing.cookieBanner.viewportCoverage, + 0 + ) / appResults.length, + thirdPartyImpact: + appResults.reduce( + (a, b) => a + b.timing.thirdParty.totalImpact, + 0 + ) / appResults.length, + mainThreadBlocking: + appResults.reduce( + (a, b) => a + b.timing.mainThreadBlocking.total, + 0 + ) / appResults.length, + cookieBannerBlocking: + appResults.reduce( + (a, b) => a + b.timing.mainThreadBlocking.cookieBannerEstimate, + 0 + ) / appResults.length, + }, + scores: scores ? transformScoresToContract(scores) : undefined, + }; + + await saveBenchmarkResult(logger, benchmarkResult); +} + diff --git a/packages/cookiebench-cli/src/index.ts b/packages/cookiebench-cli/src/index.ts index 099e635..bce469e 100644 --- a/packages/cookiebench-cli/src/index.ts +++ b/packages/cookiebench-cli/src/index.ts @@ -3,15 +3,19 @@ import * as p from '@clack/prompts'; import color from 'picocolors'; import { benchmarkCommand } from './commands/benchmark'; import { resultsCommand } from './commands/results'; -import { dbCommand } from './commands/db'; import { scoresCommand } from './commands/scores'; -import { createCliLogger, type CliLogger } from './utils/logger'; +import { saveCommand } from './commands/save'; +import { dbCommand } from './commands/db'; +import { createCliLogger, type CliLogger, isAdminUser } from './utils'; import { displayIntro } from './components/intro'; // Get log level from env or default to info const logLevel = (process.env.LOG_LEVEL as 'error' | 'warn' | 'info' | 'debug') || 'info'; const logger: CliLogger = createCliLogger(logLevel); +// Check admin access for restricted commands +const isAdmin = isAdminUser(); + function onCancel() { p.cancel('Operation cancelled.'); process.exit(0); @@ -38,44 +42,66 @@ async function main() { await benchmarkCommand(logger); break; case 'results': - await resultsCommand(logger); + await resultsCommand(logger, args[1]); break; case 'scores': await scoresCommand(logger, args[1]); break; + case 'save': + if (!isAdmin) { + logger.error('This command requires admin access'); + process.exit(1); + } + await saveCommand(logger, args[1]); + break; case 'db': + if (!isAdmin) { + logger.error('This command requires admin access'); + process.exit(1); + } await dbCommand(logger, args[1]); break; default: logger.error(`Unknown command: ${command}`); - logger.info('Available commands: benchmark, results, scores, db'); + const availableCommands = ['benchmark', 'results', 'scores']; + if (isAdmin) { + availableCommands.push('save', 'db'); + } + logger.info(`Available commands: ${availableCommands.join(', ')}`); process.exit(1); } } else { + // Build options based on admin access + const options = [ + { + value: 'benchmark', + label: 'Run a benchmark', + hint: 'Run a performance benchmark on a URL', + }, + { + value: 'results', + label: 'Results', + hint: 'View detailed benchmark results', + }, + ]; + + // Add admin-only commands + if (isAdmin) { + options.push({ + value: 'save', + label: 'Save to database', + hint: '🔒 Admin: Sync benchmark results to database', + }); + options.push({ + value: 'db', + label: 'Database', + hint: '🔒 Admin: Manage database schema and migrations', + }); + } + const selectedCommand = await p.select({ message: 'What would you like to do?', - options: [ - { - value: 'benchmark', - label: 'Run a benchmark', - hint: 'Run a performance benchmark on a URL', - }, - { - value: 'results', - label: 'Results', - hint: 'Combine and display benchmark results', - }, - { - value: 'scores', - label: 'View scores', - hint: 'View scores from existing benchmark results', - }, - { - value: 'db', - label: 'Database', - hint: 'Manage database schema and migrations', - }, - ], + options, }); if (p.isCancel(selectedCommand)) { @@ -90,8 +116,8 @@ async function main() { case 'results': await resultsCommand(logger); break; - case 'scores': - await scoresCommand(logger); + case 'save': + await saveCommand(logger); break; case 'db': await dbCommand(logger); diff --git a/packages/cookiebench-cli/src/utils/auth.ts b/packages/cookiebench-cli/src/utils/auth.ts new file mode 100644 index 0000000..c458910 --- /dev/null +++ b/packages/cookiebench-cli/src/utils/auth.ts @@ -0,0 +1,12 @@ +/** + * Check if the user has admin access for database operations + * This is gated by an admin flag to prevent unauthorized database writes + */ +export function isAdminUser(): boolean { + // Check for admin flag in environment + const adminFlag = process.env.CONSENT_ADMIN; + + // Accept 'true', '1', 'yes' as valid values + return adminFlag === 'true' || adminFlag === '1' || adminFlag === 'yes'; +} + diff --git a/packages/cookiebench-cli/src/utils/index.ts b/packages/cookiebench-cli/src/utils/index.ts index d3abec4..72fb575 100644 --- a/packages/cookiebench-cli/src/utils/index.ts +++ b/packages/cookiebench-cli/src/utils/index.ts @@ -4,6 +4,7 @@ import type { Config } from "../types"; export * from './logger'; export * from './scoring'; +export * from './auth'; export function readConfig(configName = "config"): Config | null { try { From 3f15183016a5631a7568c5ba31e0ab484dcb48f2 Mon Sep 17 00:00:00 2001 From: burnedchris Date: Wed, 29 Oct 2025 22:58:03 -0700 Subject: [PATCH 04/21] Update benchmark configurations and dependencies across multiple projects. Refactor TypeScript settings, enhance package.json scripts, and improve logging utilities. Introduce new constants for better code maintainability and streamline project structure. Update documentation to reflect recent changes and ensure consistency in coding standards. --- .cursor/rules/ultracite.mdc | 227 + benchmarks/baseline/app/layout.tsx | 6 +- benchmarks/baseline/config.json | 56 +- benchmarks/baseline/next-env.d.ts | 1 + benchmarks/baseline/next.config.ts | 2 +- benchmarks/baseline/package.json | 49 +- benchmarks/baseline/tsconfig.json | 18 +- benchmarks/with-c15t-nextjs/app/layout.tsx | 12 +- benchmarks/with-c15t-nextjs/config.json | 92 +- benchmarks/with-c15t-nextjs/next-env.d.ts | 1 + benchmarks/with-c15t-nextjs/next.config.ts | 2 +- benchmarks/with-c15t-nextjs/package.json | 51 +- benchmarks/with-c15t-nextjs/tsconfig.json | 18 +- benchmarks/with-c15t-react/app/layout.tsx | 12 +- benchmarks/with-c15t-react/config.json | 92 +- benchmarks/with-c15t-react/next.config.ts | 2 +- benchmarks/with-c15t-react/package.json | 51 +- benchmarks/with-c15t-react/tsconfig.json | 18 +- benchmarks/with-cookie-control/app/layout.tsx | 6 +- benchmarks/with-cookie-control/config.json | 76 +- benchmarks/with-cookie-control/next.config.ts | 2 +- benchmarks/with-cookie-control/package.json | 47 +- benchmarks/with-cookie-control/tsconfig.json | 18 +- benchmarks/with-cookie-yes/app/layout.tsx | 6 +- benchmarks/with-cookie-yes/config.json | 84 +- benchmarks/with-cookie-yes/next.config.ts | 2 +- benchmarks/with-cookie-yes/package.json | 47 +- benchmarks/with-cookie-yes/tsconfig.json | 18 +- benchmarks/with-didomi/config.json | 94 +- benchmarks/with-didomi/next.config.ts | 2 +- benchmarks/with-didomi/package.json | 47 +- benchmarks/with-didomi/tsconfig.json | 18 +- benchmarks/with-enzuzo/app/layout.tsx | 8 +- benchmarks/with-enzuzo/config.json | 76 +- benchmarks/with-enzuzo/next.config.ts | 2 +- benchmarks/with-enzuzo/package.json | 47 +- benchmarks/with-enzuzo/tsconfig.json | 18 +- benchmarks/with-iubenda/app/layout.tsx | 6 +- benchmarks/with-iubenda/config.json | 86 +- benchmarks/with-iubenda/next.config.ts | 2 +- benchmarks/with-iubenda/package.json | 47 +- benchmarks/with-iubenda/tsconfig.json | 18 +- benchmarks/with-ketch/app/layout.tsx | 6 +- benchmarks/with-ketch/config.json | 74 +- benchmarks/with-ketch/next.config.ts | 2 +- benchmarks/with-ketch/package.json | 45 +- benchmarks/with-ketch/tsconfig.json | 18 +- benchmarks/with-onetrust/app/layout.tsx | 8 +- benchmarks/with-onetrust/config.json | 88 +- benchmarks/with-onetrust/next.config.ts | 2 +- benchmarks/with-onetrust/package.json | 47 +- benchmarks/with-onetrust/tsconfig.json | 18 +- benchmarks/with-osano/app/layout.tsx | 6 +- benchmarks/with-osano/config.json | 76 +- benchmarks/with-osano/next.config.ts | 2 +- benchmarks/with-osano/package.json | 47 +- benchmarks/with-osano/tsconfig.json | 18 +- benchmarks/with-usercentrics/app/layout.tsx | 6 +- benchmarks/with-usercentrics/config.json | 76 +- benchmarks/with-usercentrics/next.config.ts | 2 +- benchmarks/with-usercentrics/package.json | 47 +- benchmarks/with-usercentrics/tsconfig.json | 18 +- biome.jsonc | 112 +- package.json | 19 +- packages/benchmark/package.json | 62 +- packages/benchmark/rslib.config.ts | 9 +- packages/benchmark/src/bundle-strategy.ts | 5 +- packages/benchmark/src/constants.ts | 18 +- .../benchmark/src/cookie-banner-collector.ts | 328 +- packages/benchmark/src/index.ts | 29 +- packages/benchmark/src/network-monitor.ts | 39 +- packages/benchmark/src/perfume-collector.ts | 50 +- .../src/resource-timing-collector.ts | 92 +- packages/benchmark/src/types.ts | 43 +- packages/benchmark/tsconfig.json | 27 +- packages/cookiebench-cli/README.md | 4 +- packages/cookiebench-cli/base.json | 34 +- packages/cookiebench-cli/package.json | 72 +- packages/cookiebench-cli/rslib.config.ts | 12 +- .../cookiebench-cli/src/commands/benchmark.ts | 465 +- packages/cookiebench-cli/src/commands/db.ts | 247 +- .../cookiebench-cli/src/commands/results.ts | 2008 ++++--- packages/cookiebench-cli/src/commands/save.ts | 303 +- .../cookiebench-cli/src/commands/scores.ts | 123 +- .../cookiebench-cli/src/components/intro.ts | 32 +- packages/cookiebench-cli/src/index.ts | 93 +- packages/cookiebench-cli/src/types/index.ts | 17 +- packages/cookiebench-cli/src/utils/auth.ts | 5 +- .../cookiebench-cli/src/utils/constants.ts | 34 + packages/cookiebench-cli/src/utils/index.ts | 93 +- packages/cookiebench-cli/src/utils/logger.ts | 80 +- packages/cookiebench-cli/src/utils/scoring.ts | 765 ++- packages/cookiebench-cli/tsconfig.json | 26 +- packages/runner/package.json | 62 +- packages/runner/rslib.config.ts | 9 +- packages/runner/src/benchmark-runner.ts | 75 +- packages/runner/src/index.ts | 32 +- packages/runner/src/performance-aggregator.ts | 209 +- packages/runner/src/server.ts | 40 +- packages/runner/src/types.ts | 43 +- packages/runner/src/utils.ts | 41 +- packages/runner/tsconfig.json | 27 +- pnpm-lock.yaml | 5097 ++++++++--------- tsconfig.json | 20 +- turbo.json | 63 +- 105 files changed, 6682 insertions(+), 6375 deletions(-) create mode 100644 .cursor/rules/ultracite.mdc create mode 100644 packages/cookiebench-cli/src/utils/constants.ts diff --git a/.cursor/rules/ultracite.mdc b/.cursor/rules/ultracite.mdc new file mode 100644 index 0000000..0b3bf93 --- /dev/null +++ b/.cursor/rules/ultracite.mdc @@ -0,0 +1,227 @@ +--- +description: Ultracite Rules - AI-Ready Formatter and Linter +globs: "**/*.{ts,tsx,js,jsx,json,jsonc,html,vue,svelte,astro,css,yaml,yml,graphql,gql,md,mdx,grit}" +alwaysApply: false +--- + +Avoid `accessKey` attr and distracting els +No `aria-hidden="true"` on focusable els +No ARIA roles, states, props on unsupported els +Use `scope` prop only on `` els +No non-interactive ARIA roles on interactive els +Label els need text and associated input +No event handlers on non-interactive els +No interactive ARIA roles on non-interactive els +No `tabIndex` on non-interactive els +No positive integers on `tabIndex` prop +No `image`, `picture`, or `photo` in img alt props +No explicit role matching implicit role +Valid role attrs on static, visible els w/ click handlers +Use `title` el for `svg` els +Provide meaningful alt text for all els requiring it +Anchors need accessible content +Assign `tabIndex` to non-interactive els w/ `aria-activedescendant` +Include all required ARIA attrs for els w/ ARIA roles +Use valid ARIA props for the el's role +Use `type` attr on `button` els +Make els w/ interactive roles and handlers focusable +Heading els need accessible content +Add `lang` attr to `html` el +Use `title` attr on `iframe` els +Pair `onClick` w/ `onKeyUp`, `onKeyDown`, or `onKeyPress` +Pair `onMouseOver`/`onMouseOut` w/ `onFocus`/`onBlur` +Add caption tracks to audio and video els +Use semantic els vs role attrs +All anchors must be valid and navigable +Use valid, non-abstract ARIA props, roles, states, and values +Use valid values for `autocomplete` attr +Use correct ISO language codes in `lang` attr +Include generic font family in font families +No consecutive spaces in regex literals +Avoid `arguments`, comma op, and primitive type aliases +No empty type params in type aliases and interfaces +Keep fns under Cognitive Complexity limit +Limit nesting depth of `describe()` in tests +No unnecessary boolean casts or callbacks on `flatMap` +Use `for...of` vs `Array.forEach` +No classes w/ only static members +No `this` and `super` in static contexts +No unnecessary catch clauses, ctors, `continue`, escape sequences in regex literals, fragments, labels, or nested blocks +No empty exports +No renaming imports, exports, or destructured assignments to same name +No unnecessary string/template literal concatenation or useless cases in switch stmts, `this` aliasing, or `String.raw` without escape sequences +Use simpler alternatives to ternary ops if possible +No `any` or `unknown` as type constraints or initializing vars to `undefined` +Avoid `void` op +Use arrow fns vs function exprs +Use `Date.now()` for milliseconds since Unix Epoch +Use `.flatMap()` vs `map().flat()` +Use `indexOf`/`lastIndexOf` vs `findIndex`/`findLastIndex` for simple lookups +Use literal property access vs computed property access +Use binary, octal, or hex literals vs `parseInt()` +Use concise optional chains vs chained logical exprs +Use regex literals vs `RegExp` ctor +Use base 10 or underscore separators for number literal object member names +Remove redundant terms from logical exprs +Use `while` loops vs `for` loops if initializer and update aren't needed +No reassigning `const` vars or constant exprs in conditions +No `Math.min`/`Math.max` to clamp values where result is constant +No return values from ctors or setters +No empty character classes in regex literals or destructuring patterns +No `__dirname` and `__filename` in global scope +No calling global object props as fns or declaring fns and `var` accessible outside their block +Instantiate builtins correctly +Use `super()` correctly in classes +Use standard direction values for linear gradient fns +Use valid named grid areas in CSS Grid Layouts +Use `@import` at-rules in valid positions +No vars and params before their decl +Include `var` fn for CSS vars +No `\8` and `\9` escape sequences in strings +No literal numbers that lose precision, configured els, or assigning where both sides are same +Compare string case modifications w/ compliant values +No lexical decls in switch clauses or undeclared vars +No unknown CSS value fns, media feature names, props, pseudo-class/pseudo-element selectors, type selectors, or units +No unmatchable An+B selectors or unreachable code +Call `super()` exactly once before accessing `this` in ctors +No control flow stmts in `finally` blocks +No optional chaining where `undefined` is not allowed +No unused fn params, imports, labels, private class members, or vars +No return values from fns w/ return type `void` +Specify all dependencies correctly in React hooks and names for GraphQL operations +Call React hooks from top level of component fns +Use `isNaN()` when checking for NaN +Use `{ type: "json" }` for JSON module imports +Use radix arg w/ `parseInt()` +Start JSDoc comment lines w/ single asterisk +Move `for` loop counters in right direction +Compare `typeof` exprs to valid values +Include `yield` in generator fns +No importing deprecated exports, duplicate dependencies, or Promises where they're likely a mistake +No non-null assertions after optional chaining or shadowing vars from outer scope +No expr stmts that aren't fn calls or assignments or useless `undefined` +Add `href` attr to `` els and `width`/`height` attrs to `` els +Use consistent arrow fn bodies and either `interface` or `type` consistently +Specify deletion date w/ `@deprecated` directive +Make switch-case stmts exhaustive and limit number of fn params +Sort CSS utility classes +No spread syntax on accumulators, barrel files, `delete` op, dynamic namespace import access, namespace imports, or duplicate polyfills from Polyfill.io +Use `preconnect` attr w/ Google Fonts +Declare regex literals at top level +Add `rel="noopener"` when using `target="_blank"` +No dangerous JSX props +No both `children` and `dangerouslySetInnerHTML` props +No global `eval()` +No callbacks in async tests and hooks, TS enums, exporting imported vars, type annotations for vars initialized w/ literals, magic numbers without named constants, or TS namespaces +No negating `if` conditions when there's an `else` clause, nested ternary exprs, non-null assertions (`!`), reassigning fn params, parameter props in class ctors, specified global var names, importing specified modules, or specified user-defined types +No constants where value is upper-case version of name, template literals without interpolation or special chars, `else` blocks when `if` block breaks early, yoda exprs, or `Array` ctors +Use `String.slice()` vs `String.substr()` and `String.substring()` +Use `as const` vs literal type annotations and `at()` vs integer index access +Follow curly brace conventions +Use `else if` vs nested `if` in `else` clauses and single `if` vs nested `if` clauses +Use `T[]` vs `Array` +Use `new` for all builtins except `String`, `Number`, and `Boolean` +Use consistent accessibility modifiers on class props and methods +Declare object literals consistently +Use `const` for vars only assigned once +Put default and optional fn params last +Include `default` clause in switch stmts +Specify reason arg w/ `@deprecated` directive +Explicitly initialize each enum member value +Use `**` op vs `Math.pow` +Use `export type` and `import type` for types +Use kebab-case, ASCII filenames +Use `for...of` vs `for` loops w/ array index access +Use `<>...` vs `...` +Capitalize all enum values +Place getters and setters for same prop adjacent +Use literal values for all enum members +Use `node:assert/strict` vs `node:assert` +Use `node:` protocol for Node.js builtin modules +Use `Number` props vs global ones +Use numeric separators in numeric literals +Use object spread vs `Object.assign()` for new objects +Mark members `readonly` if never modified outside ctor +No extra closing tags for comps without children +Use assignment op shorthand +Use fn types vs object types w/ call signatures +Add description param to `Symbol()` +Use template literals vs string concatenation +Use `new` when throwing an error +No throwing non-`Error` values +Use `String.trimStart()`/`String.trimEnd()` vs `String.trimLeft()`/`String.trimRight()` +No overload signatures that can be unified +No lower specificity selectors after higher specificity selectors +No `@value` rule in CSS modules +No `alert`, `confirm`, and `prompt` +Use standard constants vs approximated literals +No assigning in exprs +No async fns as Promise executors +No `!` pattern in first position of `files.includes` +No bitwise ops +No reassigning exceptions in catch clauses +No reassigning class members +No inserting comments as text nodes +No comparing against `-0` +No labeled stmts that aren't loops +No `void` type outside generic or return types +No `console` +No TS const enums +No exprs where op doesn't affect value +No control chars in regex literals +No `debugger` +No assigning directly to `document.cookie` +Use `===` and `!==` +No duplicate `@import` rules, case labels, class members, custom props, conditions in if-else-if chains, GraphQL fields, font family names, object keys, fn param names, decl block props, keyframe selectors, or describe hooks +No empty CSS blocks, block stmts, static blocks, or interfaces +No letting vars evolve into `any` type through reassignments +No `any` type +No `export` or `module.exports` in test files +No misusing non-null assertion op (`!`) +No fallthrough in switch clauses +No focused or disabled tests +No reassigning fn decls +No assigning to native objects and read-only global vars +Use `Number.isFinite` and `Number.isNaN` vs global `isFinite` and `isNaN` +No implicit `any` type on var decls +No assigning to imported bindings +No `!important` within keyframe decls +No irregular whitespace chars +No labels that share name w/ var +No chars made w/ multiple code points in char classes +Use `new` and `constructor` properly +Place assertion fns inside `it()` fn calls +No shorthand assign when var appears on both sides +No octal escape sequences in strings +No `Object.prototype` builtins directly +No `quickfix.biome` in editor settings +No redeclaring vars, fns, classes, and types in same scope +No redundant `use strict` +No comparing where both sides are same +No shadowing restricted names +No shorthand props that override related longhand props +No sparse arrays +No template literal placeholder syntax in regular strings +No `then` prop +No `@ts-ignore` directive +No `let` or `var` vars that are read but never assigned +No unknown at-rules +No merging interface and class decls unsafely +No unsafe negation (`!`) +No unnecessary escapes in strings or useless backreferences in regex literals +No `var` +No `with` stmts +No separating overload signatures +Use `await` in async fns +Use correct syntax for ignoring folders in config +Put default clauses in switch stmts last +Pass message value when creating built-in errors +Return value from get methods +Use recommended display strategy w/ Google Fonts +Include `if` stmt in for-in loops +Use `Array.isArray()` vs `instanceof Array` +Return consistent values in iterable callbacks +Use `namespace` keyword vs `module` keyword +Use digits arg w/ `Number#toFixed()` +Use static `Response` methods vs `new Response()` +Use `use strict` directive in script files \ No newline at end of file diff --git a/benchmarks/baseline/app/layout.tsx b/benchmarks/baseline/app/layout.tsx index 9ceb42c..92f2d50 100644 --- a/benchmarks/baseline/app/layout.tsx +++ b/benchmarks/baseline/app/layout.tsx @@ -1,14 +1,14 @@ -import type { Metadata } from 'next'; +import type { Metadata } from "next"; /* * If you're using Next.js, we recommend installing the @c15t/nextjs package. * The Next.js package is a wrapper around the React package that provides * additional features for Next.js. */ -import type { ReactNode } from 'react'; +import type { ReactNode } from "react"; export const metadata: Metadata = { - title: 'benchmark', + title: "benchmark", }; export default function RootLayout({ diff --git a/benchmarks/baseline/config.json b/benchmarks/baseline/config.json index c7f541e..271d784 100644 --- a/benchmarks/baseline/config.json +++ b/benchmarks/baseline/config.json @@ -1,30 +1,30 @@ { - "$schema": "./node_modules/@cookiebench/benchmark-schema/schema.json", - "name": "baseline", - "iterations": 1, - "baseline": true, - "cookieBanner": { - "selectors": [], - "serviceHosts": [], - "waitForVisibility": false, - "measureViewportCoverage": false, - "expectedLayoutShift": false, - "serviceName": "None (Baseline)" - }, - "techStack": { - "bundler": "webpack", - "bundleType": "esm", - "frameworks": ["nextjs"], - "languages": ["javascript"], - "packageManager": "npm", - "typescript": false - }, - "source": { - "isOpenSource": true, - "license": "MIT" - }, - "includes": { - "components": ["javascript"] - }, - "tags": ["baseline"] + "$schema": "./node_modules/@cookiebench/benchmark-schema/schema.json", + "name": "baseline", + "iterations": 1, + "baseline": true, + "cookieBanner": { + "selectors": [], + "serviceHosts": [], + "waitForVisibility": false, + "measureViewportCoverage": false, + "expectedLayoutShift": false, + "serviceName": "None (Baseline)" + }, + "techStack": { + "bundler": "webpack", + "bundleType": "esm", + "frameworks": ["nextjs"], + "languages": ["javascript"], + "packageManager": "npm", + "typescript": false + }, + "source": { + "isOpenSource": true, + "license": "MIT" + }, + "includes": { + "components": ["javascript"] + }, + "tags": ["baseline"] } diff --git a/benchmarks/baseline/next-env.d.ts b/benchmarks/baseline/next-env.d.ts index 1b3be08..9edff1c 100644 --- a/benchmarks/baseline/next-env.d.ts +++ b/benchmarks/baseline/next-env.d.ts @@ -1,5 +1,6 @@ /// /// +import "./.next/types/routes.d.ts"; // NOTE: This file should not be edited // see https://nextjs.org/docs/app/api-reference/config/typescript for more information. diff --git a/benchmarks/baseline/next.config.ts b/benchmarks/baseline/next.config.ts index a67a28b..7921f35 100644 --- a/benchmarks/baseline/next.config.ts +++ b/benchmarks/baseline/next.config.ts @@ -1,4 +1,4 @@ -import type { NextConfig } from 'next'; +import type { NextConfig } from "next"; const nextConfig: NextConfig = { /* config options here */ diff --git a/benchmarks/baseline/package.json b/benchmarks/baseline/package.json index 7ab3910..d9888b3 100644 --- a/benchmarks/baseline/package.json +++ b/benchmarks/baseline/package.json @@ -1,27 +1,26 @@ { - "name": "benchmark-baseline", - "version": "0.1.0", - "private": true, - "scripts": { - "benchmark": "pnpm exec cookiebench benchmark", - "build": "next build --turbopack", - "dev": "next dev --port 3000 --turbopack", - "fmt": "pnpm biome format --write . && pnpm biome check --formatter-enabled=false --linter-enabled=false --organize-imports-enabled=true --write", - "lint": "next lint", - "start": "next start" - }, - "dependencies": { - "next": "15.3.3", - "react": "^19.0.0", - "react-dom": "^19.0.0" - }, - "devDependencies": { - "@cookiebench/benchmark-schema": "workspace:*", - "@cookiebench/ts-config": "workspace:*", - "@types/node": "^22.15.30", - "@types/react": "^19.1.6", - "@types/react-dom": "^19.1.6", - "cookiebench": "workspace:*", - "typescript": "^5.8.3" - } + "name": "benchmark-baseline", + "version": "0.1.0", + "private": true, + "scripts": { + "build": "next build", + "dev": "next dev --port 3000", + "fmt": "biome format . --write", + "lint": "biome lint .", + "start": "next start" + }, + "dependencies": { + "next": "16.0.1", + "react": "^19.2.0", + "react-dom": "^19.2.0" + }, + "devDependencies": { + "@cookiebench/benchmark-schema": "workspace:*", + "@cookiebench/ts-config": "workspace:*", + "@types/node": "^24.9.2", + "@types/react": "^19.2.2", + "@types/react-dom": "^19.2.2", + "cookiebench": "workspace:*", + "typescript": "^5.9.3" + } } diff --git a/benchmarks/baseline/tsconfig.json b/benchmarks/baseline/tsconfig.json index 99f37b5..787e3f6 100644 --- a/benchmarks/baseline/tsconfig.json +++ b/benchmarks/baseline/tsconfig.json @@ -1,11 +1,11 @@ { - "extends": "@cookiebench/ts-config/nextjs.json", - "compilerOptions": { - "baseUrl": ".", - "paths": { - "@/*": ["./*"] - } - }, - "include": ["next-env.d.ts", ".next/types/**/*.ts", "**/*.ts", "**/*.tsx"], - "exclude": ["node_modules"] + "extends": "@cookiebench/ts-config/nextjs.json", + "compilerOptions": { + "baseUrl": ".", + "paths": { + "@/*": ["./*"] + } + }, + "include": ["next-env.d.ts", ".next/types/**/*.ts", "**/*.ts", "**/*.tsx"], + "exclude": ["node_modules"] } diff --git a/benchmarks/with-c15t-nextjs/app/layout.tsx b/benchmarks/with-c15t-nextjs/app/layout.tsx index ee8b713..03438c4 100644 --- a/benchmarks/with-c15t-nextjs/app/layout.tsx +++ b/benchmarks/with-c15t-nextjs/app/layout.tsx @@ -1,4 +1,4 @@ -import type { Metadata } from 'next'; +import type { Metadata } from "next"; /* * If you're using Next.js, we recommend installing the @c15t/nextjs package. @@ -10,11 +10,11 @@ import { ConsentManagerDialog, ConsentManagerProvider, CookieBanner, -} from '@c15t/nextjs'; -import type { ReactNode } from 'react'; +} from "@c15t/nextjs"; +import type { ReactNode } from "react"; export const metadata: Metadata = { - title: 'benchmark', + title: "benchmark", }; export default function RootLayout({ @@ -27,8 +27,8 @@ export default function RootLayout({ diff --git a/benchmarks/with-c15t-nextjs/config.json b/benchmarks/with-c15t-nextjs/config.json index 80a6f08..9aabb6b 100644 --- a/benchmarks/with-c15t-nextjs/config.json +++ b/benchmarks/with-c15t-nextjs/config.json @@ -1,48 +1,48 @@ { - "$schema": "./node_modules/@cookiebench/benchmark-schema/schema.json", - "name": "with-c15t-nextjs", - "iterations": 20, - "cookieBanner": { - "selectors": [ - "[data-testid=\"cookie-banner-root\"]", - ".c15t-banner", - "[data-c15t-banner]", - ".consent-banner", - "[data-consent-banner]" - ], - "serviceHosts": ["c15t.com", "consent.io"], - "waitForVisibility": true, - "measureViewportCoverage": true, - "expectedLayoutShift": false, - "serviceName": "C15T Next.js" - }, - "techStack": { - "bundler": "rslib", - "bundleType": ["esm", "cjs"], - "frameworks": ["react", "nextjs"], - "languages": ["typescript", "javascript"], - "packageManager": "pnpm", - "typescript": true - }, - "internationalization": { - "detection": "browser", - "stringLoading": "server" - }, - "source": { - "github": "github.com/c15t/c15t", - "isOpenSource": true, - "license": "GPL-3.0-only", - "npm": "@c15t/nextjs", - "website": "https://www.c15t.com" - }, - "includes": { - "backend": ["nodejs", "typescript"], - "components": ["react", "javascript"] - }, - "company": { - "name": "c15t", - "website": "https://c15t.com", - "avatar": "https://zxlypdluowixfd7j.public.blob.vercel-storage.com/c15t-icon-z6gQxO0ogxWgY51dluWPhNfAoPyELT.png" - }, - "tags": ["c15t"] + "$schema": "./node_modules/@cookiebench/benchmark-schema/schema.json", + "name": "with-c15t-nextjs", + "iterations": 20, + "cookieBanner": { + "selectors": [ + "[data-testid=\"cookie-banner-root\"]", + ".c15t-banner", + "[data-c15t-banner]", + ".consent-banner", + "[data-consent-banner]" + ], + "serviceHosts": ["c15t.com", "consent.io"], + "waitForVisibility": true, + "measureViewportCoverage": true, + "expectedLayoutShift": false, + "serviceName": "C15T Next.js" + }, + "techStack": { + "bundler": "rslib", + "bundleType": ["esm", "cjs"], + "frameworks": ["react", "nextjs"], + "languages": ["typescript", "javascript"], + "packageManager": "pnpm", + "typescript": true + }, + "internationalization": { + "detection": "browser", + "stringLoading": "server" + }, + "source": { + "github": "github.com/c15t/c15t", + "isOpenSource": true, + "license": "GPL-3.0-only", + "npm": "@c15t/nextjs", + "website": "https://www.c15t.com" + }, + "includes": { + "backend": ["nodejs", "typescript"], + "components": ["react", "javascript"] + }, + "company": { + "name": "c15t", + "website": "https://c15t.com", + "avatar": "https://zxlypdluowixfd7j.public.blob.vercel-storage.com/c15t-icon-z6gQxO0ogxWgY51dluWPhNfAoPyELT.png" + }, + "tags": ["c15t"] } diff --git a/benchmarks/with-c15t-nextjs/next-env.d.ts b/benchmarks/with-c15t-nextjs/next-env.d.ts index 1b3be08..9edff1c 100644 --- a/benchmarks/with-c15t-nextjs/next-env.d.ts +++ b/benchmarks/with-c15t-nextjs/next-env.d.ts @@ -1,5 +1,6 @@ /// /// +import "./.next/types/routes.d.ts"; // NOTE: This file should not be edited // see https://nextjs.org/docs/app/api-reference/config/typescript for more information. diff --git a/benchmarks/with-c15t-nextjs/next.config.ts b/benchmarks/with-c15t-nextjs/next.config.ts index a67a28b..7921f35 100644 --- a/benchmarks/with-c15t-nextjs/next.config.ts +++ b/benchmarks/with-c15t-nextjs/next.config.ts @@ -1,4 +1,4 @@ -import type { NextConfig } from 'next'; +import type { NextConfig } from "next"; const nextConfig: NextConfig = { /* config options here */ diff --git a/benchmarks/with-c15t-nextjs/package.json b/benchmarks/with-c15t-nextjs/package.json index bc46b0d..98557a9 100644 --- a/benchmarks/with-c15t-nextjs/package.json +++ b/benchmarks/with-c15t-nextjs/package.json @@ -1,28 +1,27 @@ { - "name": "with-c15t-nextjs", - "version": "0.1.0", - "private": true, - "scripts": { - "benchmark": "pnpm exec cookiebench benchmark", - "build": "next build --turbopack", - "dev": "next dev --port 3001 --turbopack", - "fmt": "pnpm biome format --write . && pnpm biome check --formatter-enabled=false --linter-enabled=false --organize-imports-enabled=true --write", - "lint": "next lint", - "start": "next start" - }, - "dependencies": { - "@c15t/nextjs": "1.2.2-canary-20250603153501", - "next": "15.3.3", - "react": "^19.0.0", - "react-dom": "^19.0.0" - }, - "devDependencies": { - "@cookiebench/benchmark-schema": "workspace:*", - "@cookiebench/ts-config": "workspace:*", - "@types/node": "^22.15.30", - "@types/react": "^19.1.6", - "@types/react-dom": "^19.1.6", - "cookiebench": "workspace:*", - "typescript": "^5.8.3" - } + "name": "with-c15t-nextjs", + "version": "0.1.0", + "private": true, + "scripts": { + "build": "next build", + "dev": "next dev --port 3001", + "fmt": "biome format . --write", + "lint": "biome lint .", + "start": "next start" + }, + "dependencies": { + "@c15t/nextjs": "1.7.1", + "next": "16.0.1", + "react": "^19.2.0", + "react-dom": "^19.2.0" + }, + "devDependencies": { + "@cookiebench/benchmark-schema": "workspace:*", + "@cookiebench/ts-config": "workspace:*", + "@types/node": "^24.9.2", + "@types/react": "^19.2.2", + "@types/react-dom": "^19.2.2", + "cookiebench": "workspace:*", + "typescript": "^5.9.3" + } } diff --git a/benchmarks/with-c15t-nextjs/tsconfig.json b/benchmarks/with-c15t-nextjs/tsconfig.json index 99f37b5..787e3f6 100644 --- a/benchmarks/with-c15t-nextjs/tsconfig.json +++ b/benchmarks/with-c15t-nextjs/tsconfig.json @@ -1,11 +1,11 @@ { - "extends": "@cookiebench/ts-config/nextjs.json", - "compilerOptions": { - "baseUrl": ".", - "paths": { - "@/*": ["./*"] - } - }, - "include": ["next-env.d.ts", ".next/types/**/*.ts", "**/*.ts", "**/*.tsx"], - "exclude": ["node_modules"] + "extends": "@cookiebench/ts-config/nextjs.json", + "compilerOptions": { + "baseUrl": ".", + "paths": { + "@/*": ["./*"] + } + }, + "include": ["next-env.d.ts", ".next/types/**/*.ts", "**/*.ts", "**/*.tsx"], + "exclude": ["node_modules"] } diff --git a/benchmarks/with-c15t-react/app/layout.tsx b/benchmarks/with-c15t-react/app/layout.tsx index 4c8297d..0d80c78 100644 --- a/benchmarks/with-c15t-react/app/layout.tsx +++ b/benchmarks/with-c15t-react/app/layout.tsx @@ -1,4 +1,4 @@ -import type { Metadata } from 'next'; +import type { Metadata } from "next"; /* * If you're using Next.js, we recommend installing the @c15t/nextjs package. @@ -10,11 +10,11 @@ import { ConsentManagerDialog, ConsentManagerProvider, CookieBanner, -} from '@c15t/react'; -import type { ReactNode } from 'react'; +} from "@c15t/react"; +import type { ReactNode } from "react"; export const metadata: Metadata = { - title: 'benchmark', + title: "benchmark", }; export default function RootLayout({ @@ -27,8 +27,8 @@ export default function RootLayout({ diff --git a/benchmarks/with-c15t-react/config.json b/benchmarks/with-c15t-react/config.json index 9d524b4..b174fc7 100644 --- a/benchmarks/with-c15t-react/config.json +++ b/benchmarks/with-c15t-react/config.json @@ -1,48 +1,48 @@ { - "$schema": "./node_modules/@cookiebench/benchmark-schema/schema.json", - "name": "with-c15t-react", - "iterations": 20, - "cookieBanner": { - "selectors": [ - "[data-testid=\"cookie-banner-root\"]", - ".c15t-banner", - "[data-c15t-banner]", - ".consent-banner", - "[data-consent-banner]" - ], - "serviceHosts": ["c15t.com", "consent.io"], - "waitForVisibility": true, - "measureViewportCoverage": true, - "expectedLayoutShift": false, - "serviceName": "C15T React" - }, - "techStack": { - "bundler": "rslib", - "bundleType": ["esm", "cjs"], - "frameworks": ["react"], - "languages": ["typescript", "javascript"], - "packageManager": "pnpm", - "typescript": true - }, - "source": { - "github": "github.com/c15t/c15t", - "isOpenSource": true, - "license": "GPL-3.0-only", - "npm": "@c15t/nextjs", - "website": "https://www.c15t.com" - }, - "includes": { - "backend": ["nodejs", "typescript"], - "components": ["react", "javascript"] - }, - "internationalization": { - "detection": "browser", - "stringLoading": "server" - }, - "company": { - "name": "c15t", - "website": "https://c15t.com", - "avatar": "https://zxlypdluowixfd7j.public.blob.vercel-storage.com/c15t-icon-z6gQxO0ogxWgY51dluWPhNfAoPyELT.png" - }, - "tags": ["c15t"] + "$schema": "./node_modules/@cookiebench/benchmark-schema/schema.json", + "name": "with-c15t-react", + "iterations": 20, + "cookieBanner": { + "selectors": [ + "[data-testid=\"cookie-banner-root\"]", + ".c15t-banner", + "[data-c15t-banner]", + ".consent-banner", + "[data-consent-banner]" + ], + "serviceHosts": ["c15t.com", "consent.io"], + "waitForVisibility": true, + "measureViewportCoverage": true, + "expectedLayoutShift": false, + "serviceName": "C15T React" + }, + "techStack": { + "bundler": "rslib", + "bundleType": ["esm", "cjs"], + "frameworks": ["react"], + "languages": ["typescript", "javascript"], + "packageManager": "pnpm", + "typescript": true + }, + "source": { + "github": "github.com/c15t/c15t", + "isOpenSource": true, + "license": "GPL-3.0-only", + "npm": "@c15t/nextjs", + "website": "https://www.c15t.com" + }, + "includes": { + "backend": ["nodejs", "typescript"], + "components": ["react", "javascript"] + }, + "internationalization": { + "detection": "browser", + "stringLoading": "server" + }, + "company": { + "name": "c15t", + "website": "https://c15t.com", + "avatar": "https://zxlypdluowixfd7j.public.blob.vercel-storage.com/c15t-icon-z6gQxO0ogxWgY51dluWPhNfAoPyELT.png" + }, + "tags": ["c15t"] } diff --git a/benchmarks/with-c15t-react/next.config.ts b/benchmarks/with-c15t-react/next.config.ts index a67a28b..7921f35 100644 --- a/benchmarks/with-c15t-react/next.config.ts +++ b/benchmarks/with-c15t-react/next.config.ts @@ -1,4 +1,4 @@ -import type { NextConfig } from 'next'; +import type { NextConfig } from "next"; const nextConfig: NextConfig = { /* config options here */ diff --git a/benchmarks/with-c15t-react/package.json b/benchmarks/with-c15t-react/package.json index f31d5b6..9805f5e 100644 --- a/benchmarks/with-c15t-react/package.json +++ b/benchmarks/with-c15t-react/package.json @@ -1,28 +1,27 @@ { - "name": "with-c15t-react", - "private": true, - "scripts": { - "benchmark": "pnpm exec cookiebench benchmark", - "build": "next build --turbopack", - "dev": "next dev --port 3003 --turbopack", - "fmt": "pnpm biome format --write . && pnpm biome check --formatter-enabled=false --linter-enabled=false --organize-imports-enabled=true --write", - "lint": "next lint", - "start": "next start --port 3003" - }, - "dependencies": { - "@c15t/react": "1.2.2-canary-20250603153501", - "@c15t/translations": "^1.0.0", - "next": "15.3.3", - "react": "^19.0.0", - "react-dom": "^19.0.0" - }, - "devDependencies": { - "@cookiebench/benchmark-schema": "workspace:*", - "@cookiebench/ts-config": "workspace:*", - "@types/node": "^22.15.30", - "@types/react": "^19.1.6", - "@types/react-dom": "^19.1.6", - "cookiebench": "workspace:*", - "typescript": "^5.8.3" - } + "name": "with-c15t-react", + "private": true, + "scripts": { + "build": "next build", + "dev": "next dev --port 3003", + "fmt": "biome format . --write", + "lint": "biome lint .", + "start": "next start --port 3003" + }, + "dependencies": { + "@c15t/react": "1.7.1", + "@c15t/translations": "^1.7.0", + "next": "16.0.1", + "react": "^19.2.0", + "react-dom": "^19.2.0" + }, + "devDependencies": { + "@cookiebench/benchmark-schema": "workspace:*", + "@cookiebench/ts-config": "workspace:*", + "@types/node": "^24.9.2", + "@types/react": "^19.2.2", + "@types/react-dom": "^19.2.2", + "cookiebench": "workspace:*", + "typescript": "^5.9.3" + } } diff --git a/benchmarks/with-c15t-react/tsconfig.json b/benchmarks/with-c15t-react/tsconfig.json index 99f37b5..787e3f6 100644 --- a/benchmarks/with-c15t-react/tsconfig.json +++ b/benchmarks/with-c15t-react/tsconfig.json @@ -1,11 +1,11 @@ { - "extends": "@cookiebench/ts-config/nextjs.json", - "compilerOptions": { - "baseUrl": ".", - "paths": { - "@/*": ["./*"] - } - }, - "include": ["next-env.d.ts", ".next/types/**/*.ts", "**/*.ts", "**/*.tsx"], - "exclude": ["node_modules"] + "extends": "@cookiebench/ts-config/nextjs.json", + "compilerOptions": { + "baseUrl": ".", + "paths": { + "@/*": ["./*"] + } + }, + "include": ["next-env.d.ts", ".next/types/**/*.ts", "**/*.ts", "**/*.tsx"], + "exclude": ["node_modules"] } diff --git a/benchmarks/with-cookie-control/app/layout.tsx b/benchmarks/with-cookie-control/app/layout.tsx index 505cff4..7239ff5 100644 --- a/benchmarks/with-cookie-control/app/layout.tsx +++ b/benchmarks/with-cookie-control/app/layout.tsx @@ -1,8 +1,8 @@ -import type { Metadata } from 'next'; -import type { ReactNode } from 'react'; +import type { Metadata } from "next"; +import type { ReactNode } from "react"; export const metadata: Metadata = { - title: 'benchmark', + title: "benchmark", }; export default function RootLayout({ diff --git a/benchmarks/with-cookie-control/config.json b/benchmarks/with-cookie-control/config.json index 502dd42..c2ca2d0 100644 --- a/benchmarks/with-cookie-control/config.json +++ b/benchmarks/with-cookie-control/config.json @@ -1,40 +1,40 @@ { - "$schema": "./node_modules/@cookiebench/benchmark-schema/schema.json", - "name": "with-cookie-control", - "id": "cookie-control", - "iterations": 20, - "cookieBanner": { - "selectors": [".ccc-module--slideout"], - "serviceHosts": ["civiccomputing.com"], - "waitForVisibility": true, - "measureViewportCoverage": true, - "expectedLayoutShift": true, - "serviceName": "Cookie Control" - }, - "internationalization": { - "detection": "none", - "stringLoading": "none" - }, - "techStack": { - "bundler": "unknown", - "bundleType": "iffe", - "frameworks": [], - "languages": ["javascript"], - "packageManager": "unknown", - "typescript": false - }, - "source": { - "isOpenSource": false, - "license": "proprietary", - "website": "https://www.civicuk.com/cookie-control/" - }, - "company": { - "name": "Cookie Control", - "website": "https://www.civicuk.com/cookie-control/", - "avatar": "https://www.civicuk.com/cookie-control/images/favicon.png" - }, - "includes": { - "backend": ["proprietary"], - "components": ["javascript"] - } + "$schema": "./node_modules/@cookiebench/benchmark-schema/schema.json", + "name": "with-cookie-control", + "id": "cookie-control", + "iterations": 20, + "cookieBanner": { + "selectors": [".ccc-module--slideout"], + "serviceHosts": ["civiccomputing.com"], + "waitForVisibility": true, + "measureViewportCoverage": true, + "expectedLayoutShift": true, + "serviceName": "Cookie Control" + }, + "internationalization": { + "detection": "none", + "stringLoading": "none" + }, + "techStack": { + "bundler": "unknown", + "bundleType": "iffe", + "frameworks": [], + "languages": ["javascript"], + "packageManager": "unknown", + "typescript": false + }, + "source": { + "isOpenSource": false, + "license": "proprietary", + "website": "https://www.civicuk.com/cookie-control/" + }, + "company": { + "name": "Cookie Control", + "website": "https://www.civicuk.com/cookie-control/", + "avatar": "https://www.civicuk.com/cookie-control/images/favicon.png" + }, + "includes": { + "backend": ["proprietary"], + "components": ["javascript"] + } } diff --git a/benchmarks/with-cookie-control/next.config.ts b/benchmarks/with-cookie-control/next.config.ts index a67a28b..7921f35 100644 --- a/benchmarks/with-cookie-control/next.config.ts +++ b/benchmarks/with-cookie-control/next.config.ts @@ -1,4 +1,4 @@ -import type { NextConfig } from 'next'; +import type { NextConfig } from "next"; const nextConfig: NextConfig = { /* config options here */ diff --git a/benchmarks/with-cookie-control/package.json b/benchmarks/with-cookie-control/package.json index 06fac9e..9266dc1 100644 --- a/benchmarks/with-cookie-control/package.json +++ b/benchmarks/with-cookie-control/package.json @@ -1,26 +1,25 @@ { - "name": "with-cookie-control", - "private": true, - "scripts": { - "benchmark": "pnpm exec cookiebench benchmark", - "build": "next build --turbopack", - "dev": "next dev --port 3001 --turbopack", - "fmt": "pnpm biome format --write . && pnpm biome check --formatter-enabled=false --linter-enabled=false --organize-imports-enabled=true --write", - "lint": "next lint", - "start": "next start" - }, - "dependencies": { - "next": "15.3.3", - "react": "^19.0.0", - "react-dom": "^19.0.0" - }, - "devDependencies": { - "@cookiebench/benchmark-schema": "workspace:*", - "@cookiebench/ts-config": "workspace:*", - "@types/node": "^22.15.30", - "@types/react": "^19.1.6", - "@types/react-dom": "^19.1.6", - "cookiebench": "workspace:*", - "typescript": "^5.8.3" - } + "name": "with-cookie-control", + "private": true, + "scripts": { + "build": "next build", + "dev": "next dev --port 3001", + "fmt": "biome format . --write", + "lint": "biome lint .", + "start": "next start" + }, + "dependencies": { + "next": "16.0.1", + "react": "^19.2.0", + "react-dom": "^19.2.0" + }, + "devDependencies": { + "@cookiebench/benchmark-schema": "workspace:*", + "@cookiebench/ts-config": "workspace:*", + "@types/node": "^24.9.2", + "@types/react": "^19.2.2", + "@types/react-dom": "^19.2.2", + "cookiebench": "workspace:*", + "typescript": "^5.9.3" + } } diff --git a/benchmarks/with-cookie-control/tsconfig.json b/benchmarks/with-cookie-control/tsconfig.json index 99f37b5..787e3f6 100644 --- a/benchmarks/with-cookie-control/tsconfig.json +++ b/benchmarks/with-cookie-control/tsconfig.json @@ -1,11 +1,11 @@ { - "extends": "@cookiebench/ts-config/nextjs.json", - "compilerOptions": { - "baseUrl": ".", - "paths": { - "@/*": ["./*"] - } - }, - "include": ["next-env.d.ts", ".next/types/**/*.ts", "**/*.ts", "**/*.tsx"], - "exclude": ["node_modules"] + "extends": "@cookiebench/ts-config/nextjs.json", + "compilerOptions": { + "baseUrl": ".", + "paths": { + "@/*": ["./*"] + } + }, + "include": ["next-env.d.ts", ".next/types/**/*.ts", "**/*.ts", "**/*.tsx"], + "exclude": ["node_modules"] } diff --git a/benchmarks/with-cookie-yes/app/layout.tsx b/benchmarks/with-cookie-yes/app/layout.tsx index 62257fe..6d7584e 100644 --- a/benchmarks/with-cookie-yes/app/layout.tsx +++ b/benchmarks/with-cookie-yes/app/layout.tsx @@ -1,8 +1,8 @@ -import type { Metadata } from 'next'; -import type { ReactNode } from 'react'; +import type { Metadata } from "next"; +import type { ReactNode } from "react"; export const metadata: Metadata = { - title: 'benchmark', + title: "benchmark", }; export default function RootLayout({ diff --git a/benchmarks/with-cookie-yes/config.json b/benchmarks/with-cookie-yes/config.json index 8fa3629..b084c19 100644 --- a/benchmarks/with-cookie-yes/config.json +++ b/benchmarks/with-cookie-yes/config.json @@ -1,44 +1,44 @@ { - "$schema": "./node_modules/@cookiebench/benchmark-schema/schema.json", - "name": "with-cookie-yes", - "id": "cookie-yes-banner", - "iterations": 20, - "remote": { - "enabled": true, - "url": "https://benchmarks-with-cookie-yes.vercel.app" - }, - "cookieBanner": { - "selectors": [".cky-consent-container"], - "serviceHosts": ["cdn-cookieyes.com"], - "waitForVisibility": true, - "measureViewportCoverage": true, - "expectedLayoutShift": true, - "serviceName": "CookieYes" - }, - "internationalization": { - "detection": "manual", - "stringLoading": "server" - }, - "techStack": { - "bundler": "unknown", - "bundleType": "iffe", - "frameworks": [], - "languages": ["javascript"], - "packageManager": "unknown", - "typescript": false - }, - "source": { - "isOpenSource": false, - "license": "proprietary", - "website": "https://www.cookieyes.com" - }, - "company": { - "name": "CookieYes", - "website": "https://www.cookieyes.com", - "avatar": "https://pbs.twimg.com/profile_images/1351391912402558976/-UN3BKAG_400x400.jpg" - }, - "includes": { - "backend": ["proprietary"], - "components": ["javascript"] - } + "$schema": "./node_modules/@cookiebench/benchmark-schema/schema.json", + "name": "with-cookie-yes", + "id": "cookie-yes-banner", + "iterations": 20, + "remote": { + "enabled": true, + "url": "https://benchmarks-with-cookie-yes.vercel.app" + }, + "cookieBanner": { + "selectors": [".cky-consent-container"], + "serviceHosts": ["cdn-cookieyes.com"], + "waitForVisibility": true, + "measureViewportCoverage": true, + "expectedLayoutShift": true, + "serviceName": "CookieYes" + }, + "internationalization": { + "detection": "manual", + "stringLoading": "server" + }, + "techStack": { + "bundler": "unknown", + "bundleType": "iffe", + "frameworks": [], + "languages": ["javascript"], + "packageManager": "unknown", + "typescript": false + }, + "source": { + "isOpenSource": false, + "license": "proprietary", + "website": "https://www.cookieyes.com" + }, + "company": { + "name": "CookieYes", + "website": "https://www.cookieyes.com", + "avatar": "https://pbs.twimg.com/profile_images/1351391912402558976/-UN3BKAG_400x400.jpg" + }, + "includes": { + "backend": ["proprietary"], + "components": ["javascript"] + } } diff --git a/benchmarks/with-cookie-yes/next.config.ts b/benchmarks/with-cookie-yes/next.config.ts index a67a28b..7921f35 100644 --- a/benchmarks/with-cookie-yes/next.config.ts +++ b/benchmarks/with-cookie-yes/next.config.ts @@ -1,4 +1,4 @@ -import type { NextConfig } from 'next'; +import type { NextConfig } from "next"; const nextConfig: NextConfig = { /* config options here */ diff --git a/benchmarks/with-cookie-yes/package.json b/benchmarks/with-cookie-yes/package.json index 934ec59..d4e9707 100644 --- a/benchmarks/with-cookie-yes/package.json +++ b/benchmarks/with-cookie-yes/package.json @@ -1,26 +1,25 @@ { - "name": "with-cookie-yes", - "private": true, - "scripts": { - "benchmark": "pnpm exec cookiebench benchmark", - "build": "next build --turbopack", - "dev": "next dev --port 3001 --turbopack", - "fmt": "pnpm biome format --write . && pnpm biome check --formatter-enabled=false --linter-enabled=false --organize-imports-enabled=true --write", - "lint": "next lint", - "start": "next start" - }, - "dependencies": { - "next": "15.3.3", - "react": "^19.0.0", - "react-dom": "^19.0.0" - }, - "devDependencies": { - "@cookiebench/benchmark-schema": "workspace:*", - "@cookiebench/ts-config": "workspace:*", - "@types/node": "^22.15.30", - "@types/react": "^19.1.6", - "@types/react-dom": "^19.1.6", - "cookiebench": "workspace:*", - "typescript": "^5.8.3" - } + "name": "with-cookie-yes", + "private": true, + "scripts": { + "build": "next build", + "dev": "next dev --port 3001", + "fmt": "biome format . --write", + "lint": "biome lint .", + "start": "next start" + }, + "dependencies": { + "next": "16.0.1", + "react": "^19.2.0", + "react-dom": "^19.2.0" + }, + "devDependencies": { + "@cookiebench/benchmark-schema": "workspace:*", + "@cookiebench/ts-config": "workspace:*", + "@types/node": "^24.9.2", + "@types/react": "^19.2.2", + "@types/react-dom": "^19.2.2", + "cookiebench": "workspace:*", + "typescript": "^5.9.3" + } } diff --git a/benchmarks/with-cookie-yes/tsconfig.json b/benchmarks/with-cookie-yes/tsconfig.json index 99f37b5..787e3f6 100644 --- a/benchmarks/with-cookie-yes/tsconfig.json +++ b/benchmarks/with-cookie-yes/tsconfig.json @@ -1,11 +1,11 @@ { - "extends": "@cookiebench/ts-config/nextjs.json", - "compilerOptions": { - "baseUrl": ".", - "paths": { - "@/*": ["./*"] - } - }, - "include": ["next-env.d.ts", ".next/types/**/*.ts", "**/*.ts", "**/*.tsx"], - "exclude": ["node_modules"] + "extends": "@cookiebench/ts-config/nextjs.json", + "compilerOptions": { + "baseUrl": ".", + "paths": { + "@/*": ["./*"] + } + }, + "include": ["next-env.d.ts", ".next/types/**/*.ts", "**/*.ts", "**/*.tsx"], + "exclude": ["node_modules"] } diff --git a/benchmarks/with-didomi/config.json b/benchmarks/with-didomi/config.json index 724d963..c007ba3 100644 --- a/benchmarks/with-didomi/config.json +++ b/benchmarks/with-didomi/config.json @@ -1,49 +1,49 @@ { - "$schema": "./node_modules/@cookiebench/benchmark-schema/schema.json", - "name": "with-didomi", - "iterations": 20, - "cookieBanner": { - "selectors": [ - ".didomi-popup-container", - "#didomi-popup", - ".didomi-notice-popup", - ".didomi-host", - "[data-testid=\"notice\"]" - ], - "serviceHosts": [ - "api.privacy-center.org", - "sdk.privacy-center.org", - "privacy-center.org" - ], - "waitForVisibility": true, - "measureViewportCoverage": true, - "expectedLayoutShift": true, - "serviceName": "Didomi" - }, - "internationalization": { - "detection": "browser", - "stringLoading": "bundled" - }, - "techStack": { - "bundler": "unknown", - "bundleType": "bundled", - "frameworks": ["react"], - "languages": ["javascript"], - "packageManager": "unknown", - "typescript": false - }, - "source": { - "isOpenSource": false, - "license": "proprietary", - "website": "https://didomi.io" - }, - "company": { - "name": "Didomi", - "website": "https://didomi.io", - "avatar": "https://cdn.brandfetch.io/idVPEL4NcB/w/400/h/400/theme/dark/icon.jpeg?c=1dxbfHSJFAPEGdCLU4o5B" - }, - "includes": { - "backend": ["proprietary"], - "components": ["javascript", "react"] - } + "$schema": "./node_modules/@cookiebench/benchmark-schema/schema.json", + "name": "with-didomi", + "iterations": 20, + "cookieBanner": { + "selectors": [ + ".didomi-popup-container", + "#didomi-popup", + ".didomi-notice-popup", + ".didomi-host", + "[data-testid=\"notice\"]" + ], + "serviceHosts": [ + "api.privacy-center.org", + "sdk.privacy-center.org", + "privacy-center.org" + ], + "waitForVisibility": true, + "measureViewportCoverage": true, + "expectedLayoutShift": true, + "serviceName": "Didomi" + }, + "internationalization": { + "detection": "browser", + "stringLoading": "bundled" + }, + "techStack": { + "bundler": "unknown", + "bundleType": "bundled", + "frameworks": ["react"], + "languages": ["javascript"], + "packageManager": "unknown", + "typescript": false + }, + "source": { + "isOpenSource": false, + "license": "proprietary", + "website": "https://didomi.io" + }, + "company": { + "name": "Didomi", + "website": "https://didomi.io", + "avatar": "https://cdn.brandfetch.io/idVPEL4NcB/w/400/h/400/theme/dark/icon.jpeg?c=1dxbfHSJFAPEGdCLU4o5B" + }, + "includes": { + "backend": ["proprietary"], + "components": ["javascript", "react"] + } } diff --git a/benchmarks/with-didomi/next.config.ts b/benchmarks/with-didomi/next.config.ts index a67a28b..7921f35 100644 --- a/benchmarks/with-didomi/next.config.ts +++ b/benchmarks/with-didomi/next.config.ts @@ -1,4 +1,4 @@ -import type { NextConfig } from 'next'; +import type { NextConfig } from "next"; const nextConfig: NextConfig = { /* config options here */ diff --git a/benchmarks/with-didomi/package.json b/benchmarks/with-didomi/package.json index 2ac50db..f42ba1f 100644 --- a/benchmarks/with-didomi/package.json +++ b/benchmarks/with-didomi/package.json @@ -1,26 +1,25 @@ { - "name": "with-dodomi", - "private": true, - "scripts": { - "benchmark": "pnpm exec cookiebench benchmark", - "build": "next build --turbopack", - "dev": "next dev --turbopack", - "fmt": "pnpm biome format --write . && pnpm biome check --formatter-enabled=false --linter-enabled=false --organize-imports-enabled=true --write", - "lint": "next lint", - "start": "next start" - }, - "dependencies": { - "@didomi/react": "^1.8.8", - "next": "15.3.3", - "react": "^19.0.0", - "react-dom": "^19.0.0" - }, - "devDependencies": { - "@cookiebench/benchmark-schema": "workspace:*", - "@cookiebench/ts-config": "workspace:*", - "@types/node": "^22.15.30", - "@types/react": "^19.1.6", - "@types/react-dom": "^19.1.6", - "typescript": "^5.8.3" - } + "name": "with-dodomi", + "private": true, + "scripts": { + "build": "next build", + "dev": "next dev", + "fmt": "biome format . --write", + "lint": "biome lint .", + "start": "next start" + }, + "dependencies": { + "@didomi/react": "^1.8.8", + "next": "16.0.1", + "react": "^19.2.0", + "react-dom": "^19.2.0" + }, + "devDependencies": { + "@cookiebench/benchmark-schema": "workspace:*", + "@cookiebench/ts-config": "workspace:*", + "@types/node": "^24.9.2", + "@types/react": "^19.2.2", + "@types/react-dom": "^19.2.2", + "typescript": "^5.9.3" + } } diff --git a/benchmarks/with-didomi/tsconfig.json b/benchmarks/with-didomi/tsconfig.json index 99f37b5..787e3f6 100644 --- a/benchmarks/with-didomi/tsconfig.json +++ b/benchmarks/with-didomi/tsconfig.json @@ -1,11 +1,11 @@ { - "extends": "@cookiebench/ts-config/nextjs.json", - "compilerOptions": { - "baseUrl": ".", - "paths": { - "@/*": ["./*"] - } - }, - "include": ["next-env.d.ts", ".next/types/**/*.ts", "**/*.ts", "**/*.tsx"], - "exclude": ["node_modules"] + "extends": "@cookiebench/ts-config/nextjs.json", + "compilerOptions": { + "baseUrl": ".", + "paths": { + "@/*": ["./*"] + } + }, + "include": ["next-env.d.ts", ".next/types/**/*.ts", "**/*.ts", "**/*.tsx"], + "exclude": ["node_modules"] } diff --git a/benchmarks/with-enzuzo/app/layout.tsx b/benchmarks/with-enzuzo/app/layout.tsx index 84fc0f6..92e9a2e 100644 --- a/benchmarks/with-enzuzo/app/layout.tsx +++ b/benchmarks/with-enzuzo/app/layout.tsx @@ -1,8 +1,8 @@ -import type { Metadata } from 'next'; -import type { ReactNode } from 'react'; +import type { Metadata } from "next"; +import type { ReactNode } from "react"; export const metadata: Metadata = { - title: 'benchmark', + title: "benchmark", }; export default function RootLayout({ @@ -15,7 +15,7 @@ export default function RootLayout({ {/* Cookie Control Script */}