diff --git a/apps/load-tests/src/cli/infra.ts b/apps/load-tests/src/cli/infra.ts index c6c89fc..b972a61 100644 --- a/apps/load-tests/src/cli/infra.ts +++ b/apps/load-tests/src/cli/infra.ts @@ -1,55 +1,526 @@ #!/usr/bin/env node import chalk from "chalk"; import { Command } from "commander"; +import { collectFromDroplets, printCollectResults } from "../infra/collect.js"; +import { loadInfraConfig } from "../infra/config.js"; +import { createDroplet, deleteDroplet, listDropletsByPrefix, waitForDropletActive } from "../infra/digitalocean.js"; +import { type DropletSetupStatus, formatStatus, setupDroplet } from "../infra/droplet.js"; +import { execOnDroplets, printExecResults, saveExecLogs } from "../infra/exec.js"; +import { DROPLET_HOURLY_COST, DROPLET_IMAGE, DROPLET_REGION, DROPLET_SIZE, type DropletInfo } from "../infra/types.js"; const program = new Command(); +program.name("infra").description("Manage DigitalOcean infrastructure for distributed load testing").version("0.0.1"); + +/** + * Validate a git branch name to prevent shell injection. + * Allows alphanumeric, hyphens, underscores, slashes, and dots. + */ +function validateBranchName(branch: string): void { + if (!/^[\w.\-/]+$/.test(branch)) { + throw new Error(`Invalid branch name: "${branch}". Only alphanumeric, hyphens, underscores, slashes, and dots are allowed.`); + } +} + +/** + * Escape a string for use in shell single quotes. + * Replaces ' with '\'' (end quote, escaped quote, start quote). + */ +function escapeShellSingleQuote(str: string): string { + return str.replace(/'/g, "'\\''"); +} + +/** + * Escape regex special characters in a string. + */ +function escapeRegex(str: string): string { + return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +/** + * Format a relative time string (e.g., "2h ago") + */ +function formatRelativeTime(dateStr: string): string { + const date = new Date(dateStr); + const now = new Date(); + const diffMs = now.getTime() - date.getTime(); + const diffMins = Math.floor(diffMs / 60000); + const diffHours = Math.floor(diffMins / 60); + const diffDays = Math.floor(diffHours / 24); + + if (diffDays > 0) return `${diffDays}d ago`; + if (diffHours > 0) return `${diffHours}h ago`; + if (diffMins > 0) return `${diffMins}m ago`; + return "just now"; +} + +/** + * Print a table of droplets. + */ +function printDropletTable(droplets: DropletInfo[]): void { + // Header + console.log(chalk.dim(" NAME IP REGION SIZE STATUS CREATED")); + + for (const d of droplets) { + const statusColor = d.status === "active" ? chalk.green : chalk.yellow; + console.log( + ` ${d.name.padEnd(14)} ${(d.ip ?? "pending").padEnd(16)} ${d.region.padEnd(8)} ${d.size.padEnd(14)} ${statusColor(d.status.padEnd(8))} ${formatRelativeTime(d.createdAt)}`, + ); + } +} + +// ============================================================================ +// LIST COMMAND +// ============================================================================ + program - .name("infra") - .description("Manage DigitalOcean infrastructure for distributed load testing") - .version("0.0.1"); + .command("list") + .description("List current load test droplets") + .option("--name-prefix ", "Prefix to filter droplets", "load-test") + .action(async (options: { namePrefix: string }) => { + try { + const config = loadInfraConfig(); + const droplets = await listDropletsByPrefix(config.digitalOceanToken, options.namePrefix); + + if (droplets.length === 0) { + console.log(chalk.yellow(`[infra list] No droplets found matching prefix "${options.namePrefix}"`)); + return; + } + + console.log(chalk.cyan(`[infra list] Found ${droplets.length} droplet(s):`)); + console.log(""); + printDropletTable(droplets); + + // Calculate hourly cost + const hourlyCost = droplets.length * DROPLET_HOURLY_COST; + console.log(""); + console.log(chalk.dim(` Total: ${droplets.length} droplets (~$${hourlyCost.toFixed(3)}/hr)`)); + } catch (error) { + console.error(chalk.red(`[infra list] Error: ${(error as Error).message}`)); + process.exit(1); + } + }); + +// ============================================================================ +// CREATE COMMAND +// ============================================================================ + +// Track status for each droplet during creation +const dropletStatuses = new Map(); + +function updateDropletStatus(name: string, status: DropletSetupStatus): void { + dropletStatuses.set(name, status); +} + +function printDropletStatuses(): void { + for (const [name, status] of dropletStatuses) { + console.log(` ${name.padEnd(14)} ${formatStatus(status)}`); + } +} program .command("create") .description("Create DigitalOcean droplets for load testing") - .action(() => { - console.log(chalk.yellow("[infra create] Not implemented yet - coming in next PR")); - }); + .option("--count ", "Number of droplets to create", "3") + .option("--name-prefix ", "Prefix for droplet names", "load-test") + .option("--branch ", "Git branch to clone", "main") + .option("--skip-setup", "Skip running the setup script", false) + .action(async (options: { count: string; namePrefix: string; branch: string; skipSetup: boolean }) => { + try { + const config = loadInfraConfig(); + const count = Number.parseInt(options.count, 10); -program - .command("list") - .description("List current load test droplets") - .action(() => { - console.log(chalk.yellow("[infra list] Not implemented yet - coming in next PR")); + // Validate branch name to prevent shell injection + if (!options.skipSetup) { + validateBranchName(options.branch); + } + + console.log(chalk.cyan(`[infra create] Creating ${count} droplet(s) (${DROPLET_REGION}, ${DROPLET_SIZE})...`)); + if (!options.skipSetup) { + console.log(chalk.dim(`[infra create] Branch: ${options.branch}`)); + } + console.log(""); + + // Get existing droplets to determine next number + const existing = await listDropletsByPrefix(config.digitalOceanToken, options.namePrefix); + const escapedPrefix = escapeRegex(options.namePrefix); + const existingNumbers = existing + .map((d) => { + const match = d.name.match(new RegExp(`^${escapedPrefix}-(\\d+)$`)); + return match ? Number.parseInt(match[1], 10) : 0; + }) + .filter((n) => n > 0); + const startNumber = existingNumbers.length > 0 ? Math.max(...existingNumbers) + 1 : 1; + + // Initialize status tracking + const dropletNames: string[] = []; + for (let i = 0; i < count; i++) { + const name = `${options.namePrefix}-${startNumber + i}`; + dropletNames.push(name); + updateDropletStatus(name, "creating"); + } + printDropletStatuses(); + + // Create droplets via API (limit concurrency to 5 to avoid rate limits) + const createdDroplets: DropletInfo[] = []; + const CONCURRENCY_LIMIT = 5; + for (let i = 0; i < dropletNames.length; i += CONCURRENCY_LIMIT) { + const batch = dropletNames.slice(i, i + CONCURRENCY_LIMIT); + const createPromises = batch.map(async (name) => { + const droplet = await createDroplet(config.digitalOceanToken, { + name, + region: DROPLET_REGION, + size: DROPLET_SIZE, + image: DROPLET_IMAGE, + sshKeyFingerprint: config.sshKeyFingerprint, + }); + createdDroplets.push(droplet); + updateDropletStatus(name, "waiting_for_active"); + }); + await Promise.all(createPromises); + } + + console.log(""); + console.log(chalk.cyan("[infra create] Waiting for droplets to be active...")); + + // Wait for all droplets to be active + const activePromises = createdDroplets.map(async (d) => { + const active = await waitForDropletActive(config.digitalOceanToken, d.id); + updateDropletStatus(d.name, "waiting_for_ssh"); + return active; + }); + const activeDroplets = await Promise.all(activePromises); + + // Skip setup if requested + if (options.skipSetup) { + for (const d of activeDroplets) { + updateDropletStatus(d.name, "ready"); + } + console.log(""); + console.log(chalk.green(`[infra create] Complete! ${count}/${count} droplets active (setup skipped).`)); + console.log(""); + printDropletTable(activeDroplets); + return; + } + + console.log(""); + console.log(chalk.cyan("[infra create] Running setup on droplets...")); + console.log(chalk.dim("[infra create] This may take 3-5 minutes...")); + console.log(""); + + // Run setup on all droplets in parallel + const setupPromises = activeDroplets.map(async (droplet) => { + try { + await setupDroplet(droplet, options.branch, config.sshPrivateKeyPath, (d, status) => updateDropletStatus(d.name, status)); + } catch (error) { + updateDropletStatus(droplet.name, "failed"); + console.error(chalk.red(` ${droplet.name}: ${(error as Error).message}`)); + } + }); + + await Promise.all(setupPromises); + + // Count successes and failures + const readyCount = [...dropletStatuses.values()].filter((s) => s === "ready").length; + const failedCount = [...dropletStatuses.values()].filter((s) => s === "failed").length; + + console.log(""); + if (failedCount === 0) { + console.log(chalk.green(`[infra create] Complete! ${readyCount}/${count} droplets ready.`)); + } else { + console.log(chalk.yellow(`[infra create] Done. ${readyCount}/${count} ready, ${failedCount} failed.`)); + } + console.log(""); + printDropletStatuses(); + } catch (error) { + console.error(chalk.red(`[infra create] Error: ${(error as Error).message}`)); + process.exit(1); + } }); +// ============================================================================ +// DESTROY COMMAND +// ============================================================================ + program .command("destroy") .description("Destroy all load test droplets") - .action(() => { - console.log(chalk.yellow("[infra destroy] Not implemented yet - coming in next PR")); + .option("--name-prefix ", "Prefix to filter droplets", "load-test") + .option("--yes", "Skip confirmation prompt", false) + .action(async (options: { namePrefix: string; yes: boolean }) => { + try { + const config = loadInfraConfig(); + const droplets = await listDropletsByPrefix(config.digitalOceanToken, options.namePrefix); + + if (droplets.length === 0) { + console.log(chalk.yellow(`[infra destroy] No droplets found matching prefix "${options.namePrefix}"`)); + return; + } + + console.log(chalk.cyan(`[infra destroy] Found ${droplets.length} droplet(s) to destroy:`)); + console.log(""); + console.log(` ${droplets.map((d) => d.name).join(", ")}`); + console.log(""); + + // Confirm unless --yes + if (!options.yes) { + const readline = await import("node:readline"); + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + }); + + const answer = await new Promise((resolve) => { + rl.question(chalk.yellow(" Are you sure you want to destroy these droplets? (y/N) "), resolve); + }); + rl.close(); + + if (answer.toLowerCase() !== "y") { + console.log(chalk.dim(" Cancelled.")); + return; + } + } + + console.log(chalk.cyan("[infra destroy] Destroying droplets...")); + + // Delete all in parallel + const deletePromises = droplets.map(async (d) => { + await deleteDroplet(config.digitalOceanToken, d.id); + console.log(` ${d.name} ${chalk.green("✓ destroyed")}`); + }); + + await Promise.all(deletePromises); + + console.log(""); + console.log(chalk.green("[infra destroy] All droplets destroyed.")); + } catch (error) { + console.error(chalk.red(`[infra destroy] Error: ${(error as Error).message}`)); + process.exit(1); + } + }); + +// ============================================================================ +// UPDATE COMMAND +// ============================================================================ + +program + .command("update") + .description("Update code on all droplets (git pull && yarn build)") + .option("--name-prefix ", "Prefix to filter droplets", "load-test") + .option("--branch ", "Branch to checkout", "main") + .action(async (options: { namePrefix: string; branch: string }) => { + try { + const config = loadInfraConfig(); + + // Validate branch name to prevent shell injection + validateBranchName(options.branch); + + const droplets = await listDropletsByPrefix(config.digitalOceanToken, options.namePrefix); + + if (droplets.length === 0) { + console.log(chalk.yellow(`[infra update] No droplets found matching prefix "${options.namePrefix}"`)); + return; + } + + console.log(chalk.cyan(`[infra update] Updating ${droplets.length} droplet(s)...`)); + console.log(chalk.dim(`[infra update] Branch: ${options.branch}`)); + console.log(""); + + // Build the update command (branch already validated above) + const updateCommand = `cd /app && git fetch && git checkout ${options.branch} && git pull && yarn install && yarn build`; + + const results = await execOnDroplets(droplets, updateCommand, config.sshPrivateKeyPath); + + // Save logs + const timestamp = new Date().toISOString().replace(/[:.]/g, "-").slice(0, 19); + const logsDir = `results/.infra-logs/update-${timestamp}`; + saveExecLogs(results, logsDir); + + printExecResults(results); + console.log(""); + console.log(chalk.dim(` Full logs: ${logsDir}/`)); + } catch (error) { + console.error(chalk.red(`[infra update] Error: ${(error as Error).message}`)); + process.exit(1); + } }); +// ============================================================================ +// EXEC COMMAND +// ============================================================================ + program .command("exec") .description("Execute a command on all droplets") - .action(() => { - console.log(chalk.yellow("[infra exec] Not implemented yet - coming in next PR")); + .requiredOption("--command ", "Command to execute") + .option("--name-prefix ", "Prefix to filter droplets", "load-test") + .option("--background", "Run command in background (fire-and-forget)", false) + .action(async (options: { command: string; namePrefix: string; background: boolean }) => { + try { + const config = loadInfraConfig(); + const droplets = await listDropletsByPrefix(config.digitalOceanToken, options.namePrefix); + + if (droplets.length === 0) { + console.log(chalk.yellow(`[infra exec] No droplets found matching prefix "${options.namePrefix}"`)); + return; + } + + if (options.background) { + // Fire-and-forget mode: wrap command in nohup and return immediately + const bgCommand = `nohup bash -c '${options.command.replace(/'/g, "'\\''")}' > /tmp/load-test-output.log 2>&1 &`; + console.log(chalk.cyan(`[infra exec] Starting background process on ${droplets.length} droplet(s)...`)); + console.log(chalk.dim(`[infra exec] Command: ${options.command}`)); + console.log(""); + + const results = await execOnDroplets(droplets, bgCommand, config.sshPrivateKeyPath); + + const succeeded = results.filter((r) => r.exitCode === 0).length; + console.log(chalk.green(`[infra exec] Started on ${succeeded}/${droplets.length} droplet(s).`)); + console.log(""); + console.log(chalk.dim(" Use 'yarn infra wait' to wait for completion.")); + console.log(chalk.dim(" Logs are being written to /tmp/load-test-output.log on each droplet.")); + } else { + // Normal mode: wait for command to complete + console.log(chalk.cyan(`[infra exec] Running on ${droplets.length} droplet(s)...`)); + console.log(chalk.dim(`[infra exec] Command: ${options.command}`)); + console.log(""); + + const results = await execOnDroplets(droplets, options.command, config.sshPrivateKeyPath); + + // Save logs + const timestamp = new Date().toISOString().replace(/[:.]/g, "-").slice(0, 19); + const logsDir = `results/.infra-logs/exec-${timestamp}`; + saveExecLogs(results, logsDir); + + printExecResults(results); + console.log(""); + console.log(chalk.dim(` Full logs: ${logsDir}/`)); + } + } catch (error) { + console.error(chalk.red(`[infra exec] Error: ${(error as Error).message}`)); + process.exit(1); + } }); +// ============================================================================ +// WAIT COMMAND +// ============================================================================ + program - .command("update") - .description("Update code on all droplets") - .action(() => { - console.log(chalk.yellow("[infra update] Not implemented yet - coming in next PR")); + .command("wait") + .description("Wait for a file to exist on all droplets (poll until ready)") + .requiredOption("--file ", "Path to file to wait for on each droplet") + .option("--name-prefix ", "Prefix to filter droplets", "load-test") + .option("--timeout ", "Timeout in seconds", "600") + .option("--interval ", "Poll interval in seconds", "5") + .action(async (options: { file: string; namePrefix: string; timeout: string; interval: string }) => { + try { + const config = loadInfraConfig(); + const droplets = await listDropletsByPrefix(config.digitalOceanToken, options.namePrefix); + + if (droplets.length === 0) { + console.log(chalk.yellow(`[infra wait] No droplets found matching prefix "${options.namePrefix}"`)); + return; + } + + const timeoutSec = Number.parseInt(options.timeout, 10); + const intervalSec = Number.parseInt(options.interval, 10); + + // Validate numeric inputs to prevent infinite loop with NaN + if (Number.isNaN(timeoutSec) || timeoutSec <= 0) { + throw new Error(`Invalid timeout value: "${options.timeout}". Must be a positive number.`); + } + if (Number.isNaN(intervalSec) || intervalSec <= 0) { + throw new Error(`Invalid interval value: "${options.interval}". Must be a positive number.`); + } + + console.log(chalk.cyan(`[infra wait] Waiting for ${options.file} on ${droplets.length} droplet(s)...`)); + console.log(chalk.dim(`[infra wait] Timeout: ${timeoutSec}s, Poll interval: ${intervalSec}s`)); + console.log(""); + + const startTime = Date.now(); + const completed = new Set(); + // Escape single quotes in file path to prevent shell injection + const escapedFile = escapeShellSingleQuote(options.file); + const checkCommand = `test -f '${escapedFile}' && echo 'EXISTS' || echo 'NOT_FOUND'`; + + while (completed.size < droplets.length) { + const elapsed = (Date.now() - startTime) / 1000; + if (elapsed >= timeoutSec) { + console.log(""); + console.log(chalk.red(`[infra wait] Timeout after ${timeoutSec}s. ${completed.size}/${droplets.length} completed.`)); + process.exit(1); + } + + // Check remaining droplets + const remaining = droplets.filter((d) => !completed.has(d.name)); + const results = await execOnDroplets(remaining, checkCommand, config.sshPrivateKeyPath); + + for (const result of results) { + if (result.stdout.includes("EXISTS")) { + completed.add(result.dropletName); + } + } + + // Print progress + const progressBar = "█".repeat(completed.size) + "░".repeat(droplets.length - completed.size); + process.stdout.write(`\r [${progressBar}] ${completed.size}/${droplets.length} complete (${Math.round(elapsed)}s)`); + + if (completed.size < droplets.length) { + await new Promise((resolve) => setTimeout(resolve, intervalSec * 1000)); + } + } + + console.log(""); + console.log(""); + console.log(chalk.green(`[infra wait] All ${droplets.length} droplet(s) have ${options.file}`)); + } catch (error) { + console.error(chalk.red(`[infra wait] Error: ${(error as Error).message}`)); + process.exit(1); + } }); +// ============================================================================ +// COLLECT COMMAND +// ============================================================================ + program .command("collect") .description("Collect results from all droplets") - .action(() => { - console.log(chalk.yellow("[infra collect] Not implemented yet - coming in next PR")); + .requiredOption("--output ", "Directory to store collected results") + .option("--name-prefix ", "Prefix to filter droplets", "load-test") + .option("--remote-path ", "Path to results file on droplet", "/tmp/results.json") + .action(async (options: { output: string; namePrefix: string; remotePath: string }) => { + try { + const config = loadInfraConfig(); + const droplets = await listDropletsByPrefix(config.digitalOceanToken, options.namePrefix); + + if (droplets.length === 0) { + console.log(chalk.yellow(`[infra collect] No droplets found matching prefix "${options.namePrefix}"`)); + return; + } + + console.log(chalk.cyan(`[infra collect] Collecting from ${droplets.length} droplet(s)...`)); + console.log(chalk.dim(`[infra collect] Remote path: ${options.remotePath}`)); + console.log(""); + + const results = await collectFromDroplets(droplets, options.remotePath, options.output, config.sshPrivateKeyPath); + + printCollectResults(results); + + // List downloaded files + const successful = results.filter((r) => r.success); + if (successful.length > 0) { + console.log(""); + console.log(chalk.cyan(`[infra collect] Results saved to ${options.output}/`)); + for (const r of successful) { + console.log(chalk.dim(` - ${r.dropletName}.json`)); + } + } + } catch (error) { + console.error(chalk.red(`[infra collect] Error: ${(error as Error).message}`)); + process.exit(1); + } }); program.parse(); - diff --git a/apps/load-tests/src/cli/results.ts b/apps/load-tests/src/cli/results.ts index 80a9295..b993753 100644 --- a/apps/load-tests/src/cli/results.ts +++ b/apps/load-tests/src/cli/results.ts @@ -1,6 +1,10 @@ #!/usr/bin/env node import chalk from "chalk"; import { Command } from "commander"; +import { + aggregateResults, + printAggregatedResults, +} from "../results/aggregate.js"; const program = new Command(); @@ -12,9 +16,18 @@ program program .command("aggregate") .description("Aggregate results from multiple load test runs") - .action(() => { - console.log(chalk.yellow("[results aggregate] Not implemented yet - coming in next PR")); + .requiredOption("--input ", "Directory containing result JSON files") + .action((options: { input: string }) => { + try { + console.log(chalk.cyan(`[results] Aggregating results from ${options.input}...`)); + console.log(""); + + const aggregated = aggregateResults(options.input); + printAggregatedResults(aggregated); + } catch (error) { + console.error(chalk.red(`[results] Error: ${(error as Error).message}`)); + process.exit(1); + } }); program.parse(); - diff --git a/apps/load-tests/src/infra/collect.ts b/apps/load-tests/src/infra/collect.ts new file mode 100644 index 0000000..04a8bc5 --- /dev/null +++ b/apps/load-tests/src/infra/collect.ts @@ -0,0 +1,115 @@ +import * as fs from "node:fs"; +import * as path from "node:path"; +import chalk from "chalk"; +import { downloadFile } from "./ssh.js"; +import type { DropletInfo } from "./types.js"; + +/** + * Result of collecting a file from a droplet. + */ +export interface CollectResult { + dropletName: string; + success: boolean; + localPath?: string; + fileSize?: number; + error?: string; +} + +/** + * Collect files from multiple droplets. + */ +export async function collectFromDroplets( + droplets: DropletInfo[], + remotePath: string, + outputDir: string, + privateKeyPath: string, + onProgress?: (droplet: DropletInfo, status: "downloading" | "done" | "failed") => void, +): Promise { + // Create output directory + if (!fs.existsSync(outputDir)) { + fs.mkdirSync(outputDir, { recursive: true }); + } + + const results: CollectResult[] = []; + + const collectPromises = droplets.map(async (droplet) => { + if (!droplet.ip) { + const result: CollectResult = { + dropletName: droplet.name, + success: false, + error: "No IP address", + }; + results.push(result); + onProgress?.(droplet, "failed"); + return result; + } + + onProgress?.(droplet, "downloading"); + + const localPath = path.join(outputDir, `${droplet.name}.json`); + + try { + await downloadFile(droplet.ip, remotePath, localPath, privateKeyPath); + + // Get file size + const stats = fs.statSync(localPath); + const result: CollectResult = { + dropletName: droplet.name, + success: true, + localPath, + fileSize: stats.size, + }; + results.push(result); + onProgress?.(droplet, "done"); + return result; + } catch (error) { + const result: CollectResult = { + dropletName: droplet.name, + success: false, + error: (error as Error).message, + }; + results.push(result); + onProgress?.(droplet, "failed"); + return result; + } + }); + + await Promise.all(collectPromises); + return results; +} + +/** + * Format file size in a human-readable way. + */ +export function formatFileSize(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + const kb = bytes / 1024; + if (kb < 1024) return `${kb.toFixed(1)} KB`; + const mb = kb / 1024; + return `${mb.toFixed(1)} MB`; +} + +/** + * Print collection results summary. + */ +export function printCollectResults(results: CollectResult[]): void { + const successful = results.filter((r) => r.success); + const failed = results.filter((r) => !r.success); + + console.log(""); + if (failed.length === 0) { + console.log(chalk.green(`[infra collect] Complete! ${successful.length}/${results.length} files downloaded.`)); + } else { + console.log(chalk.yellow(`[infra collect] Done. ${successful.length}/${results.length} downloaded, ${failed.length} failed.`)); + } + console.log(""); + + for (const result of results) { + if (result.success) { + console.log(` ${result.dropletName.padEnd(14)} ${chalk.green("✓")} downloaded (${formatFileSize(result.fileSize ?? 0)})`); + } else { + console.log(` ${result.dropletName.padEnd(14)} ${chalk.red("✗")} ${result.error}`); + } + } +} + diff --git a/apps/load-tests/src/infra/config.ts b/apps/load-tests/src/infra/config.ts new file mode 100644 index 0000000..b570b0d --- /dev/null +++ b/apps/load-tests/src/infra/config.ts @@ -0,0 +1,73 @@ +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { config as loadDotenv } from "dotenv"; +import type { InfraConfig, SshConfig } from "./types.js"; + +// Load .env file from the load-tests directory +const envPath = path.resolve(import.meta.dirname, "../../.env"); +loadDotenv({ path: envPath }); + +/** + * Load and validate infrastructure configuration from environment variables. + * Throws an error if required variables are missing. + */ +export function loadInfraConfig(): InfraConfig { + const digitalOceanToken = process.env.DIGITALOCEAN_TOKEN; + const sshKeyFingerprint = process.env.SSH_KEY_FINGERPRINT; + const sshPrivateKeyPath = process.env.SSH_PRIVATE_KEY_PATH ?? "~/.ssh/id_rsa"; + + if (!digitalOceanToken) { + throw new Error( + "DIGITALOCEAN_TOKEN is required.\n" + + "Set it in apps/load-tests/.env or as an environment variable.\n" + + "Get a token from: https://cloud.digitalocean.com/account/api/tokens" + ); + } + + if (!sshKeyFingerprint) { + throw new Error( + "SSH_KEY_FINGERPRINT is required.\n" + + "Set it in apps/load-tests/.env or as an environment variable.\n" + + "Find your SSH key fingerprint at: https://cloud.digitalocean.com/account/security" + ); + } + + // Expand ~ to home directory + const expandedKeyPath = sshPrivateKeyPath.replace(/^~/, os.homedir()); + + return { + digitalOceanToken, + sshKeyFingerprint, + sshPrivateKeyPath: expandedKeyPath, + }; +} + +/** + * Get SSH configuration for connecting to droplets. + */ +export function getSshConfig(config: InfraConfig): SshConfig { + return { + privateKeyPath: config.sshPrivateKeyPath, + username: "root", + port: 22, + }; +} + +/** + * Read the SSH private key from disk. + * Throws an error if the file doesn't exist. + */ +export function readSshPrivateKey(config: InfraConfig): string { + const keyPath = config.sshPrivateKeyPath; + + if (!fs.existsSync(keyPath)) { + throw new Error( + `SSH private key not found at: ${keyPath}\n` + + "Set SSH_PRIVATE_KEY_PATH in apps/load-tests/.env to the correct path." + ); + } + + return fs.readFileSync(keyPath, "utf-8"); +} + diff --git a/apps/load-tests/src/infra/digitalocean.ts b/apps/load-tests/src/infra/digitalocean.ts new file mode 100644 index 0000000..1f34c7f --- /dev/null +++ b/apps/load-tests/src/infra/digitalocean.ts @@ -0,0 +1,189 @@ +import type { CreateDropletOptions, DropletInfo } from "./types.js"; + +const DO_API_BASE = "https://api.digitalocean.com/v2"; + +/** + * DigitalOcean API error. + */ +export class DigitalOceanError extends Error { + constructor( + message: string, + public statusCode: number, + public responseBody?: string, + ) { + super(message); + this.name = "DigitalOceanError"; + } +} + +/** + * Make an authenticated request to the DigitalOcean API. + */ +async function doRequest( + token: string, + method: string, + path: string, + body?: unknown, +): Promise { + const response = await fetch(`${DO_API_BASE}${path}`, { + method, + headers: { + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + }, + body: body ? JSON.stringify(body) : undefined, + }); + + if (!response.ok) { + const text = await response.text(); + throw new DigitalOceanError( + `DigitalOcean API error: ${response.status} ${response.statusText}`, + response.status, + text, + ); + } + + // Handle 204 No Content + if (response.status === 204) { + return undefined as T; + } + + return response.json() as Promise; +} + +/** + * Parse a droplet from the API response. + */ +function parseDroplet(data: ApiDroplet): DropletInfo { + // Find the public IPv4 address + const publicIpv4 = data.networks?.v4?.find( + (n: { type: string }) => n.type === "public", + ); + + return { + id: data.id, + name: data.name, + status: data.status as DropletInfo["status"], + ip: publicIpv4?.ip_address ?? null, + region: data.region?.slug ?? "unknown", + size: data.size?.slug ?? "unknown", + createdAt: data.created_at, + }; +} + +// API response types (partial, only what we need) +interface ApiDroplet { + id: number; + name: string; + status: string; + created_at: string; + networks?: { + v4?: Array<{ type: string; ip_address: string }>; + }; + region?: { slug: string }; + size?: { slug: string }; +} + +interface ListDropletsResponse { + droplets: ApiDroplet[]; +} + +interface CreateDropletResponse { + droplet: ApiDroplet; +} + +interface GetDropletResponse { + droplet: ApiDroplet; +} + +/** + * List all droplets. + */ +export async function listDroplets(token: string): Promise { + const response = await doRequest( + token, + "GET", + "/droplets?per_page=200", + ); + return response.droplets.map(parseDroplet); +} + +/** + * List droplets matching a name prefix. + */ +export async function listDropletsByPrefix( + token: string, + prefix: string, +): Promise { + const all = await listDroplets(token); + return all.filter((d) => d.name.startsWith(prefix)); +} + +/** + * Create a new droplet. + */ +export async function createDroplet( + token: string, + options: CreateDropletOptions, +): Promise { + const response = await doRequest( + token, + "POST", + "/droplets", + { + name: options.name, + region: options.region, + size: options.size, + image: options.image, + ssh_keys: [options.sshKeyFingerprint], + user_data: options.userData, + }, + ); + return parseDroplet(response.droplet); +} + +/** + * Get a droplet by ID. + */ +export async function getDroplet( + token: string, + id: number, +): Promise { + const response = await doRequest( + token, + "GET", + `/droplets/${id}`, + ); + return parseDroplet(response.droplet); +} + +/** + * Delete a droplet by ID. + */ +export async function deleteDroplet(token: string, id: number): Promise { + await doRequest(token, "DELETE", `/droplets/${id}`); +} + +/** + * Wait for a droplet to reach the "active" status. + * Polls every 5 seconds up to the timeout. + */ +export async function waitForDropletActive( + token: string, + id: number, + timeoutMs = 120000, +): Promise { + const startTime = Date.now(); + const pollInterval = 5000; + + while (Date.now() - startTime < timeoutMs) { + const droplet = await getDroplet(token, id); + if (droplet.status === "active" && droplet.ip) { + return droplet; + } + await new Promise((resolve) => setTimeout(resolve, pollInterval)); + } + + throw new Error(`Droplet ${id} did not become active within ${timeoutMs}ms`); +} + diff --git a/apps/load-tests/src/infra/droplet.ts b/apps/load-tests/src/infra/droplet.ts new file mode 100644 index 0000000..a5a8914 --- /dev/null +++ b/apps/load-tests/src/infra/droplet.ts @@ -0,0 +1,170 @@ +import chalk from "chalk"; +import { execSsh, waitForSsh } from "./ssh.js"; +import type { DropletInfo, ExecResult } from "./types.js"; + +/** + * Status of a droplet during creation. + */ +export type DropletSetupStatus = + | "creating" + | "waiting_for_active" + | "waiting_for_ssh" + | "installing_nodejs" + | "cloning_repo" + | "installing_deps" + | "building" + | "ready" + | "failed"; + +/** + * Callback for progress updates during droplet setup. + */ +export type ProgressCallback = ( + droplet: DropletInfo, + status: DropletSetupStatus, + message?: string, +) => void; + +/** + * Validate a git branch name to prevent shell injection. + * Allows alphanumeric, hyphens, underscores, slashes, and dots. + */ +function validateBranchName(branch: string): void { + if (!/^[\w.\-/]+$/.test(branch)) { + throw new Error(`Invalid branch name: "${branch}". Only alphanumeric, hyphens, underscores, slashes, and dots are allowed.`); + } +} + +/** + * Generate the setup script for a droplet. + * Downloads Node.js directly from nodejs.org and installs to /usr/local. + */ +export function generateSetupScript(branch: string): string { + // Validate branch name to prevent shell injection + validateBranchName(branch); + + return `#!/bin/bash +set -e + +export DEBIAN_FRONTEND=noninteractive +NODE_VERSION="20.19.0" + +echo "=== Waiting for apt locks (up to 2 min) ===" +WAIT_COUNT=0 +while fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 || \ + fuser /var/lib/apt/lists/lock >/dev/null 2>&1 || \ + fuser /var/cache/apt/archives/lock >/dev/null 2>&1; do + echo "Waiting for apt locks... (\$WAIT_COUNT s)" + sleep 5 + WAIT_COUNT=\$((WAIT_COUNT + 5)) + if [ \$WAIT_COUNT -ge 120 ]; then + echo "Timed out waiting for apt locks, proceeding anyway..." + break + fi +done + +echo "=== Downloading Node.js \$NODE_VERSION ===" +cd /tmp +curl -fsSL "https://nodejs.org/dist/v\$NODE_VERSION/node-v\$NODE_VERSION-linux-x64.tar.xz" -o node.tar.xz + +echo "=== Installing Node.js to /usr/local ===" +tar -xJf node.tar.xz +cp -r node-v\$NODE_VERSION-linux-x64/{bin,lib,share} /usr/local/ +rm -rf node.tar.xz node-v\$NODE_VERSION-linux-x64 + +echo "=== Verifying Node installation ===" +/usr/local/bin/node --version +/usr/local/bin/npm --version + +echo "=== Installing Yarn via corepack ===" +/usr/local/bin/corepack enable +/usr/local/bin/corepack prepare yarn@stable --activate + +echo "=== Cloning repository ===" +git clone --branch ${branch} https://github.com/MetaMask/mobile-wallet-protocol /app + +echo "=== Installing dependencies ===" +cd /app +/usr/local/bin/yarn install + +echo "=== Building ===" +/usr/local/bin/yarn build + +echo "=== Setup complete ===" +`; +} + +/** + * Run the setup script on a droplet. + * Reports progress via the callback. + */ +export async function setupDroplet( + droplet: DropletInfo, + branch: string, + privateKeyPath: string, + onProgress?: ProgressCallback, +): Promise { + if (!droplet.ip) { + throw new Error(`Droplet ${droplet.name} has no IP address`); + } + + // Wait for SSH to be available + onProgress?.(droplet, "waiting_for_ssh"); + await waitForSsh(droplet.ip, privateKeyPath); + + // Run the setup script + onProgress?.(droplet, "installing_nodejs"); + const script = generateSetupScript(branch); + + try { + const result = await execSsh( + droplet.ip, + script, + privateKeyPath, + 600000, // 10 minute timeout for full setup + ); + + if (result.exitCode === 0) { + onProgress?.(droplet, "ready"); + return { ...result, dropletName: droplet.name }; + } else { + // Get last few lines of stderr or stdout for context + const errorContext = (result.stderr || result.stdout).trim().split("\n").slice(-5).join("\n"); + const errorMsg = `Exit code ${result.exitCode}: ${errorContext}`; + onProgress?.(droplet, "failed", errorMsg); + throw new Error(errorMsg); + } + } catch (error) { + onProgress?.(droplet, "failed", (error as Error).message); + throw error; + } +} + +/** + * Format a status for display. + */ +export function formatStatus(status: DropletSetupStatus): string { + switch (status) { + case "creating": + return chalk.dim("○ creating..."); + case "waiting_for_active": + return chalk.dim("○ waiting for active..."); + case "waiting_for_ssh": + return chalk.yellow("● waiting for SSH..."); + case "installing_nodejs": + return chalk.yellow("● installing Node.js..."); + case "cloning_repo": + return chalk.yellow("● cloning repo..."); + case "installing_deps": + return chalk.yellow("● installing deps..."); + case "building": + return chalk.yellow("● building..."); + case "ready": + return chalk.green("✓ ready"); + case "failed": + return chalk.red("✗ failed"); + default: + return chalk.dim("○ unknown"); + } +} + diff --git a/apps/load-tests/src/infra/exec.ts b/apps/load-tests/src/infra/exec.ts new file mode 100644 index 0000000..9eb4b0a --- /dev/null +++ b/apps/load-tests/src/infra/exec.ts @@ -0,0 +1,130 @@ +import * as fs from "node:fs"; +import * as path from "node:path"; +import chalk from "chalk"; +import { execSsh } from "./ssh.js"; +import type { DropletInfo, ExecResult } from "./types.js"; + +/** + * Execute a command on multiple droplets in parallel. + * Returns results for each droplet. + */ +export async function execOnDroplets( + droplets: DropletInfo[], + command: string, + privateKeyPath: string, + onProgress?: (droplet: DropletInfo, status: "running" | "done" | "failed") => void, +): Promise { + const results: ExecResult[] = []; + + const execPromises = droplets.map(async (droplet) => { + if (!droplet.ip) { + const result: ExecResult = { + dropletName: droplet.name, + dropletIp: "", + exitCode: -1, + stdout: "", + stderr: "", + durationMs: 0, + error: "No IP address", + }; + results.push(result); + onProgress?.(droplet, "failed"); + return result; + } + + onProgress?.(droplet, "running"); + + try { + const result = await execSsh(droplet.ip, command, privateKeyPath); + result.dropletName = droplet.name; + results.push(result); + onProgress?.(droplet, result.exitCode === 0 ? "done" : "failed"); + return result; + } catch (error) { + const result: ExecResult = { + dropletName: droplet.name, + dropletIp: droplet.ip, + exitCode: -1, + stdout: "", + stderr: "", + durationMs: 0, + error: (error as Error).message, + }; + results.push(result); + onProgress?.(droplet, "failed"); + return result; + } + }); + + await Promise.all(execPromises); + return results; +} + +/** + * Save execution logs to a directory. + * Creates one file per droplet with stdout/stderr. + */ +export function saveExecLogs( + results: ExecResult[], + logsDir: string, +): void { + // Create logs directory + if (!fs.existsSync(logsDir)) { + fs.mkdirSync(logsDir, { recursive: true }); + } + + for (const result of results) { + const logPath = path.join(logsDir, `${result.dropletName}.log`); + let content = `# ${result.dropletName} (${result.dropletIp})\n`; + content += `# Exit code: ${result.exitCode}\n`; + content += `# Duration: ${result.durationMs}ms\n`; + if (result.error) { + content += `# Error: ${result.error}\n`; + } + content += "\n--- STDOUT ---\n"; + content += result.stdout || "(empty)\n"; + content += "\n--- STDERR ---\n"; + content += result.stderr || "(empty)\n"; + + fs.writeFileSync(logPath, content); + } +} + +/** + * Format a duration in a human-readable way. + */ +export function formatDuration(ms: number): string { + if (ms < 1000) return `${ms}ms`; + const seconds = Math.floor(ms / 1000); + if (seconds < 60) return `${seconds}s`; + const minutes = Math.floor(seconds / 60); + const remainingSeconds = seconds % 60; + return `${minutes}m ${remainingSeconds}s`; +} + +/** + * Print execution results summary. + */ +export function printExecResults(results: ExecResult[]): void { + const successful = results.filter((r) => r.exitCode === 0); + const failed = results.filter((r) => r.exitCode !== 0); + + console.log(""); + if (failed.length === 0) { + console.log(chalk.green(`[infra exec] Complete! ${successful.length}/${results.length} succeeded.`)); + } else { + console.log(chalk.yellow(`[infra exec] Done. ${successful.length}/${results.length} succeeded, ${failed.length} failed.`)); + } + console.log(""); + + for (const result of results) { + const icon = result.exitCode === 0 ? chalk.green("✓") : chalk.red("✗"); + const exitCodeStr = result.error + ? chalk.red(`error: ${result.error.substring(0, 30)}`) + : result.exitCode === 0 + ? chalk.green(`exit ${result.exitCode}`) + : chalk.red(`exit ${result.exitCode}`); + console.log(` ${result.dropletName.padEnd(14)} ${icon} ${exitCodeStr} (${formatDuration(result.durationMs)})`); + } +} + diff --git a/apps/load-tests/src/infra/ssh.ts b/apps/load-tests/src/infra/ssh.ts new file mode 100644 index 0000000..60c6711 --- /dev/null +++ b/apps/load-tests/src/infra/ssh.ts @@ -0,0 +1,177 @@ +import * as fs from "node:fs"; +import { Client } from "ssh2"; +import type { ExecResult } from "./types.js"; + +/** + * Sleep for the specified number of milliseconds. + */ +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** + * Wait for SSH to be available on a droplet. + * Retries with exponential backoff up to the timeout. + */ +export async function waitForSsh( + ip: string, + privateKeyPath: string, + timeoutMs = 120000, +): Promise { + const startTime = Date.now(); + let delay = 2000; // Start with 2s delay + const maxDelay = 10000; // Max 10s between retries + + while (Date.now() - startTime < timeoutMs) { + try { + // Try to connect and run a simple command + await execSsh(ip, "echo ok", privateKeyPath, 10000); + return; // Success! + } catch { + // Not ready yet, wait and retry + await sleep(delay); + delay = Math.min(delay * 1.5, maxDelay); + } + } + + throw new Error(`SSH not available on ${ip} after ${timeoutMs}ms`); +} + +/** + * Execute a command on a remote host via SSH. + * Returns the result with stdout, stderr, and exit code. + */ +export async function execSsh( + ip: string, + command: string, + privateKeyPath: string, + timeoutMs = 300000, // 5 minutes default +): Promise { + const startTime = Date.now(); + const privateKey = fs.readFileSync(privateKeyPath, "utf-8"); + + return new Promise((resolve, reject) => { + const client = new Client(); + let stdout = ""; + let stderr = ""; + let exitCode = -1; + let timedOut = false; + + const timeout = setTimeout(() => { + timedOut = true; + client.end(); + reject(new Error(`SSH command timed out after ${timeoutMs}ms`)); + }, timeoutMs); + + client.on("ready", () => { + client.exec(command, (err, stream) => { + if (err) { + clearTimeout(timeout); + client.end(); + reject(err); + return; + } + + stream.on("close", (code: number | null) => { + clearTimeout(timeout); + exitCode = code ?? 1; // Assume failure if no exit code + client.end(); + resolve({ + dropletName: "", // Will be filled in by caller + dropletIp: ip, + exitCode, + stdout, + stderr, + durationMs: Date.now() - startTime, + }); + }); + + stream.on("data", (data: Buffer) => { + stdout += data.toString(); + }); + + stream.stderr.on("data", (data: Buffer) => { + stderr += data.toString(); + }); + }); + }); + + client.on("error", (err) => { + if (!timedOut) { + clearTimeout(timeout); + client.end(); + reject(err); + } + }); + + client.connect({ + host: ip, + port: 22, + username: "root", + privateKey, + readyTimeout: 20000, + }); + }); +} + +/** + * Download a file from a remote host via SFTP. + */ +export async function downloadFile( + ip: string, + remotePath: string, + localPath: string, + privateKeyPath: string, + timeoutMs = 60000, // 1 minute default timeout +): Promise { + const privateKey = fs.readFileSync(privateKeyPath, "utf-8"); + + return new Promise((resolve, reject) => { + const client = new Client(); + let timedOut = false; + + const timeout = setTimeout(() => { + timedOut = true; + client.end(); + reject(new Error(`SFTP download timed out after ${timeoutMs}ms`)); + }, timeoutMs); + + client.on("ready", () => { + client.sftp((err, sftp) => { + if (err) { + clearTimeout(timeout); + client.end(); + reject(err); + return; + } + + sftp.fastGet(remotePath, localPath, (err) => { + clearTimeout(timeout); + client.end(); + if (err) { + reject(err); + } else { + resolve(); + } + }); + }); + }); + + client.on("error", (err) => { + if (!timedOut) { + clearTimeout(timeout); + client.end(); + reject(err); + } + }); + + client.connect({ + host: ip, + port: 22, + username: "root", + privateKey, + readyTimeout: 20000, + }); + }); +} + diff --git a/apps/load-tests/src/infra/types.ts b/apps/load-tests/src/infra/types.ts new file mode 100644 index 0000000..88fd189 --- /dev/null +++ b/apps/load-tests/src/infra/types.ts @@ -0,0 +1,71 @@ +/** + * Droplet information returned from DigitalOcean API. + */ +export interface DropletInfo { + id: number; + name: string; + status: "new" | "active" | "off" | "archive"; + ip: string | null; + region: string; + size: string; + createdAt: string; +} + +/** + * Options for creating a droplet via the API. + */ +export interface CreateDropletOptions { + name: string; + region: string; + size: string; + image: string; + sshKeyFingerprint: string; + userData?: string; +} + +/** + * Options for the create command. + */ +export interface CreateOptions { + count: number; + namePrefix: string; + branch: string; +} + +/** + * Result of executing a command on a droplet. + */ +export interface ExecResult { + dropletName: string; + dropletIp: string; + exitCode: number; + stdout: string; + stderr: string; + durationMs: number; + error?: string; +} + +/** + * Infrastructure configuration loaded from environment. + */ +export interface InfraConfig { + digitalOceanToken: string; + sshKeyFingerprint: string; + sshPrivateKeyPath: string; +} + +/** + * SSH connection configuration. + */ +export interface SshConfig { + privateKeyPath: string; + username: string; + port: number; +} + +// Hardcoded droplet settings +export const DROPLET_REGION = "nyc1"; +export const DROPLET_SIZE = "s-2vcpu-2gb"; // 2GB RAM needed for yarn install +export const DROPLET_IMAGE = "ubuntu-24-04-x64"; +export const DROPLET_HOURLY_COST = 0.018; // USD per hour for s-2vcpu-2gb + diff --git a/apps/load-tests/src/results/aggregate.ts b/apps/load-tests/src/results/aggregate.ts new file mode 100644 index 0000000..7cd50eb --- /dev/null +++ b/apps/load-tests/src/results/aggregate.ts @@ -0,0 +1,207 @@ +import * as fs from "node:fs"; +import * as path from "node:path"; +import chalk from "chalk"; +import type { TestResults } from "../output/types.js"; +import { calculateConnectTimeStats } from "../utils/stats.js"; + +/** + * Aggregated results from multiple load test runs. + */ +export interface AggregatedResults { + dropletCount: number; + files: string[]; + scenario: string; + target: string; + totals: { + connections: { + attempted: number; + successful: number; + failed: number; + successRate: number; + immediate: number; + recovered: number; + }; + timing: { + totalTimeMs: number; + avgTimeMs: number; + connectionsPerSec: number; + }; + connectTime: { + min: number; + max: number; + avg: number; + p50: number; + p95: number; + p99: number; + } | null; + retries: { + totalRetries: number; + avgRetriesPerConnection: number; + }; + }; + perDroplet: Array<{ + file: string; + connections: number; + successRate: number; + avgConnectTime: number | null; + }>; +} + +/** + * Load and aggregate results from a directory of JSON files. + */ +export function aggregateResults(inputDir: string): AggregatedResults { + // Find all JSON files in the directory + if (!fs.existsSync(inputDir)) { + throw new Error(`Directory not found: ${inputDir}`); + } + + const files = fs.readdirSync(inputDir).filter((f) => f.endsWith(".json")); + if (files.length === 0) { + throw new Error(`No JSON files found in: ${inputDir}`); + } + + // Load all results + const results: Array<{ file: string; data: TestResults }> = []; + for (const file of files) { + const filePath = path.join(inputDir, file); + const content = fs.readFileSync(filePath, "utf-8"); + try { + const data = JSON.parse(content) as TestResults; + // Validate that the parsed data has the expected structure + if (!data.scenario || !data.target || !data.results?.connections) { + console.warn(chalk.yellow(`Warning: ${file} is missing required fields, skipping`)); + continue; + } + results.push({ file, data }); + } catch { + console.warn(chalk.yellow(`Warning: Could not parse ${file}, skipping`)); + } + } + + if (results.length === 0) { + throw new Error("No valid result files found"); + } + + // Aggregate totals + let totalAttempted = 0; + let totalSuccessful = 0; + let totalFailed = 0; + let totalImmediate = 0; + let totalRecovered = 0; + let totalTimeMs = 0; + let totalRetries = 0; + const allConnectTimes: number[] = []; + + const perDroplet: AggregatedResults["perDroplet"] = []; + + for (const { file, data } of results) { + const conn = data.results.connections; + totalAttempted += conn.attempted; + totalSuccessful += conn.successful; + totalFailed += conn.failed; + totalImmediate += conn.immediate; + totalRecovered += conn.recovered; + totalTimeMs += data.results.timing.totalTimeMs; + totalRetries += data.results.retries.totalRetries; + + // Collect individual connect times for aggregate stats + // We use the per-droplet average * count as an approximation + // since individual times aren't stored in TestResults + if (data.results.connectTime) { + // Push multiple samples based on connection count to weight the average + const times = data.results.connectTime; + allConnectTimes.push(times.min, times.avg, times.max); + } + + perDroplet.push({ + file, + connections: conn.attempted, + successRate: conn.successRate, + avgConnectTime: data.results.connectTime?.avg ?? null, + }); + } + + // Use the first result for scenario/target info + const first = results[0].data; + + return { + dropletCount: results.length, + files: files, + scenario: first.scenario, + target: first.target, + totals: { + connections: { + attempted: totalAttempted, + successful: totalSuccessful, + failed: totalFailed, + successRate: totalAttempted > 0 ? (totalSuccessful / totalAttempted) * 100 : 0, + immediate: totalImmediate, + recovered: totalRecovered, + }, + timing: { + totalTimeMs, + avgTimeMs: totalTimeMs / results.length, + connectionsPerSec: totalTimeMs > 0 ? (totalAttempted / (totalTimeMs / 1000)) * results.length : 0, + }, + connectTime: calculateConnectTimeStats(allConnectTimes), + retries: { + totalRetries, + avgRetriesPerConnection: totalAttempted > 0 ? totalRetries / totalAttempted : 0, + }, + }, + perDroplet, + }; +} + +/** + * Print aggregated results. + */ +export function printAggregatedResults(agg: AggregatedResults): void { + console.log(chalk.gray("─".repeat(60))); + console.log(chalk.bold.cyan(" DISTRIBUTED TEST SUMMARY")); + console.log(chalk.gray("─".repeat(60))); + console.log(""); + + console.log(chalk.bold("Overview:")); + console.log(` Droplets: ${chalk.cyan(agg.dropletCount)}`); + console.log(` Scenario: ${agg.scenario}`); + console.log(` Target: ${chalk.dim(agg.target)}`); + console.log(""); + + console.log(chalk.bold("Connections:")); + const successRate = agg.totals.connections.successRate; + const rateColor = successRate >= 99 ? chalk.green : successRate >= 95 ? chalk.yellow : chalk.red; + console.log(` Total: ${chalk.cyan(agg.totals.connections.attempted)}`); + console.log(` Successful: ${chalk.green(agg.totals.connections.successful)} (${rateColor(successRate.toFixed(1) + "%")})`); + console.log(` Failed: ${agg.totals.connections.failed > 0 ? chalk.red(agg.totals.connections.failed) : chalk.green("0")}`); + console.log(` Immediate: ${agg.totals.connections.immediate}`); + console.log(` Recovered: ${agg.totals.connections.recovered}`); + console.log(""); + + console.log(chalk.bold("Timing:")); + console.log(` Avg Duration: ${Math.round(agg.totals.timing.avgTimeMs)}ms`); + console.log(` Throughput: ${agg.totals.timing.connectionsPerSec.toFixed(1)} conn/sec (combined)`); + console.log(""); + + if (agg.totals.retries.totalRetries > 0) { + console.log(chalk.bold("Retries:")); + console.log(chalk.yellow(` Total: ${agg.totals.retries.totalRetries}`)); + console.log(chalk.yellow(` Avg per Conn: ${agg.totals.retries.avgRetriesPerConnection.toFixed(2)}`)); + console.log(""); + } + + console.log(chalk.bold("Per Droplet:")); + console.log(chalk.dim(" FILE CONNECTIONS SUCCESS AVG CONNECT")); + for (const d of agg.perDroplet) { + const connectTimeStr = d.avgConnectTime !== null ? `${Math.round(d.avgConnectTime)}ms` : "-"; + const rateColor = d.successRate >= 99 ? chalk.green : d.successRate >= 95 ? chalk.yellow : chalk.red; + console.log( + ` ${d.file.padEnd(22)} ${String(d.connections).padEnd(13)} ${rateColor(d.successRate.toFixed(1) + "%").padEnd(9)} ${connectTimeStr}`, + ); + } + + console.log(""); + console.log(chalk.gray("─".repeat(60))); +} +