From 886c3bbcf0ee476a5079c995c0e9cfe15783c671 Mon Sep 17 00:00:00 2001 From: Conny Brunnkvist Date: Fri, 20 Feb 2026 11:06:21 +0700 Subject: [PATCH] feat: add roll-call command for batch-testing model connectivity --- packages/opencode/src/cli/cmd/roll-call.ts | 298 +++++++++++++++++++ packages/opencode/src/index.ts | 2 + packages/opencode/test/cli/roll-call.test.ts | 209 +++++++++++++ 3 files changed, 509 insertions(+) create mode 100644 packages/opencode/src/cli/cmd/roll-call.ts create mode 100644 packages/opencode/test/cli/roll-call.test.ts diff --git a/packages/opencode/src/cli/cmd/roll-call.ts b/packages/opencode/src/cli/cmd/roll-call.ts new file mode 100644 index 0000000000..7b8dd3ba89 --- /dev/null +++ b/packages/opencode/src/cli/cmd/roll-call.ts @@ -0,0 +1,298 @@ +import type { Argv } from "yargs" +import { Instance } from "../../project/instance" +import { Provider } from "../../provider/provider" +import { ProviderTransform } from "../../provider/transform" +import { cmd } from "./cmd" +import { UI } from "../ui" +import { APICallError } from "ai" +import { ProviderError } from "../../provider/error" +import { generateText, type ModelMessage } from "ai" +import { randomUUID } from "crypto" + +const HEADERS = ["Model", "Access", "Snippet", "Latency"] +const SEPARATOR_PADDING = 9 // " | " between 4 columns = 3 * 3 = 9 chars + +// Detect if stderr is a TTY for conditional color output +const isTTY = process.stderr.isTTY ?? false + +// Helper to conditionally apply colors only when output is to a TTY +function color(style: string): string { + return isTTY ? style : "" +} + +// Strip ANSI escape sequences and control characters for accurate width calculation +function sanitize(text: string): string { + return text + .replace(/\x1b\[[0-9;]*m/g, "") // ANSI color codes + .replace(/[\x00-\x1f\x7f]/g, "") // control characters (including \0, \n, etc.) +} + +function truncate(text: string, maxLen: number): string { + if (maxLen < 4) return text.substring(0, maxLen) + return text.length > maxLen ? text.substring(0, maxLen - 3) + "..." : text +} + +export function formatTable( + rows: string[][], + terminalWidth: number, +): { header: string; separator: string; rows: string[] } { + // Sanitize all cell content to strip control chars and ANSI codes + const sanitizedRows = rows.map((row) => row.map((cell) => sanitize(cell ?? ""))) + + // Calculate natural width for each column based on sanitized content + const widths = HEADERS.map((h, i) => Math.max(h.length, ...sanitizedRows.map((r) => r[i].length))) + + // Total width with separators + const totalWidth = widths.reduce((a, b) => a + b, 0) + SEPARATOR_PADDING + + // Only shrink snippet column (index 2) if total exceeds terminal width + // Minimum snippet width is header length (7) + 3 chars for meaningful content with "..." + const minSnippetWidth = HEADERS[2].length + 3 + if (totalWidth > terminalWidth && widths[2] > minSnippetWidth) { + const overflow = totalWidth - terminalWidth + widths[2] = Math.max(minSnippetWidth, widths[2] - overflow) + } + + const header = HEADERS.map((h, i) => h.padEnd(widths[i])).join(" | ") + const separator = "-".repeat(header.length) + + const formattedRows = sanitizedRows.map((row) => { + const truncatedRow = [row[0], row[1], row[2] ? truncate(row[2], widths[2]) : row[2], row[3]] + return truncatedRow.map((c, i) => c.padEnd(widths[i])).join(" | ") + }) + + return { header, separator, rows: formattedRows } +} + +export const RollCallCommand = cmd({ + command: "roll-call ", + describe: "batch-test models matching a filter for connectivity and latency", + builder: (yargs: Argv) => { + return yargs + .positional("filter", { + type: "string", + describe: "regex to filter models by provider/modelID (required)", + demandOption: true, + }) + .option("prompt", { + type: "string", + default: "Hello", + describe: "Prompt to send to each model", + }) + .option("timeout", { + type: "number", + default: 25000, + describe: "Timeout for each model call in milliseconds", + }) + .option("parallel", { + type: "number", + default: 5, + describe: "Number of parallel model calls", + }) + .option("retries", { + type: "number", + default: 0, + describe: "Number of additional retries for each model call", + }) + .option("verbose", { + type: "boolean", + default: false, + describe: "Show verbose output", + }) + .option("quiet", { + type: "boolean", + default: false, + describe: "Suppress non-error output", + }) + .option("output", { + type: "string", + choices: ["table", "json"], + default: "table", + describe: "Output format", + }) + }, + handler: async (args) => { + await rollCallHandler(args) + }, +}) + +interface RollCallResult { + model: string + access: boolean + snippet: string + latency: number | null + errorType: string | null + errorMessage: string | null +} + +export async function rollCallHandler(args: any) { + const { prompt, timeout, filter, parallel, output, verbose, quiet } = args + + if (!quiet) { + UI.println( + `${color(UI.Style.TEXT_INFO)}Starting roll call for models with prompt: "${prompt}"${color(UI.Style.TEXT_NORMAL)}`, + ) + UI.println( + `${color(UI.Style.TEXT_INFO)}Timeout per model: ${timeout}ms, Parallel calls: ${parallel}${color(UI.Style.TEXT_NORMAL)}`, + ) + } + + await Instance.provide({ + directory: process.cwd(), + async fn() { + const providers = await Provider.list() + const modelsToTest: { providerID: string; modelID: string; model: Provider.Model }[] = [] + + for (const [providerID, provider] of Object.entries(providers)) { + for (const [modelID, model] of Object.entries(provider.models)) { + const fullName = `${providerID}/${modelID}` + if (filter) { + try { + const regex = new RegExp(filter, "i") + if (!regex.test(fullName)) continue + } catch (e) { + UI.error(`Invalid filter regex: ${filter}`) + return + } + } + modelsToTest.push({ providerID, modelID, model }) + } + } + + if (modelsToTest.length === 0) { + if (!quiet) + UI.println(`${color(UI.Style.TEXT_WARNING)}No models to test after filtering.${color(UI.Style.TEXT_NORMAL)}`) + return + } + + if (!quiet) { + UI.println( + `${color(UI.Style.TEXT_INFO)}Prompting ${modelsToTest.length} models...${color(UI.Style.TEXT_NORMAL)}`, + ) + } + + const results: RollCallResult[] = [] + const queue = [...modelsToTest] + const activePromises: Promise[] = [] + + const processModel = async (item: (typeof modelsToTest)[0]) => { + const { providerID, modelID, model } = item + const fullName = `${providerID}/${modelID}` + const startTime = Date.now() + let access = false + let snippet = "" + let latency: number | null = null + let errorType: string | null = null + let errorMessage: string | null = null + + try { + const languageModel = await Provider.getLanguage(model) + + // Build provider options similar to how session/index.ts does it + const sessionID = randomUUID() + const baseOptions = ProviderTransform.options({ model, sessionID }) + const providerOptions = ProviderTransform.providerOptions(model, baseOptions) + const maxTokens = ProviderTransform.maxOutputTokens(model) + const temperature = ProviderTransform.temperature(model) + const topP = ProviderTransform.topP(model) + const topK = ProviderTransform.topK(model) + + const messages: ModelMessage[] = [{ role: "user", content: prompt }] + const transformedMessages = ProviderTransform.message(messages, model, baseOptions) + + const { text } = await generateText({ + model: languageModel, + messages: transformedMessages, + abortSignal: AbortSignal.timeout(timeout), + maxOutputTokens: maxTokens, + temperature, + topP, + topK, + providerOptions, + }) + access = true + snippet = text.replace(/\n/g, " ") + latency = Date.now() - startTime + } catch (e: any) { + latency = Date.now() - startTime + if (e instanceof APICallError) { + const parsedError = ProviderError.parseAPICallError({ + providerID, + error: e, + }) + errorType = parsedError.type + errorMessage = parsedError.message + } else { + errorType = "unknown" + errorMessage = e.message || "An unknown error occurred" + } + } + + results.push({ + model: fullName, + access, + snippet, + latency, + errorType, + errorMessage, + }) + + if (verbose && !quiet) { + if (access) { + UI.println(`${color(UI.Style.TEXT_SUCCESS)}✔${color(UI.Style.TEXT_NORMAL)} ${fullName} - ${latency}ms`) + } else { + UI.println( + `${color(UI.Style.TEXT_DANGER)}✘${color(UI.Style.TEXT_NORMAL)} ${fullName} - ${errorType}: ${errorMessage}`, + ) + } + } + } + + while (queue.length > 0 || activePromises.length > 0) { + while (queue.length > 0 && activePromises.length < parallel) { + const item = queue.shift()! + const promise = processModel(item).finally(() => { + const index = activePromises.indexOf(promise) + if (index > -1) { + activePromises.splice(index, 1) + } + }) + activePromises.push(promise) + } + if (activePromises.length > 0) { + await Promise.race(activePromises) + } + } + + if (quiet) return + + if (output === "json") { + console.log(JSON.stringify(results, null, 2)) + } else { + const rows = results.map((r) => [ + r.model, + r.access ? "YES" : "NO", + r.access ? r.snippet : r.errorMessage ? `(${r.errorMessage})` : "", + r.latency !== null ? `${r.latency}ms` : "N/A", + ]) + + const terminalWidth = parseInt(process.env.COLUMNS || "", 10) || process.stdout.columns || 80 + const table = formatTable(rows, terminalWidth) + + UI.println(table.header) + UI.println(table.separator) + table.rows.forEach((line, idx) => { + const rowColor = results[idx].access ? UI.Style.TEXT_SUCCESS : UI.Style.TEXT_DANGER + UI.println(color(rowColor) + line + color(UI.Style.TEXT_NORMAL)) + }) + + const successful = results.filter((r) => r.access).length + const failed = results.length - successful + UI.println("") + UI.println( + `${color(UI.Style.TEXT_SUCCESS)}${successful} accessible${color(UI.Style.TEXT_NORMAL)}, ${color(UI.Style.TEXT_DANGER)}${failed} failed${color(UI.Style.TEXT_NORMAL)}`, + ) + } + }, + }) +} diff --git a/packages/opencode/src/index.ts b/packages/opencode/src/index.ts index cc79306bc2..ffedfabab8 100644 --- a/packages/opencode/src/index.ts +++ b/packages/opencode/src/index.ts @@ -8,6 +8,7 @@ import { AgentCommand } from "./cli/cmd/agent" import { UpgradeCommand } from "./cli/cmd/upgrade" import { UninstallCommand } from "./cli/cmd/uninstall" import { ModelsCommand } from "./cli/cmd/models" +import { RollCallCommand } from "./cli/cmd/roll-call" import { UI } from "./cli/ui" import { Installation } from "./installation" import { NamedError } from "@opencode-ai/util/error" @@ -131,6 +132,7 @@ const cli = yargs(hideBin(process.argv)) .command(ServeCommand) .command(WebCommand) .command(ModelsCommand) + .command(RollCallCommand) .command(StatsCommand) .command(ExportCommand) .command(ImportCommand) diff --git a/packages/opencode/test/cli/roll-call.test.ts b/packages/opencode/test/cli/roll-call.test.ts new file mode 100644 index 0000000000..f9995c16a5 --- /dev/null +++ b/packages/opencode/test/cli/roll-call.test.ts @@ -0,0 +1,209 @@ +// kilocode_change - new file +import { describe, expect, test } from "bun:test" +import { formatTable } from "../../src/cli/cmd/roll-call" + +// NOTE: We test formatTable function directly. The color() helper and TTY detection +// are not directly tested as they depend on process.stderr.isTTY which changes +// based on how the process is invoked. + +describe("formatTable", () => { + describe("column width calculation", () => { + test("columns grow to fit content", () => { + const rows = [["kilo/provider/model-name", "YES", "Hi", "100ms"]] + const result = formatTable(rows, 120) + + // Model column should be wide enough for the model name + expect(result.header).toContain("Model") + expect(result.rows[0]).toContain("kilo/provider/model-name") + // All columns should be present + expect(result.rows[0]).toContain("YES") + expect(result.rows[0]).toContain("Hi") + expect(result.rows[0]).toContain("100ms") + }) + + test("short snippet does not force minimum width", () => { + const rows = [["m", "YES", "Hi", "1ms"]] + const result = formatTable(rows, 120) + + // Table should be compact when content is short + // Header is: "Model | Access | Snippet | Latency" + // Minimum widths are header lengths: Model=5, Access=6, Snippet=7, Latency=7 + // With separators: 5 + 6 + 7 + 7 + 9 = 34 + expect(result.header.length).toBeLessThan(50) + expect(result.separator.length).toBe(result.header.length) + }) + + test("separator length matches header length", () => { + const rows = [ + ["kilo/openai/gpt-4", "YES", "Hello there!", "500ms"], + ["kilo/anthropic/claude", "NO", "(Error)", "100ms"], + ] + const result = formatTable(rows, 120) + + expect(result.separator.length).toBe(result.header.length) + expect(result.separator).toMatch(/^-+$/) + }) + + test("all rows have same length as header", () => { + const rows = [ + ["short", "YES", "Hello", "10ms"], + ["kilo/very/long/provider/model-name", "NO", "(Some error message)", "1000ms"], + ] + const result = formatTable(rows, 120) + + for (const row of result.rows) { + expect(row.length).toBe(result.header.length) + } + }) + }) + + describe("snippet truncation", () => { + test("long snippet is truncated with ellipsis when exceeding terminal width", () => { + const longSnippet = "This is a very long snippet that should be truncated because it exceeds the available width" + const rows = [["kilo/provider/model", "YES", longSnippet, "100ms"]] + const result = formatTable(rows, 80) // narrow terminal + + expect(result.rows[0]).toContain("...") + expect(result.rows[0].length).toBeLessThanOrEqual(80) + }) + + test("snippet is not truncated when table fits terminal width", () => { + const snippet = "Short response" + const rows = [["m", "YES", snippet, "1ms"]] + const result = formatTable(rows, 120) + + expect(result.rows[0]).toContain(snippet) + expect(result.rows[0]).not.toContain("...") + }) + + test("table fits within terminal width when content exceeds available space", () => { + // This test reproduces the "off by 2" bug where table was 2 chars too wide + const rows = [ + ["kilo/openrouter/free", "YES", "Le temps passe vi...", "802ms"], + ["kilo/arcee-ai/trinity-large-preview:free", "YES", '"Le soleil brille...', "1527ms"], + ["kilo/minimax/minimax-m2.5:free", "YES", "Voici une phrase ...", "2615ms"], + ["kilo/stepfun/step-3.5-flash:free", "YES", "Aujourd'hui, je s...", "3561ms"], + ["kilo/corethink:free", "NO", "(Invalid JSON res...", "18490ms"], + ["kilo/z-ai/glm-5:free", "NO", "(The operation ti...", "25010ms"], + ] + const terminalWidth = 80 + const result = formatTable(rows, terminalWidth) + + // Header, separator, and all rows must fit within terminal width + expect(result.header.length).toBeLessThanOrEqual(terminalWidth) + expect(result.separator.length).toBeLessThanOrEqual(terminalWidth) + for (const row of result.rows) { + expect(row.length).toBeLessThanOrEqual(terminalWidth) + } + }) + + test("truncate handles very short maxLen gracefully", () => { + const rows = [["m", "YES", "Hello World", "1ms"]] + // This shouldn't crash even with extreme truncation + const result = formatTable(rows, 30) + expect(result.rows[0]).toBeDefined() + }) + }) + + describe("error messages", () => { + test("error message in parentheses is displayed", () => { + const rows = [["kilo/provider/model", "NO", "(Connection refused)", "500ms"]] + const result = formatTable(rows, 120) + + expect(result.rows[0]).toContain("(Connection refused)") + }) + + test("empty snippet cell is handled", () => { + const rows = [["kilo/provider/model", "NO", "", "500ms"]] + const result = formatTable(rows, 120) + + // Should not crash and row should be formatted + expect(result.rows[0]).toContain("kilo/provider/model") + expect(result.rows[0]).toContain("NO") + expect(result.rows[0]).toContain("500ms") + }) + }) + + describe("sanitization", () => { + test("strips ANSI color codes from snippet", () => { + const rows = [["model", "YES", "\x1b[92mGreen text\x1b[0m", "100ms"]] + const result = formatTable(rows, 120) + + expect(result.rows[0]).toContain("Green text") + expect(result.rows[0]).not.toContain("\x1b") + }) + + test("strips null bytes and control characters", () => { + const rows = [["model", "YES", "Hello\x00World\x01Test", "100ms"]] + const result = formatTable(rows, 120) + + expect(result.rows[0]).toContain("HelloWorldTest") + expect(result.rows[0]).not.toContain("\x00") + expect(result.rows[0]).not.toContain("\x01") + }) + + test("width calculation uses sanitized content", () => { + // ANSI codes add bytes but not visible width + const withAnsi = "\x1b[92mHi\x1b[0m" // "Hi" with color codes = 11 bytes but 2 visible chars + const rows = [["m", "YES", withAnsi, "1ms"]] + const result = formatTable(rows, 120) + + // Snippet column should be sized for "Hi" (2 chars), not 11 bytes + // Header "Snippet" is 7 chars, so minimum width is 7 + const snippetColStart = result.header.indexOf("Snippet") + const accessColEnd = result.header.indexOf("Access") + "Access".length + const snippetWidth = result.header.indexOf(" | Latency") - snippetColStart + + expect(snippetWidth).toBe(7) // "Snippet" header length, not inflated by ANSI codes + }) + + test("strips newlines from content", () => { + const rows = [["model", "YES", "Line1\nLine2", "100ms"]] + const result = formatTable(rows, 120) + + expect(result.rows[0]).toContain("Line1Line2") + expect(result.rows[0]).not.toContain("\n") + }) + }) + + describe("edge cases", () => { + test("empty rows array", () => { + const result = formatTable([], 120) + + expect(result.header).toContain("Model") + expect(result.rows).toHaveLength(0) + }) + + test("handles undefined cells gracefully", () => { + const rows = [["model", "YES", undefined as unknown as string, "100ms"]] + const result = formatTable(rows, 120) + + // Should not crash + expect(result.rows[0]).toContain("model") + }) + + test("very narrow terminal still produces valid output", () => { + const rows = [["kilo/provider/model", "YES", "Hello", "100ms"]] + const result = formatTable(rows, 40) + + // Should produce valid output even if truncated + expect(result.header.length).toBeGreaterThan(0) + expect(result.rows[0].length).toBe(result.header.length) + }) + + test("terminal width of 120 (default) handles typical content", () => { + const rows = [ + ["kilo/openai/gpt-4", "YES", "Hello! How can I help you today?", "500ms"], + ["kilo/anthropic/claude-3-opus", "YES", "Hi there! I'm Claude, an AI assistant...", "1200ms"], + ["kilo/google/gemini-pro", "NO", "(Rate limit exceeded)", "100ms"], + ] + const result = formatTable(rows, 120) + + // All rows should fit within 120 chars + expect(result.header.length).toBeLessThanOrEqual(120) + for (const row of result.rows) { + expect(row.length).toBeLessThanOrEqual(120) + } + }) + }) +})