From 4f77af35931f338acf50764f60e161ba6ac6fb52 Mon Sep 17 00:00:00 2001 From: kian woon Date: Fri, 3 Apr 2026 04:50:48 +0800 Subject: [PATCH] feat(proxy): make connection-level retries configurable per provider Add `connectionRetries` config field (default 3) to control how many times forwardWithRetry() retries a provider on TTFB/stall/connection errors before escalating to fallback. Operators can now set low values (1-2) to fail fast on known-bad providers, reducing worst-case latency from ~35s to ~8-17s on connection failures. - src/types.ts: add _connectionRetries to ProviderConfig - src/config.ts: add to Zod schema, peekConfig type, and loadConfig mapping - src/proxy.ts: use provider._connectionRetries ?? CONNECTION_RETRY_MAX --- src/config.ts | 7 +++++-- src/proxy.ts | 7 ++++--- src/types.ts | 2 ++ 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/config.ts b/src/config.ts index f9207cd..d4e7081 100644 --- a/src/config.ts +++ b/src/config.ts @@ -134,6 +134,7 @@ const providerSchema = z.object({ modelLimits: modelLimitsSchema, concurrentLimit: z.number().int().min(1).optional(), poolSize: z.number().int().min(1).max(100).optional(), + connectionRetries: z.number().int().min(0).max(10).optional(), circuitBreaker: z.object({ failureThreshold: z.number().int().min(1).optional(), windowSeconds: z.number().int().min(1).optional(), @@ -222,7 +223,7 @@ export function findConfigFile(cwd: string = process.cwd(), { skipGlobal = false * Used by init wizard to show existing providers and offer add/edit. */ export function peekConfig( cwd?: string, -): { configPath: string; providers: Map; server: { port: number; host: string } | null; modelRouting: Map; hedging?: { speculativeDelay: number; cvThreshold: number; maxHedge: number } } | null { +): { configPath: string; providers: Map; server: { port: number; host: string } | null; modelRouting: Map; hedging?: { speculativeDelay: number; cvThreshold: number; maxHedge: number } } | null { const configPath = findConfigFile(cwd); if (!configPath) return null; @@ -230,7 +231,7 @@ export function peekConfig( const parsed = parseYaml(raw) as Record; const providersRaw = (parsed?.providers ?? {}) as Record>; - const providers = new Map(); + const providers = new Map(); for (const [id, config] of Object.entries(providersRaw)) { const apiKey = String(config.apiKey ?? ""); @@ -254,6 +255,7 @@ export function peekConfig( concurrentLimit: config.concurrentLimit !== undefined ? Number(config.concurrentLimit) : undefined, stallTimeout: config.stallTimeout !== undefined ? Number(config.stallTimeout) : undefined, poolSize: config.poolSize !== undefined ? Number(config.poolSize) : undefined, + connectionRetries: config.connectionRetries !== undefined ? Number(config.connectionRetries) : undefined, circuitBreaker, }); } @@ -442,6 +444,7 @@ export async function loadConfig(configPath?: string, cwd?: string): Promise<{ c }); createdAgents.push(providerConfig._agent); providerConfig.poolSize = poolSize ?? 10; + providerConfig._connectionRetries = p.connectionRetries; // Create per-provider circuit breaker const cbConfig = p.circuitBreaker; providerConfig._circuitBreaker = new CircuitBreaker(cbConfig ? { diff --git a/src/proxy.ts b/src/proxy.ts index 0bad4f3..64e5b26 100644 --- a/src/proxy.ts +++ b/src/proxy.ts @@ -997,7 +997,8 @@ async function forwardWithRetry( ): Promise { let lastResult: Response | undefined; - for (let attempt = 0; attempt <= CONNECTION_RETRY_MAX; attempt++) { + const maxRetries = provider._connectionRetries ?? CONNECTION_RETRY_MAX; + for (let attempt = 0; attempt <= maxRetries; attempt++) { const result = await forwardRequest(provider, entry, ctx, incomingRequest, chainSignal, index); // Non-502 responses pass through immediately (success or upstream error) @@ -1035,7 +1036,7 @@ async function forwardWithRetry( } const delay = CONNECTION_RETRY_BASE_MS * Math.pow(2, attempt); - console.warn(`[proxy] Connection error on "${provider.name}" (attempt ${attempt + 1}/${CONNECTION_RETRY_MAX}), retrying in ${delay}ms: ${body.slice(0, 200)}`); + console.warn(`[proxy] Connection error on "${provider.name}" (attempt ${attempt + 1}/${maxRetries}), retrying in ${delay}ms: ${body.slice(0, 200)}`); // Reset stream state for retry ctx._streamState = "start"; @@ -1056,7 +1057,7 @@ async function forwardWithRetry( } // Stall errors are recorded in handleStall() (per-request, no retry amplification). - console.warn(`[proxy] All ${CONNECTION_RETRY_MAX + 1} attempts failed for "${provider.name}" — escalating to fallback`); + console.warn(`[proxy] All ${maxRetries + 1} attempts failed for "${provider.name}" — escalating to fallback`); return lastResult!; } diff --git a/src/types.ts b/src/types.ts index dcefe42..0ef3930 100644 --- a/src/types.ts +++ b/src/types.ts @@ -23,6 +23,8 @@ export interface ProviderConfig { _circuitBreaker?: CircuitBreaker; _serverConfig?: ServerConfig; poolSize?: number; + /** Max connection-level retries (TTFB timeout/stall/connection failure) before escalating to fallback. Default: 3 */ + _connectionRetries?: number; } export interface RoutingEntry {