From 6e25021625d08d28c7d4ff08266060a97ea1325e Mon Sep 17 00:00:00 2001 From: Chromie Bot Date: Tue, 27 Jan 2026 23:51:37 +0000 Subject: [PATCH 1/2] test: add vitest tests for model format deprecation Add unit tests to verify: - UnsupportedModelError includes guidance about provider/model format - LLMProvider logs deprecation warning for legacy model names - Legacy model names still work (non-breaking) - Provider/model format works without deprecation warning - UnsupportedAISDKModelProviderError for invalid providers Co-Authored-By: Claude Opus 4.5 --- packages/core/tests/model-deprecation.test.ts | 196 ++++++++++++++++++ 1 file changed, 196 insertions(+) create mode 100644 packages/core/tests/model-deprecation.test.ts diff --git a/packages/core/tests/model-deprecation.test.ts b/packages/core/tests/model-deprecation.test.ts new file mode 100644 index 000000000..85c1c4ec3 --- /dev/null +++ b/packages/core/tests/model-deprecation.test.ts @@ -0,0 +1,196 @@ +import { describe, expect, it, vi } from "vitest"; +import { LLMProvider } from "../lib/v3/llm/LLMProvider"; +import { + UnsupportedModelError, + UnsupportedAISDKModelProviderError, +} from "../lib/v3/types/public/sdkErrors"; +import type { LogLine } from "../lib/v3/types/public/logs"; + +// Mock client options with fake API keys for testing +const mockClientOptions = { apiKey: "test-api-key-for-testing" }; + +describe("Model format deprecation", () => { + describe("UnsupportedModelError", () => { + it("includes guidance to use provider/model format for unknown model names", () => { + const error = new UnsupportedModelError([ + "gpt-4o", + "claude-3-5-sonnet-latest", + ]); + + // Should mention the new format + expect(error.message).toContain("provider/model"); + // Should still list supported models + expect(error.message).toContain("gpt-4o"); + }); + + it("includes example of provider/model format", () => { + const error = new UnsupportedModelError(["gpt-4o"]); + + // Should provide an example like openai/gpt-4o + expect(error.message).toMatch(/openai\/gpt-4o|provider\/model/i); + }); + + it("works with feature parameter", () => { + const error = new UnsupportedModelError(["gpt-4o"], "extract"); + + expect(error.message).toContain("extract"); + expect(error.message).toContain("provider/model"); + }); + }); + + describe("LLMProvider.getClient deprecation warning", () => { + it("logs deprecation warning for legacy model names", () => { + const logs: LogLine[] = []; + const logger = (line: LogLine) => logs.push(line); + const provider = new LLMProvider(logger); + + // Using a legacy model name like "gpt-4o" instead of "openai/gpt-4o" + // Should not throw, but should log a deprecation warning + const client = provider.getClient("gpt-4o" as any, mockClientOptions); + + // Should return a client (not throw) + expect(client).toBeDefined(); + + // Should have logged a deprecation warning at level 0 + const deprecationWarning = logs.find( + (log) => + log.message.toLowerCase().includes("deprecated") || + log.message.toLowerCase().includes("deprecation"), + ); + expect(deprecationWarning).toBeDefined(); + expect(deprecationWarning!.level).toBe(0); + }); + + it("deprecation warning mentions provider/model format", () => { + const logs: LogLine[] = []; + const logger = (line: LogLine) => logs.push(line); + const provider = new LLMProvider(logger); + + provider.getClient("gpt-4o" as any, mockClientOptions); + + const deprecationWarning = logs.find( + (log) => + log.message.toLowerCase().includes("deprecated") || + log.message.toLowerCase().includes("deprecation"), + ); + + expect(deprecationWarning).toBeDefined(); + const message = deprecationWarning!.message; + // Should mention the provider/model format + expect(message).toContain("provider/model"); + // Should give an example + expect(message).toContain("openai/gpt-4o"); + }); + + it("returns OpenAIClient for legacy OpenAI model names", () => { + const logs: LogLine[] = []; + const logger = (line: LogLine) => logs.push(line); + const provider = new LLMProvider(logger); + + const client = provider.getClient("gpt-4o" as any, mockClientOptions); + + // Should return a client + expect(client).toBeDefined(); + // The client should be an OpenAIClient (check constructor name) + expect(client.constructor.name).toBe("OpenAIClient"); + }); + + it("returns AnthropicClient for legacy Anthropic model names", () => { + const logs: LogLine[] = []; + const logger = (line: LogLine) => logs.push(line); + const provider = new LLMProvider(logger); + + const client = provider.getClient( + "claude-3-5-sonnet-latest" as any, + mockClientOptions, + ); + + // Should return a client + expect(client).toBeDefined(); + // The client should be an AnthropicClient + expect(client.constructor.name).toBe("AnthropicClient"); + }); + }); + + describe("LLMProvider.getClient error handling", () => { + it("throws UnsupportedModelError for unknown model without slash", () => { + const logs: LogLine[] = []; + const logger = (line: LogLine) => logs.push(line); + const provider = new LLMProvider(logger); + + // Unknown model without slash should throw UnsupportedModelError + expect(() => { + provider.getClient("some-unknown-model" as any, mockClientOptions); + }).toThrow(UnsupportedModelError); + }); + + it("UnsupportedModelError includes provider/model format guidance", () => { + const logs: LogLine[] = []; + const logger = (line: LogLine) => logs.push(line); + const provider = new LLMProvider(logger); + + try { + provider.getClient("some-unknown-model" as any, mockClientOptions); + } catch (error) { + expect((error as Error).message).toContain("provider/model"); + } + }); + + it("throws UnsupportedAISDKModelProviderError for invalid provider in provider/model format", () => { + const logs: LogLine[] = []; + const logger = (line: LogLine) => logs.push(line); + const provider = new LLMProvider(logger); + + // Invalid provider but correct format + expect(() => { + provider.getClient( + "invalid-provider/some-model" as any, + mockClientOptions, + ); + }).toThrow(UnsupportedAISDKModelProviderError); + }); + + it("UnsupportedAISDKModelProviderError lists valid providers", () => { + const logs: LogLine[] = []; + const logger = (line: LogLine) => logs.push(line); + const provider = new LLMProvider(logger); + + try { + provider.getClient( + "invalid-provider/some-model" as any, + mockClientOptions, + ); + } catch (error) { + const message = (error as Error).message; + // Should list valid providers + expect(message).toContain("openai"); + expect(message).toContain("anthropic"); + expect(message).toContain("google"); + } + }); + }); + + describe("new provider/model format", () => { + it("does not log deprecation warning for provider/model format", () => { + const logs: LogLine[] = []; + const logger = (line: LogLine) => logs.push(line); + const provider = new LLMProvider(logger); + + // Using the new format + const client = provider.getClient( + "openai/gpt-4o" as any, + mockClientOptions, + ); + + expect(client).toBeDefined(); + + // Should NOT have a deprecation warning + const deprecationWarning = logs.find( + (log) => + log.message.toLowerCase().includes("deprecated") || + log.message.toLowerCase().includes("deprecation"), + ); + expect(deprecationWarning).toBeUndefined(); + }); + }); +}); From ef92609040e696df88adc3ee61975b75706502d7 Mon Sep 17 00:00:00 2001 From: Chromie Bot Date: Tue, 27 Jan 2026 23:51:44 +0000 Subject: [PATCH 2/2] feat: improve model deprecation messaging Updates to inform users about the new provider/model format: 1. UnsupportedModelError now includes a note encouraging users to use the provider/model format (e.g., "openai/gpt-4o") instead of the legacy format (e.g., "gpt-4o") 2. Fixed the default case in LLMProvider switch to throw UnsupportedModelProviderError (for internal consistency issues) instead of UnsupportedModelError (which is for user-facing errors) This is a non-breaking change - legacy model names still work but users are now informed about the preferred format. Co-Authored-By: Claude Opus 4.5 --- packages/core/lib/v3/llm/LLMProvider.ts | 11 +++++++++++ packages/core/lib/v3/types/public/sdkErrors.ts | 9 +++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/packages/core/lib/v3/llm/LLMProvider.ts b/packages/core/lib/v3/llm/LLMProvider.ts index aea9ac2eb..2df026e61 100644 --- a/packages/core/lib/v3/llm/LLMProvider.ts +++ b/packages/core/lib/v3/llm/LLMProvider.ts @@ -169,10 +169,19 @@ export class LLMProvider { }); } + // Model name doesn't include "/" - this format is deprecated + // Log a deprecation warning at level 0 (always shown) but continue with legacy functionality const provider = modelToProviderMap[modelName]; if (!provider) { throw new UnsupportedModelError(Object.keys(modelToProviderMap)); } + + this.logger({ + category: "llm", + message: `Deprecation warning: Model format "${modelName}" is deprecated. Please use the provider/model format (e.g., "openai/gpt-4o" or "anthropic/claude-3-5-sonnet-latest").`, + level: 0, + }); + const availableModel = modelName as AvailableModel; switch (provider) { case "openai": @@ -206,6 +215,8 @@ export class LLMProvider { clientOptions, }); default: + // This default case handles unknown providers that exist in modelToProviderMap + // but aren't implemented in the switch. This is an internal consistency issue. throw new UnsupportedModelProviderError([ ...new Set(Object.values(modelToProviderMap)), ]); diff --git a/packages/core/lib/v3/types/public/sdkErrors.ts b/packages/core/lib/v3/types/public/sdkErrors.ts index ca8443c26..2d743ad0b 100644 --- a/packages/core/lib/v3/types/public/sdkErrors.ts +++ b/packages/core/lib/v3/types/public/sdkErrors.ts @@ -48,10 +48,15 @@ export class MissingEnvironmentVariableError extends StagehandError { export class UnsupportedModelError extends StagehandError { constructor(supportedModels: string[], feature?: string) { + const modelList = supportedModels.join(", "); + const deprecationNote = + `\n\nNote: The legacy model format (e.g., "gpt-4o") is deprecated. ` + + `Please use the provider/model format instead (e.g., "openai/gpt-4o", "anthropic/claude-3-5-sonnet-latest").`; + super( feature - ? `${feature} requires one of the following models: ${supportedModels}` - : `please use one of the supported models: ${supportedModels}`, + ? `${feature} requires one of the following models: ${modelList}${deprecationNote}` + : `Please use one of the supported models: ${modelList}${deprecationNote}`, ); } }