diff --git a/package.json b/package.json index eec05a232..fe60ccad1 100644 --- a/package.json +++ b/package.json @@ -67,7 +67,7 @@ "valibot": "1.2.0" }, "devDependencies": { - "@ai-sdk/mcp": "^0.0.12", + "@ai-sdk/mcp": "^1.0.1", "@ai-sdk/openai": "3.0.0-beta.101", "@iconify-json/vscode-icons": "^1.2.37", "@nuxt/eslint": "^1.12.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c6cafeaad..2483dfdf1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -149,8 +149,8 @@ importers: version: 1.2.0(typescript@5.9.3) devDependencies: '@ai-sdk/mcp': - specifier: ^0.0.12 - version: 0.0.12(zod@4.1.13) + specifier: ^1.0.1 + version: 1.0.1(zod@4.1.13) '@ai-sdk/openai': specifier: 3.0.0-beta.101 version: 3.0.0-beta.101(zod@4.1.13) @@ -250,8 +250,8 @@ packages: peerDependencies: zod: ^3.25.76 || ^4.1.8 - '@ai-sdk/mcp@0.0.12': - resolution: {integrity: sha512-hyf31U2CmgGexqOLgLfno525pjbqidJLu9pU+XcEwW/PkMcfTFuRq1iD3wbqtAmURRW0qJITiKV+in1B4I23gA==} + '@ai-sdk/mcp@1.0.1': + resolution: {integrity: sha512-4ejg3Nol3ma7r8Otj9m05+K0mpCoGq1gY1vp4IMS23uEflFHPRuZqq4PIogRZrD+r9FlZmbmwnxhJEcEob22nw==} engines: {node: '>=18'} peerDependencies: zod: ^3.25.76 || ^4.1.8 @@ -262,12 +262,6 @@ packages: peerDependencies: zod: ^3.25.76 || ^4.1.8 - '@ai-sdk/provider-utils@3.0.19': - resolution: {integrity: sha512-W41Wc9/jbUVXVwCN/7bWa4IKe8MtxO3EyA0Hfhx6grnmiYlCvpI8neSYWFE0zScXJkgA/YK3BRybzgyiXuu6JA==} - engines: {node: '>=18'} - peerDependencies: - zod: ^3.25.76 || ^4.1.8 - '@ai-sdk/provider-utils@4.0.0-beta.51': resolution: {integrity: sha512-7NEGTKMRbOOI6n7Xm1w7XiFqHaDcNxORql3iUIjPyXgYhnvF48JaB8cxySmtdDqpeR1DD4d6B6Cbjhn45DVR0g==} engines: {node: '>=18'} @@ -284,8 +278,14 @@ packages: effect: optional: true - '@ai-sdk/provider@2.0.0': - resolution: {integrity: sha512-6o7Y2SeO9vFKB8lArHXehNuusnpddKPk7xqL7T2/b+OvXMRIXUO1rR4wcv1hAFUAT9avGZshty3Wlua/XA7TvA==} + '@ai-sdk/provider-utils@4.0.1': + resolution: {integrity: sha512-de2v8gH9zj47tRI38oSxhQIewmNc+OZjYIOOaMoVWKL65ERSav2PYYZHPSPCrfOeLMkv+Dyh8Y0QGwkO29wMWQ==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.25.76 || ^4.1.8 + + '@ai-sdk/provider@3.0.0': + resolution: {integrity: sha512-m9ka3ptkPQbaHHZHqDXDF9C9B5/Mav0KTdky1k2HZ3/nrW2t1AgObxIVPyGDWQNS9FXT/FS6PIoSjpcP/No8rQ==} engines: {node: '>=18'} '@ai-sdk/provider@3.0.0-beta.26': @@ -3187,8 +3187,8 @@ packages: resolution: {integrity: sha512-Qig2Wso7gPkU1PtXwFzndh+CTRzrIFxVGqv6eCetjU7YqxlHItj+GvQYwYTppCRgAPawtRN/4AJcEgB9xDHGug==} hasBin: true - '@standard-schema/spec@1.0.0': - resolution: {integrity: sha512-m2bOd0f2RT9k8QJx1JN85cZYyH1RqFBdlwtkSlf4tBDYLCiiZnv1fIIwacK6cqwXavOydf0NPToMQgpKq+dVlA==} + '@standard-schema/spec@1.1.0': + resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} '@stricli/auto-complete@1.2.4': resolution: {integrity: sha512-MxGgeBbFyH+YtzhIjJeOr68NSx0r8mn4+hWRGgWn31YTG9u0Oa2y9oGHPY+PHeoTfvS7vhgllFvOrHcTVpwTrA==} @@ -9354,10 +9354,10 @@ snapshots: - arktype - effect - '@ai-sdk/mcp@0.0.12(zod@4.1.13)': + '@ai-sdk/mcp@1.0.1(zod@4.1.13)': dependencies: - '@ai-sdk/provider': 2.0.0 - '@ai-sdk/provider-utils': 3.0.19(zod@4.1.13) + '@ai-sdk/provider': 3.0.0 + '@ai-sdk/provider-utils': 4.0.1(zod@4.1.13) pkce-challenge: 5.0.1 zod: 4.1.13 @@ -9371,21 +9371,21 @@ snapshots: - arktype - effect - '@ai-sdk/provider-utils@3.0.19(zod@4.1.13)': + '@ai-sdk/provider-utils@4.0.0-beta.51(zod@4.1.13)': dependencies: - '@ai-sdk/provider': 2.0.0 - '@standard-schema/spec': 1.0.0 + '@ai-sdk/provider': 3.0.0-beta.26 + '@standard-schema/spec': 1.1.0 eventsource-parser: 3.0.6 zod: 4.1.13 - '@ai-sdk/provider-utils@4.0.0-beta.51(zod@4.1.13)': + '@ai-sdk/provider-utils@4.0.1(zod@4.1.13)': dependencies: - '@ai-sdk/provider': 3.0.0-beta.26 - '@standard-schema/spec': 1.0.0 + '@ai-sdk/provider': 3.0.0 + '@standard-schema/spec': 1.1.0 eventsource-parser: 3.0.6 zod: 4.1.13 - '@ai-sdk/provider@2.0.0': + '@ai-sdk/provider@3.0.0': dependencies: json-schema: 0.4.0 @@ -10705,7 +10705,7 @@ snapshots: '@nuxtjs/mdc': 0.19.1(patch_hash=db93ab1d8c350f97add01adf4e65cf6abc756b9d037a4789a617c9b9f9d3331a)(magicast@0.5.1) '@shikijs/langs': 3.20.0 '@sqlite.org/sqlite-wasm': 3.50.4-build1 - '@standard-schema/spec': 1.0.0 + '@standard-schema/spec': 1.1.0 '@webcontainer/env': 1.1.1 c12: 3.3.2(magicast@0.5.1) chokidar: 3.6.0 @@ -11320,7 +11320,7 @@ snapshots: '@nuxt/kit': 4.2.2(magicast@0.5.1) '@nuxt/schema': 4.2.2 '@nuxtjs/color-mode': 3.5.2(magicast@0.5.1) - '@standard-schema/spec': 1.0.0 + '@standard-schema/spec': 1.1.0 '@tailwindcss/postcss': 4.1.17 '@tailwindcss/vite': 4.1.17(vite@8.0.0-beta.2(@types/node@24.10.2)(esbuild@0.27.1)(jiti@2.6.1)(terser@5.44.1)(yaml@2.8.2)) '@tanstack/vue-table': 8.21.3(vue@3.5.25(typescript@5.9.3)) @@ -12493,7 +12493,7 @@ snapshots: '@sqlite.org/sqlite-wasm@3.50.4-build1': {} - '@standard-schema/spec@1.0.0': {} + '@standard-schema/spec@1.1.0': {} '@stricli/auto-complete@1.2.4': dependencies: @@ -13197,7 +13197,7 @@ snapshots: '@vitest/expect@4.0.15': dependencies: - '@standard-schema/spec': 1.0.0 + '@standard-schema/spec': 1.1.0 '@types/chai': 5.2.3 '@vitest/spy': 4.0.15 '@vitest/utils': 4.0.15 diff --git a/test/mcp.eval.ts b/test/mcp.eval.ts index 724a8520b..3fbebe3dd 100644 --- a/test/mcp.eval.ts +++ b/test/mcp.eval.ts @@ -1,7 +1,10 @@ -import { experimental_createMCPClient as createMCPClient } from '@ai-sdk/mcp' +import { createMCPClient } from '@ai-sdk/mcp' import { generateText } from 'ai' import { evalite } from 'evalite' import { toolCallAccuracy } from 'evalite/scorers' +import type { Evalite } from 'evalite' + +type ToolCall = Evalite.Scorers.ToolCall /** * MCP Evaluation Tests @@ -43,6 +46,7 @@ evalite('Evaluate Nuxt MCP Documentation Tools', { const result = await generateText({ model, prompt: input, + // @ts-expect-error - MCP tools type mismatch with ai SDK ToolSet tools: await mcpClient.tools() }) return result.toolCalls ?? [] @@ -50,7 +54,7 @@ evalite('Evaluate Nuxt MCP Documentation Tools', { await mcpClient.close() } }, - scorers: [async ({ output, expected }) => toolCallAccuracy({ actualCalls: output, expectedCalls: expected })] + scorers: [async ({ output, expected }) => toolCallAccuracy({ actualCalls: output as ToolCall[], expectedCalls: expected as ToolCall[] })] }) evalite('Evaluate Nuxt MCP Blog Tools', { @@ -66,6 +70,7 @@ evalite('Evaluate Nuxt MCP Blog Tools', { const result = await generateText({ model, prompt: input, + // @ts-expect-error - MCP tools type mismatch with ai SDK ToolSet tools: await mcpClient.tools() }) return result.toolCalls ?? [] @@ -73,7 +78,7 @@ evalite('Evaluate Nuxt MCP Blog Tools', { await mcpClient.close() } }, - scorers: [async ({ output, expected }) => toolCallAccuracy({ actualCalls: output, expectedCalls: expected })] + scorers: [async ({ output, expected }) => toolCallAccuracy({ actualCalls: output as ToolCall[], expectedCalls: expected as ToolCall[] })] }) evalite('Evaluate Nuxt MCP Deploy Tools', { @@ -90,6 +95,7 @@ evalite('Evaluate Nuxt MCP Deploy Tools', { const result = await generateText({ model, prompt: input, + // @ts-expect-error - MCP tools type mismatch with ai SDK ToolSet tools: await mcpClient.tools() }) return result.toolCalls ?? [] @@ -97,10 +103,10 @@ evalite('Evaluate Nuxt MCP Deploy Tools', { await mcpClient.close() } }, - scorers: [async ({ output, expected }) => toolCallAccuracy({ actualCalls: output, expectedCalls: expected })] + scorers: [async ({ output, expected }) => toolCallAccuracy({ actualCalls: output as ToolCall[], expectedCalls: expected as ToolCall[] })] }) -evalite('Evaluate Nuxt MCP Module Tools', { +evalite('Evaluate Nuxt MCP Module Tools', { data: async () => [ { input: 'I need to add authentication with social login providers to my app. Find me a suitable module.', expected: [{ toolName: 'list_modules', input: { category: 'authentication' } }] }, { input: 'What modules are available for image optimization and lazy loading?', expected: [{ toolName: 'list_modules', input: { category: 'media' } }] }, @@ -111,8 +117,14 @@ evalite('Evaluate Nuxt MCP Module Tools', { task: async (input) => { const mcpClient = await createMCPClient({ transport: { type: 'http', url: MCP_URL } }) try { - const result = await generateText({ model, prompt: input, tools: await mcpClient.tools(), maxSteps: 3 }) - return result.toolCalls ?? [] + const result = await generateText({ + model, + prompt: input, + // @ts-expect-error - MCP tools type mismatch with ai SDK ToolSet + tools: await mcpClient.tools(), + maxSteps: 3 + }) + return result.toolCalls as ToolCall[] ?? [] } finally { await mcpClient.close() } @@ -120,7 +132,7 @@ evalite('Evaluate Nuxt MCP Module Tools', { scorers: [async ({ output, expected }) => toolCallAccuracy({ actualCalls: output, expectedCalls: expected })] }) -evalite('Evaluate Nuxt MCP Cross-Tool Workflows', { +evalite('Evaluate Nuxt MCP Cross-Tool Workflows', { data: async () => [ { input: 'I want to build an e-commerce site with Nuxt 4. What modules do I need and where should I deploy it?', expected: [{ toolName: 'list_modules' }, { toolName: 'list_deploy_providers' }] }, { input: 'Show me the latest features in Nuxt 4 and link to the relevant documentation', expected: [{ toolName: 'list_blog_posts' }, { toolName: 'get_documentation_page', input: { path: '/docs/4.x/getting-started/introduction' } }] } @@ -128,8 +140,14 @@ evalite('Evaluate Nuxt MCP Cross-Tool Workflows', { task: async (input) => { const mcpClient = await createMCPClient({ transport: { type: 'http', url: MCP_URL } }) try { - const result = await generateText({ model, prompt: input, tools: await mcpClient.tools(), maxSteps: 5 }) - return result.toolCalls ?? [] + const result = await generateText({ + model, + prompt: input, + // @ts-expect-error - MCP tools type mismatch with ai SDK ToolSet + tools: await mcpClient.tools(), + maxSteps: 5 + }) + return result.toolCalls as ToolCall[] ?? [] } finally { await mcpClient.close() } diff --git a/test/tsconfig.json b/test/tsconfig.json new file mode 100644 index 000000000..59b3fb796 --- /dev/null +++ b/test/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "module": "NodeNext", + "moduleResolution": "NodeNext", + "target": "ESNext", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "resolveJsonModule": true, + "isolatedModules": true, + "verbatimModuleSyntax": true, + "noEmit": true + }, + "include": [ + "*.eval.ts" + ] +}