diff --git a/src/runner/templates/agents/node/manual/template.njk b/src/runner/templates/agents/node/manual/template.njk index 44f8e8f..40996b8 100644 --- a/src/runner/templates/agents/node/manual/template.njk +++ b/src/runner/templates/agents/node/manual/template.njk @@ -45,46 +45,50 @@ function trimMessages(messages, maxSize = 10000) { } function buildSentryMessages(messages) { - const sentryMessages = []; - const legacyMessages = []; let systemInstructions = null; for (const msg of messages) { - const { role, content } = msg; - - if (role === "system") { - systemInstructions = typeof content === "string" ? content : JSON.stringify(content); + if (msg.role === "system") { + systemInstructions = + typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content); } + } - const parts = []; - if (typeof content === "string") { - parts.push({ type: "text", text: content }); - } else if (Array.isArray(content)) { - for (const part of content) { - if (part.type === "text") { - parts.push({ type: "text", text: part.text }); - } else if (part.type === "image") { - parts.push({ type: "text", text: "[Blob substitute]" }); - } - } - } + const lastMessage = messages[messages.length - 1]; + if (!lastMessage) { + return { sentryMessages: [], legacyMessages: [], systemInstructions }; + } - sentryMessages.push({ - role, - content: typeof content === "string" ? content : "[multimodal]", - parts, - }); - - if (Array.isArray(content)) { - const redacted = content.map((part) => - part.type === "image" ? { type: "text", text: "[Blob substitute]" } : part - ); - legacyMessages.push({ role, content: redacted }); - } else { - legacyMessages.push({ role, content }); + const { role, content } = lastMessage; + + const parts = []; + if (typeof content === "string") { + parts.push({ type: "text", text: content }); + } else if (Array.isArray(content)) { + for (const part of content) { + if (part.type === "text") { + parts.push({ type: "text", text: part.text }); + } else if (part.type === "image") { + parts.push({ type: "text", text: "[Blob substitute]" }); + } } } + const sentryMessages = [{ + role, + content: typeof content === "string" ? content : "[multimodal]", + parts, + }]; + + const legacyMessages = [Array.isArray(content) + ? { + role, + content: content.map((part) => + part.type === "image" ? { type: "text", text: "[Blob substitute]" } : part, + ), + } + : { role, content }]; + return { sentryMessages, legacyMessages, systemInstructions }; } @@ -150,6 +154,10 @@ const TOOLS = {{ agent.tools | dump }}; agentSpan.setAttribute("gen_ai.usage.output_tokens", outputTokens{{ loop.index }}); agentSpan.setAttribute("gen_ai.usage.output_tokens.reasoning", Math.max(0, Math.floor(outputTokens{{ loop.index }} / 4))); agentSpan.setAttribute("gen_ai.usage.total_tokens", totalTokens{{ loop.index }}); + agentSpan.setAttribute("gen_ai.input.messages", otelJson{{ loop.index }}); + if (system{{ loop.index }} !== null) { + agentSpan.setAttribute("gen_ai.system_instructions", system{{ loop.index }}); + } // Chat span (child of agent) const chatDesc{{ loop.index }} = `chat ${model{{ loop.index }}}`; diff --git a/src/runner/templates/agents/python/manual/template.njk b/src/runner/templates/agents/python/manual/template.njk index 37b85aa..e863ef9 100644 --- a/src/runner/templates/agents/python/manual/template.njk +++ b/src/runner/templates/agents/python/manual/template.njk @@ -48,46 +48,47 @@ def trim_messages(messages, max_size=10000): def build_sentry_messages(messages): - """Build gen_ai.input.messages and gen_ai.request.messages from raw messages.""" - sentry_messages = [] - legacy_messages = [] + """Build gen_ai.input.messages and gen_ai.request.messages from the last raw message.""" system_instructions = None for msg in messages: - role = msg["role"] content = msg.get("content", "") - - if role == "system": + if msg["role"] == "system": system_instructions = content if isinstance(content, str) else json.dumps(content) - # New OTel format with parts - parts = [] - if isinstance(content, str): - parts.append({"type": "text", "text": content}) - elif isinstance(content, list): - for part in content: - if part.get("type") == "text": - parts.append({"type": "text", "text": part["text"]}) - elif part.get("type") == "image": - parts.append({"type": "text", "text": "[Blob substitute]"}) - - sentry_messages.append({ - "role": role, - "content": content if isinstance(content, str) else "[multimodal]", - "parts": parts, - }) - - # Legacy format with binary redaction - if isinstance(content, list): - redacted = [] - for part in content: - if part.get("type") == "text": - redacted.append(part) - elif part.get("type") == "image": - redacted.append({"type": "text", "text": "[Blob substitute]"}) - legacy_messages.append({"role": role, "content": redacted}) - else: - legacy_messages.append({"role": role, "content": content}) + if not messages: + return [], [], system_instructions + + last_message = messages[-1] + role = last_message["role"] + content = last_message.get("content", "") + + parts = [] + if isinstance(content, str): + parts.append({"type": "text", "text": content}) + elif isinstance(content, list): + for part in content: + if part.get("type") == "text": + parts.append({"type": "text", "text": part["text"]}) + elif part.get("type") == "image": + parts.append({"type": "text", "text": "[Blob substitute]"}) + + sentry_messages = [{ + "role": role, + "content": content if isinstance(content, str) else "[multimodal]", + "parts": parts, + }] + + if isinstance(content, list): + redacted = [] + for part in content: + if part.get("type") == "text": + redacted.append(part) + elif part.get("type") == "image": + redacted.append({"type": "text", "text": "[Blob substitute]"}) + legacy_messages = [{"role": role, "content": redacted}] + else: + legacy_messages = [{"role": role, "content": content}] return sentry_messages, legacy_messages, system_instructions @@ -151,6 +152,9 @@ TOOLS = {{ agent.tools | dump }} agent_span.set_data("gen_ai.usage.output_tokens", output_tokens) agent_span.set_data("gen_ai.usage.output_tokens.reasoning", max(0, output_tokens // 4)) agent_span.set_data("gen_ai.usage.total_tokens", total_tokens) + agent_span.set_data("gen_ai.input.messages", otel_messages_json) + if system_instructions is not None: + agent_span.set_data("gen_ai.system_instructions", system_instructions) # Chat span (child of agent) chat_desc = f"chat {model}" diff --git a/src/runner/templates/llm/node/manual/template.njk b/src/runner/templates/llm/node/manual/template.njk index ee56df6..114d48f 100644 --- a/src/runner/templates/llm/node/manual/template.njk +++ b/src/runner/templates/llm/node/manual/template.njk @@ -45,48 +45,50 @@ function trimMessages(messages, maxSize = 10000) { } function buildSentryMessages(messages) { - const sentryMessages = []; - const legacyMessages = []; let systemInstructions = null; for (const msg of messages) { - const { role, content } = msg; - - if (role === "system") { - systemInstructions = typeof content === "string" ? content : JSON.stringify(content); + if (msg.role === "system") { + systemInstructions = + typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content); } + } - // New OTel format with parts - const parts = []; - if (typeof content === "string") { - parts.push({ type: "text", text: content }); - } else if (Array.isArray(content)) { - for (const part of content) { - if (part.type === "text") { - parts.push({ type: "text", text: part.text }); - } else if (part.type === "image") { - parts.push({ type: "text", text: "[Blob substitute]" }); - } - } - } + const lastMessage = messages[messages.length - 1]; + if (!lastMessage) { + return { sentryMessages: [], legacyMessages: [], systemInstructions }; + } - sentryMessages.push({ - role, - content: typeof content === "string" ? content : "[multimodal]", - parts, - }); - - // Legacy format with binary redaction - if (Array.isArray(content)) { - const redacted = content.map((part) => - part.type === "image" ? { type: "text", text: "[Blob substitute]" } : part - ); - legacyMessages.push({ role, content: redacted }); - } else { - legacyMessages.push({ role, content }); + const { role, content } = lastMessage; + + const parts = []; + if (typeof content === "string") { + parts.push({ type: "text", text: content }); + } else if (Array.isArray(content)) { + for (const part of content) { + if (part.type === "text") { + parts.push({ type: "text", text: part.text }); + } else if (part.type === "image") { + parts.push({ type: "text", text: "[Blob substitute]" }); + } } } + const sentryMessages = [{ + role, + content: typeof content === "string" ? content : "[multimodal]", + parts, + }]; + + const legacyMessages = [Array.isArray(content) + ? { + role, + content: content.map((part) => + part.type === "image" ? { type: "text", text: "[Blob substitute]" } : part, + ), + } + : { role, content }]; + return { sentryMessages, legacyMessages, systemInstructions }; } {% endblock %} diff --git a/src/runner/templates/llm/python/manual/template.njk b/src/runner/templates/llm/python/manual/template.njk index b2a8925..77b2f0a 100644 --- a/src/runner/templates/llm/python/manual/template.njk +++ b/src/runner/templates/llm/python/manual/template.njk @@ -44,46 +44,47 @@ def trim_messages(messages, max_size=10000): def build_sentry_messages(messages): - """Build gen_ai.input.messages and gen_ai.request.messages from raw messages.""" - sentry_messages = [] - legacy_messages = [] + """Build gen_ai.input.messages and gen_ai.request.messages from the last raw message.""" system_instructions = None for msg in messages: - role = msg["role"] content = msg.get("content", "") - - if role == "system": + if msg["role"] == "system": system_instructions = content if isinstance(content, str) else json.dumps(content) - # New OTel format with parts - parts = [] - if isinstance(content, str): - parts.append({"type": "text", "text": content}) - elif isinstance(content, list): - for part in content: - if part.get("type") == "text": - parts.append({"type": "text", "text": part["text"]}) - elif part.get("type") == "image": - parts.append({"type": "text", "text": "[Blob substitute]"}) - - sentry_messages.append({ - "role": role, - "content": content if isinstance(content, str) else "[multimodal]", - "parts": parts, - }) - - # Legacy format with binary redaction - if isinstance(content, list): - redacted = [] - for part in content: - if part.get("type") == "text": - redacted.append(part) - elif part.get("type") == "image": - redacted.append({"type": "text", "text": "[Blob substitute]"}) - legacy_messages.append({"role": role, "content": redacted}) - else: - legacy_messages.append({"role": role, "content": content}) + if not messages: + return [], [], system_instructions + + last_message = messages[-1] + role = last_message["role"] + content = last_message.get("content", "") + + parts = [] + if isinstance(content, str): + parts.append({"type": "text", "text": content}) + elif isinstance(content, list): + for part in content: + if part.get("type") == "text": + parts.append({"type": "text", "text": part["text"]}) + elif part.get("type") == "image": + parts.append({"type": "text", "text": "[Blob substitute]"}) + + sentry_messages = [{ + "role": role, + "content": content if isinstance(content, str) else "[multimodal]", + "parts": parts, + }] + + if isinstance(content, list): + redacted = [] + for part in content: + if part.get("type") == "text": + redacted.append(part) + elif part.get("type") == "image": + redacted.append({"type": "text", "text": "[Blob substitute]"}) + legacy_messages = [{"role": role, "content": redacted}] + else: + legacy_messages = [{"role": role, "content": content}] return sentry_messages, legacy_messages, system_instructions {% endblock %} diff --git a/src/test-cases/checks.ts b/src/test-cases/checks.ts index 4bd09ef..a2a4ab4 100644 --- a/src/test-cases/checks.ts +++ b/src/test-cases/checks.ts @@ -108,6 +108,220 @@ export function checkAISpanCount( // Span Type Attribute Checks // ============================================================================= +function parseInputMessages( + span: CapturedSpan, +): { messages?: unknown[]; attribute?: string; error?: string } { + const result = getAttributeWithFallback( + span, + "gen_ai.input.messages", + "gen_ai.request.messages", + ); + + if (result.value === undefined) { + return { + attribute: result.usedAttribute ?? "gen_ai.input.messages", + error: "Missing messages attribute", + }; + } + + if (typeof result.value === "string") { + try { + const parsed = JSON.parse(result.value); + if (!Array.isArray(parsed)) { + return { + attribute: result.usedAttribute!, + error: `${result.usedAttribute} should be an array`, + }; + } + return { messages: parsed, attribute: result.usedAttribute! }; + } catch { + return { + attribute: result.usedAttribute!, + error: `Invalid JSON in ${result.usedAttribute}`, + }; + } + } + + if (!Array.isArray(result.value)) { + return { + attribute: result.usedAttribute!, + error: `${result.usedAttribute} should be an array`, + }; + } + + return { messages: result.value as unknown[], attribute: result.usedAttribute! }; +} + +function getMessageText(message: unknown): string | undefined { + if (typeof message !== "object" || message === null) { + return undefined; + } + + const msg = message as { + content?: unknown; + parts?: Array<{ type?: unknown; text?: unknown; content?: unknown }>; + }; + + if (typeof msg.content === "string") { + return msg.content; + } + + if (Array.isArray(msg.content)) { + const textParts = msg.content + .map((part) => { + if (typeof part === "string") { + return part; + } + if (typeof part === "object" && part !== null) { + const candidate = part as { text?: unknown; content?: unknown; type?: unknown }; + if (candidate.type === "text" && typeof candidate.text === "string") { + return candidate.text; + } + if (typeof candidate.text === "string") { + return candidate.text; + } + if (typeof candidate.content === "string") { + return candidate.content; + } + } + return undefined; + }) + .filter((part): part is string => typeof part === "string"); + + if (textParts.length > 0) { + return textParts.join("\n"); + } + } + + if (typeof msg.content === "object" && msg.content !== null) { + const candidate = msg.content as { text?: unknown; content?: unknown }; + if (typeof candidate.text === "string") { + return candidate.text; + } + if (typeof candidate.content === "string") { + return candidate.content; + } + } + + if (Array.isArray(msg.parts)) { + const textParts = msg.parts + .map((part) => { + if (part?.type === "text" && typeof part.text === "string") { + return part.text; + } + if (typeof part?.text === "string") { + return part.text; + } + if (typeof part?.content === "string") { + return part.content; + } + return undefined; + }) + .filter((part): part is string => typeof part === "string"); + + if (textParts.length > 0) { + return textParts.join("\n"); + } + } + + return undefined; +} + +function messagesMatchLoosely(actual: unknown, expected: unknown): boolean { + if (typeof actual !== "object" || actual === null) { + return false; + } + if (typeof expected !== "object" || expected === null) { + return false; + } + + const actualMessage = actual as { role?: unknown }; + const expectedMessage = expected as { role?: unknown }; + + if (actualMessage.role !== expectedMessage.role) { + return false; + } + + const actualText = getMessageText(actual); + const expectedText = getMessageText(expected); + + if (actualText !== undefined || expectedText !== undefined) { + return actualText === expectedText; + } + + return JSON.stringify(actual) === JSON.stringify(expected); +} + +function assertOnlyLastInputMessage( + spans: CapturedSpan[], + testDef: { inputs: Array<{ messages?: unknown[] }> }, + spanType: "chat" | "agent", +): void { + if (testDef.inputs.length <= 1) { + return; + } + + const sortedSpans = [...spans].sort( + (a, b) => a.start_timestamp - b.start_timestamp, + ); + + if (sortedSpans.length < testDef.inputs.length) { + throw new CheckError( + `Expected at least ${testDef.inputs.length} ${spanType} span(s) for last-message validation, found ${sortedSpans.length}`, + ); + } + + const errors: string[] = []; + const locations: ErrorLocation[] = []; + + for (let i = 0; i < testDef.inputs.length; i++) { + const span = sortedSpans[i]; + const expectedMessages = testDef.inputs[i]?.messages; + const expectedLastMessage = expectedMessages?.[expectedMessages.length - 1]; + + if (expectedLastMessage === undefined) { + continue; + } + + const parsed = parseInputMessages(span); + if (parsed.error || !parsed.attribute || !parsed.messages) { + const message = parsed.error ?? "Missing messages attribute"; + errors.push(`${spanType} span ${i} ${message}`); + locations.push({ + spanId: span.span_id, + attribute: parsed.attribute ?? "gen_ai.input.messages", + message, + }); + continue; + } + + if (parsed.messages.length !== 1) { + const message = `${spanType} span ${i} should keep only the last input message, found ${parsed.messages.length} message(s)`; + errors.push(message); + locations.push({ + spanId: span.span_id, + attribute: parsed.attribute, + message, + }); + continue; + } + + if (!messagesMatchLoosely(parsed.messages[0], expectedLastMessage)) { + const message = `${spanType} span ${i} should keep only the last input message`; + errors.push(message); + locations.push({ + spanId: span.span_id, + attribute: parsed.attribute, + message, + }); + } + } + + if (errors.length > 0) { + throw new CheckError(errors.join("\n"), locations); + } +} + /** * Check attributes on chat/completion spans (LLM API calls) * @@ -177,6 +391,8 @@ export const checkChatSpanAttributes: Check = { if (errors.length > 0) { throw new CheckError(errors.join("\n"), locations); } + + assertOnlyLastInputMessage(chatSpans, testDef, "chat"); }, }; @@ -216,6 +432,7 @@ export const checkAgentSpanAttributes: Check = { } assertAttributes(agentSpans, attrs); + assertOnlyLastInputMessage(agentSpans, testDef, "agent"); }, };