diff --git a/backend/src/Taskdeck.Application/Services/GeminiLlmProvider.cs b/backend/src/Taskdeck.Application/Services/GeminiLlmProvider.cs index 534aeb381..1ce8762f3 100644 --- a/backend/src/Taskdeck.Application/Services/GeminiLlmProvider.cs +++ b/backend/src/Taskdeck.Application/Services/GeminiLlmProvider.cs @@ -81,12 +81,42 @@ public async Task CompleteAsync(ChatCompletionRequest reque return BuildFallbackResult(lastUserMessage, "Live provider request failed.", GetConfiguredModelOrDefault()); } - if (!TryParseResponse(body, out var content, out var tokensUsed)) + if (!TryParseResponse(body, out var content, out var tokensUsed, out var finishReason)) { _logger.LogWarning("Gemini completion response could not be parsed."); return BuildFallbackResult(lastUserMessage, "Live provider response parsing failed.", GetConfiguredModelOrDefault()); } + // Detect truncation: Gemini returns finishReason "MAX_TOKENS" when the + // response was cut off by the maxOutputTokens limit. + if (string.Equals(finishReason, "MAX_TOKENS", StringComparison.OrdinalIgnoreCase)) + { + _logger.LogWarning("Gemini response was truncated (finishReason=MAX_TOKENS)."); + return new LlmCompletionResult( + content, + tokensUsed, + IsActionable: false, + Provider: "Gemini", + Model: GetConfiguredModelOrDefault(), + IsDegraded: true, + DegradedReason: "Response was truncated"); + } + + // When JSON mode was requested and the response starts with '{' but + // does not parse as valid JSON, the output was likely truncated. + if (useInstructionExtraction && LooksLikeTruncatedJson(content)) + { + _logger.LogWarning("Gemini JSON-mode response is not valid JSON; treating as truncated."); + return new LlmCompletionResult( + content, + tokensUsed, + IsActionable: false, + Provider: "Gemini", + Model: GetConfiguredModelOrDefault(), + IsDegraded: true, + DegradedReason: "Response was truncated"); + } + // Try to parse structured instruction extraction from the LLM response if (LlmInstructionExtractionPrompt.TryParseStructuredResponse( content, @@ -225,10 +255,11 @@ private static object MapMessage(ChatCompletionMessage message) }; } - private static bool TryParseResponse(string responseBody, out string content, out int tokensUsed) + private static bool TryParseResponse(string responseBody, out string content, out int tokensUsed, out string? finishReason) { content = string.Empty; tokensUsed = 0; + finishReason = null; if (string.IsNullOrWhiteSpace(responseBody)) { @@ -248,6 +279,13 @@ private static bool TryParseResponse(string responseBody, out string content, ou } var firstCandidate = candidates[0]; + + if (firstCandidate.TryGetProperty("finishReason", out var finishReasonElement) && + finishReasonElement.ValueKind == JsonValueKind.String) + { + finishReason = finishReasonElement.GetString(); + } + if (!firstCandidate.TryGetProperty("content", out var candidateContent) || !candidateContent.TryGetProperty("parts", out var parts) || parts.ValueKind != JsonValueKind.Array || @@ -324,6 +362,30 @@ private static LlmCompletionResult BuildFallbackResult(string userMessage, strin Instructions: instructions); } + /// + /// Returns true when starts with '{' but does not + /// parse as valid JSON — a strong signal the response was cut off mid-output. + /// + internal static bool LooksLikeTruncatedJson(string text) + { + if (string.IsNullOrWhiteSpace(text)) + return false; + + var trimmed = text.TrimStart(); + if (!trimmed.StartsWith('{')) + return false; + + try + { + using var doc = JsonDocument.Parse(trimmed); + return false; + } + catch (JsonException) + { + return true; + } + } + private static int EstimateTokens(string text) { if (string.IsNullOrWhiteSpace(text)) diff --git a/backend/src/Taskdeck.Application/Services/ILlmProvider.cs b/backend/src/Taskdeck.Application/Services/ILlmProvider.cs index 946a91309..3c8e14ca0 100644 --- a/backend/src/Taskdeck.Application/Services/ILlmProvider.cs +++ b/backend/src/Taskdeck.Application/Services/ILlmProvider.cs @@ -10,7 +10,7 @@ public interface ILlmProvider public record ChatCompletionRequest( List Messages, - int MaxTokens = 1024, + int MaxTokens = 2048, double Temperature = 0.7, LlmRequestAttribution? Attribution = null, string? SystemPrompt = null, diff --git a/backend/src/Taskdeck.Application/Services/OpenAiLlmProvider.cs b/backend/src/Taskdeck.Application/Services/OpenAiLlmProvider.cs index e77f590a4..c8a161bb3 100644 --- a/backend/src/Taskdeck.Application/Services/OpenAiLlmProvider.cs +++ b/backend/src/Taskdeck.Application/Services/OpenAiLlmProvider.cs @@ -52,12 +52,43 @@ public async Task CompleteAsync(ChatCompletionRequest reque return BuildFallbackResult(lastUserMessage, "Live provider request failed.", GetConfiguredModelOrDefault()); } - if (!TryParseResponse(body, out var content, out var tokensUsed)) + if (!TryParseResponse(body, out var content, out var tokensUsed, out var finishReason)) { _logger.LogWarning("OpenAI completion response could not be parsed."); return BuildFallbackResult(lastUserMessage, "Live provider response parsing failed.", GetConfiguredModelOrDefault()); } + // Detect truncation: OpenAI returns finish_reason "length" when the + // response was cut off by the max_tokens limit. + if (string.Equals(finishReason, "length", StringComparison.OrdinalIgnoreCase)) + { + _logger.LogWarning("OpenAI response was truncated (finish_reason=length)."); + return new LlmCompletionResult( + content, + tokensUsed, + IsActionable: false, + Provider: "OpenAI", + Model: GetConfiguredModelOrDefault(), + IsDegraded: true, + DegradedReason: "Response was truncated"); + } + + // When JSON mode was requested and the response starts with '{' but + // does not parse as valid JSON, the output was likely truncated. + var useInstructionExtraction = request.SystemPrompt is null; + if (useInstructionExtraction && LooksLikeTruncatedJson(content)) + { + _logger.LogWarning("OpenAI JSON-mode response is not valid JSON; treating as truncated."); + return new LlmCompletionResult( + content, + tokensUsed, + IsActionable: false, + Provider: "OpenAI", + Model: GetConfiguredModelOrDefault(), + IsDegraded: true, + DegradedReason: "Response was truncated"); + } + // Try to parse structured instruction extraction from the LLM response if (LlmInstructionExtractionPrompt.TryParseStructuredResponse( content, @@ -227,10 +258,11 @@ private object BuildRequestPayload(ChatCompletionRequest request) return payload; } - private static bool TryParseResponse(string responseBody, out string content, out int tokensUsed) + private static bool TryParseResponse(string responseBody, out string content, out int tokensUsed, out string? finishReason) { content = string.Empty; tokensUsed = 0; + finishReason = null; if (string.IsNullOrWhiteSpace(responseBody)) { @@ -259,6 +291,12 @@ private static bool TryParseResponse(string responseBody, out string content, ou return false; } + if (first.TryGetProperty("finish_reason", out var finishReasonElement) && + finishReasonElement.ValueKind == JsonValueKind.String) + { + finishReason = finishReasonElement.GetString(); + } + if (root.TryGetProperty("usage", out var usage) && usage.TryGetProperty("total_tokens", out var totalTokens) && totalTokens.TryGetInt32(out var parsedTokens)) @@ -308,6 +346,30 @@ private static LlmCompletionResult BuildFallbackResult(string userMessage, strin Instructions: instructions); } + /// + /// Returns true when starts with '{' but does not + /// parse as valid JSON — a strong signal the response was cut off mid-output. + /// + internal static bool LooksLikeTruncatedJson(string text) + { + if (string.IsNullOrWhiteSpace(text)) + return false; + + var trimmed = text.TrimStart(); + if (!trimmed.StartsWith('{')) + return false; + + try + { + using var doc = JsonDocument.Parse(trimmed); + return false; + } + catch (JsonException) + { + return true; + } + } + private static int EstimateTokens(string text) { if (string.IsNullOrWhiteSpace(text)) diff --git a/backend/tests/Taskdeck.Application.Tests/Services/GeminiLlmProviderTests.cs b/backend/tests/Taskdeck.Application.Tests/Services/GeminiLlmProviderTests.cs index 3971b5272..9d5d7021e 100644 --- a/backend/tests/Taskdeck.Application.Tests/Services/GeminiLlmProviderTests.cs +++ b/backend/tests/Taskdeck.Application.Tests/Services/GeminiLlmProviderTests.cs @@ -432,6 +432,55 @@ await provider.CompleteAsync(new ChatCompletionRequest( hasSystemInstruction.Should().BeFalse("empty system prompt should not produce system_instruction field"); } + [Fact] + public async Task CompleteAsync_ShouldReturnDegraded_WhenFinishReasonIsMaxTokens() + { + var settings = BuildSettings(); + var handler = new StubHttpMessageHandler(_ => + { + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent( + """ + { + "candidates": [ + { + "content": { + "parts": [{ "text": "partial response" }] + }, + "finishReason": "MAX_TOKENS" + } + ], + "usageMetadata": { "totalTokenCount": 50 } + } + """, + Encoding.UTF8, + "application/json") + }; + }); + + var provider = new GeminiLlmProvider(new HttpClient(handler), settings, NullLogger.Instance); + var result = await provider.CompleteAsync(new ChatCompletionRequest( + [new ChatCompletionMessage("User", "tell me something")], + SystemPrompt: string.Empty)); + + result.IsDegraded.Should().BeTrue(); + result.DegradedReason.Should().Be("Response was truncated"); + result.Content.Should().Be("partial response"); + result.IsActionable.Should().BeFalse(); + } + + [Theory] + [InlineData("{\"reply\":\"incomplete", true)] + [InlineData("{}", false)] + [InlineData("plain text response", false)] + [InlineData("", false)] + [InlineData(" { broken json", true)] + public void LooksLikeTruncatedJson_ShouldDetectPartialJson(string input, bool expected) + { + GeminiLlmProvider.LooksLikeTruncatedJson(input).Should().Be(expected); + } + private static LlmProviderSettings BuildSettings() { return new LlmProviderSettings diff --git a/backend/tests/Taskdeck.Application.Tests/Services/OpenAiLlmProviderTests.cs b/backend/tests/Taskdeck.Application.Tests/Services/OpenAiLlmProviderTests.cs index c29c75273..df3dca0c5 100644 --- a/backend/tests/Taskdeck.Application.Tests/Services/OpenAiLlmProviderTests.cs +++ b/backend/tests/Taskdeck.Application.Tests/Services/OpenAiLlmProviderTests.cs @@ -269,6 +269,85 @@ public async Task CompleteAsync_ShouldRedactSensitiveDetails_WhenUnexpectedExcep message.Should().Contain($"Authorization: Bearer {SensitiveDataRedactor.RedactedValue}"); } + [Fact] + public async Task CompleteAsync_ShouldReturnDegraded_WhenFinishReasonIsLength() + { + var settings = BuildSettings(); + var handler = new StubHttpMessageHandler(_ => + { + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent( + """ + { + "choices": [ + { + "message": { "content": "partial response" }, + "finish_reason": "length" + } + ], + "usage": { "total_tokens": 50 } + } + """, + Encoding.UTF8, + "application/json") + }; + }); + + var provider = new OpenAiLlmProvider(new HttpClient(handler), settings, NullLogger.Instance); + var result = await provider.CompleteAsync(new ChatCompletionRequest( + [new ChatCompletionMessage("User", "tell me something")], + SystemPrompt: string.Empty)); + + result.IsDegraded.Should().BeTrue(); + result.DegradedReason.Should().Be("Response was truncated"); + result.Content.Should().Be("partial response"); + result.IsActionable.Should().BeFalse(); + } + + [Fact] + public async Task CompleteAsync_ShouldReturnDegraded_WhenJsonModeResponseIsInvalidJson() + { + var settings = BuildSettings(); + // Build a valid OpenAI response whose content value is truncated JSON. + // The inner content must be JSON-escaped so the outer envelope parses. + var truncatedContent = "{\\\"reply\\\":\\\"this is cut off"; + var responseBody = $@"{{ + ""choices"": [{{ + ""message"": {{ ""content"": ""{truncatedContent}"" }}, + ""finish_reason"": ""stop"" + }}], + ""usage"": {{ ""total_tokens"": 50 }} +}}"; + var handler = new StubHttpMessageHandler(_ => + { + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(responseBody, Encoding.UTF8, "application/json") + }; + }); + + var provider = new OpenAiLlmProvider(new HttpClient(handler), settings, NullLogger.Instance); + + // SystemPrompt defaults to null -> JSON mode is requested + var result = await provider.CompleteAsync(new ChatCompletionRequest( + [new ChatCompletionMessage("User", "tell me something")])); + + result.IsDegraded.Should().BeTrue(); + result.DegradedReason.Should().Be("Response was truncated"); + } + + [Theory] + [InlineData("{\"reply\":\"incomplete", true)] + [InlineData("{}", false)] + [InlineData("plain text response", false)] + [InlineData("", false)] + [InlineData(" { broken json", true)] + public void LooksLikeTruncatedJson_ShouldDetectPartialJson(string input, bool expected) + { + OpenAiLlmProvider.LooksLikeTruncatedJson(input).Should().Be(expected); + } + private static LlmProviderSettings BuildSettings() { return new LlmProviderSettings diff --git a/frontend/taskdeck-web/src/views/AutomationChatView.vue b/frontend/taskdeck-web/src/views/AutomationChatView.vue index 33cc09f54..fd1a14176 100644 --- a/frontend/taskdeck-web/src/views/AutomationChatView.vue +++ b/frontend/taskdeck-web/src/views/AutomationChatView.vue @@ -28,6 +28,20 @@ function renderMarkdown(content: string): string { return DOMPurify.sanitize(marked.parse(content, { async: false })) } +function isTruncatedJson(content: string): boolean { + if (!content) return false + const trimmed = content.trim() + if (!trimmed.startsWith('{')) return false + try { + JSON.parse(trimmed) + return false + } catch { + return true + } +} + +const truncationNotice = 'This response was cut short. Try a simpler question or rephrase.' + const router = useRouter() const route = useRoute() const toast = useToastStore() @@ -661,7 +675,13 @@ watch(