Skip to content

Fix chat response truncation and raw JSON display #635

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Chris0Jeky merged 6 commits into from
Mar 31, 2026
Merged

Fix chat response truncation and raw JSON display #635

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
a56a3be
Increase default MaxTokens from 1024 to 2048
Chris0Jeky Mar 31, 2026
ca2fabc
Add finish_reason and JSON truncation detection to OpenAI provider
Chris0Jeky Mar 31, 2026
1eb8548
Add finish_reason and JSON truncation detection to Gemini provider
Chris0Jeky Mar 31, 2026
d6bc411
Detect truncated JSON in chat and show friendly notice
Chris0Jeky Mar 31, 2026
a463641
Refine JSON truncation check to only flag partial JSON objects
Chris0Jeky Mar 31, 2026
5d51b68
Add tests for finish_reason and truncated JSON detection
Chris0Jeky Mar 31, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 64 additions & 2 deletions backend/src/Taskdeck.Application/Services/GeminiLlmProvider.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,12 +81,42 @@ public async Task<LlmCompletionResult> CompleteAsync(ChatCompletionRequest reque
return BuildFallbackResult(lastUserMessage, "Live provider request failed.", GetConfiguredModelOrDefault());
}

if (!TryParseResponse(body, out var content, out var tokensUsed))
if (!TryParseResponse(body, out var content, out var tokensUsed, out var finishReason))
{
_logger.LogWarning("Gemini completion response could not be parsed.");
return BuildFallbackResult(lastUserMessage, "Live provider response parsing failed.", GetConfiguredModelOrDefault());
}

// Detect truncation: Gemini returns finishReason "MAX_TOKENS" when the
// response was cut off by the maxOutputTokens limit.
if (string.Equals(finishReason, "MAX_TOKENS", StringComparison.OrdinalIgnoreCase))
{
_logger.LogWarning("Gemini response was truncated (finishReason=MAX_TOKENS).");
return new LlmCompletionResult(
content,
tokensUsed,
IsActionable: false,
Provider: "Gemini",
Model: GetConfiguredModelOrDefault(),
IsDegraded: true,
DegradedReason: "Response was truncated");
}

// When JSON mode was requested and the response starts with '{' but
// does not parse as valid JSON, the output was likely truncated.
if (useInstructionExtraction && LooksLikeTruncatedJson(content))
{
_logger.LogWarning("Gemini JSON-mode response is not valid JSON; treating as truncated.");
return new LlmCompletionResult(
content,
tokensUsed,
IsActionable: false,
Provider: "Gemini",
Model: GetConfiguredModelOrDefault(),
IsDegraded: true,
DegradedReason: "Response was truncated");
}

// Try to parse structured instruction extraction from the LLM response
if (LlmInstructionExtractionPrompt.TryParseStructuredResponse(
content,
Expand Down Expand Up @@ -225,10 +255,11 @@ private static object MapMessage(ChatCompletionMessage message)
};
}

private static bool TryParseResponse(string responseBody, out string content, out int tokensUsed)
private static bool TryParseResponse(string responseBody, out string content, out int tokensUsed, out string? finishReason)
{
content = string.Empty;
tokensUsed = 0;
finishReason = null;

if (string.IsNullOrWhiteSpace(responseBody))
{
Expand All @@ -248,6 +279,13 @@ private static bool TryParseResponse(string responseBody, out string content, ou
}

var firstCandidate = candidates[0];

if (firstCandidate.TryGetProperty("finishReason", out var finishReasonElement) &&
finishReasonElement.ValueKind == JsonValueKind.String)
{
finishReason = finishReasonElement.GetString();
}

if (!firstCandidate.TryGetProperty("content", out var candidateContent) ||
!candidateContent.TryGetProperty("parts", out var parts) ||
parts.ValueKind != JsonValueKind.Array ||
Expand Down Expand Up @@ -324,6 +362,30 @@ private static LlmCompletionResult BuildFallbackResult(string userMessage, strin
Instructions: instructions);
}

/// <summary>
/// Returns true when <paramref name="text"/> starts with '{' but does not
/// parse as valid JSON — a strong signal the response was cut off mid-output.
/// </summary>
internal static bool LooksLikeTruncatedJson(string text)
{
if (string.IsNullOrWhiteSpace(text))
return false;

var trimmed = text.TrimStart();
if (!trimmed.StartsWith('{'))
return false;

try
{
using var doc = JsonDocument.Parse(trimmed);
return false;
}
catch (JsonException)
{
return true;
}
}
Comment on lines +369 to +387
Copy link
Copy Markdown

@gemini-code-assist gemini-code-assist bot Mar 31, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

The current implementation of LooksLikeTruncatedJson is fragile because it assumes the response starts exactly with a {. LLMs occasionally include conversational prefixes or markdown code blocks (e.g., "Here is the JSON: ```json ...") even when instructed to return JSON.

Additionally, this logic is duplicated in OpenAiLlmProvider.cs. Consider moving this utility to a shared location like LlmInstructionExtractionPrompt and making it more robust by searching for the first and last braces, similar to the logic used in TryParseStructuredResponse.

    internal static bool LooksLikeTruncatedJson(string text)
    {
        if (string.IsNullOrWhiteSpace(text)) return false;
        var trimmed = text.Trim();
        var first = trimmed.IndexOf('{');
        var last = trimmed.LastIndexOf('}');

        if (first >= 0 && last > first)
        {
            try
            {
                using var doc = JsonDocument.Parse(trimmed[first..(last + 1)]);
                return false;
            }
            catch (JsonException) { }
        }

        return first >= 0;
    }


private static int EstimateTokens(string text)
{
if (string.IsNullOrWhiteSpace(text))
Expand Down
2 changes: 1 addition & 1 deletion backend/src/Taskdeck.Application/Services/ILlmProvider.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ public interface ILlmProvider

public record ChatCompletionRequest(
List<ChatCompletionMessage> Messages,
int MaxTokens = 1024,
int MaxTokens = 2048,
double Temperature = 0.7,
LlmRequestAttribution? Attribution = null,
string? SystemPrompt = null,
Expand Down
66 changes: 64 additions & 2 deletions backend/src/Taskdeck.Application/Services/OpenAiLlmProvider.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,12 +52,43 @@ public async Task<LlmCompletionResult> CompleteAsync(ChatCompletionRequest reque
return BuildFallbackResult(lastUserMessage, "Live provider request failed.", GetConfiguredModelOrDefault());
}

if (!TryParseResponse(body, out var content, out var tokensUsed))
if (!TryParseResponse(body, out var content, out var tokensUsed, out var finishReason))
{
_logger.LogWarning("OpenAI completion response could not be parsed.");
return BuildFallbackResult(lastUserMessage, "Live provider response parsing failed.", GetConfiguredModelOrDefault());
}

// Detect truncation: OpenAI returns finish_reason "length" when the
// response was cut off by the max_tokens limit.
if (string.Equals(finishReason, "length", StringComparison.OrdinalIgnoreCase))
{
_logger.LogWarning("OpenAI response was truncated (finish_reason=length).");
return new LlmCompletionResult(
content,
tokensUsed,
IsActionable: false,
Provider: "OpenAI",
Model: GetConfiguredModelOrDefault(),
IsDegraded: true,
DegradedReason: "Response was truncated");
}

// When JSON mode was requested and the response starts with '{' but
// does not parse as valid JSON, the output was likely truncated.
var useInstructionExtraction = request.SystemPrompt is null;
if (useInstructionExtraction && LooksLikeTruncatedJson(content))
{
_logger.LogWarning("OpenAI JSON-mode response is not valid JSON; treating as truncated.");
return new LlmCompletionResult(
content,
tokensUsed,
IsActionable: false,
Provider: "OpenAI",
Model: GetConfiguredModelOrDefault(),
IsDegraded: true,
DegradedReason: "Response was truncated");
}

// Try to parse structured instruction extraction from the LLM response
if (LlmInstructionExtractionPrompt.TryParseStructuredResponse(
content,
Expand Down Expand Up @@ -227,10 +258,11 @@ private object BuildRequestPayload(ChatCompletionRequest request)
return payload;
}

private static bool TryParseResponse(string responseBody, out string content, out int tokensUsed)
private static bool TryParseResponse(string responseBody, out string content, out int tokensUsed, out string? finishReason)
{
content = string.Empty;
tokensUsed = 0;
finishReason = null;

if (string.IsNullOrWhiteSpace(responseBody))
{
Expand Down Expand Up @@ -259,6 +291,12 @@ private static bool TryParseResponse(string responseBody, out string content, ou
return false;
}

if (first.TryGetProperty("finish_reason", out var finishReasonElement) &&
finishReasonElement.ValueKind == JsonValueKind.String)
{
finishReason = finishReasonElement.GetString();
}

if (root.TryGetProperty("usage", out var usage) &&
usage.TryGetProperty("total_tokens", out var totalTokens) &&
totalTokens.TryGetInt32(out var parsedTokens))
Expand Down Expand Up @@ -308,6 +346,30 @@ private static LlmCompletionResult BuildFallbackResult(string userMessage, strin
Instructions: instructions);
}

/// <summary>
/// Returns true when <paramref name="text"/> starts with '{' but does not
/// parse as valid JSON — a strong signal the response was cut off mid-output.
/// </summary>
internal static bool LooksLikeTruncatedJson(string text)
{
if (string.IsNullOrWhiteSpace(text))
return false;

var trimmed = text.TrimStart();
if (!trimmed.StartsWith('{'))
return false;

try
{
using var doc = JsonDocument.Parse(trimmed);
return false;
}
catch (JsonException)
{
return true;
}
}
Comment on lines +353 to +371
Copy link
Copy Markdown

@gemini-code-assist gemini-code-assist bot Mar 31, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

This method is identical to the one in GeminiLlmProvider.cs. To improve maintainability and robustness, consider moving this logic to a shared utility class (e.g., LlmInstructionExtractionPrompt) and updating it to handle cases where the JSON is wrapped in markdown or preceded by text.

    internal static bool LooksLikeTruncatedJson(string text)
    {
        if (string.IsNullOrWhiteSpace(text)) return false;
        var trimmed = text.Trim();
        var first = trimmed.IndexOf('{');
        var last = trimmed.LastIndexOf('}');

        if (first >= 0 && last > first)
        {
            try
            {
                using var doc = JsonDocument.Parse(trimmed[first..(last + 1)]);
                return false;
            }
            catch (JsonException) { }
        }

        return first >= 0;
    }


private static int EstimateTokens(string text)
{
if (string.IsNullOrWhiteSpace(text))
Expand Down
49 changes: 49 additions & 0 deletions backend/tests/Taskdeck.Application.Tests/Services/GeminiLlmProviderTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -432,6 +432,55 @@ await provider.CompleteAsync(new ChatCompletionRequest(
hasSystemInstruction.Should().BeFalse("empty system prompt should not produce system_instruction field");
}

[Fact]
public async Task CompleteAsync_ShouldReturnDegraded_WhenFinishReasonIsMaxTokens()
{
var settings = BuildSettings();
var handler = new StubHttpMessageHandler(_ =>
{
return new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent(
"""
{
"candidates": [
{
"content": {
"parts": [{ "text": "partial response" }]
},
"finishReason": "MAX_TOKENS"
}
],
"usageMetadata": { "totalTokenCount": 50 }
}
""",
Encoding.UTF8,
"application/json")
};
});

var provider = new GeminiLlmProvider(new HttpClient(handler), settings, NullLogger<GeminiLlmProvider>.Instance);
var result = await provider.CompleteAsync(new ChatCompletionRequest(
[new ChatCompletionMessage("User", "tell me something")],
SystemPrompt: string.Empty));

result.IsDegraded.Should().BeTrue();
result.DegradedReason.Should().Be("Response was truncated");
result.Content.Should().Be("partial response");
result.IsActionable.Should().BeFalse();
}

[Theory]
[InlineData("{\"reply\":\"incomplete", true)]
[InlineData("{}", false)]
[InlineData("plain text response", false)]
[InlineData("", false)]
[InlineData(" { broken json", true)]
public void LooksLikeTruncatedJson_ShouldDetectPartialJson(string input, bool expected)
{
GeminiLlmProvider.LooksLikeTruncatedJson(input).Should().Be(expected);
}

private static LlmProviderSettings BuildSettings()
{
return new LlmProviderSettings
Expand Down
79 changes: 79 additions & 0 deletions backend/tests/Taskdeck.Application.Tests/Services/OpenAiLlmProviderTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,85 @@ public async Task CompleteAsync_ShouldRedactSensitiveDetails_WhenUnexpectedExcep
message.Should().Contain($"Authorization: Bearer {SensitiveDataRedactor.RedactedValue}");
}

[Fact]
public async Task CompleteAsync_ShouldReturnDegraded_WhenFinishReasonIsLength()
{
var settings = BuildSettings();
var handler = new StubHttpMessageHandler(_ =>
{
return new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent(
"""
{
"choices": [
{
"message": { "content": "partial response" },
"finish_reason": "length"
}
],
"usage": { "total_tokens": 50 }
}
""",
Encoding.UTF8,
"application/json")
};
});

var provider = new OpenAiLlmProvider(new HttpClient(handler), settings, NullLogger<OpenAiLlmProvider>.Instance);
var result = await provider.CompleteAsync(new ChatCompletionRequest(
[new ChatCompletionMessage("User", "tell me something")],
SystemPrompt: string.Empty));

result.IsDegraded.Should().BeTrue();
result.DegradedReason.Should().Be("Response was truncated");
result.Content.Should().Be("partial response");
result.IsActionable.Should().BeFalse();
}

[Fact]
public async Task CompleteAsync_ShouldReturnDegraded_WhenJsonModeResponseIsInvalidJson()
{
var settings = BuildSettings();
// Build a valid OpenAI response whose content value is truncated JSON.
// The inner content must be JSON-escaped so the outer envelope parses.
var truncatedContent = "{\\\"reply\\\":\\\"this is cut off";
var responseBody = $@"{{
""choices"": [{{
""message"": {{ ""content"": ""{truncatedContent}"" }},
""finish_reason"": ""stop""
}}],
""usage"": {{ ""total_tokens"": 50 }}
}}";
var handler = new StubHttpMessageHandler(_ =>
{
return new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent(responseBody, Encoding.UTF8, "application/json")
};
});

var provider = new OpenAiLlmProvider(new HttpClient(handler), settings, NullLogger<OpenAiLlmProvider>.Instance);

// SystemPrompt defaults to null -> JSON mode is requested
var result = await provider.CompleteAsync(new ChatCompletionRequest(
[new ChatCompletionMessage("User", "tell me something")]));

result.IsDegraded.Should().BeTrue();
result.DegradedReason.Should().Be("Response was truncated");
}

[Theory]
[InlineData("{\"reply\":\"incomplete", true)]
[InlineData("{}", false)]
[InlineData("plain text response", false)]
[InlineData("", false)]
[InlineData(" { broken json", true)]
public void LooksLikeTruncatedJson_ShouldDetectPartialJson(string input, bool expected)
{
OpenAiLlmProvider.LooksLikeTruncatedJson(input).Should().Be(expected);
}

private static LlmProviderSettings BuildSettings()
{
return new LlmProviderSettings
Expand Down
Loading
Loading