Merge pull request #773 from Chris0Jeky/feat/tool-calling-refinements-651

feat: tool-calling Phase 3 — cost tracking, feature flag, token budget

Author: Chris0Jeky

backend/src/Taskdeck.Api/Extensions/LlmProviderRegistration.cs

@@ -29,6 +29,10 @@ public static IServiceCollection AddLlmProviders(
             return new AbuseDetectionService(settings, state, usageRecords);
         });
 
+        // Tool-calling feature flag and budget settings
+        var llmToolCallingSettings = configuration.GetSection("LlmToolCalling").Get<LlmToolCallingSettings>() ?? new LlmToolCallingSettings();
+        services.AddSingleton(llmToolCallingSettings);
+
         // LLM provider settings and deterministic provider selection policy
         var llmProviderSettings = configuration.GetSection("Llm").Get<LlmProviderSettings>() ?? new LlmProviderSettings();
         services.AddSingleton(llmProviderSettings);

backend/src/Taskdeck.Api/appsettings.json

@@ -73,6 +73,10 @@
     "XFrameOptions": "DENY",
     "ReferrerPolicy": "no-referrer"
   },
+  "LlmToolCalling": {
+    "Enabled": true,
+    "MaxToolResultBytes": 8000
+  },
   "GitHubOAuth": {
     "ClientId": "",
     "ClientSecret": ""

backend/src/Taskdeck.Application/Services/ChatService.cs

@@ -34,6 +34,7 @@ public class ChatService : IChatService
     private readonly ILlmKillSwitchService? _killSwitchService;
     private readonly IBoardContextBuilder? _boardContextBuilder;
     private readonly ToolCallingChatOrchestrator? _toolCallingOrchestrator;
+    private readonly LlmToolCallingSettings _toolCallingSettings;
 
     public ChatService(
         IUnitOfWork unitOfWork,
@@ -46,7 +47,8 @@ public ChatService(
         ILlmQuotaService? quotaService = null,
         ILlmKillSwitchService? killSwitchService = null,
         IBoardContextBuilder? boardContextBuilder = null,
-        ToolCallingChatOrchestrator? toolCallingOrchestrator = null)
+        ToolCallingChatOrchestrator? toolCallingOrchestrator = null,
+        LlmToolCallingSettings? toolCallingSettings = null)
     {
         _unitOfWork = unitOfWork;
         _llmProvider = llmProvider;
@@ -59,6 +61,7 @@ public ChatService(
         _killSwitchService = killSwitchService;
         _boardContextBuilder = boardContextBuilder;
         _toolCallingOrchestrator = toolCallingOrchestrator;
+        _toolCallingSettings = toolCallingSettings ?? new LlmToolCallingSettings();
     }
 
     public async Task<Result<ChatSessionDto>> CreateSessionAsync(Guid userId, CreateChatSessionDto dto, CancellationToken ct = default)
@@ -208,7 +211,9 @@ public async Task<Result<ChatMessageDto>> SendMessageAsync(Guid sessionId, Guid
                 LlmCompletionResult? reusableNoToolResponse = null;
 
                 // Try tool-calling path for board-scoped sessions with orchestrator.
-                if (_toolCallingOrchestrator != null && session.BoardId.HasValue)
+                // The feature flag allows disabling the orchestrator without code changes
+                // (e.g. for cost control). When disabled, falls through to single-turn.
+                if (_toolCallingOrchestrator != null && _toolCallingSettings.Enabled && session.BoardId.HasValue)
                 {
                     var toolChatMessages = session.Messages
                         .Select(m => new ChatCompletionMessage(m.Role.ToString(), m.Content))

backend/src/Taskdeck.Application/Services/LlmToolCallingSettings.cs

@@ -0,0 +1,24 @@
+namespace Taskdeck.Application.Services;
+
+/// <summary>
+/// Configuration for LLM tool-calling behaviour.
+/// Bound from the "LlmToolCalling" configuration section.
+/// </summary>
+public class LlmToolCallingSettings
+{
+    /// <summary>
+    /// Enables or disables the multi-turn tool-calling orchestrator.
+    /// When false, <see cref="ChatService"/> falls through to the single-turn
+    /// <see cref="ILlmProvider.CompleteAsync"/> path for every request.
+    /// Default is true so existing behaviour is preserved.
+    /// </summary>
+    public bool Enabled { get; set; } = true;
+
+    /// <summary>
+    /// Maximum byte length of a single tool result before it is truncated.
+    /// Keeps oversized responses within the provider's context window.
+    /// 0 = no truncation limit (not recommended for production).
+    /// Default is 8 000 bytes (roughly a few thousand tokens depending on content and tokenizer).
+    /// </summary>
+    public int MaxToolResultBytes { get; set; } = 8_000;
+}

backend/src/Taskdeck.Application/Services/ToolCallingChatOrchestrator.cs

@@ -36,17 +36,20 @@ public sealed class ToolCallingChatOrchestrator
     private readonly ToolExecutorRegistry _executorRegistry;
     private readonly ILogger<ToolCallingChatOrchestrator> _logger;
     private readonly IToolStatusNotifier? _statusNotifier;
+    private readonly LlmToolCallingSettings _settings;
 
     public ToolCallingChatOrchestrator(
         ILlmProvider provider,
         ToolExecutorRegistry executorRegistry,
         ILogger<ToolCallingChatOrchestrator> logger,
-        IToolStatusNotifier? statusNotifier = null)
+        IToolStatusNotifier? statusNotifier = null,
+        LlmToolCallingSettings? settings = null)
     {
         _provider = provider;
         _executorRegistry = executorRegistry;
         _logger = logger;
         _statusNotifier = statusNotifier;
+        _settings = settings ?? new LlmToolCallingSettings();
     }
 
     /// <summary>
@@ -232,6 +235,11 @@ await _statusNotifier.NotifyToolStatusAsync(
                     }
                 }
 
+                // Enforce token budget: truncate oversized tool results before they
+                // are fed back to the LLM.  This keeps the conversation within the
+                // provider's context window even when a tool returns a large payload.
+                resultContent = TruncateToolResult(resultContent, _settings.MaxToolResultBytes);
+
                 results.Add(new ToolCallResult(
                     toolCall.CallId, toolCall.ToolName, resultContent, isError, toolCall.Arguments));
 
@@ -409,6 +417,79 @@ private static ToolCallingResult BuildDegradedResult(
             DegradedReason: reason ?? "Tool calling is not available; falling back to single-turn.");
     }
 
+    /// <summary>
+    /// Truncates a tool result string to the configured byte budget so oversized
+    /// payloads do not blow out the provider's context window.
+    /// When <paramref name="maxBytes"/> is 0 or negative, no truncation is applied.
+    /// A "...(truncated)" marker is appended so the LLM knows the result was cut short.
+    /// The returned string is always within <paramref name="maxBytes"/> UTF-8 bytes.
+    /// </summary>
+    internal static string TruncateToolResult(string content, int maxBytes)
+    {
+        if (maxBytes <= 0 || string.IsNullOrEmpty(content))
+            return content;
+
+        var utf8 = System.Text.Encoding.UTF8;
+        var encoded = utf8.GetByteCount(content);
+        if (encoded <= maxBytes)
+            return content;
+
+        const string marker = "...(truncated)";
+        var markerBytes = utf8.GetByteCount(marker);
+
+        // If the budget cannot even hold the marker, return as many bytes as fit
+        // from the marker itself so the result is always <= maxBytes.
+        if (maxBytes <= markerBytes)
+            return marker[..FindCharCountFittingBytes(marker, maxBytes, utf8)];
+
+        var maxContentBytes = maxBytes - markerBytes;
+
+        // Binary search for the longest prefix whose UTF-8 encoding fits the budget.
+        // Avoids the O(n) worst case of a decrementing walk and makes no heap
+        // allocations during the search (GetByteCount accepts ReadOnlySpan<char>).
+        var span = content.AsSpan();
+        var low = 0;
+        var high = content.Length;
+        var best = 0;
+        while (low <= high)
+        {
+            var mid = low + ((high - low) / 2);
+            if (utf8.GetByteCount(span[..mid]) <= maxContentBytes)
+            {
+                best = mid;
+                low = mid + 1;
+            }
+            else
+            {
+                high = mid - 1;
+            }
+        }
+
+        return best > 0 ? content[..best] + marker : marker;
+    }
+
+    private static int FindCharCountFittingBytes(string s, int maxBytes, System.Text.Encoding utf8)
+    {
+        var low = 0;
+        var high = s.Length;
+        var best = 0;
+        var span = s.AsSpan();
+        while (low <= high)
+        {
+            var mid = low + ((high - low) / 2);
+            if (utf8.GetByteCount(span[..mid]) <= maxBytes)
+            {
+                best = mid;
+                low = mid + 1;
+            }
+            else
+            {
+                high = mid - 1;
+            }
+        }
+        return best;
+    }
+
     private static string TruncateForLog(string content)
     {
         const int maxLength = 200;