Skip to content

Commit 0d79159

Browse files
Chris0JekyChris0Jeky
authored
Merge pull request #747 from Chris0Jeky/test/709-llm-provider-tool-calling-edge
Test: LLM provider abstraction and tool-calling edge cases (#709)
2 parents 6e25da8 + b20d2fe commit 0d79159

4 files changed

Lines changed: 1465 additions & 0 deletions

File tree

‎backend/tests/Taskdeck.Application.Tests/Services/LlmIntentClassifierEdgeCaseTests.cs‎

Lines changed: 249 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,249 @@
1+
using FluentAssertions;
2+
using Xunit;
3+
using Taskdeck.Application.Services;
4+
5+
namespace Taskdeck.Application.Tests.Services;
6+
7+
/// <summary>
8+
/// Edge case tests for LlmIntentClassifier expanding on the existing fuzz tests.
9+
/// Covers: negation filtering, other-tool questions, ambiguous inputs,
10+
/// very long inputs, prompt injection patterns, mixed casing,
11+
/// and multi-intent detection gaps.
12+
/// </summary>
13+
public class LlmIntentClassifierEdgeCaseTests
14+
{
15+
// ── Negation filtering ───────────────────────────────────────
16+
17+
[Theory]
18+
[InlineData("Don't add a card")]
19+
[InlineData("do not create a new task")]
20+
[InlineData("never move the card to done")]
21+
[InlineData("stop create new tasks")]
22+
[InlineData("cancel the delete of card 5")]
23+
[InlineData("don't remove that task")]
24+
[InlineData("avoid create a task please")] // negation regex: "avoid" followed by verbs within word distance
25+
public void Classify_NegatedInput_IsNotActionable(string input)
26+
{
27+
var (isActionable, _) = LlmIntentClassifier.Classify(input);
28+
29+
isActionable.Should().BeFalse(
30+
$"negated input '{input}' should not be classified as actionable");
31+
}
32+
33+
// ── Other-tool questions ─────────────────────────────────────
34+
35+
[Theory]
36+
[InlineData("How do I add a card in Trello?")]
37+
[InlineData("How do I create a task in Jira?")]
38+
[InlineData("Where do I move cards in Asana?")]
39+
[InlineData("Can I create boards in Notion?")]
40+
public void Classify_OtherToolQuestion_IsNotActionable(string input)
41+
{
42+
var (isActionable, _) = LlmIntentClassifier.Classify(input);
43+
44+
isActionable.Should().BeFalse(
45+
$"question about another tool '{input}' should not be actionable");
46+
}
47+
48+
// ── Positive detection ───────────────────────────────────────
49+
50+
[Theory]
51+
[InlineData("create a new card called Test", "card.create")]
52+
[InlineData("add a task for the meeting", "card.create")]
53+
[InlineData("make a new task for sprint review", "card.create")]
54+
[InlineData("move card to done column", "card.move")]
55+
[InlineData("archive the old task", "card.archive")]
56+
[InlineData("delete card number 5", "card.archive")]
57+
[InlineData("remove the finished task", "card.archive")]
58+
[InlineData("update card title to new name", "card.update")]
59+
[InlineData("rename task to better name", "card.update")]
60+
[InlineData("edit card description", "card.update")]
61+
[InlineData("create a new board for the project", "board.create")]
62+
[InlineData("rename board to Sprint 42", "board.update")]
63+
[InlineData("reorder columns on the board", "column.reorder")]
64+
public void Classify_ActionableInput_DetectsCorrectIntent(string input, string expectedIntent)
65+
{
66+
var (isActionable, actionIntent) = LlmIntentClassifier.Classify(input);
67+
68+
isActionable.Should().BeTrue($"'{input}' should be detected as actionable");
69+
actionIntent.Should().Be(expectedIntent);
70+
}
71+
72+
// ── Non-actionable inputs ────────────────────────────────────
73+
74+
[Theory]
75+
[InlineData("hello")]
76+
[InlineData("what is the weather?")]
77+
[InlineData("tell me about the project")]
78+
[InlineData("how are my tasks doing?")]
79+
[InlineData("show me a summary")]
80+
[InlineData("what's the status?")]
81+
public void Classify_NonActionableInput_ReturnsFalse(string input)
82+
{
83+
var (isActionable, actionIntent) = LlmIntentClassifier.Classify(input);
84+
85+
isActionable.Should().BeFalse(
86+
$"non-actionable input '{input}' should not be classified as actionable");
87+
actionIntent.Should().BeNull();
88+
}
89+
90+
// ── Edge cases ───────────────────────────────────────────────
91+
92+
[Fact]
93+
public void Classify_NullInput_ReturnsFalse()
94+
{
95+
var (isActionable, _) = LlmIntentClassifier.Classify(null!);
96+
97+
isActionable.Should().BeFalse();
98+
}
99+
100+
[Fact]
101+
public void Classify_EmptyString_ReturnsFalse()
102+
{
103+
var (isActionable, _) = LlmIntentClassifier.Classify("");
104+
105+
isActionable.Should().BeFalse();
106+
}
107+
108+
[Fact]
109+
public void Classify_WhitespaceOnly_ReturnsFalse()
110+
{
111+
var (isActionable, _) = LlmIntentClassifier.Classify(" \t\n ");
112+
113+
isActionable.Should().BeFalse();
114+
}
115+
116+
[Fact]
117+
public void Classify_VeryLongInput_DoesNotThrow()
118+
{
119+
// 10,000 character message should be handled gracefully
120+
var longInput = new string('a', 9990) + " create a card";
121+
122+
var act = () => LlmIntentClassifier.Classify(longInput);
123+
124+
act.Should().NotThrow();
125+
}
126+
127+
[Fact]
128+
public void Classify_VeryLongInput_WithActionableContent_StillDetects()
129+
{
130+
// Actionable content at the start should be detected even in long messages
131+
var longInput = "create a new task called test " + new string('x', 5000);
132+
133+
var (isActionable, actionIntent) = LlmIntentClassifier.Classify(longInput);
134+
135+
isActionable.Should().BeTrue();
136+
actionIntent.Should().Be("card.create");
137+
}
138+
139+
[Theory]
140+
[InlineData("CREATE A NEW CARD")]
141+
[InlineData("Create A New Card")]
142+
[InlineData("cReAtE a NeW cArD")]
143+
public void Classify_MixedCase_StillDetects(string input)
144+
{
145+
var (isActionable, _) = LlmIntentClassifier.Classify(input);
146+
147+
isActionable.Should().BeTrue(
148+
$"mixed case input '{input}' should still be detected");
149+
}
150+
151+
[Theory]
152+
[InlineData("create a card\nand some other text")]
153+
[InlineData("create\na\ncard")]
154+
public void Classify_NewlinesInInput_DoesNotThrow(string input)
155+
{
156+
// Verify that newlines in input do not cause exceptions.
157+
// The classifier may or may not detect the intent depending on
158+
// whether the regex matches across line boundaries, but it must
159+
// never crash.
160+
var act = () => LlmIntentClassifier.Classify(input);
161+
act.Should().NotThrow("newlines in input must not cause exceptions");
162+
}
163+
164+
[Fact]
165+
public void Classify_NewlinesSeparatingActionablePhrase_DetectsWhenOnOneLine()
166+
{
167+
// "create a card" on a single line should be detected even with trailing newlines
168+
var input = "create a card\nsome other text after";
169+
170+
var (isActionable, actionIntent) = LlmIntentClassifier.Classify(input);
171+
172+
isActionable.Should().BeTrue("actionable phrase on first line should be detected");
173+
actionIntent.Should().Be("card.create");
174+
}
175+
176+
[Fact]
177+
public void Classify_PromptInjection_DoesNotCrashAndStillClassifies()
178+
{
179+
// Injection payloads that contain "create a card" should still be classified
180+
// as actionable — the classifier is a regex-based intent detector, not a
181+
// sanitizer. The key guarantee is no crashes and correct classification.
182+
var injections = new[]
183+
{
184+
"create a card'; DROP TABLE cards;--",
185+
"create a card with <script>alert('xss')</script>",
186+
"create a card\0with null bytes",
187+
"create a card\\nwith escaped newlines"
188+
};
189+
190+
foreach (var input in injections)
191+
{
192+
var act = () => LlmIntentClassifier.Classify(input);
193+
act.Should().NotThrow($"input '{input}' should not cause an exception");
194+
195+
var (isActionable, actionIntent) = LlmIntentClassifier.Classify(input);
196+
isActionable.Should().BeTrue(
197+
$"injection input '{input}' still contains 'create a card' and should be actionable");
198+
actionIntent.Should().Be("card.create");
199+
}
200+
}
201+
202+
// ── Archive vs Move disambiguation ───────────────────────────
203+
204+
[Fact]
205+
public void Classify_RemoveCard_ClassifiesAsArchive_NotMove()
206+
{
207+
// "remove" contains "move" as a substring. Verify archive takes priority.
208+
var (isActionable, actionIntent) = LlmIntentClassifier.Classify("remove the task from backlog");
209+
210+
isActionable.Should().BeTrue();
211+
actionIntent.Should().Be("card.archive");
212+
}
213+
214+
// ── Stemming/plural variations ───────────────────────────────
215+
216+
[Theory]
217+
[InlineData("create new cards", "card.create")]
218+
[InlineData("add tasks for the team", "card.create")]
219+
[InlineData("move tasks to done", "card.move")]
220+
[InlineData("archive cards", "card.archive")]
221+
[InlineData("update tasks", "card.update")]
222+
public void Classify_PluralNouns_StillDetects(string input, string expectedIntent)
223+
{
224+
var (isActionable, actionIntent) = LlmIntentClassifier.Classify(input);
225+
226+
isActionable.Should().BeTrue($"plural input '{input}' should be detected");
227+
actionIntent.Should().Be(expectedIntent);
228+
}
229+
230+
// ── Verb coverage ────────────────────────────────────────────
231+
232+
[Theory]
233+
[InlineData("generate a card for testing", "card.create")]
234+
[InlineData("build a task list", "card.create")]
235+
[InlineData("prepare a new task", "card.create")]
236+
[InlineData("set up a new board", "board.create")]
237+
[InlineData("modify the card title", "card.update")]
238+
[InlineData("change task priority", "card.update")]
239+
[InlineData("sort the columns", "column.reorder")]
240+
[InlineData("rearrange the columns", "column.reorder")]
241+
[InlineData("reorganize the board columns", "column.reorder")]
242+
public void Classify_AlternateVerbs_DetectedCorrectly(string input, string expectedIntent)
243+
{
244+
var (isActionable, actionIntent) = LlmIntentClassifier.Classify(input);
245+
246+
isActionable.Should().BeTrue($"verb in '{input}' should be recognized");
247+
actionIntent.Should().Be(expectedIntent);
248+
}
249+
}

0 commit comments

Comments
 (0)