-
Notifications
You must be signed in to change notification settings - Fork 0
Improve LlmIntentClassifier NLP coverage #579
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
Show all changes
6 commits
Select commit
Hold shift + click to select a range
4fc24d7
Add regex-based NLP matching and negative context filtering to LlmInt…
Chris0Jeky 57b0863
Add comprehensive tests for improved LlmIntentClassifier NLP matching
Chris0Jeky 8a4907f
Tighten regex patterns to reduce false positives
Chris0Jeky 27aed01
Fix review findings: document gerund gap, add false-positive test, fi…
Chris0Jeky 9b9f142
Remove redundant substring checks now covered by regex patterns
Chris0Jeky 09d73d2
Merge branch 'main' into enhance/571-intent-classifier-nlp
Chris0Jeky File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
220 changes: 198 additions & 22 deletions
220
backend/src/Taskdeck.Application/Services/LlmIntentClassifier.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,43 +1,219 @@ | ||
| using System.Text.RegularExpressions; | ||
|
|
||
| namespace Taskdeck.Application.Services; | ||
|
|
||
| public static class LlmIntentClassifier | ||
| { | ||
| // Timeout to prevent catastrophic backtracking | ||
| private static readonly TimeSpan RegexTimeout = TimeSpan.FromMilliseconds(100); | ||
|
|
||
| // Negative context patterns — suppress matches in these contexts | ||
| private static readonly Regex NegationPattern = new( | ||
| @"\b(don'?t|do not|never|stop|cancel|undo|avoid)\b(\s+\w+){0,6}\s+\b(create|add|make|move|archive|delete|remove|update|edit|rename|generate|build|set up|prepare)\b", | ||
| RegexOptions.Compiled | RegexOptions.IgnoreCase, | ||
| RegexTimeout); | ||
|
|
||
| private static readonly Regex OtherToolPattern = new( | ||
| @"\b(in|for|with|using|on)\s+(jira|trello|asana|notion|monday|clickup|linear|github issues|azure devops)\b", | ||
| RegexOptions.Compiled | RegexOptions.IgnoreCase, | ||
| RegexTimeout); | ||
|
|
||
| private static readonly Regex QuestionAboutHowPattern = new( | ||
| @"^\s*(how|what|where|when|why|can\s+i|is\s+it)\b.*\?$", | ||
| RegexOptions.Compiled | RegexOptions.IgnoreCase, | ||
| RegexTimeout); | ||
|
|
||
| // Card creation — verbs followed by optional words then card/task nouns | ||
| private static readonly Regex CardCreatePattern = new( | ||
| @"\b(create|add|make|generate|build|prepare|set\s+up)\b(\s+\w+){0,5}\s+\b(cards?|tasks?|items?)\b", | ||
| RegexOptions.Compiled | RegexOptions.IgnoreCase, | ||
| RegexTimeout); | ||
|
|
||
| // "new card/task" with optional words between | ||
| private static readonly Regex NewCardPattern = new( | ||
| @"\b(new)\b(\s+\w+){0,4}\s+\b(cards?|tasks?|items?)\b", | ||
| RegexOptions.Compiled | RegexOptions.IgnoreCase, | ||
| RegexTimeout); | ||
|
|
||
| // Card move — "move" + optional words + "card/task" | ||
| private static readonly Regex CardMovePattern = new( | ||
| @"\bmove\b(\s+\w+){0,4}\s+\b(cards?|tasks?)\b", | ||
| RegexOptions.Compiled | RegexOptions.IgnoreCase, | ||
| RegexTimeout); | ||
|
|
||
| // Card archive — "archive/delete/remove" + optional words + "card/task" | ||
| private static readonly Regex CardArchivePattern = new( | ||
| @"\b(archive|delete|remove)\b(\s+\w+){0,4}\s+\b(cards?|tasks?)\b", | ||
| RegexOptions.Compiled | RegexOptions.IgnoreCase, | ||
| RegexTimeout); | ||
|
|
||
| // Card update — "update/edit/rename" + optional words + "card/task" | ||
| private static readonly Regex CardUpdatePattern = new( | ||
| @"\b(update|edit|rename|modify|change)\b(\s+\w+){0,4}\s+\b(cards?|tasks?)\b", | ||
| RegexOptions.Compiled | RegexOptions.IgnoreCase, | ||
| RegexTimeout); | ||
|
|
||
| // Board creation — verbs + optional words + "board" | ||
| private static readonly Regex BoardCreatePattern = new( | ||
| @"\b(create|add|make|generate|build|prepare|set\s+up|new)\b(\s+\w+){0,4}\s+\b(boards?|project\s+boards?)\b", | ||
| RegexOptions.Compiled | RegexOptions.IgnoreCase, | ||
| RegexTimeout); | ||
|
|
||
| // Board rename | ||
| private static readonly Regex BoardRenamePattern = new( | ||
| @"\b(rename|update|edit)\b(\s+\w+){0,4}\s+\bboards?\b", | ||
| RegexOptions.Compiled | RegexOptions.IgnoreCase, | ||
| RegexTimeout); | ||
|
|
||
| // Column reorder / sort | ||
| private static readonly Regex ReorderPattern = new( | ||
| @"\b(reorder|sort|rearrange|reorganize)\b(\s+\w+){0,4}\s+\b(cards?|columns?|boards?)\b", | ||
| RegexOptions.Compiled | RegexOptions.IgnoreCase, | ||
| RegexTimeout); | ||
|
|
||
| public static (bool IsActionable, string? ActionIntent) Classify(string message) | ||
| { | ||
| if (string.IsNullOrWhiteSpace(message)) | ||
| return (false, null); | ||
|
|
||
| var lower = message.ToLowerInvariant(); | ||
|
|
||
| // Card creation — explicit commands and natural language | ||
| if (lower.Contains("create card") || lower.Contains("add card") | ||
| || lower.Contains("create a card") || lower.Contains("add a card") | ||
| || lower.Contains("create task") || lower.Contains("add task") | ||
| || lower.Contains("create a task") || lower.Contains("add a task") | ||
| || lower.Contains("new card") || lower.Contains("new task") | ||
| || lower.Contains("make a card") || lower.Contains("make a task") | ||
| || lower.Contains("make card") || lower.Contains("make task")) | ||
| return (true, "card.create"); | ||
| // Check negative context first — suppress if negated or about another tool | ||
| if (IsNegativeContext(lower, message)) | ||
| return (false, null); | ||
|
|
||
| if (lower.Contains("move card")) | ||
| return (true, "card.move"); | ||
| if (lower.Contains("archive card") || lower.Contains("delete card") | ||
| || lower.Contains("remove card")) | ||
| // Archive/delete/remove must be checked BEFORE move to fix the | ||
| // "remove card" substring bug (remove contains "move") | ||
| if (MatchesCardArchive(lower)) | ||
| return (true, "card.archive"); | ||
| if (lower.Contains("update card") || lower.Contains("edit card") | ||
| || lower.Contains("rename card")) | ||
|
|
||
| if (MatchesCardMove(lower)) | ||
| return (true, "card.move"); | ||
|
|
||
| if (MatchesCardUpdate(lower)) | ||
| return (true, "card.update"); | ||
| if (lower.Contains("create board") || lower.Contains("add board") | ||
| || lower.Contains("new board")) | ||
|
|
||
| if (MatchesCardCreate(lower)) | ||
| return (true, "card.create"); | ||
|
|
||
| if (MatchesBoardCreate(lower)) | ||
| return (true, "board.create"); | ||
| if (lower.Contains("rename board")) | ||
|
|
||
| if (MatchesBoardRename(lower)) | ||
| return (true, "board.update"); | ||
| if (lower.Contains("reorder cards") || lower.Contains("reorder column") | ||
| || lower.Contains("reorder columns") || lower.Contains("reorder board") | ||
| || lower.Contains("sort cards") || lower.Contains("sort column") | ||
| || lower.Contains("sort columns") || lower.Contains("sort board")) | ||
|
|
||
| if (MatchesReorder(lower)) | ||
| return (true, "column.reorder"); | ||
|
|
||
| return (false, null); | ||
| } | ||
|
|
||
| private static bool IsNegativeContext(string lower, string original) | ||
| { | ||
| try | ||
| { | ||
| // Negation: "don't create task yet" | ||
| if (NegationPattern.IsMatch(lower)) | ||
| return true; | ||
|
|
||
| // Asking about another tool: "how do I create a card in Jira?" | ||
| if (OtherToolPattern.IsMatch(lower) && QuestionAboutHowPattern.IsMatch(original.Trim())) | ||
| return true; | ||
| } | ||
| catch (RegexMatchTimeoutException) | ||
| { | ||
| // On timeout, fall through to normal classification | ||
| } | ||
|
|
||
| return false; | ||
| } | ||
|
|
||
| private static bool MatchesCardCreate(string lower) | ||
| { | ||
| try | ||
| { | ||
| if (CardCreatePattern.IsMatch(lower)) | ||
| return true; | ||
| if (NewCardPattern.IsMatch(lower)) | ||
| return true; | ||
| } | ||
| catch (RegexMatchTimeoutException) | ||
| { | ||
| // Fall through — don't match on timeout | ||
| } | ||
|
|
||
| return false; | ||
| } | ||
|
|
||
| private static bool MatchesCardMove(string lower) | ||
| { | ||
| try | ||
| { | ||
| if (CardMovePattern.IsMatch(lower)) | ||
| return true; | ||
| } | ||
| catch (RegexMatchTimeoutException) { } | ||
|
|
||
| return false; | ||
| } | ||
|
|
||
| private static bool MatchesCardArchive(string lower) | ||
| { | ||
| try | ||
| { | ||
| if (CardArchivePattern.IsMatch(lower)) | ||
| return true; | ||
| } | ||
| catch (RegexMatchTimeoutException) { } | ||
|
|
||
| return false; | ||
| } | ||
|
|
||
| private static bool MatchesCardUpdate(string lower) | ||
| { | ||
| try | ||
| { | ||
| if (CardUpdatePattern.IsMatch(lower)) | ||
| return true; | ||
| } | ||
| catch (RegexMatchTimeoutException) { } | ||
|
|
||
| return false; | ||
| } | ||
|
|
||
| private static bool MatchesBoardCreate(string lower) | ||
| { | ||
| try | ||
| { | ||
| if (BoardCreatePattern.IsMatch(lower)) | ||
| return true; | ||
| } | ||
| catch (RegexMatchTimeoutException) { } | ||
|
|
||
| return false; | ||
| } | ||
|
|
||
| private static bool MatchesBoardRename(string lower) | ||
| { | ||
| try | ||
| { | ||
| if (BoardRenamePattern.IsMatch(lower)) | ||
| return true; | ||
| } | ||
| catch (RegexMatchTimeoutException) { } | ||
|
|
||
| return false; | ||
| } | ||
|
|
||
| private static bool MatchesReorder(string lower) | ||
| { | ||
| try | ||
| { | ||
| if (ReorderPattern.IsMatch(lower)) | ||
| return true; | ||
| } | ||
| catch (RegexMatchTimeoutException) { } | ||
|
|
||
| return false; | ||
| } | ||
| } | ||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This method, and the other
Matches...methods that follow, can be significantly simplified.The initial
ifblock withlower.Contains(...)is now redundant. The new regular expressions are supersets that cover these exact-match cases while also providing broader NLP capabilities. Removing these checks simplifies the code without losing backward compatibility.The
try-catchlogic can be made more direct by combining the regex checks and returningfalsefrom within thecatchblock.This refactoring makes the intent clearer and reduces code duplication across all the
Matches...methods. The same pattern can be applied toMatchesCardMove,MatchesCardArchive, etc.Here is a suggested simplification for this method: