From 24056ca1286fbfcfe7148659133008e5c0b4cbb1 Mon Sep 17 00:00:00 2001 From: Mariusz Sabath Date: Mon, 16 Mar 2026 17:01:47 -0400 Subject: [PATCH] fix: resolve Ollama function calling failures with crewai 1.10.1 crewai 1.10.1's instructor integration requires LLMs to produce structured tool calls for output_pydantic parsing. Ollama models (granite4, granite3.3, llama3.2) don't reliably generate these, causing "Instructor does not support multiple tool calls" errors and empty tool_calls responses. Changes: - Remove output_pydantic from prereq task, parse raw JSON output instead - Add _parse_prereq_from_raw() fallback for extracting JSON from LLM text - Switch default model prefix from ollama/ to ollama_chat/ for better litellm chat completions routing - Pass num_ctx=8192 to Ollama models (default 2048 is too small) - Fix config parameter bug: GitIssueAgent now uses passed config instead of module-level settings singleton - Add field_validator for IssueSearchInfo.issue_numbers to handle string-encoded arrays from small LLMs - Add max_retry_limit=3 and respect_context_window=True to issue researcher Tested locally with granite4, granite3.3:8b, and llama3.2:3b-instruct-fp16. Fixes kagenti/agent-examples#173 Assisted-By: Claude (Anthropic AI) Signed-off-by: Mariusz Sabath --- a2a/git_issue_agent/.env.ollama | 2 +- a2a/git_issue_agent/.env.template | 2 +- a2a/git_issue_agent/git_issue_agent/agents.py | 9 ++-- a2a/git_issue_agent/git_issue_agent/config.py | 2 +- .../git_issue_agent/data_types.py | 17 ++++++- a2a/git_issue_agent/git_issue_agent/llm.py | 16 ++++--- a2a/git_issue_agent/git_issue_agent/main.py | 45 +++++++++++++++++-- 7 files changed, 78 insertions(+), 15 deletions(-) diff --git a/a2a/git_issue_agent/.env.ollama b/a2a/git_issue_agent/.env.ollama index 61509211..33fa0862 100644 --- a/a2a/git_issue_agent/.env.ollama +++ b/a2a/git_issue_agent/.env.ollama @@ -5,7 +5,7 @@ # ollama pull ibm/granite4:latest # LLM configuration -TASK_MODEL_ID=ollama/ibm/granite4:latest +TASK_MODEL_ID=ollama_chat/ibm/granite4:latest # Ollama API base URL. Required by litellm (used by crewai >=1.10). # For Docker Desktop / Kind: http://host.docker.internal:11434 # For in-cluster Ollama: http://ollama.ollama.svc:11434 diff --git a/a2a/git_issue_agent/.env.template b/a2a/git_issue_agent/.env.template index e00a2203..cf94681b 100644 --- a/a2a/git_issue_agent/.env.template +++ b/a2a/git_issue_agent/.env.template @@ -1,4 +1,4 @@ -TASK_MODEL_ID = "ollama/ibm/granite4:latest" +TASK_MODEL_ID = "ollama_chat/ibm/granite4:latest" LLM_API_BASE = "http://host.docker.internal:11434" LLM_API_KEY = "ollama" MODEL_TEMPERATURE = 0 diff --git a/a2a/git_issue_agent/git_issue_agent/agents.py b/a2a/git_issue_agent/git_issue_agent/agents.py index 8de15ec9..6f819dd8 100644 --- a/a2a/git_issue_agent/git_issue_agent/agents.py +++ b/a2a/git_issue_agent/git_issue_agent/agents.py @@ -1,6 +1,5 @@ from crewai import Agent, Crew, Process, Task from git_issue_agent.config import Settings -from git_issue_agent.data_types import IssueSearchInfo from git_issue_agent.llm import CrewLLM from git_issue_agent.prompts import TOOL_CALL_PROMPT, INFO_PARSER_PROMPT @@ -23,8 +22,10 @@ def __init__(self, config: Settings, issue_tools): self.prereq_identifier_task = Task( description=("User query: {request}"), agent=self.prereq_identifier, - output_pydantic=IssueSearchInfo, - expected_output=("A pydantic object representing the extracted relevant information."), + expected_output=( + 'A JSON object with keys "owner", "repo", and "issue_numbers". ' + "Example: {\"owner\": \"kagenti\", \"repo\": \"kagenti\", \"issue_numbers\": null}" + ), ) self.prereq_id_crew = Crew( @@ -49,6 +50,8 @@ def __init__(self, config: Settings, issue_tools): llm=self.llm.llm, inject_date=True, max_iter=6, + max_retry_limit=3, + respect_context_window=True, ) # --- A generic task template ------------------------------------------------- diff --git a/a2a/git_issue_agent/git_issue_agent/config.py b/a2a/git_issue_agent/git_issue_agent/config.py index 39c6748e..3f688285 100644 --- a/a2a/git_issue_agent/git_issue_agent/config.py +++ b/a2a/git_issue_agent/git_issue_agent/config.py @@ -12,7 +12,7 @@ class Settings(BaseSettings): description="Application log level", ) TASK_MODEL_ID: str = Field( - os.getenv("TASK_MODEL_ID", "ollama/ibm/granite4:latest"), + os.getenv("TASK_MODEL_ID", "ollama_chat/ibm/granite4:latest"), description="The ID of the task model", ) LLM_API_BASE: str = Field( diff --git a/a2a/git_issue_agent/git_issue_agent/data_types.py b/a2a/git_issue_agent/git_issue_agent/data_types.py index 37c53230..7673ebd9 100644 --- a/a2a/git_issue_agent/git_issue_agent/data_types.py +++ b/a2a/git_issue_agent/git_issue_agent/data_types.py @@ -1,4 +1,6 @@ -from pydantic import BaseModel, Field +import json + +from pydantic import BaseModel, Field, field_validator ############ # Pydantic types for LLM response formats @@ -11,3 +13,16 @@ class IssueSearchInfo(BaseModel): issue_numbers: list[int] | None = Field( None, description="Specific issue number(s) mentioned by the user. If none mentioned leave blank." ) + + @field_validator("issue_numbers", mode="before") + @classmethod + def coerce_string_to_list(cls, v): + """Small LLMs often serialize arrays as strings in tool call args.""" + if isinstance(v, str): + try: + parsed = json.loads(v) + if isinstance(parsed, list): + return parsed + except (json.JSONDecodeError, ValueError): + pass + return v diff --git a/a2a/git_issue_agent/git_issue_agent/llm.py b/a2a/git_issue_agent/git_issue_agent/llm.py index e2169a0e..3a19d68f 100644 --- a/a2a/git_issue_agent/git_issue_agent/llm.py +++ b/a2a/git_issue_agent/git_issue_agent/llm.py @@ -4,13 +4,19 @@ class CrewLLM: def __init__(self, config: Settings): + kwargs = {} + if config.EXTRA_HEADERS is not None and None not in config.EXTRA_HEADERS: + kwargs["extra_headers"] = config.EXTRA_HEADERS + + # For Ollama models, pass num_ctx to set the context window size. + # Ollama defaults to 2048 tokens which is too small for agent workflows. + if config.TASK_MODEL_ID.startswith(("ollama/", "ollama_chat/")): + kwargs["num_ctx"] = 8192 + self.llm = LLM( model=config.TASK_MODEL_ID, base_url=config.LLM_API_BASE, api_key=config.LLM_API_KEY, - **( - {"extra_headers": config.EXTRA_HEADERS} - if config.EXTRA_HEADERS is not None and None not in config.EXTRA_HEADERS - else {} - ), + temperature=config.MODEL_TEMPERATURE, + **kwargs, ) diff --git a/a2a/git_issue_agent/git_issue_agent/main.py b/a2a/git_issue_agent/git_issue_agent/main.py index 311927d2..dfbce1b9 100644 --- a/a2a/git_issue_agent/git_issue_agent/main.py +++ b/a2a/git_issue_agent/git_issue_agent/main.py @@ -1,9 +1,12 @@ from crewai_tools.adapters.tool_collection import ToolCollection +import json import logging +import re import sys from git_issue_agent.config import Settings, settings +from git_issue_agent.data_types import IssueSearchInfo from git_issue_agent.event import Event from git_issue_agent.agents import GitAgents @@ -11,6 +14,25 @@ logging.basicConfig(level=settings.LOG_LEVEL, stream=sys.stdout, format="%(levelname)s: %(message)s") +def _parse_prereq_from_raw(raw: str) -> IssueSearchInfo: + """Parse IssueSearchInfo from raw LLM text when instructor/pydantic parsing fails. + + Some Ollama models don't produce structured tool calls that crewai's instructor + integration expects. This fallback extracts JSON from the raw text output. + """ + # Try to find JSON in the raw output + json_match = re.search(r'\{[^{}]*\}', raw) + if json_match: + try: + data = json.loads(json_match.group()) + return IssueSearchInfo(**data) + except (json.JSONDecodeError, ValueError): + pass + + # Fallback: return empty IssueSearchInfo (no pre-identified fields) + return IssueSearchInfo() + + class GitIssueAgent: def __init__( self, @@ -19,7 +41,7 @@ def __init__( mcp_toolkit: ToolCollection = None, logger=None, ): - self.agents = GitAgents(settings, mcp_toolkit) + self.agents = GitAgents(config, mcp_toolkit) self.eventer = eventer self.logger = logger or logging.getLogger(__name__) @@ -45,11 +67,28 @@ def extract_user_input(self, body): return latest_content + async def _get_prereq_output(self, query: str) -> IssueSearchInfo: + """Run the prereq crew and extract IssueSearchInfo from raw text output. + + We avoid using crewai's output_pydantic because it relies on instructor's + tool-call-based parsing, which fails with Ollama models that don't produce + structured tool calls. Instead, we ask the LLM for JSON and parse it ourselves. + """ + try: + await self.agents.prereq_id_crew.kickoff_async( + inputs={"request": query, "repo": "", "owner": "", "issues": []} + ) + raw = self.agents.prereq_identifier_task.output.raw + self.logger.info(f"Prereq raw output: {raw}") + return _parse_prereq_from_raw(raw) + except Exception as e: + self.logger.warning(f"Prereq crew failed: {e}") + return IssueSearchInfo() + async def execute(self, user_input): query = self.extract_user_input(user_input) await self._send_event("🧐 Evaluating requirements...") - await self.agents.prereq_id_crew.kickoff_async(inputs={"request": query, "repo": "", "owner": "", "issues": []}) - repo_id_task_output = self.agents.prereq_identifier_task.output.pydantic + repo_id_task_output = await self._get_prereq_output(query) if repo_id_task_output.issue_numbers: if not repo_id_task_output.owner or not repo_id_task_output.repo: