Testing LLM Tool Calls With Pytest: Asserting the Right Tool, Arguments, and Order

from __future__ import annotations import json from dataclasses import dataclass, field from typing import Any, Callable, Protocol from pydantic import BaseModel, ValidationError @dataclass class ToolCall: name: str arguments: dict[str, Any] @dataclass class FinalAnswer: text: str Step = ToolCall | FinalAnswer class LLMClient(Protocol): def next_step(self, messages: list[dict[str, Any]]) -> Step: ... class ToolError(Exception): """Raised by a tool when the underlying system fails.""" @dataclass class Tool: args_model: type[BaseModel] func: Callable[..., Any] requires: str | None = None # name of a tool that must succeed first @dataclass class CallRecord: name: str arguments: dict[str, Any] ok: bool @dataclass class AgentResult: answer: str calls: list[CallRecord] = field(default_factory=list) def _serialise(result: Any) -> str: """Tool results go to the model as JSON, not as a Python repr.""" return json.dumps(result, default=str) def run_agent(client: LLMClient, tools: dict[str, Tool], user_message: str, max_steps: int = 5) -> AgentResult: messages: list[dict[str, Any]] = [{"role": "user", "content": user_message}] calls: list[CallRecord] = [] for _ in range(max_steps): step = client.next_step(messages) if isinstance(step, FinalAnswer): return AgentResult(answer=step.text, calls=calls) tool = tools.get(step.name) if tool is None: result, ok = f"unknown tool: {step.name}", False elif tool.requires and not any(c.name == tool.requires and c.ok for c in calls): result, ok = f"blocked: {step.name} needs a successful {tool.requires} first", False else: try: args = tool.args_model.model_validate(step.arguments) result, ok = _serialise(tool.func(**args.model_dump())), True except ValidationError as exc: result, ok = f"invalid arguments: {exc.errors(include_url=False)}", False except ToolError as exc: result, ok = f"tool error: {exc}", False except Exception as exc: # A tool raised something we did not plan for. Report it to the # model instead of letting it kill the request. result, ok = f"tool crashed: {type(exc).__name__}: {exc}", False # Record the assistant turn that produced the call, then its result. messages.append({"role": "assistant", "tool_calls": [{"name": step.name, "arguments": step.arguments}]}) calls.append(CallRecord(step.name, step.arguments, ok)) messages.append({"role": "tool", "name": step.name, "content": result, "is_error": not ok}) raise RuntimeError(f"agent did not finish in {max_steps} steps") 

from decimal import Decimal from typing import Literal from pydantic import BaseModel, ConfigDict, Field class LookupOrderArgs(BaseModel): model_config = ConfigDict(extra="forbid") order_id: str = Field(pattern=r"^A-\d{4}$") class IssueRefundArgs(BaseModel): model_config = ConfigDict(extra="forbid") order_id: str = Field(pattern=r"^A-\d{4}$") amount: Decimal = Field(gt=0, max_digits=8, decimal_places=2) reason: Literal["damaged", "late", "other"] 

from decimal import Decimal from typing import Any import pytest from agent import Step, Tool, ToolError from tools import IssueRefundArgs, LookupOrderArgs class ScriptedLLM: """Returns pre-recorded steps in order and keeps every prompt it saw.""" def __init__(self, steps: list[Step]): self._steps = list(steps) self.seen: list[list[dict[str, Any]]] = [] def next_step(self, messages: list[dict[str, Any]]) -> Step: self.seen.append([dict(m) for m in messages]) if not self._steps: raise AssertionError("agent asked the LLM for more steps than scripted") return self._steps.pop(0) @pytest.fixture def orders() -> dict[str, dict[str, Any]]: return {"A-1001": {"total": Decimal("49.90"), "status": "delivered"}} @pytest.fixture def refunds() -> list[dict[str, Any]]: return [] @pytest.fixture def tools(orders, refunds) -> dict[str, Tool]: def lookup_order(order_id: str) -> dict[str, Any]: if order_id not in orders: raise ToolError(f"order {order_id} not found") return orders[order_id] def issue_refund(order_id: str, amount: Decimal, reason: str) -> dict[str, str]: if order_id not in orders: # Never assume an earlier lookup covered this order id. raise ToolError(f"order {order_id} not found") if amount > orders[order_id]["total"]: raise ToolError("refund exceeds order total") refunds.append({"order_id": order_id, "amount": amount, "reason": reason}) return {"refund_id": f"R-{len(refunds)}"} return { "lookup_order": Tool(LookupOrderArgs, lookup_order), "issue_refund": Tool(IssueRefundArgs, issue_refund, requires="lookup_order"), } @pytest.fixture def llm(): """Returns the class, so each test scripts its own steps: llm([...]).""" return ScriptedLLM 

from decimal import Decimal import pytest from agent import FinalAnswer, ToolCall, run_agent def test_status_question_calls_lookup_only(llm, tools, refunds): client = llm([ ToolCall("lookup_order", {"order_id": "A-1001"}), FinalAnswer("Order A-1001 was delivered."), ]) result = run_agent(client, tools, "Where is my order A-1001?") assert [c.name for c in result.calls] == ["lookup_order"] assert result.calls[0].ok assert refunds == [] def test_unknown_tool_is_reported_not_crashed(llm, tools): client = llm([ ToolCall("cancel_subscription", {"user_id": "u1"}), FinalAnswer("I can't do that here."), ]) result = run_agent(client, tools, "Cancel my plan") assert result.calls[0].ok is False last_tool_msg = client.seen[-1][-1] assert last_tool_msg["is_error"] is True assert "unknown tool" in last_tool_msg["content"] 

@pytest.mark.parametrize("bad_args", [ {"order_id": "1001", "amount": "10.00", "reason": "late"}, # wrong id format {"order_id": "A-1001", "amount": "-5", "reason": "late"}, # negative amount {"order_id": "A-1001", "amount": "0.00", "reason": "late"}, # zero amount {"order_id": "A-1001", "amount": "10.001", "reason": "late"}, # sub-cent precision {"order_id": "A-1001", "amount": "10.00"}, # missing reason {"order_id": "A-1001", "amount": "10.00", "reason": "because"}, # not in enum {"order_id": "A-1001", "amount": "10.00", "reason": "late", "force": True}, # extra field ]) def test_invalid_refund_arguments_never_reach_the_tool(llm, tools, refunds, bad_args): client = llm([ ToolCall("lookup_order", {"order_id": "A-1001"}), ToolCall("issue_refund", bad_args), FinalAnswer("Something went wrong."), ]) result = run_agent(client, tools, "Refund A-1001") assert result.calls[1].ok is False assert refunds == [] assert "invalid arguments" in client.seen[-1][-1]["content"] 

def test_refund_values_match_the_request(llm, tools, refunds): client = llm([ ToolCall("lookup_order", {"order_id": "A-1001"}), ToolCall("issue_refund", {"order_id": "A-1001", "amount": "49.90", "reason": "damaged"}), FinalAnswer("Refunded 49.90."), ]) run_agent(client, tools, "A-1001 arrived broken, refund please") assert refunds == [{"order_id": "A-1001", "amount": Decimal("49.90"), "reason": "damaged"}] def test_refund_above_order_total_is_rejected(llm, tools, refunds): client = llm([ ToolCall("lookup_order", {"order_id": "A-1001"}), ToolCall("issue_refund", {"order_id": "A-1001", "amount": "99.00", "reason": "other"}), FinalAnswer("I couldn't process that refund."), ]) result = run_agent(client, tools, "Refund 99 for A-1001") assert result.calls[1].ok is False assert refunds == [] assert "exceeds order total" in client.seen[-1][-1]["content"] 

def test_refund_flow_runs_in_order(llm, tools): client = llm([ ToolCall("lookup_order", {"order_id": "A-1001"}), ToolCall("issue_refund", {"order_id": "A-1001", "amount": "49.90", "reason": "late"}), FinalAnswer("Done."), ]) result = run_agent(client, tools, "Refund A-1001, it was late") assert [(c.name, c.ok) for c in result.calls] == [ ("lookup_order", True), ("issue_refund",Source：https://dev.to/krasmik/testing-llm-tool-calls-with-pytest-asserting-the-right-tool-arguments-and-order-3bae
My favorites