refactor(proxy): load test_model turn 1 request bodies from JSON files

litellm/proxy/health_endpoints/model_checks.py

@@ -1,9 +1,10 @@
 """Live checks behind ``POST /health/test_model``.
 
-One two-turn tool-call conversation per selected endpoint, scripted in that endpoint's native request shape
-with no prompt-cache markers, so the second turn can only report cached tokens when the provider caches the
-repeated prefix on its own. Three verdicts come out of the two turns: the conversation itself (both turns
-parse, the tool call lands, the tool result is used), prompt caching, and cost tracking.
+One two-turn tool-call conversation per selected endpoint. Turn 1 is the JSON body in ``test_model_requests/``
+for that endpoint, in its native request shape with no prompt-cache markers, so the second turn can only report
+cached tokens when the provider caches the repeated prefix on its own. Turn 2 is built from turn 1's reply, since
+it has to carry the tool call id the model returned. Three verdicts come out of the two turns: the conversation
+itself (both turns parse, the tool call lands, the tool result is used), prompt caching, and cost tracking.
 """
 
 from __future__ import annotations
@@ -12,6 +13,7 @@ import json
 import time
 from collections.abc import Awaitable, Callable, Mapping, Sequence
 from dataclasses import dataclass
+from pathlib import Path
 from types import MappingProxyType
 from typing import Final, Literal, TypeAlias
 
@@ -27,50 +29,7 @@ EndpointStatus: TypeAlias = Literal["success", "warning", "error"]
 TEST_MODEL_ENDPOINTS: Final[tuple[TestModelEndpoint, ...]] = ("/v1/chat/completions", "/v1/responses", "/v1/messages")
 WEATHER_TOOL_NAME: Final = "get_weather"
 WEATHER_TOOL_RESULT: Final = "72F and sunny"
-WEATHER_QUESTION: Final = "What is the weather in Boston right now? Use the get_weather tool to find out."
-MAX_OUTPUT_TOKENS: Final = 2048
-THINKING_BUDGET_TOKENS: Final = 1024
-
-_WEATHER_PARAMETERS: Final[Mapping[str, object]] = MappingProxyType(
-    {
-        "type": "object",
-        "properties": {
-            "location": {"type": "string", "description": "City name, for example Boston"},
-            "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
-        },
-        "required": ["location"],
-    }
-)
-_WEATHER_DESCRIPTION: Final = "Get the current weather for a city"
-
-_CITIES: Final = ("Boston", "Denver", "Lisbon", "Osaka", "Nairobi", "Perth", "Quito", "Reykjavik")
-_SECTION_SENTENCES: Final = (
-    "Section {n}: deliveries routed through the {city} hub are scanned at intake, weighed on the calibrated "
-    "scale, and matched against the manifest before any label is printed or any pallet leaves the dock.",
-    "Agents handling {city} tickets confirm the tracking number, the declared value, and the delivery window "
-    "with the sender before promising a reroute, and record the promise in the ticket with a timestamp.",
-    "If a parcel bound for {city} misses its scheduled scan by more than {hours} hours, the agent opens a "
-    "trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift.",
-    "Refunds for {city} shipments above {amount} dollars need a supervisor code, a photo of the damage when "
-    "one exists, and the original invoice number, and are paid to the original payment method within ten days.",
-)
-_SECTION_COUNT: Final = 32
-
-SYSTEM_PROMPT: Final = "\n\n".join(
-    (
-        "You are the support assistant for Northwind Logistics. Follow the playbook below exactly.",
-        *(
-            " ".join(
-                sentence.format(n=n, city=_CITIES[n % len(_CITIES)], hours=4 + (n % 5), amount=100 * (1 + (n % 7)))
-                for sentence in _SECTION_SENTENCES
-            )
-            for n in range(1, _SECTION_COUNT + 1)
-        ),
-        f"When the user asks about the weather, call the {WEATHER_TOOL_NAME} tool with the city name. After a "
-        "tool result arrives, answer in one short sentence that repeats the temperature exactly as the tool "
-        "reported it.",
-    )
-)
+TEST_MODEL_REQUESTS_DIR: Final = Path(__file__).resolve().parent / "test_model_requests"
 
 
 @dataclass(frozen=True, slots=True)
@@ -232,6 +191,18 @@ _JSON_DOCUMENT: Final = TypeAdapter(object)
 _EMPTY_ASSISTANT: Final[Mapping[str, object]] = MappingProxyType({"role": "assistant", "content": None})
 
 
+def _request_file(name: str) -> Callable[[], Mapping[str, object]]:
+    """Turn 1 of each script is a plain JSON request body on disk, replayable with curl, parsed fresh per call so
+    the provider code never touches a shared copy."""
+    text: Final = (TEST_MODEL_REQUESTS_DIR / f"{name}.json").read_text(encoding="utf-8")
+    return lambda: _PLAIN_OBJECT.validate_json(text)
+
+
+_CHAT_TURN_ONE: Final = _request_file("chat_completions")
+_RESPONSES_TURN_ONE: Final = _request_file("responses")
+_MESSAGES_TURN_ONE: Final = _request_file("messages")
+
+
 def _plain(response: object) -> Mapping[str, object] | None:
     if isinstance(response, BaseModel):
         return _PLAIN_OBJECT.validate_python(response.model_dump(exclude_none=True))
@@ -254,29 +225,8 @@ def _arguments(raw: str) -> Mapping[str, object] | None:
     return MappingProxyType(_PLAIN_OBJECT.validate_python(parsed)) if isinstance(parsed, dict) else None
 
 
-def _chat_turn_one() -> Mapping[str, object]:
-    return {
-        "messages": [
-            {"role": "system", "content": SYSTEM_PROMPT},
-            {"role": "user", "content": WEATHER_QUESTION},
-        ],
-        "tools": [
-            {
-                "type": "function",
-                "function": {
-                    "name": WEATHER_TOOL_NAME,
-                    "description": _WEATHER_DESCRIPTION,
-                    "parameters": dict(_WEATHER_PARAMETERS),
-                },
-            }
-        ],
-        "reasoning_effort": "low",
-        "max_tokens": MAX_OUTPUT_TOKENS,
-    }
-
-
 def _chat_turn_two(turn_one: ParsedTurn) -> Mapping[str, object]:
-    first: Final = _chat_turn_one()
+    first: Final = _CHAT_TURN_ONE()
     tool_call_id: Final = turn_one.tool_call.id if turn_one.tool_call is not None else ""
     return {
         **first,
@@ -332,25 +282,8 @@ def _parse_chat(response: object) -> ParsedTurn | str:
     )
 
 
-def _responses_turn_one() -> Mapping[str, object]:
-    return {
-        "instructions": SYSTEM_PROMPT,
-        "input": [{"role": "user", "content": WEATHER_QUESTION}],
-        "tools": [
-            {
-                "type": "function",
-                "name": WEATHER_TOOL_NAME,
-                "description": _WEATHER_DESCRIPTION,
-                "parameters": dict(_WEATHER_PARAMETERS),
-            }
-        ],
-        "reasoning": {"effort": "low"},
-        "max_output_tokens": MAX_OUTPUT_TOKENS,
-    }
-
-
 def _responses_turn_two(turn_one: ParsedTurn) -> Mapping[str, object]:
-    first: Final = _responses_turn_one()
+    first: Final = _RESPONSES_TURN_ONE()
     call_id: Final = turn_one.tool_call.id if turn_one.tool_call is not None else ""
     return {
         **first,
@@ -398,24 +331,8 @@ def _parse_responses(response: object) -> ParsedTurn | str:
     )
 
 
-def _messages_turn_one() -> Mapping[str, object]:
-    return {
-        "system": SYSTEM_PROMPT,
-        "messages": [{"role": "user", "content": WEATHER_QUESTION}],
-        "tools": [
-            {
-                "name": WEATHER_TOOL_NAME,
-                "description": _WEATHER_DESCRIPTION,
-                "input_schema": dict(_WEATHER_PARAMETERS),
-            }
-        ],
-        "thinking": {"type": "enabled", "budget_tokens": THINKING_BUDGET_TOKENS},
-        "max_tokens": MAX_OUTPUT_TOKENS,
-    }
-
-
 def _messages_turn_two(turn_one: ParsedTurn) -> Mapping[str, object]:
-    first: Final = _messages_turn_one()
+    first: Final = _MESSAGES_TURN_ONE()
     tool_use_id: Final = turn_one.tool_call.id if turn_one.tool_call is not None else ""
     return {
         **first,
@@ -463,7 +380,7 @@ CHAT_COMPLETIONS_SCRIPT: Final = EndpointScript(
     endpoint="/v1/chat/completions",
     route_type="acompletion",
     metadata_key="metadata",
-    turn_one=_chat_turn_one,
+    turn_one=_CHAT_TURN_ONE,
     turn_two=_chat_turn_two,
     parse=_parse_chat,
 )
@@ -471,7 +388,7 @@ RESPONSES_SCRIPT: Final = EndpointScript(
     endpoint="/v1/responses",
     route_type="aresponses",
     metadata_key="litellm_metadata",
-    turn_one=_responses_turn_one,
+    turn_one=_RESPONSES_TURN_ONE,
     turn_two=_responses_turn_two,
     parse=_parse_responses,
 )
@@ -479,7 +396,7 @@ MESSAGES_SCRIPT: Final = EndpointScript(
     endpoint="/v1/messages",
     route_type="anthropic_messages",
     metadata_key="litellm_metadata",
-    turn_one=_messages_turn_one,
+    turn_one=_MESSAGES_TURN_ONE,
     turn_two=_messages_turn_two,
     parse=_parse_messages,
 )

pyproject.toml

@@ -332,6 +332,7 @@ include = [
     "litellm/router_strategy/complexity_router/artifacts/*.json",
     "litellm/router_strategy/complexity_router/fuse_presets.json",
     "litellm/proxy/model_insights_tasks.json",
+    "litellm/proxy/health_endpoints/test_model_requests/*.json",
     "litellm/proxy/public_endpoints/whats_new.json",
     "litellm/proxy/common_utils/codex_base_instructions.md",
     "litellm/proxy/common_utils/codex_bundled_models_0.159.3.json",

tests/integration/management/test_model_health_check.py

@@ -27,6 +27,8 @@ from integration._support.wire import Reply, Request, Wire, wire_server
 from integration.cost_calculation.cost_tracking_case import JsonResponse
 from pydantic import JsonValue
 
+from litellm.proxy.health_endpoints.model_checks import TEST_MODEL_REQUESTS_DIR
+
 _DECISIONS_PROBE_REPLY: Final = JsonResponse(
     content_type="application/json",
     body={
@@ -1438,6 +1440,9 @@ def test_health_check_of_gpt_transcribe_probes_without_verbose_json_and_reports_
 
 
 _TEST_MODEL_TOOL_CALL_ID: Final = "call_boston_wire"
+_TEST_MODEL_CHAT_REQUEST: Final = JSON_OBJECT.validate_json(
+    (TEST_MODEL_REQUESTS_DIR / "chat_completions.json").read_text(encoding="utf-8")
+)
 
 
 def _test_model_chat_reply(request: Request, *, cached_tokens: int) -> Reply:
@@ -1446,14 +1451,14 @@ def _test_model_chat_reply(request: Request, *, cached_tokens: int) -> Reply:
     assert "cache_control" not in request.body.decode(), request.body.decode()
     messages: Final = body["messages"]
     assert isinstance(messages, list) and isinstance(messages[0], dict), body
-    assert messages[0]["role"] == "system", messages[0]
     tool_turn: Final = any(isinstance(message, dict) and message.get("role") == "tool" for message in messages)
+    assert messages[: len(_TEST_MODEL_CHAT_REQUEST["messages"])] == _TEST_MODEL_CHAT_REQUEST["messages"], messages
+    assert body["tools"] == _TEST_MODEL_CHAT_REQUEST["tools"], body["tools"]
     if tool_turn:
         assert isinstance(messages[-1], dict), messages
         assert messages[-1]["tool_call_id"] == _TEST_MODEL_TOOL_CALL_ID, messages[-1]
         message: dict[str, JsonValue] = {"role": "assistant", "content": "Boston is 72F and sunny."}
     else:
-        assert body["tools"], body
         message = {
             "role": "assistant",
             "content": None,

tests/unit/proxy/health_endpoints/test_model_checks.py

@@ -9,6 +9,7 @@ from litellm.proxy.health_endpoints.model_checks import (
     CHAT_COMPLETIONS_SCRIPT,
     MESSAGES_SCRIPT,
     RESPONSES_SCRIPT,
+    TEST_MODEL_REQUESTS_DIR,
     WEATHER_TOOL_RESULT,
     CalledTurn,
     EndpointScript,
@@ -371,3 +372,18 @@ def test_payload_lists_the_checks_in_order_with_their_details():
     assert all(check["status"] == "pass" for check in payload["checks"])
     assert payload["cache_read_percent"] == 96.0
     assert isinstance(payload["duration_ms"], int)
+
+
+def test_the_three_request_files_send_one_system_prompt_and_one_tool_so_endpoint_verdicts_are_comparable():
+    bodies: Final = {
+        path.stem: json.loads(path.read_text(encoding="utf-8")) for path in TEST_MODEL_REQUESTS_DIR.glob("*.json")
+    }
+    chat, messages, responses = bodies["chat_completions"], bodies["messages"], bodies["responses"]
+
+    assert set(bodies) == {"chat_completions", "messages", "responses"}, bodies.keys()
+    assert chat["messages"][0]["content"] == responses["instructions"] == messages["system"]
+    assert chat["tools"][0]["function"]["parameters"] == responses["tools"][0]["parameters"]
+    assert chat["tools"][0]["function"]["parameters"] == messages["tools"][0]["input_schema"]
+    assert CHAT_COMPLETIONS_SCRIPT.turn_one() == chat
+    assert RESPONSES_SCRIPT.turn_one() == responses
+    assert MESSAGES_SCRIPT.turn_one() == messages