diff --git a/litellm/proxy/health_endpoints/model_checks.py b/litellm/proxy/health_endpoints/model_checks.py index 719243cc032..4296d798ca7 100644 --- a/litellm/proxy/health_endpoints/model_checks.py +++ b/litellm/proxy/health_endpoints/model_checks.py @@ -1,9 +1,10 @@ """Live checks behind ``POST /health/test_model``. -One two-turn tool-call conversation per selected endpoint, scripted in that endpoint's native request shape -with no prompt-cache markers, so the second turn can only report cached tokens when the provider caches the -repeated prefix on its own. Three verdicts come out of the two turns: the conversation itself (both turns -parse, the tool call lands, the tool result is used), prompt caching, and cost tracking. +One two-turn tool-call conversation per selected endpoint. Turn 1 is the JSON body in ``test_model_requests/`` +for that endpoint, in its native request shape with no prompt-cache markers, so the second turn can only report +cached tokens when the provider caches the repeated prefix on its own. Turn 2 is built from turn 1's reply, since +it has to carry the tool call id the model returned. Three verdicts come out of the two turns: the conversation +itself (both turns parse, the tool call lands, the tool result is used), prompt caching, and cost tracking. """ from __future__ import annotations @@ -12,6 +13,7 @@ import json import time from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass +from pathlib import Path from types import MappingProxyType from typing import Final, Literal, TypeAlias @@ -27,50 +29,7 @@ EndpointStatus: TypeAlias = Literal["success", "warning", "error"] TEST_MODEL_ENDPOINTS: Final[tuple[TestModelEndpoint, ...]] = ("/v1/chat/completions", "/v1/responses", "/v1/messages") WEATHER_TOOL_NAME: Final = "get_weather" WEATHER_TOOL_RESULT: Final = "72F and sunny" -WEATHER_QUESTION: Final = "What is the weather in Boston right now? Use the get_weather tool to find out." -MAX_OUTPUT_TOKENS: Final = 2048 -THINKING_BUDGET_TOKENS: Final = 1024 - -_WEATHER_PARAMETERS: Final[Mapping[str, object]] = MappingProxyType( - { - "type": "object", - "properties": { - "location": {"type": "string", "description": "City name, for example Boston"}, - "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, - }, - "required": ["location"], - } -) -_WEATHER_DESCRIPTION: Final = "Get the current weather for a city" - -_CITIES: Final = ("Boston", "Denver", "Lisbon", "Osaka", "Nairobi", "Perth", "Quito", "Reykjavik") -_SECTION_SENTENCES: Final = ( - "Section {n}: deliveries routed through the {city} hub are scanned at intake, weighed on the calibrated " - "scale, and matched against the manifest before any label is printed or any pallet leaves the dock.", - "Agents handling {city} tickets confirm the tracking number, the declared value, and the delivery window " - "with the sender before promising a reroute, and record the promise in the ticket with a timestamp.", - "If a parcel bound for {city} misses its scheduled scan by more than {hours} hours, the agent opens a " - "trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift.", - "Refunds for {city} shipments above {amount} dollars need a supervisor code, a photo of the damage when " - "one exists, and the original invoice number, and are paid to the original payment method within ten days.", -) -_SECTION_COUNT: Final = 32 - -SYSTEM_PROMPT: Final = "\n\n".join( - ( - "You are the support assistant for Northwind Logistics. Follow the playbook below exactly.", - *( - " ".join( - sentence.format(n=n, city=_CITIES[n % len(_CITIES)], hours=4 + (n % 5), amount=100 * (1 + (n % 7))) - for sentence in _SECTION_SENTENCES - ) - for n in range(1, _SECTION_COUNT + 1) - ), - f"When the user asks about the weather, call the {WEATHER_TOOL_NAME} tool with the city name. After a " - "tool result arrives, answer in one short sentence that repeats the temperature exactly as the tool " - "reported it.", - ) -) +TEST_MODEL_REQUESTS_DIR: Final = Path(__file__).resolve().parent / "test_model_requests" @dataclass(frozen=True, slots=True) @@ -232,6 +191,18 @@ _JSON_DOCUMENT: Final = TypeAdapter(object) _EMPTY_ASSISTANT: Final[Mapping[str, object]] = MappingProxyType({"role": "assistant", "content": None}) +def _request_file(name: str) -> Callable[[], Mapping[str, object]]: + """Turn 1 of each script is a plain JSON request body on disk, replayable with curl, parsed fresh per call so + the provider code never touches a shared copy.""" + text: Final = (TEST_MODEL_REQUESTS_DIR / f"{name}.json").read_text(encoding="utf-8") + return lambda: _PLAIN_OBJECT.validate_json(text) + + +_CHAT_TURN_ONE: Final = _request_file("chat_completions") +_RESPONSES_TURN_ONE: Final = _request_file("responses") +_MESSAGES_TURN_ONE: Final = _request_file("messages") + + def _plain(response: object) -> Mapping[str, object] | None: if isinstance(response, BaseModel): return _PLAIN_OBJECT.validate_python(response.model_dump(exclude_none=True)) @@ -254,29 +225,8 @@ def _arguments(raw: str) -> Mapping[str, object] | None: return MappingProxyType(_PLAIN_OBJECT.validate_python(parsed)) if isinstance(parsed, dict) else None -def _chat_turn_one() -> Mapping[str, object]: - return { - "messages": [ - {"role": "system", "content": SYSTEM_PROMPT}, - {"role": "user", "content": WEATHER_QUESTION}, - ], - "tools": [ - { - "type": "function", - "function": { - "name": WEATHER_TOOL_NAME, - "description": _WEATHER_DESCRIPTION, - "parameters": dict(_WEATHER_PARAMETERS), - }, - } - ], - "reasoning_effort": "low", - "max_tokens": MAX_OUTPUT_TOKENS, - } - - def _chat_turn_two(turn_one: ParsedTurn) -> Mapping[str, object]: - first: Final = _chat_turn_one() + first: Final = _CHAT_TURN_ONE() tool_call_id: Final = turn_one.tool_call.id if turn_one.tool_call is not None else "" return { **first, @@ -332,25 +282,8 @@ def _parse_chat(response: object) -> ParsedTurn | str: ) -def _responses_turn_one() -> Mapping[str, object]: - return { - "instructions": SYSTEM_PROMPT, - "input": [{"role": "user", "content": WEATHER_QUESTION}], - "tools": [ - { - "type": "function", - "name": WEATHER_TOOL_NAME, - "description": _WEATHER_DESCRIPTION, - "parameters": dict(_WEATHER_PARAMETERS), - } - ], - "reasoning": {"effort": "low"}, - "max_output_tokens": MAX_OUTPUT_TOKENS, - } - - def _responses_turn_two(turn_one: ParsedTurn) -> Mapping[str, object]: - first: Final = _responses_turn_one() + first: Final = _RESPONSES_TURN_ONE() call_id: Final = turn_one.tool_call.id if turn_one.tool_call is not None else "" return { **first, @@ -398,24 +331,8 @@ def _parse_responses(response: object) -> ParsedTurn | str: ) -def _messages_turn_one() -> Mapping[str, object]: - return { - "system": SYSTEM_PROMPT, - "messages": [{"role": "user", "content": WEATHER_QUESTION}], - "tools": [ - { - "name": WEATHER_TOOL_NAME, - "description": _WEATHER_DESCRIPTION, - "input_schema": dict(_WEATHER_PARAMETERS), - } - ], - "thinking": {"type": "enabled", "budget_tokens": THINKING_BUDGET_TOKENS}, - "max_tokens": MAX_OUTPUT_TOKENS, - } - - def _messages_turn_two(turn_one: ParsedTurn) -> Mapping[str, object]: - first: Final = _messages_turn_one() + first: Final = _MESSAGES_TURN_ONE() tool_use_id: Final = turn_one.tool_call.id if turn_one.tool_call is not None else "" return { **first, @@ -463,7 +380,7 @@ CHAT_COMPLETIONS_SCRIPT: Final = EndpointScript( endpoint="/v1/chat/completions", route_type="acompletion", metadata_key="metadata", - turn_one=_chat_turn_one, + turn_one=_CHAT_TURN_ONE, turn_two=_chat_turn_two, parse=_parse_chat, ) @@ -471,7 +388,7 @@ RESPONSES_SCRIPT: Final = EndpointScript( endpoint="/v1/responses", route_type="aresponses", metadata_key="litellm_metadata", - turn_one=_responses_turn_one, + turn_one=_RESPONSES_TURN_ONE, turn_two=_responses_turn_two, parse=_parse_responses, ) @@ -479,7 +396,7 @@ MESSAGES_SCRIPT: Final = EndpointScript( endpoint="/v1/messages", route_type="anthropic_messages", metadata_key="litellm_metadata", - turn_one=_messages_turn_one, + turn_one=_MESSAGES_TURN_ONE, turn_two=_messages_turn_two, parse=_parse_messages, ) diff --git a/litellm/proxy/health_endpoints/test_model_requests/chat_completions.json b/litellm/proxy/health_endpoints/test_model_requests/chat_completions.json new file mode 100644 index 00000000000..9b03f3758ab --- /dev/null +++ b/litellm/proxy/health_endpoints/test_model_requests/chat_completions.json @@ -0,0 +1,42 @@ +{ + "messages": [ + { + "role": "system", + "content": "You are the support assistant for Northwind Logistics. Follow the playbook below exactly.\n\nSection 1: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 2: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 3: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 4: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 5: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 6: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 7: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 8: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 9: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 10: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 11: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 12: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 13: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 14: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 15: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 16: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 17: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 18: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 19: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 20: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 21: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 22: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 23: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 24: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 25: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 26: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 27: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 28: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 29: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 30: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 31: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 32: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nWhen the user asks about the weather, call the get_weather tool with the city name. After a tool result arrives, answer in one short sentence that repeats the temperature exactly as the tool reported it." + }, + { + "role": "user", + "content": "What is the weather in Boston right now? Use the get_weather tool to find out." + } + ], + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a city", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "City name, for example Boston" + }, + "unit": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ] + } + }, + "required": [ + "location" + ] + } + } + } + ], + "reasoning_effort": "low", + "max_tokens": 2048 +} diff --git a/litellm/proxy/health_endpoints/test_model_requests/messages.json b/litellm/proxy/health_endpoints/test_model_requests/messages.json new file mode 100644 index 00000000000..514a8226caf --- /dev/null +++ b/litellm/proxy/health_endpoints/test_model_requests/messages.json @@ -0,0 +1,39 @@ +{ + "system": "You are the support assistant for Northwind Logistics. Follow the playbook below exactly.\n\nSection 1: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 2: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 3: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 4: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 5: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 6: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 7: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 8: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 9: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 10: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 11: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 12: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 13: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 14: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 15: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 16: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 17: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 18: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 19: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 20: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 21: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 22: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 23: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 24: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 25: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 26: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 27: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 28: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 29: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 30: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 31: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 32: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nWhen the user asks about the weather, call the get_weather tool with the city name. After a tool result arrives, answer in one short sentence that repeats the temperature exactly as the tool reported it.", + "messages": [ + { + "role": "user", + "content": "What is the weather in Boston right now? Use the get_weather tool to find out." + } + ], + "tools": [ + { + "name": "get_weather", + "description": "Get the current weather for a city", + "input_schema": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "City name, for example Boston" + }, + "unit": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ] + } + }, + "required": [ + "location" + ] + } + } + ], + "thinking": { + "type": "enabled", + "budget_tokens": 1024 + }, + "max_tokens": 2048 +} diff --git a/litellm/proxy/health_endpoints/test_model_requests/responses.json b/litellm/proxy/health_endpoints/test_model_requests/responses.json new file mode 100644 index 00000000000..f394f7a224d --- /dev/null +++ b/litellm/proxy/health_endpoints/test_model_requests/responses.json @@ -0,0 +1,39 @@ +{ + "instructions": "You are the support assistant for Northwind Logistics. Follow the playbook below exactly.\n\nSection 1: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 2: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 3: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 4: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 5: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 6: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 7: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 8: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 9: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 10: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 11: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 12: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 13: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 14: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 15: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 16: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 17: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 18: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 19: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 20: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 21: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 22: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 23: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 24: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 25: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 26: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 27: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 28: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 29: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 30: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 31: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 32: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nWhen the user asks about the weather, call the get_weather tool with the city name. After a tool result arrives, answer in one short sentence that repeats the temperature exactly as the tool reported it.", + "input": [ + { + "role": "user", + "content": "What is the weather in Boston right now? Use the get_weather tool to find out." + } + ], + "tools": [ + { + "type": "function", + "name": "get_weather", + "description": "Get the current weather for a city", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "City name, for example Boston" + }, + "unit": { + "type": "string", + "enum": [ + "celsius", + "fahrenheit" + ] + } + }, + "required": [ + "location" + ] + } + } + ], + "reasoning": { + "effort": "low" + }, + "max_output_tokens": 2048 +} diff --git a/pyproject.toml b/pyproject.toml index 6e9e75a5b7f..ef78006a002 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -332,6 +332,7 @@ include = [ "litellm/router_strategy/complexity_router/artifacts/*.json", "litellm/router_strategy/complexity_router/fuse_presets.json", "litellm/proxy/model_insights_tasks.json", + "litellm/proxy/health_endpoints/test_model_requests/*.json", "litellm/proxy/public_endpoints/whats_new.json", "litellm/proxy/common_utils/codex_base_instructions.md", "litellm/proxy/common_utils/codex_bundled_models_0.159.3.json", diff --git a/tests/integration/management/test_model_health_check.py b/tests/integration/management/test_model_health_check.py index 36572d9d5b8..8cb2e3016da 100644 --- a/tests/integration/management/test_model_health_check.py +++ b/tests/integration/management/test_model_health_check.py @@ -27,6 +27,8 @@ from integration._support.wire import Reply, Request, Wire, wire_server from integration.cost_calculation.cost_tracking_case import JsonResponse from pydantic import JsonValue +from litellm.proxy.health_endpoints.model_checks import TEST_MODEL_REQUESTS_DIR + _DECISIONS_PROBE_REPLY: Final = JsonResponse( content_type="application/json", body={ @@ -1438,6 +1440,9 @@ def test_health_check_of_gpt_transcribe_probes_without_verbose_json_and_reports_ _TEST_MODEL_TOOL_CALL_ID: Final = "call_boston_wire" +_TEST_MODEL_CHAT_REQUEST: Final = JSON_OBJECT.validate_json( + (TEST_MODEL_REQUESTS_DIR / "chat_completions.json").read_text(encoding="utf-8") +) def _test_model_chat_reply(request: Request, *, cached_tokens: int) -> Reply: @@ -1446,14 +1451,14 @@ def _test_model_chat_reply(request: Request, *, cached_tokens: int) -> Reply: assert "cache_control" not in request.body.decode(), request.body.decode() messages: Final = body["messages"] assert isinstance(messages, list) and isinstance(messages[0], dict), body - assert messages[0]["role"] == "system", messages[0] tool_turn: Final = any(isinstance(message, dict) and message.get("role") == "tool" for message in messages) + assert messages[: len(_TEST_MODEL_CHAT_REQUEST["messages"])] == _TEST_MODEL_CHAT_REQUEST["messages"], messages + assert body["tools"] == _TEST_MODEL_CHAT_REQUEST["tools"], body["tools"] if tool_turn: assert isinstance(messages[-1], dict), messages assert messages[-1]["tool_call_id"] == _TEST_MODEL_TOOL_CALL_ID, messages[-1] message: dict[str, JsonValue] = {"role": "assistant", "content": "Boston is 72F and sunny."} else: - assert body["tools"], body message = { "role": "assistant", "content": None, diff --git a/tests/unit/proxy/health_endpoints/test_model_checks.py b/tests/unit/proxy/health_endpoints/test_model_checks.py index 35094b8f32e..9ae29833f87 100644 --- a/tests/unit/proxy/health_endpoints/test_model_checks.py +++ b/tests/unit/proxy/health_endpoints/test_model_checks.py @@ -9,6 +9,7 @@ from litellm.proxy.health_endpoints.model_checks import ( CHAT_COMPLETIONS_SCRIPT, MESSAGES_SCRIPT, RESPONSES_SCRIPT, + TEST_MODEL_REQUESTS_DIR, WEATHER_TOOL_RESULT, CalledTurn, EndpointScript, @@ -371,3 +372,18 @@ def test_payload_lists_the_checks_in_order_with_their_details(): assert all(check["status"] == "pass" for check in payload["checks"]) assert payload["cache_read_percent"] == 96.0 assert isinstance(payload["duration_ms"], int) + + +def test_the_three_request_files_send_one_system_prompt_and_one_tool_so_endpoint_verdicts_are_comparable(): + bodies: Final = { + path.stem: json.loads(path.read_text(encoding="utf-8")) for path in TEST_MODEL_REQUESTS_DIR.glob("*.json") + } + chat, messages, responses = bodies["chat_completions"], bodies["messages"], bodies["responses"] + + assert set(bodies) == {"chat_completions", "messages", "responses"}, bodies.keys() + assert chat["messages"][0]["content"] == responses["instructions"] == messages["system"] + assert chat["tools"][0]["function"]["parameters"] == responses["tools"][0]["parameters"] + assert chat["tools"][0]["function"]["parameters"] == messages["tools"][0]["input_schema"] + assert CHAT_COMPLETIONS_SCRIPT.turn_one() == chat + assert RESPONSES_SCRIPT.turn_one() == responses + assert MESSAGES_SCRIPT.turn_one() == messages