From ee4cc1c3be12587c101b3c5fbfa930af2c5be95b Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 31 Oct 2026 18:52:17 +0000 Subject: [PATCH] feat(proxy): fail test_model prompt caching when turn 3 reads fewer cached tokens than turn 2 --- .../proxy/health_endpoints/model_checks.py | 22 +++++++-- .../test_model_requests/messages.json | 35 +++++++++----- .../health_endpoints/test_model_checks.py | 48 ++++++++++++++----- 3 files changed, 78 insertions(+), 27 deletions(-) diff --git a/litellm/proxy/health_endpoints/model_checks.py b/litellm/proxy/health_endpoints/model_checks.py index 21dbe1502d3..89537b5454b 100644 --- a/litellm/proxy/health_endpoints/model_checks.py +++ b/litellm/proxy/health_endpoints/model_checks.py @@ -3,9 +3,9 @@ One three-turn tool-call conversation per selected endpoint, read from that endpoint's file in ``test_model_requests/``. ``turn_1`` is the full first request body in the endpoint's native shape. Chat Completions and Responses carry no prompt-cache markers, so later turns can only report cached tokens when the provider caches -the repeated prefix on its own. Messages mirrors Claude Code: ``cache_control`` on the system prompt and a -mid-conversation ``role: system`` reminder, so a proxy that hoists that reminder into the system prompt shows up as a -cache miss. ``turn_2_appended_messages`` lists what the second request adds after turn 1's messages (the model's tool +the repeated prefix on its own. Messages mirrors Claude Code: ``cache_control`` on the latest user message and a +mid-conversation ``role: system`` reminder, so a proxy that hoists that reminder into the system prompt changes the +cached prefix and shows up as a cache miss. ``turn_2_appended_messages`` lists what the second request adds after turn 1's messages (the model's tool call echoed back plus the tool result), ``turn_3_appended_messages`` what the third adds after that (the model's answer echoed back, the reminder, a user follow-up), with ````, ```` and ```` filled in from the model's replies. @@ -564,6 +564,19 @@ def _cache_miss(turn: RepeatedTurn, parsed: ParsedTurn) -> CheckResult: ) +def _cache_shrank(second: ParsedTurn | None, third: ParsedTurn | None) -> CheckResult | None: + if second is None or third is None or third.cache_read_tokens >= second.cache_read_tokens: + return None + return CheckResult( + "prompt_caching", + "fail", + f"turn 3 read {third.cache_read_tokens} cached tokens, fewer than turn 2's {second.cache_read_tokens}, so the " + "prompt the provider saw on turn 3 no longer starts with the one it cached on turn 2 (something between the " + "client and the provider rewrote the earlier messages, for example a mid-conversation system message moved " + "into the system prompt)", + ) + + def _prompt_caching_check(second: RepeatedTurn, third: RepeatedTurn) -> CheckResult: reached: Final = tuple((turn, parsed) for turn in (second, third) if (parsed := _parsed(turn)) is not None) if not reached: @@ -571,6 +584,9 @@ def _prompt_caching_check(second: RepeatedTurn, third: RepeatedTurn) -> CheckRes missed: Final = next((_cache_miss(turn, parsed) for turn, parsed in reached if parsed.cache_read_tokens == 0), None) if missed is not None: return missed + shrank: Final = _cache_shrank(_parsed(second), _parsed(third)) + if shrank is not None: + return shrank return CheckResult("prompt_caching", "pass", ", ".join(_cache_read_note(turn, parsed) for turn, parsed in reached)) diff --git a/litellm/proxy/health_endpoints/test_model_requests/messages.json b/litellm/proxy/health_endpoints/test_model_requests/messages.json index 84ff9821268..b55e89507f0 100644 --- a/litellm/proxy/health_endpoints/test_model_requests/messages.json +++ b/litellm/proxy/health_endpoints/test_model_requests/messages.json @@ -1,18 +1,18 @@ { "turn_1": { - "system": [ - { - "type": "text", - "text": "You are the support assistant for Northwind Logistics. Follow the playbook below exactly.\n\nSection 1: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 2: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 3: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 4: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 5: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 6: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 7: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 8: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 9: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 10: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 11: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 12: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 13: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 14: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 15: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 16: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 17: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 18: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 19: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 20: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 21: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 22: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 23: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 24: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 25: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 26: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 27: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 28: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 29: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 30: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 31: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 32: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nWhen the user asks about the weather, call the get_weather tool with the city name. After a tool result arrives, answer in one short sentence that repeats the temperature exactly as the tool reported it.", - "cache_control": { - "type": "ephemeral" - } - } - ], + "system": "You are the support assistant for Northwind Logistics. Follow the playbook below exactly.\n\nSection 1: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 2: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 3: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 4: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 5: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 6: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 7: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 8: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 9: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 10: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 11: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 12: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 13: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 14: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 15: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 16: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 17: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 18: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 19: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 20: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 21: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 22: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 23: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 24: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 25: deliveries routed through the Denver hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Denver tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Denver misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Denver shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 26: deliveries routed through the Lisbon hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Lisbon tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Lisbon misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Lisbon shipments above 600 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 27: deliveries routed through the Osaka hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Osaka tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Osaka misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Osaka shipments above 700 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 28: deliveries routed through the Nairobi hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Nairobi tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Nairobi misses its scheduled scan by more than 7 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Nairobi shipments above 100 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 29: deliveries routed through the Perth hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Perth tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Perth misses its scheduled scan by more than 8 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Perth shipments above 200 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 30: deliveries routed through the Quito hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Quito tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Quito misses its scheduled scan by more than 4 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Quito shipments above 300 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 31: deliveries routed through the Reykjavik hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Reykjavik tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Reykjavik misses its scheduled scan by more than 5 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Reykjavik shipments above 400 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nSection 32: deliveries routed through the Boston hub are scanned at intake, weighed on the calibrated scale, and matched against the manifest before any label is printed or any pallet leaves the dock. Agents handling Boston tickets confirm the tracking number, the declared value, and the delivery window with the sender before promising a reroute, and record the promise in the ticket with a timestamp. If a parcel bound for Boston misses its scheduled scan by more than 6 hours, the agent opens a trace request, notifies the recipient in plain language, and schedules a follow-up for the next shift. Refunds for Boston shipments above 500 dollars need a supervisor code, a photo of the damage when one exists, and the original invoice number, and are paid to the original payment method within ten days.\n\nWhen the user asks about the weather, call the get_weather tool with the city name. After a tool result arrives, answer in one short sentence that repeats the temperature exactly as the tool reported it.", "messages": [ { "role": "user", - "content": "What is the weather in Boston right now? First tell me in one sentence what you are about to do, then use the get_weather tool to find out." + "content": [ + { + "type": "text", + "text": "What is the weather in Boston right now? First tell me in one sentence what you are about to do, then use the get_weather tool to find out.", + "cache_control": { + "type": "ephemeral" + } + } + ] } ], "tools": [ @@ -57,7 +57,10 @@ { "type": "tool_result", "tool_use_id": "", - "content": "72F and sunny" + "content": "72F and sunny", + "cache_control": { + "type": "ephemeral" + } } ] } @@ -69,7 +72,15 @@ }, { "role": "user", - "content": "Thanks. Is that warm enough for a short walk? Repeat the temperature you found in your answer." + "content": [ + { + "type": "text", + "text": "Thanks. Is that warm enough for a short walk? Repeat the temperature you found in your answer.", + "cache_control": { + "type": "ephemeral" + } + } + ] }, { "role": "system", diff --git a/tests/unit/proxy/health_endpoints/test_model_checks.py b/tests/unit/proxy/health_endpoints/test_model_checks.py index 7b2815e6e4a..8787e33b299 100644 --- a/tests/unit/proxy/health_endpoints/test_model_checks.py +++ b/tests/unit/proxy/health_endpoints/test_model_checks.py @@ -184,7 +184,7 @@ def test_every_endpoint_passes_all_three_checks_and_feeds_each_reply_back_in_its CalledTurn(reply(tool_call=True), 0.004), [ CalledTurn(reply(tool_call=False, text=_TURN_TWO_ANSWER, cached_tokens=4800), 0.0005), - CalledTurn(reply(tool_call=False, text=_TURN_THREE_ANSWER, cached_tokens=4900), 0.0004), + CalledTurn(reply(tool_call=False, text=_TURN_THREE_ANSWER, cached_tokens=4800), 0.0004), ], ) @@ -200,7 +200,7 @@ def test_every_endpoint_passes_all_three_checks_and_feeds_each_reply_back_in_its assert checks["prompt_caching"] == ( "pass", "turn 2 read 4800 of 5000 input tokens from cache (96.0%), " - "turn 3 read 4900 of 5000 input tokens from cache (98.0%)", + "turn 3 read 4800 of 5000 input tokens from cache (96.0%)", ), checks assert checks["cost_tracking"] == ( "pass", @@ -216,7 +216,9 @@ def test_every_endpoint_passes_all_three_checks_and_feeds_each_reply_back_in_its sent_json: Final = json.dumps(provider.sent[1:]) assert _TOOL_CALL_ID in sent_json and WEATHER_TOOL_RESULT in sent_json, sent_json if script is MESSAGES_SCRIPT: - assert turn_two["system"][0]["cache_control"] == {"type": "ephemeral"}, turn_two["system"] + assert turn_two["messages"][-1]["content"][0]["cache_control"] == {"type": "ephemeral"}, turn_two["messages"][ + -1 + ] else: assert "cache_control" not in sent_json and "cachePoint" not in sent_json, sent_json assert "" in third, (name, third) assert "user" in {m["role"] for m in file["turn_3_appended_messages"][-2:]}, (name, third) - assert chat["messages"][0]["content"] == responses["instructions"] == messages["system"][0]["text"] - assert messages["system"][0]["cache_control"] == {"type": "ephemeral"}, messages["system"] + assert chat["messages"][0]["content"] == responses["instructions"] == messages["system"] + assert messages["messages"][-1]["content"][0]["cache_control"] == {"type": "ephemeral"}, messages["messages"][-1] assert chat["tools"][0]["function"]["parameters"] == responses["tools"][0]["parameters"] assert chat["tools"][0]["function"]["parameters"] == messages["tools"][0]["input_schema"] assert CHAT_COMPLETIONS_SCRIPT.turn_one() == chat