feat(proxy): fail test_model prompt caching when turn 3 reads fewer cached tokens than turn 2

litellm/proxy/health_endpoints/model_checks.py

@@ -3,9 +3,9 @@
 One three-turn tool-call conversation per selected endpoint, read from that endpoint's file in
 ``test_model_requests/``. ``turn_1`` is the full first request body in the endpoint's native shape. Chat Completions
 and Responses carry no prompt-cache markers, so later turns can only report cached tokens when the provider caches
-the repeated prefix on its own. Messages mirrors Claude Code: ``cache_control`` on the system prompt and a
-mid-conversation ``role: system`` reminder, so a proxy that hoists that reminder into the system prompt shows up as a
-cache miss. ``turn_2_appended_messages`` lists what the second request adds after turn 1's messages (the model's tool
+the repeated prefix on its own. Messages mirrors Claude Code: ``cache_control`` on the latest user message and a
+mid-conversation ``role: system`` reminder, so a proxy that hoists that reminder into the system prompt changes the
+cached prefix and shows up as a cache miss. ``turn_2_appended_messages`` lists what the second request adds after turn 1's messages (the model's tool
 call echoed back plus the tool result), ``turn_3_appended_messages`` what the third adds after that (the model's
 answer echoed back, the reminder, a user follow-up), with ``<turn 1 response: assistant output>``,
 ``<turn 1 response: tool call id>`` and ``<turn 2 response: assistant output>`` filled in from the model's replies.
@@ -564,6 +564,19 @@ def _cache_miss(turn: RepeatedTurn, parsed: ParsedTurn) -> CheckResult:
     )
 
 
+def _cache_shrank(second: ParsedTurn | None, third: ParsedTurn | None) -> CheckResult | None:
+    if second is None or third is None or third.cache_read_tokens >= second.cache_read_tokens:
+        return None
+    return CheckResult(
+        "prompt_caching",
+        "fail",
+        f"turn 3 read {third.cache_read_tokens} cached tokens, fewer than turn 2's {second.cache_read_tokens}, so the "
+        "prompt the provider saw on turn 3 no longer starts with the one it cached on turn 2 (something between the "
+        "client and the provider rewrote the earlier messages, for example a mid-conversation system message moved "
+        "into the system prompt)",
+    )
+
+
 def _prompt_caching_check(second: RepeatedTurn, third: RepeatedTurn) -> CheckResult:
     reached: Final = tuple((turn, parsed) for turn in (second, third) if (parsed := _parsed(turn)) is not None)
     if not reached:
@@ -571,6 +584,9 @@ def _prompt_caching_check(second: RepeatedTurn, third: RepeatedTurn) -> CheckRes
     missed: Final = next((_cache_miss(turn, parsed) for turn, parsed in reached if parsed.cache_read_tokens == 0), None)
     if missed is not None:
         return missed
+    shrank: Final = _cache_shrank(_parsed(second), _parsed(third))
+    if shrank is not None:
+        return shrank
     return CheckResult("prompt_caching", "pass", ", ".join(_cache_read_note(turn, parsed) for turn, parsed in reached))
 
 

tests/unit/proxy/health_endpoints/test_model_checks.py

@@ -184,7 +184,7 @@ def test_every_endpoint_passes_all_three_checks_and_feeds_each_reply_back_in_its
         CalledTurn(reply(tool_call=True), 0.004),
         [
             CalledTurn(reply(tool_call=False, text=_TURN_TWO_ANSWER, cached_tokens=4800), 0.0005),
-            CalledTurn(reply(tool_call=False, text=_TURN_THREE_ANSWER, cached_tokens=4900), 0.0004),
+            CalledTurn(reply(tool_call=False, text=_TURN_THREE_ANSWER, cached_tokens=4800), 0.0004),
         ],
     )
 
@@ -200,7 +200,7 @@ def test_every_endpoint_passes_all_three_checks_and_feeds_each_reply_back_in_its
     assert checks["prompt_caching"] == (
         "pass",
         "turn 2 read 4800 of 5000 input tokens from cache (96.0%), "
-        "turn 3 read 4900 of 5000 input tokens from cache (98.0%)",
+        "turn 3 read 4800 of 5000 input tokens from cache (96.0%)",
     ), checks
     assert checks["cost_tracking"] == (
         "pass",
@@ -216,7 +216,9 @@ def test_every_endpoint_passes_all_three_checks_and_feeds_each_reply_back_in_its
     sent_json: Final = json.dumps(provider.sent[1:])
     assert _TOOL_CALL_ID in sent_json and WEATHER_TOOL_RESULT in sent_json, sent_json
     if script is MESSAGES_SCRIPT:
-        assert turn_two["system"][0]["cache_control"] == {"type": "ephemeral"}, turn_two["system"]
+        assert turn_two["messages"][-1]["content"][0]["cache_control"] == {"type": "ephemeral"}, turn_two["messages"][
+            -1
+        ]
     else:
         assert "cache_control" not in sent_json and "cachePoint" not in sent_json, sent_json
     assert "<turn " not in sent_json, sent_json
@@ -242,7 +244,12 @@ def test_every_endpoint_passes_all_three_checks_and_feeds_each_reply_back_in_its
         assert turn_three["input"][-3]["content"][0]["text"] == _TURN_TWO_ANSWER, turn_three
     else:
         assert turn_two["messages"][-1]["content"] == [
-            {"type": "tool_result", "tool_use_id": _TOOL_CALL_ID, "content": WEATHER_TOOL_RESULT}
+            {
+                "type": "tool_result",
+                "tool_use_id": _TOOL_CALL_ID,
+                "content": WEATHER_TOOL_RESULT,
+                "cache_control": {"type": "ephemeral"},
+            }
         ]
         assert turn_two["messages"][-2]["content"][0]["type"] == "thinking"
         assert [m["role"] for m in turn_three["messages"][-3:]] == ["assistant", "user", "system"], turn_three
@@ -282,7 +289,7 @@ def test_an_answer_that_ignores_the_tool_result_fails_the_conversation():
         CalledTurn(_chat_reply(tool_call=True), 0.004),
         [
             CalledTurn(_chat_reply(tool_call=False, text="I cannot tell.", cached_tokens=4800), 0.0005),
-            CalledTurn(_chat_reply(tool_call=False, text=_TURN_THREE_ANSWER, cached_tokens=4900), 0.0004),
+            CalledTurn(_chat_reply(tool_call=False, text=_TURN_THREE_ANSWER, cached_tokens=4800), 0.0004),
         ],
     )
 
@@ -297,7 +304,7 @@ def test_a_follow_up_answer_that_forgets_the_earlier_turn_fails_the_conversation
         CalledTurn(_chat_reply(tool_call=True), 0.004),
         [
             CalledTurn(_chat_reply(tool_call=False, text=_TURN_TWO_ANSWER, cached_tokens=4800), 0.0005),
-            CalledTurn(_chat_reply(tool_call=False, text="I do not know the temperature.", cached_tokens=4900), 0.0004),
+            CalledTurn(_chat_reply(tool_call=False, text="I do not know the temperature.", cached_tokens=4800), 0.0004),
         ],
     )
 
@@ -335,7 +342,7 @@ def test_zero_cached_tokens_is_retried_once_and_fails_when_the_retry_misses_too(
         [
             CalledTurn(_chat_reply(tool_call=False, text="72F", cached_tokens=0), 0.004),
             CalledTurn(_chat_reply(tool_call=False, text="72F", cached_tokens=0), 0.004),
-            CalledTurn(_chat_reply(tool_call=False, text="72F", cached_tokens=4900), 0.0004),
+            CalledTurn(_chat_reply(tool_call=False, text="72F", cached_tokens=4800), 0.0004),
         ],
     )
 
@@ -358,7 +365,7 @@ def test_a_cache_hit_on_the_retry_passes_and_says_so():
             CalledTurn(_chat_reply(tool_call=False, text="72F", cached_tokens=0), 0.004),
             CalledTurn(_chat_reply(tool_call=False, text="72F", cached_tokens=5000), 0.0005),
             CalledTurn(_chat_reply(tool_call=False, text="72F", cached_tokens=0), 0.004),
-            CalledTurn(_chat_reply(tool_call=False, text="72F", cached_tokens=4900), 0.0004),
+            CalledTurn(_chat_reply(tool_call=False, text="72F", cached_tokens=5000), 0.0004),
         ],
     )
 
@@ -368,11 +375,28 @@ def test_a_cache_hit_on_the_retry_passes_and_says_so():
     assert _checks(result)["prompt_caching"] == (
         "pass",
         "turn 2 read 5000 of 5000 input tokens from cache (100.0%) on the second attempt, "
-        "turn 3 read 4900 of 5000 input tokens from cache (98.0%) on the second attempt",
+        "turn 3 read 5000 of 5000 input tokens from cache (100.0%) on the second attempt",
     )
     assert result.spend_usd == pytest.approx(0.0049)
 
 
+def test_turn_three_reading_fewer_cached_tokens_than_turn_two_fails_prompt_caching():
+    provider: Final = _ScriptedProvider(
+        CalledTurn(_chat_reply(tool_call=True), 0.004),
+        [
+            CalledTurn(_chat_reply(tool_call=False, text="72F", cached_tokens=4800), 0.0005),
+            CalledTurn(_chat_reply(tool_call=False, text="72F", cached_tokens=4600), 0.0005),
+        ],
+    )
+
+    result: Final = _run(CHAT_COMPLETIONS_SCRIPT, provider)
+
+    assert len(provider.sent) == 3
+    status, detail = _checks(result)["prompt_caching"]
+    assert status == "fail", detail
+    assert "turn 3 read 4600 cached tokens, fewer than turn 2's 4800" in detail, detail
+
+
 def test_a_cache_miss_on_turn_three_fails_prompt_caching_after_its_own_retry():
     provider: Final = _ScriptedProvider(
         CalledTurn(_chat_reply(tool_call=True), 0.004),
@@ -428,7 +452,7 @@ def test_a_missing_spend_log_row_fails_cost_tracking_even_when_the_response_was_
         CalledTurn(_chat_reply(tool_call=True), 0.004),
         [
             CalledTurn(_chat_reply(tool_call=False, text="72F", cached_tokens=4800), 0.0005),
-            CalledTurn(_chat_reply(tool_call=False, text="72F", cached_tokens=4900), 0.0004),
+            CalledTurn(_chat_reply(tool_call=False, text="72F", cached_tokens=4800), 0.0004),
         ],
     )
 
@@ -483,8 +507,8 @@ def test_the_three_request_files_send_one_system_prompt_and_one_tool_so_endpoint
         third: Final = json.dumps(file["turn_3_appended_messages"])
         assert "<turn 2 response: assistant output>" in third, (name, third)
         assert "user" in {m["role"] for m in file["turn_3_appended_messages"][-2:]}, (name, third)
-    assert chat["messages"][0]["content"] == responses["instructions"] == messages["system"][0]["text"]
-    assert messages["system"][0]["cache_control"] == {"type": "ephemeral"}, messages["system"]
+    assert chat["messages"][0]["content"] == responses["instructions"] == messages["system"]
+    assert messages["messages"][-1]["content"][0]["cache_control"] == {"type": "ephemeral"}, messages["messages"][-1]
     assert chat["tools"][0]["function"]["parameters"] == responses["tools"][0]["parameters"]
     assert chat["tools"][0]["function"]["parameters"] == messages["tools"][0]["input_schema"]
     assert CHAT_COMPLETIONS_SCRIPT.turn_one() == chat