fix(health): read multipart chat replies, require turn 3 for prompt_caching, drop the turn 3 cache retry, enforce budgets on /health/test_model

litellm/proxy/auth/auth_checks.py

@@ -1066,6 +1066,7 @@ BUDGET_ENFORCED_SIDE_EFFECT_ROUTES: Final = frozenset(
         "/health",
         "/health/services",
         "/health/test_connection",
+        "/health/test_model",
         "/auto_router/test_routing",
     }
 )

litellm/proxy/health_endpoints/model_checks.py

@@ -323,6 +323,14 @@ def _first_choice_message(plain: Mapping[str, object]) -> object:
     return {**_EMPTY_ASSISTANT, **_PLAIN_OBJECT.validate_python(message)}
 
 
+def _chat_text(content: str | Sequence[object] | None) -> str:
+    if isinstance(content, str):
+        return content
+    if content is None:
+        return ""
+    return " ".join(text for part in content if isinstance(part, Mapping) and isinstance(text := part.get("text"), str))
+
+
 def _parse_chat(response: object) -> ParsedTurn | str:
     plain: Final = _plain(response)
     if plain is None:
@@ -348,7 +356,7 @@ def _parse_chat(response: object) -> ParsedTurn | str:
             if first_call is not None
             else None
         ),
-        text=message.content if isinstance(message.content, str) else "",
+        text=_chat_text(message.content),
         input_tokens=usage.prompt_tokens if usage is not None else 0,
         cache_read_tokens=(details.cached_tokens or 0) if details is not None else 0,
         assistant_echo=_first_choice_message(plain),
@@ -550,8 +558,9 @@ def _cache_miss(turn: RepeatedTurn, parsed: ParsedTurn) -> CheckResult:
     return CheckResult(
         "prompt_caching",
         "fail",
-        f"turn {turn.number} reported 0 cached tokens on {parsed.input_tokens} input tokens in {turn.attempts} "
-        "attempts, the provider did not reuse the repeated system prompt",
+        f"turn {turn.number} reported 0 cached tokens on {parsed.input_tokens} input tokens"
+        f"{f' in {turn.attempts} attempts' if turn.attempts > 1 else ''}, the provider did not reuse the repeated "
+        "system prompt",
     )
 
 
@@ -570,8 +579,9 @@ def _cache_shrank(second: ParsedTurn | None, third: ParsedTurn | None) -> CheckR
 
 def _prompt_caching_check(second: RepeatedTurn, third: RepeatedTurn) -> CheckResult:
     reached: Final = tuple((turn, parsed) for turn in (second, third) if (parsed := _parsed(turn)) is not None)
-    if not reached:
-        return CheckResult("prompt_caching", "fail", "not checked, the conversation did not reach turn 2")
+    if len(reached) < 2:
+        unreached: Final = 2 if not reached else 3
+        return CheckResult("prompt_caching", "fail", f"not checked, the conversation did not reach turn {unreached}")
     missed: Final = next((_cache_miss(turn, parsed) for turn, parsed in reached if parsed.cache_read_tokens == 0), None)
     if missed is not None:
         return missed
@@ -662,7 +672,7 @@ async def _repeated_turn(
         return RepeatedTurn(number=number, outcome=None, parsed=None, attempts=0)
     first: Final = await _turn(script, build(), call, new_call_id())
     first_parsed: Final = script.parse(first.response) if first.response is not None else None
-    if not _missed_cache(first_parsed):
+    if number == 3 or not _missed_cache(first_parsed):
         return RepeatedTurn(number=number, outcome=first, parsed=first_parsed, attempts=1)
     retry: Final = await _turn(script, build(), call, new_call_id())
     retry_parsed: Final = script.parse(retry.response) if retry.response is not None else None

tests/integration/management/test_model_health_check.py

@@ -1581,7 +1581,7 @@ def test_health_test_model_names_the_check_that_failed_for_an_unsaved_model_that
         ), response.text
         assert checks["cost_tracking"][0] == "fail", response.text
         assert "no price resolved for this model" in checks["cost_tracking"][1], response.text
-        assert len(wire.drain()) == 5, response.text
+        assert len(wire.drain()) == 4, response.text
 
 
 @pytest.mark.timeout(240)

tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py

@@ -5481,8 +5481,6 @@ async def _run_centralized_checks_for_end_user(
             setattr(_proxy_server_mod, attribute, value)
 
 
-
-
 @pytest.mark.parametrize(
     ("make_token", "make_end_user", "gpt_4o_spend", "model", "route", "must_raise"),
     [
@@ -11212,7 +11210,8 @@ def registered_budgeted_route() -> Iterator[None]:
 )
 @pytest.mark.usefixtures("registered_budgeted_route")
 @pytest.mark.parametrize(
-    "route", ["/v1/messages", "/health/services", "/health/test_connection", BUDGETED_PASS_THROUGH_ROUTE]
+    "route",
+    ["/v1/messages", "/health/services", "/health/test_connection", "/health/test_model", BUDGETED_PASS_THROUGH_ROUTE],
 )
 async def test_cached_key_team_member_budget_blocks_at_exact_cap(team_member_spend, expect_blocked, route):
     """A team member counter sitting exactly at the cap (where a resized reservation

tests/unit/proxy/health_endpoints/test_model_checks.py

@@ -339,7 +339,7 @@ def test_a_turn_three_provider_error_is_reported_as_the_conversation_failure():
         "fail",
         "turn 3 request with the follow-up question failed: RuntimeError: 400 invalid assistant message",
     )
-    assert checks["prompt_caching"] == ("pass", "turn 2 read 4800 of 5000 input tokens from cache (96.0%)")
+    assert checks["prompt_caching"] == ("fail", "not checked, the conversation did not reach turn 3")
 
 
 def test_zero_cached_tokens_is_retried_once_and_fails_when_the_retry_misses_too():
@@ -370,18 +370,17 @@ def test_a_cache_hit_on_the_retry_passes_and_says_so():
         [
             CalledTurn(_chat_reply(tool_call=False, text="72F", cached_tokens=0), 0.004),
             CalledTurn(_chat_reply(tool_call=False, text="72F", cached_tokens=5000), 0.0005),
-            CalledTurn(_chat_reply(tool_call=False, text="72F", cached_tokens=0), 0.004),
             CalledTurn(_chat_reply(tool_call=False, text="72F", cached_tokens=5000), 0.0004),
         ],
     )
 
     result: Final = _run(CHAT_COMPLETIONS_SCRIPT, provider)
 
-    assert len(provider.sent) == 5
+    assert len(provider.sent) == 4
     assert _checks(result)["prompt_caching"] == (
         "pass",
         "turn 2 read 5000 of 5000 input tokens from cache (100.0%) on the second attempt, "
-        "turn 3 read 5000 of 5000 input tokens from cache (100.0%) on the second attempt",
+        "turn 3 read 5000 of 5000 input tokens from cache (100.0%)",
     )
     assert result.spend_usd == pytest.approx(0.0049)
 
@@ -403,27 +402,43 @@ def test_turn_three_reading_fewer_cached_tokens_than_turn_two_fails_prompt_cachi
     assert "turn 3 read 4600 cached tokens, fewer than turn 2's 4800" in detail, detail
 
 
-def test_a_cache_miss_on_turn_three_fails_prompt_caching_after_its_own_retry():
+def test_a_cache_miss_on_turn_three_fails_prompt_caching_without_a_retry_that_could_hit_a_rewritten_prompt():
     provider: Final = _ScriptedProvider(
         CalledTurn(_chat_reply(tool_call=True), 0.004),
         [
             CalledTurn(_chat_reply(tool_call=False, text="72F", cached_tokens=4800), 0.0005),
             CalledTurn(_chat_reply(tool_call=False, text="72F", cached_tokens=0), 0.004),
-            CalledTurn(_chat_reply(tool_call=False, text="72F", cached_tokens=0), 0.004),
+            CalledTurn(_chat_reply(tool_call=False, text="72F", cached_tokens=5000), 0.0004),
         ],
     )
 
     result: Final = _run(CHAT_COMPLETIONS_SCRIPT, provider)
 
-    assert len(provider.sent) == 4
+    assert len(provider.sent) == 3
     assert _checks(result)["prompt_caching"] == (
         "fail",
-        "turn 3 reported 0 cached tokens on 5000 input tokens in 2 attempts, the provider did not reuse the "
-        "repeated system prompt",
+        "turn 3 reported 0 cached tokens on 5000 input tokens, the provider did not reuse the repeated system prompt",
     )
     assert _checks(result)["conversation"][0] == "pass"
 
 
+def test_a_chat_reply_whose_content_is_a_list_of_text_parts_still_answers_the_conversation():
+    plain: Final = _chat_reply(tool_call=False, text="", cached_tokens=4800).model_dump()
+    multipart: Final = {
+        **plain,
+        "choices": [
+            {**plain["choices"][0], "message": {"role": "assistant", "content": [{"type": "text", "text": "72F"}]}}
+        ],
+    }
+    provider: Final = _ScriptedProvider(
+        CalledTurn(_chat_reply(tool_call=True), 0.004), [CalledTurn(multipart, 0.0005), CalledTurn(multipart, 0.0005)]
+    )
+
+    checks: Final = _checks(_run(CHAT_COMPLETIONS_SCRIPT, provider))
+
+    assert checks["conversation"][0] == "pass", checks["conversation"]
+
+
 def test_a_model_without_a_price_fails_cost_tracking_and_a_zero_priced_model_only_warns():
     unpriced: Final = _ScriptedProvider(
         CalledTurn(_chat_reply(tool_call=True), None),