diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 9d98e2c9973..e1e1200f93f 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -1066,6 +1066,7 @@ BUDGET_ENFORCED_SIDE_EFFECT_ROUTES: Final = frozenset( "/health", "/health/services", "/health/test_connection", + "/health/test_model", "/auto_router/test_routing", } ) diff --git a/litellm/proxy/health_endpoints/model_checks.py b/litellm/proxy/health_endpoints/model_checks.py index d920179dfd0..2c5ed1e4680 100644 --- a/litellm/proxy/health_endpoints/model_checks.py +++ b/litellm/proxy/health_endpoints/model_checks.py @@ -323,6 +323,14 @@ def _first_choice_message(plain: Mapping[str, object]) -> object: return {**_EMPTY_ASSISTANT, **_PLAIN_OBJECT.validate_python(message)} +def _chat_text(content: str | Sequence[object] | None) -> str: + if isinstance(content, str): + return content + if content is None: + return "" + return " ".join(text for part in content if isinstance(part, Mapping) and isinstance(text := part.get("text"), str)) + + def _parse_chat(response: object) -> ParsedTurn | str: plain: Final = _plain(response) if plain is None: @@ -348,7 +356,7 @@ def _parse_chat(response: object) -> ParsedTurn | str: if first_call is not None else None ), - text=message.content if isinstance(message.content, str) else "", + text=_chat_text(message.content), input_tokens=usage.prompt_tokens if usage is not None else 0, cache_read_tokens=(details.cached_tokens or 0) if details is not None else 0, assistant_echo=_first_choice_message(plain), @@ -550,8 +558,9 @@ def _cache_miss(turn: RepeatedTurn, parsed: ParsedTurn) -> CheckResult: return CheckResult( "prompt_caching", "fail", - f"turn {turn.number} reported 0 cached tokens on {parsed.input_tokens} input tokens in {turn.attempts} " - "attempts, the provider did not reuse the repeated system prompt", + f"turn {turn.number} reported 0 cached tokens on {parsed.input_tokens} input tokens" + f"{f' in {turn.attempts} attempts' if turn.attempts > 1 else ''}, the provider did not reuse the repeated " + "system prompt", ) @@ -570,8 +579,9 @@ def _cache_shrank(second: ParsedTurn | None, third: ParsedTurn | None) -> CheckR def _prompt_caching_check(second: RepeatedTurn, third: RepeatedTurn) -> CheckResult: reached: Final = tuple((turn, parsed) for turn in (second, third) if (parsed := _parsed(turn)) is not None) - if not reached: - return CheckResult("prompt_caching", "fail", "not checked, the conversation did not reach turn 2") + if len(reached) < 2: + unreached: Final = 2 if not reached else 3 + return CheckResult("prompt_caching", "fail", f"not checked, the conversation did not reach turn {unreached}") missed: Final = next((_cache_miss(turn, parsed) for turn, parsed in reached if parsed.cache_read_tokens == 0), None) if missed is not None: return missed @@ -662,7 +672,7 @@ async def _repeated_turn( return RepeatedTurn(number=number, outcome=None, parsed=None, attempts=0) first: Final = await _turn(script, build(), call, new_call_id()) first_parsed: Final = script.parse(first.response) if first.response is not None else None - if not _missed_cache(first_parsed): + if number == 3 or not _missed_cache(first_parsed): return RepeatedTurn(number=number, outcome=first, parsed=first_parsed, attempts=1) retry: Final = await _turn(script, build(), call, new_call_id()) retry_parsed: Final = script.parse(retry.response) if retry.response is not None else None diff --git a/tests/integration/management/test_model_health_check.py b/tests/integration/management/test_model_health_check.py index 38f8fc126a0..7264737ecee 100644 --- a/tests/integration/management/test_model_health_check.py +++ b/tests/integration/management/test_model_health_check.py @@ -1581,7 +1581,7 @@ def test_health_test_model_names_the_check_that_failed_for_an_unsaved_model_that ), response.text assert checks["cost_tracking"][0] == "fail", response.text assert "no price resolved for this model" in checks["cost_tracking"][1], response.text - assert len(wire.drain()) == 5, response.text + assert len(wire.drain()) == 4, response.text @pytest.mark.timeout(240) diff --git a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py index 17e6c004566..68c0da0e012 100644 --- a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py +++ b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py @@ -5481,8 +5481,6 @@ async def _run_centralized_checks_for_end_user( setattr(_proxy_server_mod, attribute, value) - - @pytest.mark.parametrize( ("make_token", "make_end_user", "gpt_4o_spend", "model", "route", "must_raise"), [ @@ -11212,7 +11210,8 @@ def registered_budgeted_route() -> Iterator[None]: ) @pytest.mark.usefixtures("registered_budgeted_route") @pytest.mark.parametrize( - "route", ["/v1/messages", "/health/services", "/health/test_connection", BUDGETED_PASS_THROUGH_ROUTE] + "route", + ["/v1/messages", "/health/services", "/health/test_connection", "/health/test_model", BUDGETED_PASS_THROUGH_ROUTE], ) async def test_cached_key_team_member_budget_blocks_at_exact_cap(team_member_spend, expect_blocked, route): """A team member counter sitting exactly at the cap (where a resized reservation diff --git a/tests/unit/proxy/health_endpoints/test_model_checks.py b/tests/unit/proxy/health_endpoints/test_model_checks.py index 27527ae1f1d..c13463eb511 100644 --- a/tests/unit/proxy/health_endpoints/test_model_checks.py +++ b/tests/unit/proxy/health_endpoints/test_model_checks.py @@ -339,7 +339,7 @@ def test_a_turn_three_provider_error_is_reported_as_the_conversation_failure(): "fail", "turn 3 request with the follow-up question failed: RuntimeError: 400 invalid assistant message", ) - assert checks["prompt_caching"] == ("pass", "turn 2 read 4800 of 5000 input tokens from cache (96.0%)") + assert checks["prompt_caching"] == ("fail", "not checked, the conversation did not reach turn 3") def test_zero_cached_tokens_is_retried_once_and_fails_when_the_retry_misses_too(): @@ -370,18 +370,17 @@ def test_a_cache_hit_on_the_retry_passes_and_says_so(): [ CalledTurn(_chat_reply(tool_call=False, text="72F", cached_tokens=0), 0.004), CalledTurn(_chat_reply(tool_call=False, text="72F", cached_tokens=5000), 0.0005), - CalledTurn(_chat_reply(tool_call=False, text="72F", cached_tokens=0), 0.004), CalledTurn(_chat_reply(tool_call=False, text="72F", cached_tokens=5000), 0.0004), ], ) result: Final = _run(CHAT_COMPLETIONS_SCRIPT, provider) - assert len(provider.sent) == 5 + assert len(provider.sent) == 4 assert _checks(result)["prompt_caching"] == ( "pass", "turn 2 read 5000 of 5000 input tokens from cache (100.0%) on the second attempt, " - "turn 3 read 5000 of 5000 input tokens from cache (100.0%) on the second attempt", + "turn 3 read 5000 of 5000 input tokens from cache (100.0%)", ) assert result.spend_usd == pytest.approx(0.0049) @@ -403,27 +402,43 @@ def test_turn_three_reading_fewer_cached_tokens_than_turn_two_fails_prompt_cachi assert "turn 3 read 4600 cached tokens, fewer than turn 2's 4800" in detail, detail -def test_a_cache_miss_on_turn_three_fails_prompt_caching_after_its_own_retry(): +def test_a_cache_miss_on_turn_three_fails_prompt_caching_without_a_retry_that_could_hit_a_rewritten_prompt(): provider: Final = _ScriptedProvider( CalledTurn(_chat_reply(tool_call=True), 0.004), [ CalledTurn(_chat_reply(tool_call=False, text="72F", cached_tokens=4800), 0.0005), CalledTurn(_chat_reply(tool_call=False, text="72F", cached_tokens=0), 0.004), - CalledTurn(_chat_reply(tool_call=False, text="72F", cached_tokens=0), 0.004), + CalledTurn(_chat_reply(tool_call=False, text="72F", cached_tokens=5000), 0.0004), ], ) result: Final = _run(CHAT_COMPLETIONS_SCRIPT, provider) - assert len(provider.sent) == 4 + assert len(provider.sent) == 3 assert _checks(result)["prompt_caching"] == ( "fail", - "turn 3 reported 0 cached tokens on 5000 input tokens in 2 attempts, the provider did not reuse the " - "repeated system prompt", + "turn 3 reported 0 cached tokens on 5000 input tokens, the provider did not reuse the repeated system prompt", ) assert _checks(result)["conversation"][0] == "pass" +def test_a_chat_reply_whose_content_is_a_list_of_text_parts_still_answers_the_conversation(): + plain: Final = _chat_reply(tool_call=False, text="", cached_tokens=4800).model_dump() + multipart: Final = { + **plain, + "choices": [ + {**plain["choices"][0], "message": {"role": "assistant", "content": [{"type": "text", "text": "72F"}]}} + ], + } + provider: Final = _ScriptedProvider( + CalledTurn(_chat_reply(tool_call=True), 0.004), [CalledTurn(multipart, 0.0005), CalledTurn(multipart, 0.0005)] + ) + + checks: Final = _checks(_run(CHAT_COMPLETIONS_SCRIPT, provider)) + + assert checks["conversation"][0] == "pass", checks["conversation"] + + def test_a_model_without_a_price_fails_cost_tracking_and_a_zero_priced_model_only_warns(): unpriced: Final = _ScriptedProvider( CalledTurn(_chat_reply(tool_call=True), None),