From 369f3cb5f0b4284ab6b7d8faba2019137fb145d0 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 20:08:45 -0700 Subject: [PATCH 01/12] feat(cost-map): add baseten joey-1 decision model (#49123) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 9 +++++++++ model_prices_and_context_window.json | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 1a1982fddd7..77ac4f49e8b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -81059,5 +81059,14 @@ "supports_response_schema": true, "supports_tool_choice": false, "supports_vision": true + }, + "baseten/decision-models/baseten-joey-2b-latest": { + "input_cost_per_token": 1e-08, + "litellm_provider": "baseten", + "max_input_tokens": 16384, + "mode": "decisions", + "output_cost_per_token": 0.0, + "source": "https://inference.baseten.co/v1/models", + "supports_vision": true } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 1a1982fddd7..77ac4f49e8b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -81059,5 +81059,14 @@ "supports_response_schema": true, "supports_tool_choice": false, "supports_vision": true + }, + "baseten/decision-models/baseten-joey-2b-latest": { + "input_cost_per_token": 1e-08, + "litellm_provider": "baseten", + "max_input_tokens": 16384, + "mode": "decisions", + "output_cost_per_token": 0.0, + "source": "https://inference.baseten.co/v1/models", + "supports_vision": true } } From 733cfeba935284056e77baf2dcc9517fdc96a2ba Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Thu, 29 Oct 2026 20:30:42 -0700 Subject: [PATCH 02/12] fix(cost): bill a provider reported usage.cost on every path unless the deployment is priced (#48953) * fix(cost): bill a provider reported usage.cost on every path unless the deployment is priced * test(integration): a reported cost is attributed to output in the gateway parity cases * fix(cost): skip a chat usage.cost that already holds litellm's stamped bill and Responses API objects * fix(cost): record litellm's bill on the Responses terminal event so a later pricing pass does not read it as the upstream's charge * fix(cost): mark usage.cost litellm wrote as its own bill and keep a breakdown for a reported cost * fix(cost): keep the token breakdown when a header reported cost repeats an already priced bill * fix(cost): read the stored breakdown total through the logging object's typed field * refactor(cost): name the providers billed from their own usage fields for what they are * fix(cost): keep the upstream's usage.cost beside litellm's bill and stamp a zero operator price on the stream frame --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/cost_calculator.py | 21 +- litellm/litellm_core_utils/core_helpers.py | 37 ++- .../streaming_chunk_builder_utils.py | 23 +- litellm/responses/streaming_iterator.py | 3 +- .../cost_calculation/cases/openai.py | 225 ++++++++++++++++++ .../integration/cost_calculation/cost_map.py | 6 + .../litellm_core_utils/test_core_helpers.py | 78 +++--- .../test_streaming_handler.py | 6 +- .../llms/openai_like/test_kyma_provider.py | 8 +- tests/unit/test_main.py | 40 ++++ 10 files changed, 387 insertions(+), 60 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 6677b7327dc..ee102e82ca4 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1544,6 +1544,12 @@ def _response_time_ms_for_cost( return litellm_logging_obj.get_response_ms() +def _breakdown_total(litellm_logging_obj: LitellmLoggingObject | None) -> float | None: + breakdown: Final = None if litellm_logging_obj is None else litellm_logging_obj.cost_breakdown + total: Final[object] = None if breakdown is None else breakdown.get("total_cost") + return float(total) if isinstance(total, (int, float)) and not isinstance(total, bool) else None + + def _adjusted_provider_billed_cost( reported_cost: float, custom_llm_provider: str | None, @@ -1561,9 +1567,9 @@ def _adjusted_provider_billed_cost( ) _store_cost_breakdown_in_logging_obj( litellm_logging_obj=litellm_logging_obj, - prompt_tokens_cost_usd_dollar=None, - completion_tokens_cost_usd_dollar=None, - cost_for_built_in_tools_cost_usd_dollar=None, + prompt_tokens_cost_usd_dollar=0.0, + completion_tokens_cost_usd_dollar=adjusted_cost, + cost_for_built_in_tools_cost_usd_dollar=0.0, total_cost_usd_dollar=adjusted_cost, original_cost=reported_cost, discount_percent=discount_percent, @@ -2376,6 +2382,15 @@ def response_cost_calculator( ) provider_response_cost: Final = get_response_cost_from_hidden_params(hidden_params) if provider_response_cost is not None and not custom_pricing: + if _breakdown_total(litellm_logging_obj) != provider_response_cost: + _store_cost_breakdown_in_logging_obj( + litellm_logging_obj=litellm_logging_obj, + prompt_tokens_cost_usd_dollar=0.0, + completion_tokens_cost_usd_dollar=provider_response_cost, + cost_for_built_in_tools_cost_usd_dollar=0.0, + total_cost_usd_dollar=provider_response_cost, + original_cost=provider_response_cost, + ) return provider_response_cost response_cost = completion_cost( diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index d940baf8b56..28f45d2bb2c 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -1017,13 +1017,44 @@ def chunk_usage(chunk: object) -> object: return getattr(chunk, "usage", None) -USAGE_COST_BILLED_PROVIDERS: Final = frozenset({"kyma"}) +PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS: Final = frozenset({"xai", "perplexity", "opencode_go"}) + + +UPSTREAM_USAGE_COST: Final = "upstream_usage_cost" + + +def _numeric(value: object) -> float | None: + return float(value) if isinstance(value, (int, float)) and not isinstance(value, bool) else None + + +def stamp_litellm_bill_on_usage(response: object, usage: object, bill: float) -> None: + """Writes litellm's bill into ``usage.cost`` and the hidden ``response_cost`` cache, keeping the upstream's own + ``usage.cost`` under ``upstream_usage_cost`` so a later pricing pass still bills the upstream's charge instead of + reading litellm's bill back as if the upstream had reported it.""" + hidden: Final[object] = getattr(response, "_hidden_params", None) + if isinstance(hidden, dict): + hidden_params: Final = cast(dict[str, object], hidden) # cast-ok: hidden params ride on the response as a dict + upstream: Final = ( + hidden_params[UPSTREAM_USAGE_COST] + if UPSTREAM_USAGE_COST in hidden_params + else _numeric(getattr(usage, "cost", None)) + ) + hidden_params.update({"response_cost": bill, UPSTREAM_USAGE_COST: upstream}) + setattr(usage, "cost", bill) def billed_usage_cost(response: object, custom_llm_provider: str | None) -> float | None: - if custom_llm_provider not in USAGE_COST_BILLED_PROVIDERS: + """The charge an OpenAI-compatible upstream reports in ``usage.cost``, skipped for providers billed from their own + usage fields. Once litellm wrote its own bill into ``usage.cost`` (the stream builder and the Responses terminal + event do), the upstream's figure is read from ``upstream_usage_cost`` instead.""" + if custom_llm_provider in PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS: return None - cost: Final = provider_reported_usage_cost(_usage_field(response, "usage"), custom_llm_provider) + hidden: Final = as_str_mapping(getattr(response, "_hidden_params", None)) + cost: Final = ( + _numeric(hidden[UPSTREAM_USAGE_COST]) + if hidden is not None and UPSTREAM_USAGE_COST in hidden + else provider_reported_usage_cost(_usage_field(response, "usage"), custom_llm_provider) + ) return cost if cost is not None and math.isfinite(cost) and cost >= 0 else None diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index c69d9277d65..94351e72ab1 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -13,9 +13,10 @@ from typing_extensions import ReadOnly, Required import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.core_helpers import ( - USAGE_COST_BILLED_PROVIDERS, + PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS, chunk_usage, provider_reported_usage_cost, + stamp_litellm_bill_on_usage, ) from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import BEDROCK_TRACE_FIELDS, object_items, object_mapping @@ -1380,9 +1381,6 @@ def stream_chunk_builder_text_completion( return TextCompletionResponse(**response) -_CALCULATOR_PRICED_REPORTED_COST_PROVIDERS: Final = frozenset({LlmProviders.XAI.value}) | USAGE_COST_BILLED_PROVIDERS - - def _logged_provider(logging_obj: Optional["Logging"]) -> object: if logging_obj is None: return None @@ -1390,10 +1388,6 @@ def _logged_provider(logging_obj: Optional["Logging"]) -> object: return provider -def _reported_cost_is_priced_by_calculator(logging_obj: Optional["Logging"]) -> bool: - return _logged_provider(logging_obj) in _CALCULATOR_PRICED_REPORTED_COST_PROVIDERS - - def _stream_builder_response_cost( response: ModelResponse, logging_obj: Optional["Logging"], chunks: Sequence[object] ) -> float | None: @@ -1404,9 +1398,6 @@ def _stream_builder_response_cost( for chunk in reversed(chunks) ) return next((cost for cost in reported if cost is not None), None) - usage_cost: Final = getattr(getattr(response, "usage", None), "cost", None) - if isinstance(usage_cost, (int, float)) and not _reported_cost_is_priced_by_calculator(logging_obj): - return float(usage_cost) if logging_obj is not None: return None provider_hint: Final = response._hidden_params.get( # pyright: ignore[reportPrivateUsage] # no public accessor @@ -1453,13 +1444,15 @@ def _set_stream_builder_response_cost( def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None: if logging_obj is None: return - if isinstance(getattr(usage, "cost", None), (int, float)): + if _logged_provider(logging_obj) in PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS and isinstance( + getattr(usage, "cost", None), (int, float) + ): return computed_cost: Final = logging_obj.response_cost_calculator(result=response) - if not isinstance(computed_cost, (int, float)) or computed_cost <= 0: + if isinstance(computed_cost, bool) or not isinstance(computed_cost, (int, float)) or computed_cost < 0: return - setattr(usage, "cost", computed_cost) - _set_hidden_response_cost(response, float(computed_cost)) + if computed_cost > 0 or isinstance(getattr(usage, "cost", None), (int, float)): + stamp_litellm_bill_on_usage(response, usage, float(computed_cost)) def _field_of(value: object, name: str) -> object: diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 6a82304b76a..f0bb2201162 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -43,6 +43,7 @@ from litellm.litellm_core_utils.core_helpers import ( process_response_headers, provider_reported_usage_cost, set_response_cost_in_hidden_params, + stamp_litellm_bill_on_usage, ) from litellm.litellm_core_utils.exception_mapping_utils import with_deployment_api_base from litellm.litellm_core_utils.internal_call_metadata import ( @@ -1869,7 +1870,7 @@ def stamp_responses_usage_cost( if isinstance(bill, bool) or not isinstance(bill, (int, float)): return if bill > 0 or isinstance(getattr(usage_obj, "cost", None), (int, float)): - setattr(usage_obj, "cost", float(bill)) + stamp_litellm_bill_on_usage(response_obj, usage_obj, float(bill)) def opening_item_payload(payload: Mapping[str, object]) -> Mapping[str, object]: diff --git a/tests/integration/cost_calculation/cases/openai.py b/tests/integration/cost_calculation/cases/openai.py index 53fd5c1cec2..1443cc5ab59 100644 --- a/tests/integration/cost_calculation/cases/openai.py +++ b/tests/integration/cost_calculation/cases/openai.py @@ -2,6 +2,7 @@ PARITY pairs the streamed case with its plain twin where both bill the same row.""" +from collections.abc import Mapping from typing import Final, cast from integration.cost_calculation.cost_tracking_case import ( @@ -11878,7 +11879,215 @@ GPT_5_6_TERRA_MESSAGES_BRIDGED_STREAM: Final = GPT_5_6_TERRA_MESSAGES_BRIDGED_JS } ) + +# An OpenAI-compatible gateway that reports what it charged under usage.cost (#47533). The reported total is the bill, +# split across input and output by the ratio of the cost map rates (100 * 1e-6 : 20 * 3e-6), on plain and streamed +# replies alike. A deployment the operator priced keeps the operator's rates on both. +_GATEWAY_REPORTED_USAGE: Final = {"prompt_tokens": 100, "completion_tokens": 20, "total_tokens": 120, "cost": 0.037} +_GATEWAY_REPORTED_EXPECTED: Final = ExactExpected( + spend=0.037, input_cost=0.0, output_cost=0.037, prompt_tokens=100, completion_tokens=20 +) +_GATEWAY_OPERATOR_PRICED: Final = Deployment(input_cost_per_token=2e-06, output_cost_per_token=6e-06) +_GATEWAY_OPERATOR_PRICED_EXPECTED: Final = ExactExpected( + spend=0.00032, input_cost=0.0002, output_cost=0.00012, prompt_tokens=100, completion_tokens=20 +) +_GATEWAY_CHUNK: Final = { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion.chunk", + "created": 1, + "model": "gateway-reported-cost-fixture", +} + +GATEWAY_REPORTED_COST: Final = CostTrackingTestCase( + name="gateway-reported-cost", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="gateway-reported-cost-fixture", + endpoint="/v1/chat/completions", + request={"model": "$MODEL", "messages": [{"role": "user", "content": "reported cost"}], "stream": False}, + response=JsonResponse( + content_type="application/json", + body={ + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1, + "model": "gateway-reported-cost-fixture", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": _GATEWAY_REPORTED_USAGE, + }, + ), + expected=_GATEWAY_REPORTED_EXPECTED, +) + +GATEWAY_REPORTED_COST_STREAM: Final = CostTrackingTestCase( + name="gateway-reported-cost-stream", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="gateway-reported-cost-fixture", + endpoint="/v1/chat/completions", + request={ + "model": "$MODEL", + "messages": [{"role": "user", "content": "reported cost"}], + "stream": True, + "stream_options": {"include_usage": True}, + }, + response=SseResponse( + content_type="text/event-stream", + frames=sse_frames( + {**_GATEWAY_CHUNK, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "ok"}}]}, + {**_GATEWAY_CHUNK, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}, + {**_GATEWAY_CHUNK, "choices": [], "usage": _GATEWAY_REPORTED_USAGE}, + done=True, + ), + ), + expected=_GATEWAY_REPORTED_EXPECTED, +) + +GATEWAY_REPORTED_COST_OPERATOR_PRICED: Final = GATEWAY_REPORTED_COST.model_copy( + update={ + "name": "gateway-reported-cost-operator_priced", + "deployment": _GATEWAY_OPERATOR_PRICED, + "expected": _GATEWAY_OPERATOR_PRICED_EXPECTED, + } +) + +GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM: Final = GATEWAY_REPORTED_COST_STREAM.model_copy( + update={ + "name": "gateway-reported-cost-operator_priced-stream", + "deployment": _GATEWAY_OPERATOR_PRICED, + "expected": _GATEWAY_OPERATOR_PRICED_EXPECTED, + } +) + +_GATEWAY_RESPONSES_USAGE: Final[Mapping[str, JsonValue]] = {"input_tokens": 100, "output_tokens": 20, "total_tokens": 120, "cost": 0.037} +_GATEWAY_RESPONSES_ITEM: Final[Mapping[str, JsonValue]] = { + "id": "msg_gateway", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "ok", "annotations": []}], +} +_GATEWAY_RESPONSES_BODY: Final[Mapping[str, JsonValue]] = { + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gateway-reported-cost-fixture", + "output": [_GATEWAY_RESPONSES_ITEM], + "usage": _GATEWAY_RESPONSES_USAGE, +} + +GATEWAY_REPORTED_COST_RESPONSES: Final = CostTrackingTestCase( + name="gateway-reported-cost-responses", + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="gateway-reported-cost-fixture", + endpoint="/v1/responses", + request={"model": "$MODEL", "input": "reported cost", "cache": {"no-cache": True}}, + response=JsonResponse(content_type="application/json", body=_GATEWAY_RESPONSES_BODY), + expected=_GATEWAY_REPORTED_EXPECTED, +) + +GATEWAY_REPORTED_COST_RESPONSES_STREAM: Final = GATEWAY_REPORTED_COST_RESPONSES.model_copy( + update={ + "name": "gateway-reported-cost-responses-stream", + "request": {**GATEWAY_REPORTED_COST_RESPONSES.request, "stream": True}, + "response": SseResponse( + content_type="text/event-stream", + frames=sse_frames( + { + "type": "response.created", + "sequence_number": 0, + "response": {**_GATEWAY_RESPONSES_BODY, "status": "in_progress", "output": [], "usage": None}, + }, + { + "type": "response.output_text.delta", + "sequence_number": 1, + "item_id": "msg_gateway", + "output_index": 0, + "content_index": 0, + "delta": "ok", + }, + {"type": "response.completed", "sequence_number": 2, "response": _GATEWAY_RESPONSES_BODY}, + ), + ), + } +) + +GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED: Final = GATEWAY_REPORTED_COST_RESPONSES.model_copy( + update={ + "name": "gateway-reported-cost-responses-operator_priced", + "deployment": _GATEWAY_OPERATOR_PRICED, + "expected": _GATEWAY_OPERATOR_PRICED_EXPECTED, + } +) + +GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED_STREAM: Final = GATEWAY_REPORTED_COST_RESPONSES_STREAM.model_copy( + update={ + "name": "gateway-reported-cost-responses-operator_priced-stream", + "deployment": _GATEWAY_OPERATOR_PRICED, + "expected": _GATEWAY_OPERATOR_PRICED_EXPECTED, + } +) + + +_GATEWAY_ZERO_USAGE: Final = {**_GATEWAY_REPORTED_USAGE, "cost": 0} + +GATEWAY_REPORTED_ZERO_OPERATOR_PRICED: Final = GATEWAY_REPORTED_COST_OPERATOR_PRICED.model_copy( + update={ + "name": "gateway-reported-zero-operator_priced", + "response": JsonResponse( + content_type="application/json", + body={**GATEWAY_REPORTED_COST.response.body, "usage": _GATEWAY_ZERO_USAGE}, + ), + } +) + +GATEWAY_REPORTED_ZERO_OPERATOR_PRICED_STREAM: Final = GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM.model_copy( + update={ + "name": "gateway-reported-zero-operator_priced-stream", + "response": SseResponse( + content_type="text/event-stream", + frames=sse_frames( + {**_GATEWAY_CHUNK, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "ok"}}]}, + {**_GATEWAY_CHUNK, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}, + {**_GATEWAY_CHUNK, "choices": [], "usage": _GATEWAY_ZERO_USAGE}, + done=True, + ), + ), + } +) + +_GATEWAY_OPENROUTER_OPERATOR_PRICED: Final = _GATEWAY_OPERATOR_PRICED.model_copy( + update={"model": "openrouter/vendor/gateway-reported-cost-fixture"} +) + +GATEWAY_REPORTED_COST_OPENROUTER_OPERATOR_PRICED: Final = GATEWAY_REPORTED_COST_OPERATOR_PRICED.model_copy( + update={ + "name": "gateway-reported-cost-openrouter-operator_priced", + "deployment": _GATEWAY_OPENROUTER_OPERATOR_PRICED, + } +) + +GATEWAY_REPORTED_COST_OPENROUTER_OPERATOR_PRICED_STREAM: Final = ( + GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM.model_copy( + update={ + "name": "gateway-reported-cost-openrouter-operator_priced-stream", + "deployment": _GATEWAY_OPENROUTER_OPERATOR_PRICED, + } + ) +) + CASES: Final[tuple[CostTrackingTestCase, ...]] = ( + GATEWAY_REPORTED_COST, + GATEWAY_REPORTED_COST_STREAM, + GATEWAY_REPORTED_COST_OPERATOR_PRICED, + GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM, + GATEWAY_REPORTED_COST_RESPONSES, + GATEWAY_REPORTED_COST_RESPONSES_STREAM, + GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED, + GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED_STREAM, + GATEWAY_REPORTED_ZERO_OPERATOR_PRICED, + GATEWAY_REPORTED_ZERO_OPERATOR_PRICED_STREAM, + GATEWAY_REPORTED_COST_OPENROUTER_OPERATOR_PRICED, + GATEWAY_REPORTED_COST_OPENROUTER_OPERATOR_PRICED_STREAM, GPT_5_6_SLASH_ALIAS, GPT_5_6_SLASH_ALIAS_STREAM, GPT_6_SOL_RESPONSES_DEPLOYMENT_JSON, @@ -12120,6 +12329,22 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( ) PARITY: Final[tuple[StreamParityTestCase, ...]] = ( + StreamParityTestCase(plain=GATEWAY_REPORTED_COST, streamed=GATEWAY_REPORTED_COST_STREAM), + StreamParityTestCase( + plain=GATEWAY_REPORTED_COST_OPERATOR_PRICED, streamed=GATEWAY_REPORTED_COST_OPERATOR_PRICED_STREAM + ), + StreamParityTestCase( + plain=GATEWAY_REPORTED_ZERO_OPERATOR_PRICED, streamed=GATEWAY_REPORTED_ZERO_OPERATOR_PRICED_STREAM + ), + StreamParityTestCase( + plain=GATEWAY_REPORTED_COST_OPENROUTER_OPERATOR_PRICED, + streamed=GATEWAY_REPORTED_COST_OPENROUTER_OPERATOR_PRICED_STREAM, + ), + StreamParityTestCase(plain=GATEWAY_REPORTED_COST_RESPONSES, streamed=GATEWAY_REPORTED_COST_RESPONSES_STREAM), + StreamParityTestCase( + plain=GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED, + streamed=GATEWAY_REPORTED_COST_RESPONSES_OPERATOR_PRICED_STREAM, + ), StreamParityTestCase(plain=GPT_5_3_CODEX_INPUT_TEXT, streamed=GPT_5_3_CODEX_STREAM), StreamParityTestCase(plain=GPT_5_3_CODEX_INPUT_TEXT, streamed=GPT_5_3_CODEX_STREAM_INCOMPLETE), StreamParityTestCase(plain=GPT_5_3_CODEX_INPUT_TEXT, streamed=GPT_5_3_CODEX_STREAM_UNVALIDATED), diff --git a/tests/integration/cost_calculation/cost_map.py b/tests/integration/cost_calculation/cost_map.py index 32722b37c9b..da4c2b6e859 100644 --- a/tests/integration/cost_calculation/cost_map.py +++ b/tests/integration/cost_calculation/cost_map.py @@ -26,6 +26,12 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = { "input_cost_per_token_above_128k_tokens": 2.2e-07, "output_cost_per_token_above_128k_tokens": 8.8e-07, }, + "gateway-reported-cost-fixture": { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 3e-06, + }, "openrouter/anthropic/claude-sonnet-5": { "litellm_provider": "openrouter", "mode": "chat", diff --git a/tests/unit/litellm_core_utils/test_core_helpers.py b/tests/unit/litellm_core_utils/test_core_helpers.py index a4a3ae285b1..1c86c4e30d8 100644 --- a/tests/unit/litellm_core_utils/test_core_helpers.py +++ b/tests/unit/litellm_core_utils/test_core_helpers.py @@ -69,9 +69,7 @@ def test_preserve_upstream_non_openai_attributes_keeps_scalar_and_structured_val scalar_attribute="scalar", structured_attribute={"items": ["value"]}, ) - model_response: Final = ModelResponseStream( - choices=[StreamingChoices(index=0, delta=Delta(content="hello"))] - ) + model_response: Final = ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(content="hello"))]) preserve_upstream_non_openai_attributes(model_response, original_chunk) @@ -262,9 +260,7 @@ class TestMapFinishReasonAnthropic: ("content_filtered", "content_filter"), ], ) - def test_anthropic_finish_reasons( - self, provider_reason: str, expected: str - ) -> None: + def test_anthropic_finish_reasons(self, provider_reason: str, expected: str) -> None: assert map_finish_reason(provider_reason) == expected def test_refusal(self): @@ -336,9 +332,7 @@ class TestMapFinishReasonZhipu: class TestMapFinishReasonOpenAIPassthrough: - @pytest.mark.parametrize( - "reason", ["stop", "length", "tool_calls", "function_call", "content_filter"] - ) + @pytest.mark.parametrize("reason", ["stop", "length", "tool_calls", "function_call", "content_filter"]) def test_openai_values_pass_through(self, reason): assert map_finish_reason(reason) == reason @@ -349,9 +343,7 @@ class TestMapFinishReasonGenericError: assert map_finish_reason("error") == "stop" def test_lowercase_error_does_not_warn(self, mocker): - warn = mocker.patch( - "litellm.litellm_core_utils.core_helpers.verbose_logger.warning" - ) + warn = mocker.patch("litellm.litellm_core_utils.core_helpers.verbose_logger.warning") assert map_finish_reason("error") == "stop" warn.assert_not_called() @@ -369,8 +361,7 @@ class TestFinishReasonMapOutputsAreValid: """Every value in _FINISH_REASON_MAP must be a valid OpenAI finish reason.""" for provider_reason, openai_reason in _FINISH_REASON_MAP.items(): assert openai_reason in VALID_OPENAI_FINISH_REASONS, ( - f"Mapped value '{openai_reason}' (from '{provider_reason}') " - f"is not a valid OpenAI finish reason" + f"Mapped value '{openai_reason}' (from '{provider_reason}') is not a valid OpenAI finish reason" ) @@ -380,28 +371,18 @@ class TestRedactNestedMatchAndRegexKeys: "assessments": [ { "sensitiveInformationPolicy": { - "piiEntities": [ - {"type": "NAME", "match": "secret-name", "action": "BLOCKED"} - ] - }, - "wordPolicy": { - "customWords": [{"match": "badword", "action": "BLOCKED"}] + "piiEntities": [{"type": "NAME", "match": "secret-name", "action": "BLOCKED"}] }, + "wordPolicy": {"customWords": [{"match": "badword", "action": "BLOCKED"}]}, } ], "regex": "should-redact-key-named-regex", } out = redact_nested_match_and_regex_keys(payload) - assert out["assessments"][0]["sensitiveInformationPolicy"]["piiEntities"][0][ - "match" - ] == "[REDACTED]" - assert out["assessments"][0]["wordPolicy"]["customWords"][0]["match"] == ( - "[REDACTED]" - ) + assert out["assessments"][0]["sensitiveInformationPolicy"]["piiEntities"][0]["match"] == "[REDACTED]" + assert out["assessments"][0]["wordPolicy"]["customWords"][0]["match"] == ("[REDACTED]") assert out["regex"] == "[REDACTED]" - assert payload["assessments"][0]["sensitiveInformationPolicy"]["piiEntities"][ - 0 - ]["match"] == "secret-name" + assert payload["assessments"][0]["sensitiveInformationPolicy"]["piiEntities"][0]["match"] == "secret-name" def test_passes_through_none_and_str(self): assert redact_nested_match_and_regex_keys(None) is None @@ -564,13 +545,17 @@ class TestIsExpectedClientError: assert is_expected_client_error(over_budget) is True litellm_limit = RateLimitError( - message="key over rpm", llm_provider="anthropic", model="claude-haiku-4-5", + message="key over rpm", + llm_provider="anthropic", + model="claude-haiku-4-5", category=RateLimitErrorCategory.LITELLM_RATE_LIMIT, ) assert is_expected_client_error(litellm_limit) is True vendor_limit = RateLimitError( - message="rate limited upstream", llm_provider="anthropic", model="claude-haiku-4-5", + message="rate limited upstream", + llm_provider="anthropic", + model="claude-haiku-4-5", category=RateLimitErrorCategory.VENDOR_RATE_LIMIT, ) assert is_expected_client_error(vendor_limit) is False @@ -803,3 +788,34 @@ class TestReceiveStampSurvivesASecondMetadataBucket: } assert get_litellm_metadata_from_kwargs(kwargs)["litellm_received_at"] is own + + +def test_billed_usage_cost_bills_any_chat_upstream_without_its_own_reader() -> None: + from types import SimpleNamespace + + from litellm.litellm_core_utils.core_helpers import billed_usage_cost + from litellm.types.utils import ModelResponse, Usage + + usage = Usage(prompt_tokens=100, completion_tokens=20, total_tokens=120, cost=0.037) + chat = ModelResponse(usage=usage) + assert billed_usage_cost(chat, "openai") == 0.037 + assert billed_usage_cost(chat, "openrouter") == 0.037 + assert billed_usage_cost(chat, "xai") is None + assert billed_usage_cost(SimpleNamespace(usage=usage), "openai") == 0.037 + + +def test_billed_usage_cost_keeps_the_upstream_charge_after_litellm_wrote_its_bill_into_usage_cost() -> None: + from litellm.litellm_core_utils.core_helpers import billed_usage_cost, stamp_litellm_bill_on_usage + from litellm.types.utils import ModelResponse, Usage + + reported = ModelResponse(usage=Usage(prompt_tokens=100, completion_tokens=20, total_tokens=120, cost=0.037)) + stamp_litellm_bill_on_usage(reported, reported.usage, 0.0407) + stamp_litellm_bill_on_usage(reported, reported.usage, 0.0407) + assert reported.usage.cost == 0.0407 + assert reported._hidden_params["response_cost"] == 0.0407 + assert billed_usage_cost(reported, "openai") == 0.037 + + unreported = ModelResponse(usage=Usage(prompt_tokens=100, completion_tokens=20, total_tokens=120)) + stamp_litellm_bill_on_usage(unreported, unreported.usage, 0.0015) + assert unreported.usage.cost == 0.0015 + assert billed_usage_cost(unreported, "openai") is None diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index f4502937fce..41e8c50fa85 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -2659,7 +2659,7 @@ def test_perplexity_streaming_dict_cost_bills_through_its_own_calculator(): ) == pytest.approx(0.00503) -def test_openai_compatible_streaming_cost_is_priced_from_the_cost_map(): +def test_openai_compatible_streaming_reported_cost_is_billed_instead_of_the_cost_map(): import litellm from litellm.cost_calculator import ( get_response_cost_from_hidden_params, @@ -2681,7 +2681,7 @@ def test_openai_compatible_streaming_cost_is_priced_from_the_cost_map(): id="chatcmpl-openai-compatible", model=model, choices=[], - usage=Usage(completion_tokens=5, prompt_tokens=10, total_tokens=15, cost=3_144_000), + usage=Usage(completion_tokens=5, prompt_tokens=10, total_tokens=15, cost=0.037), ) CustomStreamWrapper._propagate_usage_cost_to_hidden_params(complete_response, "openai") @@ -2693,7 +2693,7 @@ def test_openai_compatible_streaming_cost_is_priced_from_the_cost_map(): custom_llm_provider="openai", call_type="completion", optional_params={}, - ) == pytest.approx(2e-5) + ) == pytest.approx(0.037) def test_xai_streaming_reported_cost_still_takes_the_margin(monkeypatch): diff --git a/tests/unit/llms/openai_like/test_kyma_provider.py b/tests/unit/llms/openai_like/test_kyma_provider.py index 2b3740ca676..e834e0fc0e9 100644 --- a/tests/unit/llms/openai_like/test_kyma_provider.py +++ b/tests/unit/llms/openai_like/test_kyma_provider.py @@ -254,7 +254,7 @@ def test_interaction_on_kyma_is_billed_at_the_cost_kyma_reports(): @pytest.mark.parametrize("model", ("openrouter/openai/gpt-4o-mini", "deepseek/deepseek-chat")) -def test_interaction_on_another_provider_keeps_its_upstream_cost_out_of_the_body(model: str): +def test_interaction_on_another_openai_compatible_provider_is_billed_at_its_reported_cost(model: str): chat_usage: Final = {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7, "cost": 0.00721} responses_body: Final = { "id": "resp_upstream", @@ -278,7 +278,7 @@ def test_interaction_on_another_provider_keeps_its_upstream_cost_out_of_the_body upstream.route(path__regex=r".*/responses$").respond(200, json=responses_body) interaction: Final = litellm.interactions.create(model=model, input="ping", api_key="k") - assert "cost" not in interaction.model_dump(exclude_none=True)["usage"] + assert interaction.model_dump(exclude_none=True)["usage"]["cost"] == pytest.approx(0.00721) def test_sdk_completion_cost_prices_kyma_at_the_cost_kyma_reports(): @@ -410,8 +410,8 @@ def test_cost_headers_show_the_discount_and_margin_taken_from_the_cost_kyma_repo assert float(headers["x-litellm-response-cost-discount-amount"] or "nan") == pytest.approx(0.004) assert float(headers["x-litellm-response-cost-margin-amount"] or "nan") == pytest.approx(0.0004) assert float(headers["x-litellm-response-cost"] or "nan") == pytest.approx(0.0044) - assert headers["x-litellm-response-cost-input"] is None - assert headers["x-litellm-response-cost-output"] is None + assert float(headers["x-litellm-response-cost-input"] or "nan") == 0.0 + assert float(headers["x-litellm-response-cost-output"] or "nan") == pytest.approx(0.0044) @pytest.mark.parametrize("thinking", ({"type": "enabled", "budget_tokens": 1024}, {"type": "adaptive"})) diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 571f890a4e6..b2ae6b43d26 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -5443,6 +5443,46 @@ def test_stream_chunk_builder_keeps_provider_reported_usage_cost(): assert response._hidden_params["response_cost"] == pytest.approx(0.5) +def test_completion_cost_on_a_stamped_stream_still_bills_the_reported_charge(): + usage_chunk: Final = _stream_builder_text_chunk("gpt-4o", "") + usage_chunk.usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15, cost=0.5) + chunks: Final = [ + _stream_builder_text_chunk("gpt-4o", "Hello ", finish_reason="stop"), + usage_chunk, + ] + + response: Final = litellm.stream_chunk_builder( + chunks=chunks, messages=[{"role": "user", "content": "hi"}], logging_obj=_stream_builder_logging_obj() + ) + + assert response is not None + assert litellm.completion_cost(completion_response=response) == pytest.approx(0.5) + + +def test_stream_chunk_builder_stamps_a_zero_operator_price_over_a_reported_cost(): + logging_obj: Final = _stream_builder_logging_obj() + logging_obj.litellm_params = { + **logging_obj.litellm_params, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + } + logging_obj.custom_cost_per_token = {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0} + usage_chunk: Final = _stream_builder_text_chunk("gpt-4o", "") + usage_chunk.usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15, cost=0.5) + chunks: Final = [ + _stream_builder_text_chunk("gpt-4o", "Hello ", finish_reason="stop"), + usage_chunk, + ] + + response: Final = litellm.stream_chunk_builder( + chunks=chunks, messages=[{"role": "user", "content": "hi"}], logging_obj=logging_obj + ) + + assert response is not None + assert getattr(response.usage, "cost", None) == 0.0 + assert response._hidden_params["response_cost"] == 0.0 + + def test_stream_chunk_builder_prices_alias_from_openai_sdk_usage_chunk(): from openai.types.completion_usage import CompletionUsage From 342ce6ae4a4549cacaf96e47aa80a096ae1035bd Mon Sep 17 00:00:00 2001 From: DewellNaa Date: Thu, 29 Oct 2026 20:30:50 -0700 Subject: [PATCH 03/12] feat(zai): support response_format and max_completion_tokens, add glm-5v-turbo pricing (#48335) * feat(zai): fall back to BIGMODEL/ZHIPUAI api keys and add GLM-5.3 aliases and glm-5v-turbo Resolve the zai api key from ZAI_API_KEY, BIGMODEL_API_KEY or ZHIPUAI_API_KEY so deployments predating the zai rename keep working. Accept response_format and max_completion_tokens as supported openai params. Register zai/GLM-5.3 and zai/GLM-5.3-Flash aliases plus the glm-5v-turbo vision model in both the main and backup cost maps * fix(zai): read only ZAI_API_KEY, dropping the BigModel key fallbacks * fix(zai): map max_completion_tokens to max_tokens, correct glm-5v-turbo pricing, drop redundant aliases * fix(zai): reject json_schema response_format, add glm-5v-turbo cache read price * fix(zai): forward every response_format type so allowed_openai_params keeps working * fix(zai): follow json_schema natively on glm-5.3, prompt other models, drop over-limit max_tokens under drop_params * fix(zai): keep BigModel's json_schema as sent and never drop the caller's max_tokens under drop_params * fix(zai): write the prompted JSON schema as JSON so models answer with valid JSON * test(zai): type the zai provider tests and mark their locals Final --------- Co-authored-by: DewellNaa <210961864+DewellNaa@users.noreply.github.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../prompt_templates/factory.py | 34 ++- litellm/llms/bigmodel/chat/transformation.py | 4 + litellm/llms/zai/chat/transformation.py | 94 ++++++++- ...odel_prices_and_context_window_backup.json | 17 ++ model_prices_and_context_window.json | 17 ++ .../llms/bigmodel/test_bigmodel_provider.py | 23 ++ tests/unit/llms/zai/test_zai_provider.py | 197 ++++++++++++++++++ 7 files changed, 364 insertions(+), 22 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index fad4caa59a0..834d26b4fe8 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -5878,38 +5878,36 @@ def function_call_prompt( ] -def response_schema_prompt(model: str, response_schema: dict, llm_provider: str) -> str: +def response_schema_prompt(model: str, response_schema: Mapping[str, object], llm_provider: str) -> str: """ Decides if a user-defined custom prompt or default needs to be used Returns the prompt str that's passed to the model as a user message """ - custom_prompt_details: dict | None = None - response_schema_as_message: Final = [{"role": "user", "content": f"{response_schema}"}] - if f"{model}/response_schema_prompt" in litellm.custom_prompt_dict: - custom_prompt_details = litellm.custom_prompt_dict[ - f"{model}/response_schema_prompt" - ] # allow user to define custom response schema prompt by model - elif "response_schema_prompt" in litellm.custom_prompt_dict: - custom_prompt_details = litellm.custom_prompt_dict["response_schema_prompt"] - - if custom_prompt_details is not None: - return render_custom_prompt_template( - template=custom_prompt_details, messages=response_schema_as_message, model=model, llm_provider=llm_provider - ) - else: + prompts: Final[Mapping[str, Mapping[str, object]]] = litellm.custom_prompt_dict + model_prompt_key: Final = f"{model}/response_schema_prompt" + custom_prompt_details: Final = prompts.get( + model_prompt_key if model_prompt_key in prompts else "response_schema_prompt" + ) + if custom_prompt_details is None: return default_response_schema_prompt(response_schema=response_schema) + return render_custom_prompt_template( + template=custom_prompt_details, + messages=[{"role": "user", "content": f"{response_schema}"}], + model=model, + llm_provider=llm_provider, + ) -def default_response_schema_prompt(response_schema: dict) -> str: +def default_response_schema_prompt(response_schema: Mapping[str, object]) -> str: """ Used if provider/model doesn't support 'response_schema' param. This is the default prompt. Allow user to override this with a custom_prompt. """ prompt_str: Final = f"""Use this JSON schema: - ```json - {response_schema} + ```json + {json.dumps(response_schema, default=str)} ```""" return prompt_str diff --git a/litellm/llms/bigmodel/chat/transformation.py b/litellm/llms/bigmodel/chat/transformation.py index 12d5e912c15..93e9e24fe7b 100644 --- a/litellm/llms/bigmodel/chat/transformation.py +++ b/litellm/llms/bigmodel/chat/transformation.py @@ -26,6 +26,10 @@ class BigModelChatConfig(ZAIChatConfig): def custom_llm_provider(self) -> str: return "bigmodel" + @property + def prompts_unsupported_json_schema(self) -> bool: + return False + def get_openai_compatible_provider_info(self, api_base: str | None, api_key: str | None) -> tuple[str, str | None]: return get_bigmodel_url(api_base, "chat"), get_bigmodel_api_key(api_key) diff --git a/litellm/llms/zai/chat/transformation.py b/litellm/llms/zai/chat/transformation.py index e0e42d772f8..9e2b97d2bed 100644 --- a/litellm/llms/zai/chat/transformation.py +++ b/litellm/llms/zai/chat/transformation.py @@ -1,11 +1,43 @@ +from collections.abc import Mapping from typing import Final +from typing_extensions import TypeIs # noqa: TID251 # TypeIs reaches typing only on 3.13 + +import litellm +from litellm.litellm_core_utils.prompt_templates.factory import response_schema_prompt from litellm.secret_managers.main import get_secret_str -from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam +from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam, ChatCompletionUserMessage +from litellm.utils import ModelInfoLookupMiss, lookup_model_info_for_registration from ...openai.chat.gpt_transformation import OpenAIGPTConfig ZAI_API_BASE: Final = "https://api.z.ai/api/paas/v4" +TOKEN_LIMIT_PARAMS: Final = frozenset({"max_tokens", "max_completion_tokens"}) + + +def _is_json_object( + value: object, +) -> TypeIs[Mapping[str, object]]: # guard-ok: cost-map rows and request params are JSON + return isinstance(value, Mapping) + + +def _json_field(value: object, key: str) -> object: + return value.get(key) if _is_json_object(value) else None + + +def _exceeds_output_limit(model: str, max_tokens: object) -> bool: + info: Final = lookup_model_info_for_registration(model=model, custom_llm_provider="zai") + limit: Final = None if isinstance(info, ModelInfoLookupMiss) else info.get("max_output_tokens") + return isinstance(max_tokens, int) and isinstance(limit, int) and max_tokens > limit + + +def _schema_without_native_support(model: str, response_format: object) -> Mapping[str, object] | None: + if _json_field(response_format, "type") != "json_schema": + return None + if litellm.supports_response_schema(model=model, custom_llm_provider="zai"): + return None + schema: Final = _json_field(_json_field(response_format, "json_schema"), "schema") + return schema if _is_json_object(schema) else None class ZAIChatConfig(OpenAIGPTConfig): @@ -13,6 +45,10 @@ class ZAIChatConfig(OpenAIGPTConfig): def custom_llm_provider(self) -> str | None: return "zai" + @property + def prompts_unsupported_json_schema(self) -> bool: + return True + def _get_openai_compatible_provider_info( self, api_base: str | None, api_key: str | None ) -> tuple[str | None, str | None]: @@ -40,9 +76,11 @@ class ZAIChatConfig(OpenAIGPTConfig): # GLM/ZAI supports cache_control, so return messages and tools unchanged return messages, tools - def get_supported_openai_params(self, model: str) -> list: + def get_supported_openai_params(self, model: str) -> list[str]: base_params: Final = [ "max_tokens", + "max_completion_tokens", + "response_format", "stream", "stream_options", "temperature", @@ -52,8 +90,6 @@ class ZAIChatConfig(OpenAIGPTConfig): "tool_choice", ] - import litellm - try: if litellm.supports_reasoning(model=model, custom_llm_provider=self.custom_llm_provider): base_params.append("thinking") @@ -61,3 +97,53 @@ class ZAIChatConfig(OpenAIGPTConfig): pass return base_params + + def map_openai_params( + self, + non_default_params: Mapping[str, object], + optional_params: Mapping[str, object], + model: str, + drop_params: bool, + ) -> dict[str, object]: # mutable-ok: request pipeline adds parameters to the returned body + supported_params: Final = self.get_supported_openai_params(model) + sent: Final = {param: value for param, value in non_default_params.items() if param in supported_params} + max_completion_tokens: Final = sent.get("max_completion_tokens") + falls_back_to_max_tokens: Final = max_completion_tokens is None or ( + (litellm.drop_params or drop_params) and _exceeds_output_limit(model, max_completion_tokens) + ) + max_tokens: Final = sent.get("max_tokens") if falls_back_to_max_tokens else max_completion_tokens + return { + **optional_params, + **{param: value for param, value in sent.items() if param not in TOKEN_LIMIT_PARAMS}, + **({} if max_tokens is None else {"max_tokens": max_tokens}), + } + + def transform_request( + self, + model: str, + messages: list[AllMessageValues], # mutable-ok: OpenAIGPTConfig.transform_request signature + optional_params: dict[str, object], # mutable-ok: OpenAIGPTConfig.transform_request signature + litellm_params: dict[str, object], # mutable-ok: OpenAIGPTConfig.transform_request signature + headers: dict[str, str], # mutable-ok: OpenAIGPTConfig.transform_request signature + ) -> dict[str, object]: # mutable-ok: inherited contract returns the request body + schema: Final = ( + _schema_without_native_support(model, optional_params.get("response_format")) + if self.prompts_unsupported_json_schema + else None + ) + schema_message: Final = ( + () + if schema is None + else ( + ChatCompletionUserMessage( + role="user", content=response_schema_prompt(model=model, response_schema=schema, llm_provider="zai") + ), + ) + ) + return super().transform_request( + model, + [*messages, *schema_message], + optional_params if schema is None else {**optional_params, "response_format": {"type": "json_object"}}, + litellm_params, + headers, + ) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 77ac4f49e8b..96a04c96f44 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -54512,6 +54512,7 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, + "supports_response_schema": true, "supports_tool_choice": true }, "zai/glm-5.3-flash": { @@ -54526,10 +54527,26 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, + "supports_response_schema": true, "supports_tool_choice": true, "source": "https://docs.z.ai/guides/overview/pricing", "supports_vision": true }, + "zai/glm-5v-turbo": { + "cache_read_input_token_cost": 2.4e-07, + "input_cost_per_token": 1.2e-06, + "litellm_provider": "zai", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://vercel.com/ai-gateway/models/glm-5v-turbo", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, "zai/glm-5.1": { "cache_creation_input_token_cost": 0, "cache_read_input_token_cost": 2.6e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 77ac4f49e8b..96a04c96f44 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -54512,6 +54512,7 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, + "supports_response_schema": true, "supports_tool_choice": true }, "zai/glm-5.3-flash": { @@ -54526,10 +54527,26 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, + "supports_response_schema": true, "supports_tool_choice": true, "source": "https://docs.z.ai/guides/overview/pricing", "supports_vision": true }, + "zai/glm-5v-turbo": { + "cache_read_input_token_cost": 2.4e-07, + "input_cost_per_token": 1.2e-06, + "litellm_provider": "zai", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://vercel.com/ai-gateway/models/glm-5v-turbo", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, "zai/glm-5.1": { "cache_creation_input_token_cost": 0, "cache_read_input_token_cost": 2.6e-07, diff --git a/tests/unit/llms/bigmodel/test_bigmodel_provider.py b/tests/unit/llms/bigmodel/test_bigmodel_provider.py index cdf98c8f9a4..6abfcb060d0 100644 --- a/tests/unit/llms/bigmodel/test_bigmodel_provider.py +++ b/tests/unit/llms/bigmodel/test_bigmodel_provider.py @@ -197,6 +197,29 @@ def test_chat_does_not_send_the_openai_organization_to_bigmodel(monkeypatch: pyt assert "openai-organization" not in route.calls.last.request.headers +@respx.mock +def test_chat_sends_json_schema_response_format_as_given() -> None: + route: Final = respx.post(f"{BASE}/api/paas/v4/chat/completions").respond( + json={ + "id": "chat_test", + "object": "chat.completion", + "created": 1, + "model": "glm-4.6", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "{}"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + ) + messages: Final = [{"role": "user", "content": "Ann likes 7"}] + response_format: Final = { + "type": "json_schema", + "json_schema": {"name": "profile", "schema": {"type": "object", "properties": {"n": {"type": "integer"}}}}, + } + _ = litellm.completion(model="bigmodel/glm-4.6", messages=messages, response_format=response_format) + body: Final = json.loads(route.calls.last.request.content) + assert body["response_format"] == response_format + assert body["messages"] == messages + + @pytest.mark.parametrize("is_async", (False, True)) @pytest.mark.asyncio @respx.mock diff --git a/tests/unit/llms/zai/test_zai_provider.py b/tests/unit/llms/zai/test_zai_provider.py index 32849d5eef1..fb126bd8f41 100644 --- a/tests/unit/llms/zai/test_zai_provider.py +++ b/tests/unit/llms/zai/test_zai_provider.py @@ -2,13 +2,27 @@ Tests for Z.AI (Zhipu AI) provider - GLM models """ +import json import math +from collections.abc import Mapping +from typing import Final import pytest +import respx import litellm from litellm import completion from litellm.cost_calculator import cost_per_token +from litellm.llms.zai.chat.transformation import ZAIChatConfig +from litellm.types.utils import PromptTokensDetailsWrapper, Usage + +JSON_SCHEMA_RESPONSE_FORMAT: Final = { + "type": "json_schema", + "json_schema": { + "name": "profile", + "schema": {"type": "object", "properties": {"favorite_number": {"type": "integer"}}}, + }, +} @pytest.fixture @@ -99,3 +113,186 @@ def test_zai_sync_completion(respx_mock, zai_response, monkeypatch): assert response.choices[0].message.content == "Hello! How can I help you today?" assert response.usage.total_tokens == 25 + + +def test_zai_sends_response_format_and_maps_max_completion_tokens( + respx_mock: respx.MockRouter, zai_response: Mapping[str, object], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("ZAI_API_KEY", "test-api-key") + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://api.z.ai/api/paas/v4/chat/completions").respond(json=zai_response) + + completion( + model="zai/glm-5.3", + messages=[{"role": "user", "content": "Reply in JSON"}], + response_format={"type": "json_object"}, + max_completion_tokens=64, + ) + + body: Final = json.loads(respx_mock.calls[0].request.content) + assert body["response_format"] == {"type": "json_object"} + assert body["max_tokens"] == 64 + assert "max_completion_tokens" not in body + + +@pytest.mark.parametrize( + "token_limits", + [{"max_tokens": 300, "max_completion_tokens": 15}, {"max_completion_tokens": 15, "max_tokens": 300}], +) +def test_zai_max_completion_tokens_wins_over_max_tokens(token_limits: dict[str, int]) -> None: + mapped: Final = ZAIChatConfig().map_openai_params(token_limits, {}, "glm-5.3", drop_params=False) + + assert mapped["max_tokens"] == 15 + assert "max_completion_tokens" not in mapped + + +@pytest.mark.parametrize( + ("over_limit_by", "drop_params", "forwarded"), + [(1, True, False), (1, False, True), (0, True, True)], +) +def test_zai_drop_params_drops_max_completion_tokens_above_model_limit( + local_model_cost_map: None, + respx_mock: respx.MockRouter, + zai_response: Mapping[str, object], + monkeypatch: pytest.MonkeyPatch, + over_limit_by: int, + drop_params: bool, + forwarded: bool, +) -> None: + monkeypatch.setenv("ZAI_API_KEY", "test-api-key") + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://api.z.ai/api/paas/v4/chat/completions").respond(json=zai_response) + requested: Final = litellm.model_cost["zai/glm-4.5-flash"]["max_output_tokens"] + over_limit_by + + completion( + model="zai/glm-4.5-flash", + messages=[{"role": "user", "content": "Count"}], + max_completion_tokens=requested, + drop_params=drop_params, + ) + + body: Final = json.loads(respx_mock.calls[0].request.content) + assert body.get("max_tokens") == (requested if forwarded else None) + + +@pytest.mark.parametrize("with_over_limit_max_completion_tokens", [False, True]) +def test_zai_drop_params_keeps_max_tokens_above_model_limit( + local_model_cost_map: None, + respx_mock: respx.MockRouter, + zai_response: Mapping[str, object], + monkeypatch: pytest.MonkeyPatch, + with_over_limit_max_completion_tokens: bool, +) -> None: + monkeypatch.setenv("ZAI_API_KEY", "test-api-key") + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://api.z.ai/api/paas/v4/chat/completions").respond(json=zai_response) + over_limit: Final = litellm.model_cost["zai/glm-4.5-flash"]["max_output_tokens"] + 1 + over_limit_max_completion_tokens: Final = ( + {"max_completion_tokens": over_limit + 1} if with_over_limit_max_completion_tokens else {} + ) + + completion( + model="zai/glm-4.5-flash", + messages=[{"role": "user", "content": "Count"}], + max_tokens=over_limit, + drop_params=True, + **over_limit_max_completion_tokens, + ) + + body: Final = json.loads(respx_mock.calls[0].request.content) + assert body["max_tokens"] == over_limit + + +@pytest.mark.parametrize("pass_through", [{}, {"allowed_openai_params": ["response_format"]}]) +def test_zai_forwards_json_schema_to_models_that_enforce_it( + local_model_cost_map: None, + respx_mock: respx.MockRouter, + zai_response: Mapping[str, object], + monkeypatch: pytest.MonkeyPatch, + pass_through: dict[str, list[str]], +) -> None: + monkeypatch.setenv("ZAI_API_KEY", "test-api-key") + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://api.z.ai/api/paas/v4/chat/completions").respond(json=zai_response) + messages: Final = [{"role": "user", "content": "Ann likes 7"}] + + completion(model="zai/glm-5.3", messages=messages, response_format=JSON_SCHEMA_RESPONSE_FORMAT, **pass_through) + + body: Final = json.loads(respx_mock.calls[0].request.content) + assert body["response_format"] == JSON_SCHEMA_RESPONSE_FORMAT + assert body["messages"] == messages + + +@pytest.mark.parametrize("pass_through", [{}, {"allowed_openai_params": ["response_format"]}]) +def test_zai_puts_json_schema_in_prompt_for_models_that_ignore_it( + local_model_cost_map: None, + respx_mock: respx.MockRouter, + zai_response: Mapping[str, object], + monkeypatch: pytest.MonkeyPatch, + pass_through: dict[str, list[str]], +) -> None: + monkeypatch.setenv("ZAI_API_KEY", "test-api-key") + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://api.z.ai/api/paas/v4/chat/completions").respond(json=zai_response) + + completion( + model="zai/glm-4.6", + messages=[{"role": "user", "content": "Ann likes 7"}], + response_format=JSON_SCHEMA_RESPONSE_FORMAT, + **pass_through, + ) + + body: Final = json.loads(respx_mock.calls[0].request.content) + assert body["response_format"] == {"type": "json_object"} + assert body["messages"][0] == {"role": "user", "content": "Ann likes 7"} + assert body["messages"][-1]["role"] == "user" + prompt: Final = body["messages"][-1]["content"] + rendered_schema: Final = prompt[prompt.index("{") : prompt.rindex("}") + 1] + assert json.loads(rendered_schema) == JSON_SCHEMA_RESPONSE_FORMAT["json_schema"]["schema"] + + +def test_zai_glm_5v_turbo_bills_cached_input_at_cache_read_price(local_model_cost_map: None) -> None: + usage: Final = Usage( + prompt_tokens=1000, + completion_tokens=0, + total_tokens=1000, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=800), + ) + + prompt_cost, _ = cost_per_token(model="glm-5v-turbo", custom_llm_provider="zai", usage_object=usage) + + entry: Final = litellm.model_cost["zai/glm-5v-turbo"] + assert math.isclose( + prompt_cost, 200 * entry["input_cost_per_token"] + 800 * entry["cache_read_input_token_cost"] + ) + + +@pytest.mark.parametrize("model", ["glm-5v-turbo", "GLM-5V-Turbo", "GLM-5.3", "GLM-5.3-Flash"]) +def test_zai_cost_resolves_case_insensitively(local_model_cost_map: None, model: str) -> None: + prompt_cost, completion_cost = cost_per_token( + model=model, custom_llm_provider="zai", prompt_tokens=1000, completion_tokens=1000 + ) + + entry: Final = litellm.model_cost[f"zai/{model.lower()}"] + assert math.isclose(prompt_cost, 1000 * entry["input_cost_per_token"]) + assert math.isclose(completion_cost, 1000 * entry["output_cost_per_token"]) + + +def test_zai_glm_5v_turbo_accepts_thinking( + local_model_cost_map: None, + respx_mock: respx.MockRouter, + zai_response: Mapping[str, object], + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("ZAI_API_KEY", "test-api-key") + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://api.z.ai/api/paas/v4/chat/completions").respond(json=zai_response) + + completion( + model="zai/glm-5v-turbo", + messages=[{"role": "user", "content": "Hello"}], + thinking={"type": "disabled"}, + ) + + body: Final = json.loads(respx_mock.calls[0].request.content) + assert body["thinking"] == {"type": "disabled"} From b91fc92b6d0815df038e3496b669871fd917c056 Mon Sep 17 00:00:00 2001 From: paveln83 <349044532+paveln83@users.noreply.github.com> Date: Thu, 29 Oct 2026 20:32:04 -0700 Subject: [PATCH 04/12] fix(router): cap the default OTPM reservation and reconcile token counters after each call (#49054) * fix(router): cap the default OTPM reservation for requests without max_tokens * fix(router): reconcile ITPM and OTPM reservations after every successful call * fix(router): read the default OTPM reservation from the served model and make its fraction configurable The default reservation now looks up max output from litellm_params.model, as main does, so a pricing base_model never changes it. NO_MAX_TOKENS_TPM_RESERVATION_FRACTION sets the fraction for both the router check and the key and team TPM floor. A max_tokens that is not a number or string counts as no cap instead of raising inside the check. * fix(router): refund a failed attempt's io token reservation in make_call The proxy passes one logging object to every retry, and it runs failure callbacks only for the first attempt, so the last attempt's ITPM and OTPM reservation stayed held until the minute ended. Each attempt that raises now gives its reservation back before the retry or fallback * fix(router): count Anthropic cache writes once in ITPM reconcile Native /v1/messages usage reports input_tokens without cache reads and writes, so the reconcile now maps it through AnthropicConfig.calculate_usage before subtracting cached tokens. * fix(router): use a deployment max_tokens, charge late usage to its finish minute, and read the reservation fraction as a fraction * fix(router): reserve with the deployment cap only through the param the request's API sends * fix(router): never let a deployment's output cap raise the reservation above the default * test(router): move the reservation fraction env var test to the integration suite The unit test started a child interpreter, which the unit guide forbids. The integration test boots a real proxy with the env var set and checks admission against the held reservation. * test(router): retry the reservation fraction scenario when the minute rolls over The OTPM counter is keyed by minute, so a rollover between the held stream and the capped requests gave the capped request a fresh counter. The control request now leaves room for usage the first attempt left in the new minute. * test(router): give the rollover retry its own deployment --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/constants.py | 4 + litellm/litellm_core_utils/env_utils.py | 10 +- .../hooks/parallel_request_limiter_v3.py | 17 +- litellm/router.py | 30 +- .../io_token_rate_limit_check.py | 185 +++--- .../pre_call_checks/model_rate_limit_check.py | 37 +- .../test_otpm_default_reservation_fraction.py | 137 +++++ .../unit/litellm_core_utils/test_env_utils.py | 10 + tests/unit/test_constants.py | 9 +- .../test_router/test_io_token_rate_limits.py | 530 +++++++++++++++++- 10 files changed, 831 insertions(+), 138 deletions(-) create mode 100644 tests/integration/routing/test_otpm_default_reservation_fraction.py diff --git a/litellm/constants.py b/litellm/constants.py index e07d727973d..cfee777f36a 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -7,6 +7,7 @@ from typing import Final, Literal from litellm.litellm_core_utils.env_utils import ( get_env_float, + get_env_float_in_range, get_env_float_or_none, get_env_int, get_env_int_in_range, @@ -149,6 +150,9 @@ DEFAULT_FAILURE_THRESHOLD_PERCENT: Final = get_env_float( "DEFAULT_FAILURE_THRESHOLD_PERCENT", 0.5 ) # default cooldown a deployment if 50% of requests fail in a given minute DEFAULT_MAX_TOKENS: Final = get_env_int("DEFAULT_MAX_TOKENS", 4096) +NO_MAX_TOKENS_TPM_RESERVATION_FRACTION: Final = get_env_float_in_range( + "NO_MAX_TOKENS_TPM_RESERVATION_FRACTION", 0.25, 0.01, 1.0 +) DEFAULT_ALLOWED_FAILS: Final = get_env_int("DEFAULT_ALLOWED_FAILS", 3) DEFAULT_REDIS_SYNC_INTERVAL: Final = get_env_int("DEFAULT_REDIS_SYNC_INTERVAL", 1) DEFAULT_COOLDOWN_TIME_SECONDS: Final = get_env_int("DEFAULT_COOLDOWN_TIME_SECONDS", 5) diff --git a/litellm/litellm_core_utils/env_utils.py b/litellm/litellm_core_utils/env_utils.py index 6883af98bbd..462b0b3df47 100644 --- a/litellm/litellm_core_utils/env_utils.py +++ b/litellm/litellm_core_utils/env_utils.py @@ -71,7 +71,15 @@ def get_env_int_in_range(env_var: str, default: int, minimum: int, maximum: int) Values outside the range fall back to the default and warn, so a misconfigured knob can neither crash the caller nor silently change the meaning of what it computes. """ - value: Final = get_env_int(env_var, default) + return _within_range(env_var, get_env_int(env_var, default), default, minimum, maximum) + + +def get_env_float_in_range(env_var: str, default: float, minimum: float, maximum: float) -> float: + """Float counterpart of `get_env_int_in_range`.""" + return _within_range(env_var, get_env_float(env_var, default), default, minimum, maximum) + + +def _within_range(env_var: str, value: _NumberT, default: _NumberT, minimum: _NumberT, maximum: _NumberT) -> _NumberT: if minimum <= value <= maximum: return value logging.getLogger("LiteLLM").warning( diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 404aff84994..14b08a09116 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -42,7 +42,11 @@ from litellm.caching.redis_batch import ( active_request_redis_batch, ) from litellm.caching.redis_cache import log_redis_failure -from litellm.constants import DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE, INTERNAL_CALL_ORIGIN_METADATA_KEY +from litellm.constants import ( + DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE, + INTERNAL_CALL_ORIGIN_METADATA_KEY, + NO_MAX_TOKENS_TPM_RESERVATION_FRACTION, +) from litellm.cost_calculator import get_usage_object from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.audio_utils.utils import estimate_audio_tokens @@ -508,11 +512,6 @@ REDIS_NODE_HASHTAG_NAME: Final = "all_keys" # *some* output budget; these define that fallback estimate. DEFAULT_MAX_TOKENS_ESTIMATE: Final = 4096 DEFAULT_CHARS_PER_TOKEN: Final = 4 -# Fraction of the available output budget reserved as the upfront floor when -# the request omits max_tokens. Applied to both DEFAULT_MAX_TOKENS_ESTIMATE -# (baseline floor) and to the smallest configured TPM limit (capped floor for -# small per-tenant TPM caps). -_TPM_FLOOR_FRACTION: Final = 4 # Both embeddings and the Responses API put their prompt in data["input"], # but only embeddings have no output tokens. Every "is this an embedding" # check on data["input"] must exclude these call types, or a Responses call @@ -1003,10 +1002,10 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): per-tenant cap can't be tripped by the floor alone. Returns the baseline floor when no limit is provided. """ - baseline: Final = DEFAULT_MAX_TOKENS_ESTIMATE // _TPM_FLOOR_FRACTION + baseline: Final = int(DEFAULT_MAX_TOKENS_ESTIMATE * NO_MAX_TOKENS_TPM_RESERVATION_FRACTION) if min_configured_tpm_limit is None: return baseline - return min(baseline, max(1, min_configured_tpm_limit // _TPM_FLOOR_FRACTION)) + return min(baseline, max(1, int(min_configured_tpm_limit * NO_MAX_TOKENS_TPM_RESERVATION_FRACTION))) @staticmethod def _is_embedding_request(data: object, call_type: str | None) -> bool: @@ -1138,7 +1137,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): if call_type in RESPONSES_API_CALL_TYPES else base_capped_floor ) - baseline_floor: Final = DEFAULT_MAX_TOKENS_ESTIMATE // _TPM_FLOOR_FRACTION + baseline_floor: Final = int(DEFAULT_MAX_TOKENS_ESTIMATE * NO_MAX_TOKENS_TPM_RESERVATION_FRACTION) is_embedding: Final = PROXY_MaxParallelRequestsHandler_v3._is_embedding_request(data, call_type) if ( capped_floor >= baseline_floor diff --git a/litellm/router.py b/litellm/router.py index 3a423b79e49..20af2e5649e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -377,6 +377,7 @@ from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check impor EncryptedContentAffinityCheck, ) from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import ( + async_io_token_refund_failure, build_io_token_rate_limit_headers, deployment_has_io_token_limits, refund_stale_reservation_before_retry, @@ -9500,7 +9501,7 @@ class Router(DeprecatedRouterMembers): raise original_exception - async def make_call(self, original_function: Any, *args, **kwargs): + async def make_call(self, original_function: Any, *args: object, **kwargs: object): """ Handler for making a call to the .completion()/.embeddings()/etc. functions. """ @@ -9509,17 +9510,26 @@ class Router(DeprecatedRouterMembers): endpoint_function: Final = ( generic_function if callable(generic_function) else _OPAQUE_VALUE_ADAPTER.validate_python(original_function) ) - with endpoint_selection( - getattr(endpoint_function, "__name__", None), - getattr(endpoint_function, "__module__", None), - model=_OPAQUE_VALUE_ADAPTER.validate_python(model_group), - ): - response = original_function(*args, **kwargs) - if coroutine_checker.is_async_callable(response) or inspect.isawaitable(response): - response = await response + try: + with endpoint_selection( + getattr(endpoint_function, "__name__", None), + getattr(endpoint_function, "__module__", None), + model=_OPAQUE_VALUE_ADAPTER.validate_python(model_group), + ): + response = original_function(*args, **kwargs) + if coroutine_checker.is_async_callable(response) or inspect.isawaitable(response): + response = await response + except Exception: + with contextlib.suppress(Exception): + await async_io_token_refund_failure( + self.cache, kwargs, parent_otel_span=get_parent_otel_span_from_kwargs(kwargs) + ) + raise await self.increment_deployment_usage_for_response(response=response, request_kwargs=kwargs) ## PROCESS RESPONSE HEADERS - response = await self.set_response_headers(response=response, model_group=model_group, request_kwargs=kwargs) + response = await self.set_response_headers( + response=response, model_group=model_group if isinstance(model_group, str) else None, request_kwargs=kwargs + ) return response diff --git a/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py b/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py index 0b3d48c9e1d..7580c7dadd3 100644 --- a/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py +++ b/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py @@ -1,7 +1,8 @@ """ Separate ITPM/OTPM (input/output tokens per minute) deployment rate limits. -- Pre-call: atomically reserve estimated_input against ITPM and max_tokens against OTPM +- Pre-call: atomically reserve estimated_input against ITPM and max_tokens against OTPM; a request + without max_tokens reserves the model's max output, capped at a fraction of the OTPM limit - Post-call: reconcile ITPM to actual input tokens and OTPM to actual output tokens - Cached prompt-read tokens are excluded from ITPM post-call accounting @@ -22,6 +23,7 @@ from litellm import token_counter from litellm._internal_context import with_service_target from litellm._logging import verbose_router_logger from litellm.caching.dual_cache import DualCache +from litellm.constants import NO_MAX_TOKENS_TPM_RESERVATION_FRACTION from litellm.litellm_core_utils.token_counter import deployment_openai_model, offload_token_count from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping from litellm.router_utils.routing_read_batch import ROUTER_USAGE_TARGET @@ -127,10 +129,15 @@ def _get_cache_keys(deployment: dict, current_minute: str) -> tuple[str, str] | return itpm_key, otpm_key -def _tokenizer_model(deployment: object) -> str: +def _served_model(deployment: object) -> str: params: Final = deployment.get("litellm_params") if is_str_keyed_mapping(deployment) else None served: Final = params.get("model") if is_str_keyed_mapping(params) else None - return deployment_openai_model(deployment) or (served if isinstance(served, str) else "") + return served if isinstance(served, str) else "" + + +def _tokenizer_and_served_models(deployment: object) -> tuple[str, str]: + served: Final = _served_model(deployment) + return deployment_openai_model(deployment) or served, served def _estimate_input_tokens(request_kwargs: dict[str, Any] | None, model: str = "") -> int: @@ -160,24 +167,61 @@ def _model_max_output_tokens(model_name: str) -> int | None: return None -def _resolve_max_tokens(request_kwargs: dict[str, Any] | None, deployment: dict) -> int: - if request_kwargs: - # An explicit max_tokens=0 must be honored, not treated as absent and - # replaced by the model default. - explicit = request_kwargs.get("max_tokens") - if explicit is None: - explicit = request_kwargs.get("max_completion_tokens") - if explicit is None: - explicit = request_kwargs.get("max_output_tokens") - if explicit is not None: - return max(0, int(explicit)) +_CHAT_OUTPUT_CAP_KEYS: Final = ("max_tokens", "max_completion_tokens") +_RESPONSES_OUTPUT_CAP_KEYS: Final = ("max_output_tokens",) +_OUTPUT_CAP_KEYS: Final = _CHAT_OUTPUT_CAP_KEYS + _RESPONSES_OUTPUT_CAP_KEYS - model_name: Final = (deployment.get("litellm_params") or {}).get("model") - if model_name: - model_max: Final = _model_max_output_tokens(model_name) - if model_max is not None: - return model_max - return 4096 + +def _explicit_max_tokens( + request_kwargs: Mapping[str, object] | None, cap_keys: tuple[str, ...] = _OUTPUT_CAP_KEYS +) -> int | None: + if not request_kwargs: + return None + explicit: Final = next( + (request_kwargs[key] for key in cap_keys if request_kwargs.get(key) is not None), + None, + ) + match explicit: + case int() | float() | str(): + return max(0, int(explicit)) + case _: + return None + + +def _sent_deployment_cap_keys(request_kwargs: Mapping[str, object] | None) -> tuple[str, ...]: + is_responses_request: Final = ( + request_kwargs is not None and "input" in request_kwargs and "messages" not in request_kwargs + ) + return _RESPONSES_OUTPUT_CAP_KEYS if is_responses_request else _CHAT_OUTPUT_CAP_KEYS + + +def _deployment_output_cap(request_kwargs: Mapping[str, object] | None, deployment: Mapping[str, object]) -> int | None: + params: Final = deployment.get("litellm_params") + return ( + _explicit_max_tokens(params, _sent_deployment_cap_keys(request_kwargs)) + if is_str_keyed_mapping(params) + else None + ) + + +def _default_otpm_reservation(model_name: str, otpm_limit: int) -> int: + model_max: Final = _model_max_output_tokens(model_name) if model_name else None + share_of_limit: Final = max(1, int(otpm_limit * NO_MAX_TOKENS_TPM_RESERVATION_FRACTION)) + return min(4096 if model_max is None else model_max, share_of_limit) + + +def _otpm_reservation( + request_kwargs: Mapping[str, object] | None, + deployment: Mapping[str, object], + model_name: str, + otpm_limit: int, +) -> int: + from_request: Final = _explicit_max_tokens(request_kwargs) + if from_request is not None: + return from_request + default: Final = _default_otpm_reservation(model_name, otpm_limit) + from_deployment: Final = _deployment_output_cap(request_kwargs, deployment) + return default if from_deployment is None else min(from_deployment, default) def _get_usage_tokens(usage: object) -> tuple[int, int, int]: @@ -210,6 +254,12 @@ def _extract_response_usage(response_obj: object) -> object: return getattr(response_obj, "usage", None) +def _openai_shaped_usage(usage: object) -> object: + if isinstance(usage, dict) and litellm.AnthropicConfig.is_anthropic_usage_object(usage): + return litellm.AnthropicConfig().calculate_usage(usage_object=usage, reasoning_content=None) + return usage + + def _usage_is_present(usage: object) -> bool: """ True only if usage carries an actual input/output breakdown. @@ -238,7 +288,7 @@ def _resolve_reconcile_usage_tokens( fields. When usage cannot be resolved, return ``usage_resolved=False`` so callers keep the pre-call reservation instead of refunding it as zero usage. """ - usage: Final = _extract_response_usage(response_obj) + usage: Final = _openai_shaped_usage(_extract_response_usage(response_obj)) if _usage_is_present(usage): prompt_tokens, completion_tokens, cached_tokens = _get_usage_tokens(usage) return max(0, prompt_tokens - cached_tokens), completion_tokens, True @@ -261,6 +311,21 @@ def _resolve_reconcile_usage_tokens( return 0, 0, False +def _key_for_this_minute(reserved_key: str) -> str: + counter, _, _ = reserved_key.rpartition(":") + return f"{counter}:{get_utc_datetime().strftime('%H-%M')}" + + +def _reconcile_increments( + reservation: tuple[int, int, str | None, str | None], billable_input: int, completion_tokens: int +) -> tuple[tuple[str, int], ...]: + itpm_reserved, otpm_reserved, itpm_key, otpm_key = reservation + deltas: Final = ((itpm_key, billable_input - itpm_reserved), (otpm_key, completion_tokens - otpm_reserved)) + return tuple( + (key if delta < 0 else _key_for_this_minute(key), delta) for key, delta in deltas if key is not None and delta + ) + + def _stash_reservation_in_metadata( request_kwargs: dict[str, object] | None, *, @@ -311,7 +376,9 @@ def _reservation_channels(kwargs: Mapping[str, object] | None) -> tuple[object, return () top_level: Final = (kwargs.get("metadata"), kwargs.get("litellm_metadata")) litellm_params: Final = _as_mutable_mapping(kwargs.get("litellm_params")) - from_params: Final = () if litellm_params is None else (litellm_params.get("metadata"),) + from_params: Final = ( + () if litellm_params is None else (litellm_params.get("metadata"), litellm_params.get("litellm_metadata")) + ) standard_logging_object: Final = _as_mutable_mapping(kwargs.get("standard_logging_object")) from_logging_object: Final = () if standard_logging_object is None else (standard_logging_object.get("metadata"),) return top_level + from_params + from_logging_object @@ -433,9 +500,11 @@ def io_token_pre_call_check( return deployment request_kwargs: Final = get_io_token_rate_limit_request_kwargs() - _model: Final = _tokenizer_model(deployment) + _model, served_model = _tokenizer_and_served_models(deployment) estimated_input: Final = _estimate_input_tokens(request_kwargs, model=_model) - max_tokens: Final = _resolve_max_tokens(request_kwargs, deployment) + otpm_to_reserve: Final = ( + _otpm_reservation(request_kwargs, deployment, served_model, otpm_limit) if otpm_limit is not None else 0 + ) dt: Final = get_utc_datetime() current_minute: Final = dt.strftime("%H-%M") @@ -458,7 +527,7 @@ def io_token_pre_call_check( ) if otpm_limit is not None: - otpm_reserved = 0 if max_tokens == 0 else _reservation_value(max_tokens, otpm_limit) + otpm_reserved = otpm_to_reserve try: _sync_increment_with_rollback( dual_cache, @@ -497,9 +566,11 @@ async def async_io_token_pre_call_check( return deployment request_kwargs: Final = get_io_token_rate_limit_request_kwargs() - _model: Final = _tokenizer_model(deployment) + _model, served_model = _tokenizer_and_served_models(deployment) estimated_input: Final = await offload_token_count(_estimate_input_tokens)(request_kwargs, model=_model) - max_tokens: Final = _resolve_max_tokens(request_kwargs, deployment) + otpm_to_reserve: Final = ( + _otpm_reservation(request_kwargs, deployment, served_model, otpm_limit) if otpm_limit is not None else 0 + ) dt: Final = get_utc_datetime() current_minute: Final = dt.strftime("%H-%M") @@ -523,7 +594,7 @@ async def async_io_token_pre_call_check( ) if otpm_limit is not None: - otpm_reserved = 0 if max_tokens == 0 else _reservation_value(max_tokens, otpm_limit) + otpm_reserved = otpm_to_reserve try: await _increment_with_rollback( dual_cache, @@ -564,7 +635,8 @@ def io_token_reconcile_success( ) -> None: request_kwargs: Final[Mapping[str, object] | None] = kwargs response: Final[object] = response_obj - itpm_reserved, otpm_reserved, itpm_key, otpm_key = _read_reservation_from_kwargs(request_kwargs) + reservation: Final = _read_reservation_from_kwargs(request_kwargs) + itpm_reserved, otpm_reserved, itpm_key, otpm_key = reservation if itpm_key is None and otpm_key is None: return @@ -572,23 +644,8 @@ def io_token_reconcile_success( try: if usage_resolved: - if itpm_key is not None: - itpm_delta: Final = billable_input - itpm_reserved - if itpm_delta != 0: - dual_cache.increment_cache( - key=itpm_key, - value=itpm_delta, - ttl=RoutingArgsTTL, - ) - - if otpm_key is not None: - otpm_delta: Final = completion_tokens - otpm_reserved - if otpm_delta != 0: - dual_cache.increment_cache( - key=otpm_key, - value=otpm_delta, - ttl=RoutingArgsTTL, - ) + for key, delta in _reconcile_increments(reservation, billable_input, completion_tokens): + dual_cache.increment_cache(key=key, value=delta, ttl=RoutingArgsTTL) else: verbose_router_logger.debug( "[IO TOKEN LIMIT] usage missing; keeping reservation (itpm_reserved=%s, otpm_reserved=%s)", @@ -618,39 +675,23 @@ async def async_io_token_reconcile_success( ) -> None: request_kwargs: Final[Mapping[str, object] | None] = kwargs response: Final[object] = response_obj - itpm_reserved, otpm_reserved, itpm_key, otpm_key = _read_reservation_from_kwargs(request_kwargs) + reservation: Final = _read_reservation_from_kwargs(request_kwargs) + itpm_reserved, otpm_reserved, itpm_key, otpm_key = reservation if itpm_key is None and otpm_key is None: return billable_input, completion_tokens, usage_resolved = _resolve_reconcile_usage_tokens(request_kwargs, response) - # Reconcile against the exact key that held the reservation (which encodes - # the reservation's minute), not a key recomputed at response time. This - # tracks actual usage even when the pre-call estimate was 0, and avoids a - # minute-boundary mismatch for calls that span into the next minute. Always - # clear the stash afterwards (even if an increment throws) so a retry or a - # duplicate success event can't re-process it. + # A refund goes back to the minute that held the reservation, while usage above the reservation is charged + # to the minute the response finished in, so a call that runs past its minute still counts against a live + # window. Always clear the stash afterwards (even if an increment throws) so a retry or a duplicate success + # event can't re-process it. try: if usage_resolved: - if itpm_key is not None: - itpm_delta: Final = billable_input - itpm_reserved - if itpm_delta != 0: - await dual_cache.async_increment_cache( - key=itpm_key, - value=itpm_delta, - ttl=RoutingArgsTTL, - parent_otel_span=parent_otel_span, - ) - - if otpm_key is not None: - otpm_delta: Final = completion_tokens - otpm_reserved - if otpm_delta != 0: - await dual_cache.async_increment_cache( - key=otpm_key, - value=otpm_delta, - ttl=RoutingArgsTTL, - parent_otel_span=parent_otel_span, - ) + for key, delta in _reconcile_increments(reservation, billable_input, completion_tokens): + await dual_cache.async_increment_cache( + key=key, value=delta, ttl=RoutingArgsTTL, parent_otel_span=parent_otel_span + ) else: verbose_router_logger.debug( "[IO TOKEN LIMIT] usage missing; keeping reservation (itpm_reserved=%s, otpm_reserved=%s)", diff --git a/litellm/router_utils/pre_call_checks/model_rate_limit_check.py b/litellm/router_utils/pre_call_checks/model_rate_limit_check.py index f4b41211c6e..f41d19cd62c 100644 --- a/litellm/router_utils/pre_call_checks/model_rate_limit_check.py +++ b/litellm/router_utils/pre_call_checks/model_rate_limit_check.py @@ -26,7 +26,6 @@ from litellm.caching.dual_cache import DualCache from litellm.caching.redis_cache import RedisCircuitBreakerOpenError from litellm.integrations.custom_logger import CustomLogger from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import ( - ITPM_RESERVED_KEY, async_io_token_pre_call_check, async_io_token_reconcile_success, async_io_token_refund_failure, @@ -485,24 +484,12 @@ class ModelRateLimitingCheck(CustomLogger): try: standard_logging_object: Final[StandardLoggingPayload | None] = kwargs.get("standard_logging_object") - - # IO token reconciliation works purely from the cache keys stashed in - # kwargs/metadata, so it must run before the model_id guard below - # (which only the TPM-tracking path needs). Otherwise a request whose - # standard_logging_object lacks model_id would never return its - # reservation, leaving the counter elevated until the TTL expires. - slo_metadata: Final = (standard_logging_object.get("metadata") or {}) if standard_logging_object else {} - kwargs_metadata: Final = kwargs.get("metadata") or {} - if ITPM_RESERVED_KEY in slo_metadata or ITPM_RESERVED_KEY in kwargs_metadata: - await async_io_token_reconcile_success( - self.dual_cache, - kwargs, - response_obj, - parent_otel_span=get_parent_otel_span_from_kwargs(kwargs), - ) - # Fall through: a deployment can also configure tpm/rpm alongside - # itpm/otpm, and that path's pre-call check reads the tpm_key - # counter tracked below, so it must still be incremented here. + await async_io_token_reconcile_success( + self.dual_cache, + kwargs, + response_obj, + parent_otel_span=get_parent_otel_span_from_kwargs(kwargs), + ) if standard_logging_object is None: return @@ -557,17 +544,7 @@ class ModelRateLimitingCheck(CustomLogger): """ try: standard_logging_object: Final[StandardLoggingPayload | None] = kwargs.get("standard_logging_object") - slo_metadata: Final = (standard_logging_object.get("metadata") or {}) if standard_logging_object else {} - kwargs_metadata: Final = kwargs.get("metadata") or {} - if ITPM_RESERVED_KEY in slo_metadata or ITPM_RESERVED_KEY in kwargs_metadata: - io_token_reconcile_success( - self.dual_cache, - kwargs, - response_obj, - ) - # Fall through: a deployment can also configure tpm/rpm alongside - # itpm/otpm, and that path's pre-call check reads the tpm_key - # counter tracked below, so it must still be incremented here. + io_token_reconcile_success(self.dual_cache, kwargs, response_obj) if standard_logging_object is None: return diff --git a/tests/integration/routing/test_otpm_default_reservation_fraction.py b/tests/integration/routing/test_otpm_default_reservation_fraction.py new file mode 100644 index 00000000000..589a2535a03 --- /dev/null +++ b/tests/integration/routing/test_otpm_default_reservation_fraction.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +import dataclasses +import threading +import uuid +from collections.abc import Callable, Iterator +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from queue import SimpleQueue +from typing import Final + +import httpx +import pytest +from integration._support.client import JSON_OBJECT, gateway_from_environment +from integration._support.openai_wire import chat_reply +from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from integration.providers._cache_control_marks_support import owned_config +from integration.routing.test_payment_required_mapping import OPENAI_MODEL, is_model_info_probe, model_list_reply + +pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120) + +GROUPS: Final = ("otpm-fraction-first", "otpm-fraction-retry") +OTPM_LIMIT: Final = 1000 +FRACTION: Final = 0.5 +DEFAULT_RESERVATION: Final = int(OTPM_LIMIT * FRACTION) + + +@dataclass(frozen=True, slots=True) +class Rig: + owned: OwnedProxy + upstream: Wire + gates: SimpleQueue[threading.Event] + + +@dataclass(frozen=True, slots=True) +class Admissions: + held_forwarded_max_tokens: bool + statuses: tuple[int, int, int] + one_minute: bool + + +def _reply_for(gates: SimpleQueue[threading.Event]) -> Callable[[Request], Reply]: + def reply(request: Request) -> Reply: + if is_model_info_probe(request): + return model_list_reply() + if JSON_OBJECT.validate_json(request.body).get("stream") is True: + return dataclasses.replace( + chat_reply("chatcmpl-held", OPENAI_MODEL, "held", stream=True), gate_before_first=gates.get(timeout=60) + ) + return chat_reply("chatcmpl-capped", OPENAI_MODEL, "capped", stream=False) + + return reply + + +def _config(directory: Path, upstream: Wire) -> Path: + return owned_config( + directory, + [ + { + "model_name": group, + "litellm_params": { + "model": OPENAI_MODEL, + "api_base": upstream.url + "/v1", + "api_key": "synthetic-openai-key", + "otpm": OTPM_LIMIT, + }, + "model_info": {"id": group}, + } + for group in GROUPS + ], + litellm_settings={"cache": False}, + router_settings={"optional_pre_call_checks": ["enforce_model_rate_limits"], "num_retries": 0}, + ) + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + directory: Final = tmp_path_factory.mktemp("otpm-fraction") + gates: Final[SimpleQueue[threading.Event]] = SimpleQueue() + with ( + gateway_from_environment() as gateway, + wire_server(_reply_for(gates)) as upstream, + owned_proxy_process( + gateway, + directory, + {"NO_MAX_TOKENS_TPM_RESERVATION_FRACTION": str(FRACTION)}, + config=_config(directory, upstream), + ) as owned, + ): + yield Rig(owned, upstream, gates) + + +def _chat(rig: Rig, group: str, **params: int | bool) -> httpx.Response: + return rig.owned.gateway.request( + "POST", + "/v1/chat/completions", + {"model": group, "messages": [{"role": "user", "content": f"fraction {uuid.uuid4().hex}"}], **params}, + ) + + +def _await_forwarded_chat(upstream: Wire) -> Request: + request: Final = upstream.received.get(timeout=60) + return _await_forwarded_chat(upstream) if is_model_info_probe(request) else request + + +def _utc_minute() -> str: + return datetime.now(UTC).strftime("%Y-%m-%d %H:%M") + + +def _admissions_beside_a_held_stream(rig: Rig, group: str) -> Admissions: + release: Final = threading.Event() + rig.gates.put(release) + started: Final = _utc_minute() + with ThreadPoolExecutor(max_workers=1) as pool: + held: Final = pool.submit(_chat, rig, group, stream=True) + try: + forwarded: Final = JSON_OBJECT.validate_json(_await_forwarded_chat(rig.upstream).body) + over: Final = _chat(rig, group, max_tokens=OTPM_LIMIT - DEFAULT_RESERVATION + 1) + fits: Final = _chat(rig, group, max_tokens=OTPM_LIMIT - DEFAULT_RESERVATION) + ended: Final = _utc_minute() + finally: + release.set() + return Admissions( + held_forwarded_max_tokens="max_tokens" in forwarded, + statuses=(over.status_code, fits.status_code, held.result(timeout=60).status_code), + one_minute=started == ended, + ) + + +def test_reservation_fraction_env_var_sets_what_a_request_without_max_tokens_holds(rig: Rig) -> None: + first: Final = _admissions_beside_a_held_stream(rig, GROUPS[0]) + admissions: Final = first if first.one_minute else _admissions_beside_a_held_stream(rig, GROUPS[1]) + + assert admissions == Admissions(held_forwarded_max_tokens=False, statuses=(429, 200, 200), one_minute=True) diff --git a/tests/unit/litellm_core_utils/test_env_utils.py b/tests/unit/litellm_core_utils/test_env_utils.py index 1ddd172e274..89b091e47c2 100644 --- a/tests/unit/litellm_core_utils/test_env_utils.py +++ b/tests/unit/litellm_core_utils/test_env_utils.py @@ -6,6 +6,7 @@ import pytest from litellm._logging import verbose_logger from litellm.litellm_core_utils.env_utils import ( get_env_float, + get_env_float_in_range, get_env_float_or_none, get_env_int, get_env_int_or_none, @@ -94,3 +95,12 @@ def test_unparsable_stdout_log_limit_does_not_break_logging( assert "first line" in messages assert "second line" in messages assert len([message for message in messages if raw in message]) == 1 + + +@pytest.mark.parametrize(("raw", "expected"), [("0.5", 0.5), ("1", 1.0), ("4", 0.25), ("0", 0.25), ("half", 0.25)]) +def test_float_in_range_keeps_values_inside_the_range_and_falls_back_otherwise( + monkeypatch: pytest.MonkeyPatch, raw: str, expected: float +) -> None: + monkeypatch.setenv("ENV_UTILS_TEST_FRACTION", raw) + + assert get_env_float_in_range("ENV_UTILS_TEST_FRACTION", 0.25, 0.01, 1.0) == expected diff --git a/tests/unit/test_constants.py b/tests/unit/test_constants.py index 99b049f1803..d9c34e95364 100644 --- a/tests/unit/test_constants.py +++ b/tests/unit/test_constants.py @@ -19,7 +19,14 @@ from unittest.mock import patch _ENV_HELPERS = frozenset( - {"get_env_int", "get_env_float", "get_env_int_in_range", "get_env_int_or_none", "get_env_float_or_none"} + { + "get_env_int", + "get_env_float", + "get_env_int_in_range", + "get_env_float_in_range", + "get_env_int_or_none", + "get_env_float_or_none", + } ) diff --git a/tests/unit/test_router/test_io_token_rate_limits.py b/tests/unit/test_router/test_io_token_rate_limits.py index 05921aa70b3..e5ff4531f84 100644 --- a/tests/unit/test_router/test_io_token_rate_limits.py +++ b/tests/unit/test_router/test_io_token_rate_limits.py @@ -3,6 +3,8 @@ Tests for separate ITPM/OTPM deployment rate limits (enforce_model_rate_limits). """ import asyncio +from collections.abc import Awaitable, Callable, Mapping +from datetime import datetime from typing import Final import pytest @@ -10,13 +12,16 @@ import pytest import litellm from litellm import Router from litellm.caching.dual_cache import DualCache +from litellm.constants import NO_MAX_TOKENS_TPM_RESERVATION_FRACTION +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import ( ITPM_CACHE_KEY, ITPM_RESERVED_KEY, OTPM_CACHE_KEY, OTPM_RESERVED_KEY, + _explicit_max_tokens, _reservation_value, - _resolve_max_tokens, async_io_token_pre_call_check, async_io_token_reconcile_success, build_io_token_rate_limit_headers, @@ -30,7 +35,16 @@ from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import ( from litellm.router_utils.pre_call_checks.model_rate_limit_check import ( ModelRateLimitingCheck, ) +from litellm.types.router import RouterCacheEnum from litellm.types.utils import ModelResponse, Usage +from litellm.utils import get_utc_datetime + +MANTLE_OPUS = "bedrock_mantle/anthropic.claude-opus-4-7" +MANTLE_OPUS_MAX_OUTPUT = litellm.get_model_info(model=MANTLE_OPUS)["max_output_tokens"] +UNMAPPED_MODEL = "openai/litellm-io-unmapped-model" +UNMAPPED_MODEL_DEFAULT_MAX_OUTPUT = 4096 +AZURE_BASE_MODEL = "azure/gpt-5" +AZURE_BASE_MODEL_MAX_OUTPUT = litellm.get_model_info(model=AZURE_BASE_MODEL)["max_output_tokens"] class TestIOTokenRateLimitHelpers: @@ -48,13 +62,23 @@ class TestIOTokenRateLimitHelpers: # A real non-zero estimate is reserved as-is. assert _reservation_value(42, 100) == 42 - def test_resolve_max_tokens_respects_explicit_zero(self): - deployment = {"litellm_params": {"model": "openai/gpt-4o-mini"}} - # An explicit max_tokens=0 is honored, not replaced by the model default. - assert _resolve_max_tokens({"max_tokens": 0}, deployment) == 0 - # max_completion_tokens is the fallback only when max_tokens is absent. - assert _resolve_max_tokens({"max_completion_tokens": 12}, deployment) == 12 - assert _resolve_max_tokens({"max_output_tokens": 9}, deployment) == 9 + @pytest.mark.parametrize( + "request_kwargs, expected", + [ + ({"max_tokens": 0}, 0), + ({"max_completion_tokens": 12}, 12), + ({"max_output_tokens": 9}, 9), + ({"max_tokens": 5, "max_completion_tokens": 12}, 5), + ({"max_tokens": None, "max_output_tokens": 9}, 9), + ({"max_tokens": [100]}, None), + ({"messages": []}, None), + (None, None), + ], + ) + def test_explicit_max_tokens_reads_every_output_cap_key( + self, request_kwargs: dict[str, object] | None, expected: int | None + ): + assert _explicit_max_tokens(request_kwargs) == expected def test_build_io_token_rate_limit_headers(self): headers = build_io_token_rate_limit_headers( @@ -386,6 +410,244 @@ class TestModelRateLimitingCheckIOTokens: normal_output_otpm_key = normal_output_kwargs["metadata"][OTPM_CACHE_KEY] assert await dual_cache.async_get_cache(key=normal_output_otpm_key) == 5 + @pytest.mark.asyncio + @pytest.mark.parametrize("use_sync_check", [False, True]) + @pytest.mark.parametrize( + "model, otpm_limit", + [ + (MANTLE_OPUS, MANTLE_OPUS_MAX_OUTPUT // 2), + (MANTLE_OPUS, MANTLE_OPUS_MAX_OUTPUT - 1), + (UNMAPPED_MODEL, 1000), + (UNMAPPED_MODEL, 3), + ], + ) + async def test_request_without_max_tokens_is_admitted_below_model_max_output( + self, model: str, otpm_limit: int, use_sync_check: bool + ): + dual_cache: Final = DualCache() + check: Final = ModelRateLimitingCheck(dual_cache=dual_cache) + deployment: Final = { + "litellm_params": {"model": model, "otpm": otpm_limit}, + "model_info": {"id": f"io-no-max-tokens-{otpm_limit}"}, + "model_name": "no-max-tokens", + } + request_kwargs: Final = {"messages": [{"role": "user", "content": "hello"}], "metadata": {}} + set_io_token_rate_limit_request_kwargs(request_kwargs) + + if use_sync_check: + check.pre_call_check(deployment) + else: + await check.async_pre_call_check(deployment) + + reserved: Final = request_kwargs["metadata"][OTPM_RESERVED_KEY] + assert isinstance(reserved, int) + assert 0 < reserved < otpm_limit + assert await dual_cache.async_get_cache(key=request_kwargs["metadata"][OTPM_CACHE_KEY]) == reserved + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "model, otpm_limit", + [(MANTLE_OPUS, MANTLE_OPUS_MAX_OUTPUT), (UNMAPPED_MODEL, UNMAPPED_MODEL_DEFAULT_MAX_OUTPUT)], + ) + async def test_concurrent_requests_without_max_tokens_share_otpm_without_overshoot( + self, model: str, otpm_limit: int + ): + dual_cache: Final = DualCache() + check: Final = ModelRateLimitingCheck(dual_cache=dual_cache) + deployment: Final = { + "litellm_params": {"model": model, "otpm": otpm_limit}, + "model_info": {"id": "io-no-max-tokens-race-id"}, + "model_name": "no-max-tokens", + } + + async def _attempt() -> dict[str, object] | None: + request_kwargs: Final = {"messages": [{"role": "user", "content": "hi"}], "metadata": {}} + set_io_token_rate_limit_request_kwargs(request_kwargs) + try: + await check.async_pre_call_check(deployment) + except litellm.RateLimitError: + return None + return request_kwargs["metadata"] + + admitted: Final = [m for m in await asyncio.gather(*[_attempt() for _ in range(8)]) if m is not None] + otpm_key: Final = admitted[0][OTPM_CACHE_KEY] + + assert len(admitted) == round(1 / NO_MAX_TOKENS_TPM_RESERVATION_FRACTION) + assert {m[OTPM_CACHE_KEY] for m in admitted} == {otpm_key} + assert await dual_cache.async_get_cache(key=otpm_key) == sum(m[OTPM_RESERVED_KEY] for m in admitted) + assert await dual_cache.async_get_cache(key=otpm_key) <= otpm_limit + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "model, model_max_output", + [(MANTLE_OPUS, MANTLE_OPUS_MAX_OUTPUT), (UNMAPPED_MODEL, UNMAPPED_MODEL_DEFAULT_MAX_OUTPUT)], + ) + async def test_request_without_max_tokens_reserves_model_max_output_when_otpm_allows( + self, model: str, model_max_output: int + ): + dual_cache: Final = DualCache() + check: Final = ModelRateLimitingCheck(dual_cache=dual_cache) + deployment: Final = { + "litellm_params": {"model": model, "otpm": model_max_output * 10}, + "model_info": {"id": "io-no-max-tokens-roomy-id"}, + "model_name": "roomy", + } + request_kwargs: Final = {"messages": [{"role": "user", "content": "hello"}], "metadata": {}} + set_io_token_rate_limit_request_kwargs(request_kwargs) + + await check.async_pre_call_check(deployment) + + assert request_kwargs["metadata"][OTPM_RESERVED_KEY] == model_max_output + + @pytest.mark.asyncio + @pytest.mark.parametrize("use_sync_check", [False, True]) + async def test_base_model_does_not_change_the_default_otpm_reservation(self, use_sync_check: bool): + check: Final = ModelRateLimitingCheck(dual_cache=DualCache()) + otpm_limit: Final = AZURE_BASE_MODEL_MAX_OUTPUT * 10 + + async def _reserved(model_info: dict[str, str]) -> int: + request_kwargs: Final = {"messages": [{"role": "user", "content": "hello"}], "metadata": {}} + set_io_token_rate_limit_request_kwargs(request_kwargs) + deployment: Final = { + "litellm_params": {"model": "azure/litellm-io-custom-deployment", "otpm": otpm_limit}, + "model_info": model_info, + "model_name": "custom-name", + } + if use_sync_check: + check.pre_call_check(deployment) + else: + await check.async_pre_call_check(deployment) + return request_kwargs["metadata"][OTPM_RESERVED_KEY] + + with_base_model: Final = await _reserved({"id": "io-with-base-model", "base_model": AZURE_BASE_MODEL}) + without_base_model: Final = await _reserved({"id": "io-without-base-model"}) + + assert with_base_model == without_base_model + + @pytest.mark.asyncio + @pytest.mark.parametrize("use_sync_check", [False, True]) + async def test_unparseable_max_tokens_leaves_itpm_uncharged(self, use_sync_check: bool): + dual_cache: Final = DualCache() + check: Final = ModelRateLimitingCheck(dual_cache=dual_cache) + deployment: Final = { + "litellm_params": {"model": MANTLE_OPUS, "itpm": 2000, "otpm": 2000}, + "model_info": {"id": "io-bad-max-tokens-id"}, + "model_name": "opus", + } + + async def _pre_call(request_kwargs: dict[str, object]) -> None: + set_io_token_rate_limit_request_kwargs(request_kwargs) + if use_sync_check: + check.pre_call_check(deployment) + else: + await check.async_pre_call_check(deployment) + + await _pre_call({"messages": [{"role": "user", "content": "hello there"}], "max_tokens": "abc", "metadata": {}}) + valid_kwargs: Final = { + "messages": [{"role": "user", "content": "hello there"}], + "max_tokens": 1, + "metadata": {}, + } + await _pre_call(valid_kwargs) + + itpm_counter: Final = await dual_cache.async_get_cache(key=valid_kwargs["metadata"][ITPM_CACHE_KEY]) + assert itpm_counter == valid_kwargs["metadata"][ITPM_RESERVED_KEY] + + @pytest.mark.asyncio + async def test_explicit_max_tokens_above_default_share_is_reserved_in_full(self): + dual_cache: Final = DualCache() + check: Final = ModelRateLimitingCheck(dual_cache=dual_cache) + deployment: Final = { + "litellm_params": {"model": MANTLE_OPUS, "otpm": 100}, + "model_info": {"id": "io-explicit-max-tokens-id"}, + "model_name": "opus", + } + request_kwargs: Final = {"messages": [{"role": "user", "content": "hello"}], "max_tokens": 100, "metadata": {}} + set_io_token_rate_limit_request_kwargs(request_kwargs) + + await check.async_pre_call_check(deployment) + + assert request_kwargs["metadata"][OTPM_RESERVED_KEY] == 100 + set_io_token_rate_limit_request_kwargs({"messages": [{"role": "user", "content": "hi"}], "metadata": {}}) + with pytest.raises(litellm.RateLimitError): + await check.async_pre_call_check(deployment) + + @pytest.mark.asyncio + @pytest.mark.parametrize("use_sync_check", [False, True]) + async def test_deployment_max_tokens_sets_the_reservation_unless_the_request_sets_one(self, use_sync_check: bool): + check: Final = ModelRateLimitingCheck(dual_cache=DualCache()) + deployment: Final = { + "litellm_params": {"model": MANTLE_OPUS, "otpm": 4000, "max_tokens": 100}, + "model_info": {"id": "io-deployment-max-tokens-id"}, + "model_name": "opus", + } + without_cap: Final = {"messages": [{"role": "user", "content": "hello"}], "metadata": {}} + with_cap: Final = {"messages": [{"role": "user", "content": "hello"}], "max_tokens": 300, "metadata": {}} + + for request_kwargs in (without_cap, with_cap): + set_io_token_rate_limit_request_kwargs(request_kwargs) + if use_sync_check: + check.pre_call_check(deployment) + else: + await check.async_pre_call_check(deployment) + + assert without_cap["metadata"][OTPM_RESERVED_KEY] == 100 + assert with_cap["metadata"][OTPM_RESERVED_KEY] == 300 + + @pytest.mark.asyncio + async def test_deployment_cap_sets_the_reservation_only_through_the_param_its_api_sends(self): + check: Final = ModelRateLimitingCheck(dual_cache=DualCache()) + chat: Final = {"messages": [{"role": "user", "content": "hello"}]} + responses: Final = {"input": "hello"} + + async def reserved(deployment_cap: Mapping[str, int], request_shape: Mapping[str, object]) -> int: + request_kwargs: Final = {**request_shape, "metadata": {}} + set_io_token_rate_limit_request_kwargs(request_kwargs) + await check.async_pre_call_check( + { + "litellm_params": {"model": MANTLE_OPUS, "otpm": 4000, **deployment_cap}, + "model_info": {"id": f"io-sent-cap-{sorted(deployment_cap)}-{sorted(request_shape)}"}, + "model_name": "opus", + } + ) + return request_kwargs["metadata"][OTPM_RESERVED_KEY] + + chat_default: Final = await reserved({}, chat) + responses_default: Final = await reserved({}, responses) + + assert chat_default == responses_default > 100 + assert await reserved({"max_tokens": 100}, chat) == 100 + assert await reserved({"max_completion_tokens": 100}, chat) == 100 + assert await reserved({"max_output_tokens": 100}, chat) == chat_default + assert await reserved({"max_output_tokens": 100}, responses) == 100 + assert await reserved({"max_tokens": 100}, responses) == responses_default + assert await reserved({"max_completion_tokens": 100}, responses) == responses_default + + @pytest.mark.asyncio + @pytest.mark.parametrize("use_sync_check", [False, True]) + async def test_deployment_cap_above_the_default_reserves_the_default(self, use_sync_check: bool): + check: Final = ModelRateLimitingCheck(dual_cache=DualCache()) + otpm_limit: Final = UNMAPPED_MODEL_DEFAULT_MAX_OUTPUT * 5 + + async def reserved(deployment_cap: Mapping[str, int]) -> int: + request_kwargs: Final = {"messages": [{"role": "user", "content": "hello"}], "metadata": {}} + set_io_token_rate_limit_request_kwargs(request_kwargs) + deployment: Final = { + "litellm_params": {"model": UNMAPPED_MODEL, "otpm": otpm_limit, **deployment_cap}, + "model_info": {"id": f"io-deployment-cap-above-default-{deployment_cap}"}, + "model_name": "unmapped", + } + if use_sync_check: + check.pre_call_check(deployment) + else: + await check.async_pre_call_check(deployment) + return request_kwargs["metadata"][OTPM_RESERVED_KEY] + + default: Final = await reserved({}) + + assert await reserved({"max_tokens": default + 1}) == default + assert await reserved({"max_tokens": otpm_limit * 2}) == default + def test_sync_io_pre_call_reserves_and_reconciles(self): from litellm.utils import get_utc_datetime @@ -415,9 +677,9 @@ class TestModelRateLimitingCheckIOTokens: "standard_logging_object": { "model_id": "io-sync-id", "hidden_params": {"litellm_model_name": "bedrock_mantle/anthropic.claude-opus-4-7"}, - "metadata": dict(request_kwargs["metadata"]), + "metadata": {}, }, - "metadata": request_kwargs["metadata"], + "litellm_params": {"metadata": request_kwargs["metadata"]}, } response = ModelResponse( choices=[{"message": {"role": "assistant", "content": "hi"}, "index": 0, "finish_reason": "stop"}], @@ -709,6 +971,32 @@ class TestModelRateLimitingCheckIOTokens: assert await dual_cache.async_get_cache(key=itpm_key) == 4 + @pytest.mark.asyncio + @pytest.mark.parametrize("use_sync_reconcile", [False, True]) + async def test_reconcile_charges_usage_above_the_reservation_to_the_minute_it_finished_in( + self, use_sync_reconcile: bool + ): + dual_cache: Final = DualCache() + counter: Final = "global_router:io-late-id:bedrock_mantle/test:otpm" + reserved_minute_key: Final = f"{counter}:99-99" + await dual_cache.async_increment_cache(key=reserved_minute_key, value=10, ttl=60) + kwargs: Final = {"metadata": {ITPM_RESERVED_KEY: 0, OTPM_RESERVED_KEY: 10, OTPM_CACHE_KEY: reserved_minute_key}} + response: Final = ModelResponse( + choices=[{"message": {"role": "assistant", "content": "hi"}, "index": 0, "finish_reason": "stop"}], + usage=Usage(prompt_tokens=4, completion_tokens=25, total_tokens=29), + ) + minute_before: Final = get_utc_datetime().strftime("%H-%M") + + if use_sync_reconcile: + io_token_reconcile_success(dual_cache, kwargs, response) + else: + await async_io_token_reconcile_success(dual_cache, kwargs, response) + + finished_minutes: Final = {minute_before, get_utc_datetime().strftime("%H-%M")} + charged: Final = [await dual_cache.async_get_cache(key=f"{counter}:{minute}") for minute in finished_minutes] + assert await dual_cache.async_get_cache(key=reserved_minute_key) == 10 + assert sum(value or 0 for value in charged) == 25 - 10 + @pytest.mark.asyncio async def test_reconcile_missing_usage_keeps_reservation(self): dual_cache = DualCache() @@ -799,16 +1087,16 @@ class TestModelRateLimitingCheckIOTokens: assert await dual_cache.async_get_cache(key=itpm_key) == 4 - def test_sync_reconcile_anthropic_dict_usage(self): + def test_sync_reconcile_anthropic_dict_usage_counts_uncached_input_once(self): dual_cache = DualCache() itpm_key = "global_router:io-anthropic:bedrock_mantle/test:itpm:00-00" otpm_key = "global_router:io-anthropic:bedrock_mantle/test:otpm:00-00" - dual_cache.set_cache(key=itpm_key, value=6, ttl=60) + dual_cache.set_cache(key=itpm_key, value=10, ttl=60) dual_cache.set_cache(key=otpm_key, value=4, ttl=60) kwargs = { "metadata": { - ITPM_RESERVED_KEY: 6, + ITPM_RESERVED_KEY: 10, OTPM_RESERVED_KEY: 4, ITPM_CACHE_KEY: itpm_key, OTPM_CACHE_KEY: otpm_key, @@ -816,12 +1104,17 @@ class TestModelRateLimitingCheckIOTokens: } response = { "type": "message", - "usage": {"input_tokens": 3, "output_tokens": 2, "cache_read_input_tokens": 1}, + "usage": { + "input_tokens": 3, + "output_tokens": 2, + "cache_read_input_tokens": 1, + "cache_creation_input_tokens": 4, + }, } io_token_reconcile_success(dual_cache, kwargs, response) - assert dual_cache.get_cache(key=itpm_key) == 2 + assert dual_cache.get_cache(key=itpm_key) == 3 + 4 assert dual_cache.get_cache(key=otpm_key) == 2 @pytest.mark.asyncio @@ -958,7 +1251,214 @@ class TestModelRateLimitingCheckIOTokens: assert current == 0 +ROUTER_IO_MODEL: Final = "anthropic/claude-haiku-5-5" +RouterCall = Callable[[Router, str | Exception], Awaitable[tuple[int, int]]] + + +def _router_with_io_limits() -> Router: + return Router( + model_list=[ + { + "model_name": "limited", + "litellm_params": {"model": ROUTER_IO_MODEL, "api_key": "sk-test", "itpm": 50000, "otpm": 50000}, + } + ], + optional_pre_call_checks=["enforce_model_rate_limits"], + num_retries=1, + ) + + +def _current_minute() -> str: + return get_utc_datetime().strftime("%H-%M") + + +def _counter(router: Router, template: RouterCacheEnum, minutes: frozenset[str]) -> int: + deployment: Final = router.model_list[0] + return sum( + router.cache.get_cache( + key=template.value.format( + id=deployment["model_info"]["id"], + model=deployment["litellm_params"]["model"], + current_minute=minute, + ) + ) + or 0 + for minute in minutes + ) + + +async def _io_counters_since(router: Router, first_minute: str) -> tuple[int, int]: + await asyncio.sleep(0) + await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10) + minutes: Final = frozenset({first_minute, _current_minute()}) + return _counter(router, RouterCacheEnum.ITPM, minutes), _counter(router, RouterCacheEnum.OTPM, minutes) + + +async def _sdk_chat(router: Router, mock_response: str | Exception) -> tuple[int, int]: + response: Final = await router.acompletion( + model="limited", + messages=[{"role": "user", "content": "hello there"}], + max_tokens=100, + mock_response=mock_response, + ) + return response.usage.prompt_tokens, response.usage.completion_tokens + + +async def _proxy_chat(router: Router, mock_response: str | Exception) -> tuple[int, int]: + response: Final = await router.acompletion( + model="limited", + messages=[{"role": "user", "content": "hello there"}], + max_tokens=100, + metadata={"user_api_key_hash": "hashed-key"}, + mock_response=mock_response, + ) + return response.usage.prompt_tokens, response.usage.completion_tokens + + +async def _proxy_responses(router: Router, mock_response: str | Exception) -> tuple[int, int]: + response: Final = await router.aresponses( + model="limited", + input="hello there", + max_output_tokens=100, + litellm_metadata={"user_api_key_hash": "hashed-key"}, + mock_response=mock_response, + ) + return response.usage.input_tokens, response.usage.output_tokens + + +async def _proxy_messages(router: Router, mock_response: str | Exception) -> tuple[int, int]: + response: Final = await router.aanthropic_messages( + model="limited", + messages=[{"role": "user", "content": "hello there"}], + max_tokens=100, + litellm_metadata={"user_api_key_hash": "hashed-key"}, + mock_response=mock_response, + ) + return response["usage"]["input_tokens"], response["usage"]["output_tokens"] + + +ProxyCallSharingLogging = Callable[[Router, str | Exception, LiteLLMLoggingObject], Awaitable[object]] +PROXY_MESSAGES: Final = [{"role": "user", "content": "hello there"}] + + +def _logging_obj_shared_across_attempts(original_function: str) -> LiteLLMLoggingObject: + logging_obj, _ = litellm.utils.function_setup( + original_function, + litellm.utils.Rules(), + datetime.now(), + model="limited", + messages=PROXY_MESSAGES, + litellm_call_id="proxy-call", + ) + return logging_obj + + +async def _proxy_chat_sharing_logging( + router: Router, mock_error: str | Exception, logging_obj: LiteLLMLoggingObject +) -> object: + return await router.acompletion( + model="limited", + messages=PROXY_MESSAGES, + max_tokens=100, + metadata={"user_api_key_hash": "hashed-key"}, + mock_response=mock_error, + litellm_call_id="proxy-call", + litellm_logging_obj=logging_obj, + ) + + +async def _proxy_responses_sharing_logging( + router: Router, mock_error: str | Exception, logging_obj: LiteLLMLoggingObject +) -> object: + return await router.aresponses( + model="limited", + input="hello there", + max_output_tokens=100, + litellm_metadata={"user_api_key_hash": "hashed-key"}, + mock_response=mock_error, + litellm_call_id="proxy-call", + litellm_logging_obj=logging_obj, + ) + + +async def _proxy_messages_sharing_logging( + router: Router, mock_error: str | Exception, logging_obj: LiteLLMLoggingObject +) -> object: + return await router.aanthropic_messages( + model="limited", + messages=PROXY_MESSAGES, + max_tokens=100, + litellm_metadata={"user_api_key_hash": "hashed-key"}, + mock_response=mock_error, + litellm_call_id="proxy-call", + litellm_logging_obj=logging_obj, + ) + + class TestRouterIOTokenIntegration: + @pytest.mark.asyncio + @pytest.mark.parametrize( + "call", + [_sdk_chat, _proxy_chat, _proxy_responses, _proxy_messages], + ids=["sdk_chat", "proxy_chat_metadata", "proxy_responses_litellm_metadata", "proxy_messages_litellm_metadata"], + ) + async def test_success_reconciles_io_counters_to_real_usage(self, call: RouterCall): + router: Final = _router_with_io_limits() + first_minute: Final = _current_minute() + + usage: Final = await call(router, "hi") + + assert await _io_counters_since(router, first_minute) == usage + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("call", "mock_error"), + [ + ( + _proxy_responses, + litellm.InternalServerError(message="mock 500", llm_provider="anthropic", model=ROUTER_IO_MODEL), + ), + (_proxy_messages, "litellm.InternalServerError"), + ], + ids=["proxy_responses_litellm_metadata", "proxy_messages_litellm_metadata"], + ) + async def test_retried_failure_refunds_each_io_reservation_once( + self, call: RouterCall, mock_error: str | Exception + ): + router: Final = _router_with_io_limits() + first_minute: Final = _current_minute() + + with pytest.raises(litellm.InternalServerError): + await call(router, mock_error) + + assert await _io_counters_since(router, first_minute) == (0, 0) + + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("call", "original_function", "mock_error"), + [ + (_proxy_chat_sharing_logging, "acompletion", "litellm.InternalServerError"), + ( + _proxy_responses_sharing_logging, + "aresponses", + litellm.InternalServerError(message="mock 500", llm_provider="anthropic", model=ROUTER_IO_MODEL), + ), + (_proxy_messages_sharing_logging, "anthropic_messages", "litellm.InternalServerError"), + ], + ids=["proxy_chat", "proxy_responses", "proxy_messages"], + ) + async def test_retried_failure_sharing_one_logging_object_refunds_every_attempt( + self, call: ProxyCallSharingLogging, original_function: str, mock_error: str | Exception + ): + router: Final = _router_with_io_limits() + first_minute: Final = _current_minute() + + with pytest.raises(litellm.InternalServerError): + await call(router, mock_error, _logging_obj_shared_across_attempts(original_function)) + + assert await _io_counters_since(router, first_minute) == (0, 0) + @pytest.mark.asyncio async def test_model_group_info_aggregates_io_limits(self): router = Router( From ccc56d27adeeece2a1069bde587c76486ba9ab39 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 21:04:22 -0700 Subject: [PATCH 05/12] fix(azure): add batch prices to the unversioned o3, o4-mini, o3-mini, gpt-4o and gpt-4o-mini rows (#49182) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../model_prices_and_context_window_backup.json | 15 +++++++++++++++ model_prices_and_context_window.json | 15 +++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 96a04c96f44..0799a298942 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -6014,11 +6014,14 @@ "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 2.5e-06, + "input_cost_per_token_batches": 1.25e-06, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, + "output_cost_per_token_batches": 5e-06, + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -6313,16 +6316,19 @@ "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 7.5e-08, "input_cost_per_token": 1.5e-07, + "input_cost_per_token_batches": 7.5e-08, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, + "output_cost_per_token_batches": 3e-07, "search_context_cost_per_query": { "search_context_size_high": 0.014, "search_context_size_low": 0.014, "search_context_size_medium": 0.014 }, + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -11303,16 +11309,19 @@ "deprecation_date": "2026-11-19", "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 2e-06, + "input_cost_per_token_batches": 1e-06, "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, "mode": "chat", "output_cost_per_token": 8e-06, + "output_cost_per_token_batches": 4e-06, "search_context_cost_per_query": { "search_context_size_high": 0.014, "search_context_size_low": 0.014, "search_context_size_medium": 0.014 }, + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -11411,11 +11420,14 @@ "deprecation_date": "2026-11-19", "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 1.1e-06, + "input_cost_per_token_batches": 5.5e-07, "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, + "output_cost_per_token_batches": 2.2e-06, + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -11513,16 +11525,19 @@ "deprecation_date": "2026-11-19", "cache_read_input_token_cost": 2.75e-07, "input_cost_per_token": 1.1e-06, + "input_cost_per_token_batches": 5.5e-07, "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, + "output_cost_per_token_batches": 2.2e-06, "search_context_cost_per_query": { "search_context_size_high": 0.014, "search_context_size_low": 0.014, "search_context_size_medium": 0.014 }, + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 96a04c96f44..0799a298942 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -6014,11 +6014,14 @@ "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 2.5e-06, + "input_cost_per_token_batches": 1.25e-06, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 1e-05, + "output_cost_per_token_batches": 5e-06, + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -6313,16 +6316,19 @@ "deprecation_date": "2027-04-14", "cache_read_input_token_cost": 7.5e-08, "input_cost_per_token": 1.5e-07, + "input_cost_per_token_batches": 7.5e-08, "litellm_provider": "azure", "max_input_tokens": 128000, "max_output_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, + "output_cost_per_token_batches": 3e-07, "search_context_cost_per_query": { "search_context_size_high": 0.014, "search_context_size_low": 0.014, "search_context_size_medium": 0.014 }, + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_prompt_caching": true, @@ -11303,16 +11309,19 @@ "deprecation_date": "2026-11-19", "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 2e-06, + "input_cost_per_token_batches": 1e-06, "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, "mode": "chat", "output_cost_per_token": 8e-06, + "output_cost_per_token_batches": 4e-06, "search_context_cost_per_query": { "search_context_size_high": 0.014, "search_context_size_low": 0.014, "search_context_size_medium": 0.014 }, + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", @@ -11411,11 +11420,14 @@ "deprecation_date": "2026-11-19", "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 1.1e-06, + "input_cost_per_token_batches": 5.5e-07, "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, + "output_cost_per_token_batches": 2.2e-06, + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, @@ -11513,16 +11525,19 @@ "deprecation_date": "2026-11-19", "cache_read_input_token_cost": 2.75e-07, "input_cost_per_token": 1.1e-06, + "input_cost_per_token_batches": 5.5e-07, "litellm_provider": "azure", "max_input_tokens": 200000, "max_output_tokens": 100000, "mode": "chat", "output_cost_per_token": 4.4e-06, + "output_cost_per_token_batches": 2.2e-06, "search_context_cost_per_query": { "search_context_size_high": 0.014, "search_context_size_low": 0.014, "search_context_size_medium": 0.014 }, + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", From f32ac94ad1fc3bce79da08752c7bb6a17ebdbfa1 Mon Sep 17 00:00:00 2001 From: mika-lt <349045450+mika-lt@users.noreply.github.com> Date: Thu, 29 Oct 2026 23:30:02 -0700 Subject: [PATCH 06/12] fix(openai): keep the upstream error body on e.response for OpenAI and Azure errors (#49079) * fix(openai): keep the upstream error response on mapped exceptions so e.response.text has the wire body * fix: keep the upstream error body on 429 and 5xx errors RateLimitError, InternalServerError, BadGatewayError, and ServiceUnavailableError rebuilt their response and dropped the upstream body. They now keep the body and exactly the upstream headers, and an unread streaming response no longer raises. * fix: keep a gzip-encoded 429 or 5xx error readable The kept body is already decoded, so the rebuilt response drops content-encoding instead of decoding the body a second time and raising DecodingError. * fix: keep the upstream headers and drop credentials on every mapped provider error The copied error response now keeps every upstream header and carries a request with only the method and the URL, so no provider key, query secret, or prompt reaches e.request. Timeout and APIError get the response too, and azure/ chat, streaming, embedding, and image errors attach it * fix: keep the upstream error body on gateway timeouts and re-raised gateway errors * test(azure): cover the 404, 500, 502, and 504 error bodies and the missing-key placeholder * fix: keep no e.response on timeouts and errors litellm raises itself * fix: keep the upstream elapsed time on error responses and the upstream body on azure_text errors * fix: keep the upstream body without credentials on Azure content filter errors and message-matched OpenAI errors --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/exceptions.py | 93 +++--- .../exception_mapping_utils.py | 57 +++- litellm/llms/azure/azure.py | 12 +- litellm/llms/azure/completion/handler.py | 22 +- litellm/llms/azure/exception_mapping.py | 5 +- litellm/llms/openai/common_utils.py | 9 + litellm/llms/openai/completion/handler.py | 23 +- .../llms/openai/image_variations/handler.py | 16 +- litellm/llms/openai/openai.py | 24 +- .../test_exception_mapping_utils.py | 82 +++++- .../azure/test_azure_exception_mapping.py | 201 +++++++++++++ .../completion/test_completion_handler.py | 24 +- .../llms/openai/image_variations/__init__.py | 0 .../test_openai_image_variations_handler.py | 51 ++++ tests/unit/llms/openai/test_openai.py | 274 +++++++++++++++++- .../proxy/test_common_request_processing.py | 63 ++++ .../test_exception_header_preservation.py | 19 ++ 17 files changed, 895 insertions(+), 80 deletions(-) create mode 100644 tests/unit/llms/openai/image_variations/__init__.py create mode 100644 tests/unit/llms/openai/image_variations/test_openai_image_variations_handler.py diff --git a/litellm/exceptions.py b/litellm/exceptions.py index 28211b3dab2..12425c45b01 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -9,6 +9,7 @@ ## LiteLLM versions of the OpenAI Exception Types +import datetime import enum from collections.abc import Mapping, Sequence from typing import Any, Final @@ -115,6 +116,54 @@ def validate_rate_limit_type(value: object) -> str | None: return None +def _upstream_body(response: object) -> bytes: + if not isinstance(response, httpx.Response): + return b"" + try: + return response.content + except httpx.ResponseNotRead: + return b"" + + +def _request_without_credentials(upstream: object, fallback_url: str) -> httpx.Request: + if not isinstance(upstream, httpx.Response): + return httpx.Request(method="POST", url=fallback_url) + try: + upstream_request: Final = upstream.request + except RuntimeError: + return httpx.Request(method="POST", url=fallback_url) + return httpx.Request( + method=upstream_request.method, + url=upstream_request.url.copy_with(userinfo=b"", query=None, fragment=None), + ) + + +def _upstream_elapsed(upstream: object) -> datetime.timedelta | None: + if not isinstance(upstream, httpx.Response): + return None + try: + return upstream.elapsed + except RuntimeError: + return None + + +def error_response_without_credentials(status_code: int, upstream: object, fallback_url: str) -> httpx.Response: + response: Final = httpx.Response( + status_code=status_code, + stream=httpx.ByteStream(_upstream_body(upstream)), + request=_request_without_credentials(upstream, fallback_url), + ) + _ = response.read() + response.headers = httpx.Headers(upstream.headers if isinstance(upstream, httpx.Response) else None) + elapsed: Final = _upstream_elapsed(upstream) + if elapsed is not None: + response.elapsed = elapsed + return response + + +_NO_UPSTREAM_REQUEST_URL: Final = "https://cloud.google.com/vertex-ai/" + + _MINIMAL_ERROR_RESPONSE: httpx.Response | None = None @@ -519,20 +568,11 @@ class RateLimitError(openai.RateLimitError): # headers stay reachable on `e.response.headers` for callers that # explicitly want them; only the proxy-supplied `headers=` kwarg # makes it onto `self.headers`. - _response_headers: Final = getattr(response, "headers", None) if response is not None else None self.headers: dict[str, str] | None = {k: str(v) for k, v in headers.items()} if headers else None # Mirrors FastAPI HTTPException.detail so the same instance can be # serialized through both the ProxyException and HTTPException paths. self.detail = detail if detail is not None else self.message - self.response = httpx.Response( - status_code=429, - headers=_response_headers, - content=response.content if response is not None else None, - request=httpx.Request( - method="POST", - url=" https://cloud.google.com/vertex-ai/", - ), - ) + self.response = error_response_without_credentials(429, response, _NO_UPSTREAM_REQUEST_URL) super().__init__( self.message, response=self.response, # pyright: ignore[reportArgumentType] # SDK accepts HTTPX at runtime @@ -757,15 +797,7 @@ class ServiceUnavailableError(openai.APIStatusError): self.litellm_debug_info = litellm_debug_info self.max_retries = max_retries self.num_retries = num_retries - _response_headers: Final = getattr(response, "headers", None) if response is not None else None - self.response = httpx.Response( - status_code=self.status_code, - headers=_response_headers, - request=httpx.Request( - method="POST", - url=" https://cloud.google.com/vertex-ai/", - ), - ) + self.response = error_response_without_credentials(self.status_code, response, _NO_UPSTREAM_REQUEST_URL) super().__init__( self.message, response=self.response, body=None ) # Call the base class constructor with the parameters it needs @@ -808,15 +840,7 @@ class BadGatewayError(openai.APIStatusError): self.litellm_debug_info = litellm_debug_info self.max_retries = max_retries self.num_retries = num_retries - _response_headers: Final = getattr(response, "headers", None) if response is not None else None - self.response = httpx.Response( - status_code=self.status_code, - headers=_response_headers, - request=httpx.Request( - method="POST", - url=" https://cloud.google.com/vertex-ai/", - ), - ) + self.response = error_response_without_credentials(self.status_code, response, _NO_UPSTREAM_REQUEST_URL) super().__init__( self.message, response=self.response, # pyright: ignore[reportArgumentType] # SDK accepts HTTPX at runtime @@ -862,15 +886,7 @@ class InternalServerError(openai.InternalServerError): self.litellm_debug_info = litellm_debug_info self.max_retries = max_retries self.num_retries = num_retries - _response_headers: Final = getattr(response, "headers", None) if response is not None else None - self.response = httpx.Response( - status_code=self.status_code, - headers=_response_headers, - request=httpx.Request( - method="POST", - url=" https://cloud.google.com/vertex-ai/", - ), - ) + self.response = error_response_without_credentials(self.status_code, response, _NO_UPSTREAM_REQUEST_URL) super().__init__( self.message, response=self.response, # pyright: ignore[reportArgumentType] # SDK accepts HTTPX at runtime @@ -908,6 +924,7 @@ class APIError(openai.APIError): max_retries: int | None = None, num_retries: int | None = None, body: object | None = None, + response: httpx.Response | None = None, ): self.status_code = status_code self.message = f"litellm.APIError: {message}" @@ -916,6 +933,8 @@ class APIError(openai.APIError): self.litellm_debug_info = litellm_debug_info self.max_retries = max_retries self.num_retries = num_retries + if response is not None: + self.response = response if request is None: request = httpx.Request(method="POST", url="https://api.openai.com/v1") super().__init__(self.message, request=request, body=body) diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 64a95eb5f28..5a27f5446ad 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -37,6 +37,7 @@ from ..exceptions import ( ServiceUnavailableError, Timeout, UnprocessableEntityError, + error_response_without_credentials, ) @@ -305,6 +306,16 @@ def _litellm_proxy_response( ) +def _without_credentials(response: httpx.Response | None) -> httpx.Response | None: + if not isinstance(response, httpx.Response): + return response + return error_response_without_credentials(response.status_code, response, "https://docs.litellm.ai/docs") + + +def _upstream_reply(response: httpx.Response | None) -> httpx.Response | None: + return response if response is not None and (response.headers or response.content) else None + + def _map_openai_exception( *, model: str, @@ -315,7 +326,7 @@ def _map_openai_exception( exception_provider: str, extra_information: str, ) -> None: - response: Final = _litellm_proxy_response(original_exception, custom_llm_provider) + response: Final = _without_credentials(_litellm_proxy_response(original_exception, custom_llm_provider)) if (cyber_policy_body := _openai_cyber_policy_body(original_exception)) is not None: raise ContentPolicyViolationError( message=f"ContentPolicyViolationError: {exception_provider} - {original_exception}", @@ -378,6 +389,7 @@ def _map_openai_exception( model=model, llm_provider=custom_llm_provider, litellm_debug_info=extra_information, + response=_upstream_reply(response), ) elif ( ("invalid_request_error" in error_str and "content_policy_violation" in error_str) @@ -449,6 +461,7 @@ def _map_openai_exception( model=model, llm_provider=custom_llm_provider, body=getattr(original_exception, "body", None), + response=_upstream_reply(response), ) elif "Request too large" in error_str: raise RateLimitError( @@ -479,6 +492,7 @@ def _map_openai_exception( model=model, request=_request, litellm_debug_info=extra_information, + response=_upstream_reply(response), ) elif hasattr(original_exception, "status_code"): if original_exception.status_code == 400: @@ -512,6 +526,7 @@ def _map_openai_exception( model=model, llm_provider=custom_llm_provider, litellm_debug_info=extra_information, + response=_upstream_reply(response), ) elif original_exception.status_code == 422: raise BadRequestError( @@ -563,6 +578,7 @@ def _map_openai_exception( llm_provider=custom_llm_provider, litellm_debug_info=extra_information, exception_status_code=original_exception.status_code, + response=_upstream_reply(response), ) else: raise APIError( @@ -572,6 +588,7 @@ def _map_openai_exception( model=model, request=getattr(original_exception, "request", None), litellm_debug_info=extra_information, + response=_upstream_reply(response), ) else: # if no status code then it is an APIConnectionError: https://github.com/openai/openai-python#handling-errors @@ -2019,6 +2036,7 @@ def _map_azure_exception( exception_provider: str, extra_information: str, ) -> None: + response: Final = _without_credentials(getattr(original_exception, "response", None)) message = get_error_message(error_obj=original_exception) if message is None: if hasattr(original_exception, "message"): @@ -2054,7 +2072,7 @@ def _map_azure_exception( llm_provider="azure", model=model, litellm_debug_info=extra_information, - response=getattr(original_exception, "response", None), + response=response, ) elif "DeploymentNotFound" in error_str: raise NotFoundError( @@ -2062,7 +2080,7 @@ def _map_azure_exception( llm_provider="azure", model=model, litellm_debug_info=extra_information, - response=getattr(original_exception, "response", None), + response=response, ) elif azure_error_code == "content_policy_violation" or ExceptionCheckers.is_azure_content_policy_violation_error( error_str @@ -2076,6 +2094,7 @@ def _map_azure_exception( model=model, extra_information=extra_information, original_exception=original_exception, + response=response, ) elif azure_error_code == "invalid_encrypted_content" or "could not be verified" in error_str: helpful_message: Final = ( @@ -2094,7 +2113,7 @@ def _map_azure_exception( llm_provider="azure", model=model, litellm_debug_info=extra_information, - response=getattr(original_exception, "response", None), + response=response, body=getattr(original_exception, "body", None), ) elif getattr(original_exception, "status_code", None) in (None, 400) and ( @@ -2106,7 +2125,7 @@ def _map_azure_exception( llm_provider="azure", model=model, litellm_debug_info=extra_information, - response=getattr(original_exception, "response", None), + response=response, body=getattr(original_exception, "body", None), ) elif "invalid_request_error" in error_str and getattr(original_exception, "status_code", None) in (None, 400): @@ -2115,7 +2134,7 @@ def _map_azure_exception( llm_provider="azure", model=model, litellm_debug_info=extra_information, - response=getattr(original_exception, "response", None), + response=response, body=getattr(original_exception, "body", None), ) elif "The api_key client option must be set either by passing api_key to the client or by setting" in error_str: @@ -2124,7 +2143,7 @@ def _map_azure_exception( llm_provider=custom_llm_provider, model=model, litellm_debug_info=extra_information, - response=getattr(original_exception, "response", None), + response=response, ) elif "Connection error" in error_str: raise APIConnectionError( @@ -2140,7 +2159,7 @@ def _map_azure_exception( llm_provider="azure", model=model, litellm_debug_info=extra_information, - response=getattr(original_exception, "response", None), + response=response, body=getattr(original_exception, "body", None), ) elif original_exception.status_code == 401: @@ -2149,7 +2168,7 @@ def _map_azure_exception( llm_provider="azure", model=model, litellm_debug_info=extra_information, - response=getattr(original_exception, "response", None), + response=response, ) elif original_exception.status_code == 404: raise NotFoundError( @@ -2157,7 +2176,7 @@ def _map_azure_exception( llm_provider="azure", model=model, litellm_debug_info=extra_information, - response=getattr(original_exception, "response", None), + response=response, ) elif original_exception.status_code == 408: raise Timeout( @@ -2165,6 +2184,7 @@ def _map_azure_exception( model=model, litellm_debug_info=extra_information, llm_provider="azure", + response=_upstream_reply(response), ) elif original_exception.status_code == 422: raise BadRequestError( @@ -2172,7 +2192,7 @@ def _map_azure_exception( model=model, llm_provider="azure", litellm_debug_info=extra_information, - response=getattr(original_exception, "response", None), + response=response, ) elif original_exception.status_code == 429: raise RateLimitError( @@ -2180,7 +2200,7 @@ def _map_azure_exception( model=model, llm_provider="azure", litellm_debug_info=extra_information, - response=getattr(original_exception, "response", None), + response=response, ) elif original_exception.status_code == 502: raise BadGatewayError( @@ -2188,7 +2208,7 @@ def _map_azure_exception( model=model, llm_provider="azure", litellm_debug_info=extra_information, - response=getattr(original_exception, "response", None), + response=response, ) elif original_exception.status_code == 503: raise ServiceUnavailableError( @@ -2196,7 +2216,7 @@ def _map_azure_exception( model=model, llm_provider="azure", litellm_debug_info=extra_information, - response=getattr(original_exception, "response", None), + response=response, ) elif original_exception.status_code == 504: # gateway timeout error raise Timeout( @@ -2205,6 +2225,7 @@ def _map_azure_exception( litellm_debug_info=extra_information, llm_provider="azure", exception_status_code=original_exception.status_code, + response=_upstream_reply(response), ) else: raise APIError( @@ -2214,6 +2235,7 @@ def _map_azure_exception( litellm_debug_info=extra_information, model=model, request=httpx.Request(method="POST", url="https://openai.com/"), + response=_upstream_reply(response), ) else: # if no status code then it is an APIConnectionError: https://github.com/openai/openai-python#handling-errors @@ -2589,17 +2611,20 @@ def exception_type( model=model, llm_provider=custom_llm_provider, litellm_debug_info=extra_information, + response=_upstream_reply( + _without_credentials(_litellm_proxy_response(mappable_exception, custom_llm_provider)) + ), ) if ( custom_llm_provider == "litellm_proxy" ): # handle special case where calling litellm proxy + exception str contains error message extract_and_raise_litellm_exception( - response=_litellm_proxy_response(mappable_exception, custom_llm_provider), + response=_without_credentials(_litellm_proxy_response(mappable_exception, custom_llm_provider)), error_str=error_str, model=model, custom_llm_provider=custom_llm_provider, - body=getattr(original_exception, "body", None), + body=getattr(mappable_exception, "body", None), ) if ( custom_llm_provider == "openai" diff --git a/litellm/llms/azure/azure.py b/litellm/llms/azure/azure.py index a1ba8de6d7f..69ffac5d6f9 100644 --- a/litellm/llms/azure/azure.py +++ b/litellm/llms/azure/azure.py @@ -42,6 +42,7 @@ from ..openai.common_utils import ( build_output_token_limit_response, is_output_token_limit_error, model_dump_provider_response, + upstream_error_response, ) from .common_utils import ( AzureOpenAIError, @@ -399,6 +400,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): raise AzureOpenAIError( status_code=status_code, message=str(e), + response=upstream_error_response(e), headers=error_headers, body=error_body, ) @@ -662,6 +664,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): raise AzureOpenAIError( status_code=status_code, message=message, + response=upstream_error_response(e), headers=error_headers, body=error_body, ) @@ -855,7 +858,12 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): if error_headers is None and error_response: error_headers = getattr(error_response, "headers", None) error_text = error_response.text - raise AzureOpenAIError(status_code=status_code, message=error_text, headers=error_headers) + raise AzureOpenAIError( + status_code=status_code, + message=error_text, + response=upstream_error_response(e), + headers=error_headers, + ) async def make_async_azure_httpx_request( self, @@ -1190,7 +1198,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM): except Exception as e: error_code: Final[object] = getattr(e, "status_code", None) if error_code is not None: - raise AzureOpenAIError(status_code=error_code, message=str(e)) + raise AzureOpenAIError(status_code=error_code, message=str(e), response=upstream_error_response(e)) else: raise AzureOpenAIError(status_code=500, message=str(e)) diff --git a/litellm/llms/azure/completion/handler.py b/litellm/llms/azure/completion/handler.py index 34e51eaf1ec..c2744e7f750 100644 --- a/litellm/llms/azure/completion/handler.py +++ b/litellm/llms/azure/completion/handler.py @@ -8,6 +8,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.prompt_templates.factory import prompt_factory from litellm.utils import CustomStreamWrapper, ModelResponse, TextCompletionResponse +from ...openai.common_utils import upstream_error_response from ...openai.completion.transformation import OpenAITextCompletionConfig from ..common_utils import AzureOpenAIError, BaseAzureLLM @@ -183,7 +184,12 @@ class AzureTextCompletion(BaseAzureLLM): error_response: Final[object] = getattr(e, "response", None) if error_headers is None and error_response: error_headers = getattr(error_response, "headers", None) - raise AzureOpenAIError(status_code=status_code, message=str(e), headers=error_headers) + raise AzureOpenAIError( + status_code=status_code, + message=str(e), + headers=error_headers, + response=upstream_error_response(e), + ) async def acompletion( self, @@ -244,7 +250,12 @@ class AzureTextCompletion(BaseAzureLLM): error_response: Final[object] = getattr(e, "response", None) if error_headers is None and error_response: error_headers = getattr(error_response, "headers", None) - raise AzureOpenAIError(status_code=status_code, message=str(e), headers=error_headers) + raise AzureOpenAIError( + status_code=status_code, + message=str(e), + headers=error_headers, + response=upstream_error_response(e), + ) def streaming( self, @@ -355,4 +366,9 @@ class AzureTextCompletion(BaseAzureLLM): error_response: Final[object] = getattr(e, "response", None) if error_headers is None and error_response: error_headers = getattr(error_response, "headers", None) - raise AzureOpenAIError(status_code=status_code, message=str(e), headers=error_headers) + raise AzureOpenAIError( + status_code=status_code, + message=str(e), + headers=error_headers, + response=upstream_error_response(e), + ) diff --git a/litellm/llms/azure/exception_mapping.py b/litellm/llms/azure/exception_mapping.py index f7146a36384..6eb5161f7e4 100644 --- a/litellm/llms/azure/exception_mapping.py +++ b/litellm/llms/azure/exception_mapping.py @@ -1,5 +1,7 @@ from typing import Final +import httpx + from litellm.exceptions import ContentPolicyViolationError @@ -14,6 +16,7 @@ class AzureOpenAIExceptionMapping: model: str, extra_information: str, original_exception: Exception, + response: httpx.Response | None, ) -> ContentPolicyViolationError: """ Create a content policy violation error @@ -39,7 +42,7 @@ class AzureOpenAIExceptionMapping: llm_provider="azure", model=model, litellm_debug_info=extra_information, - response=getattr(original_exception, "response", None), + response=response, provider_specific_fields={ # Preserve legacy key for backward compatibility. "innererror": inner_error, diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index 6c416124d1a..aa14a985afe 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -29,6 +29,7 @@ if TYPE_CHECKING: from aiohttp import ClientSession import litellm +from litellm.exceptions import error_response_without_credentials from litellm.litellm_core_utils.completion_timeout import CompletionTimeout from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object from litellm.litellm_core_utils.token_counter import token_counter, usage_tokenizer_model @@ -148,6 +149,14 @@ class OpenAIError(BaseLLMException): ) +def upstream_error_response(error: Exception) -> httpx.Response | None: + if not isinstance(error, (openai.APIStatusError, httpx.HTTPStatusError)): + return None + return error_response_without_credentials( + error.response.status_code, error.response, "https://docs.litellm.ai/docs" + ) + + _DECODED_JSON: Final = TypeAdapter(object) diff --git a/litellm/llms/openai/completion/handler.py b/litellm/llms/openai/completion/handler.py index 278427e8d77..8bfdd536f90 100644 --- a/litellm/llms/openai/completion/handler.py +++ b/litellm/llms/openai/completion/handler.py @@ -10,7 +10,7 @@ from litellm.types.llms.openai import AllMessageValues, OpenAITextCompletionUser from litellm.types.utils import LlmProviders, ModelResponse, TextCompletionResponse from litellm.utils import ProviderConfigManager -from ..common_utils import BaseOpenAILLM, OpenAIError +from ..common_utils import BaseOpenAILLM, OpenAIError, upstream_error_response from ..workload_identity import build_async_openai_client, build_openai_client from .transformation import OpenAITextCompletionConfig @@ -164,7 +164,12 @@ class OpenAITextCompletion(BaseLLM): error_response: Final[object] = getattr(e, "response", None) if error_headers is None and error_response: error_headers = getattr(error_response, "headers", None) - raise OpenAIError(status_code=status_code, message=error_text, headers=error_headers) + raise OpenAIError( + status_code=status_code, + message=error_text, + response=upstream_error_response(e), + headers=error_headers, + ) async def acompletion( self, @@ -220,7 +225,12 @@ class OpenAITextCompletion(BaseLLM): error_response: Final[object] = getattr(e, "response", None) if error_headers is None and error_response: error_headers = getattr(error_response, "headers", None) - raise OpenAIError(status_code=status_code, message=error_text, headers=error_headers) + raise OpenAIError( + status_code=status_code, + message=error_text, + response=upstream_error_response(e), + headers=error_headers, + ) def streaming( self, @@ -261,7 +271,12 @@ class OpenAITextCompletion(BaseLLM): error_response: object = getattr(e, "response", None) if error_headers is None and error_response: error_headers = getattr(error_response, "headers", None) - raise OpenAIError(status_code=status_code, message=error_text, headers=error_headers) + raise OpenAIError( + status_code=status_code, + message=error_text, + response=upstream_error_response(e), + headers=error_headers, + ) streamwrapper: Final = CustomStreamWrapper( completion_stream=response, model=model, diff --git a/litellm/llms/openai/image_variations/handler.py b/litellm/llms/openai/image_variations/handler.py index 01f01d3de1d..504dfb50015 100644 --- a/litellm/llms/openai/image_variations/handler.py +++ b/litellm/llms/openai/image_variations/handler.py @@ -14,7 +14,7 @@ from litellm.utils import ProviderConfigManager from ...base_llm.image_variations.transformation import BaseImageVariationConfig from ...custom_httpx.llm_http_handler import LiteLLMLoggingObj -from ..common_utils import OpenAIError +from ..common_utils import OpenAIError, upstream_error_response from ..workload_identity import build_async_openai_client, build_openai_client @@ -129,7 +129,12 @@ class OpenAIImageVariationsHandler: error_response: Final[object] = getattr(e, "response", None) if error_headers is None and error_response: error_headers = getattr(error_response, "headers", None) - raise OpenAIError(status_code=status_code, message=error_text, headers=error_headers) + raise OpenAIError( + status_code=status_code, + message=error_text, + response=upstream_error_response(e), + headers=error_headers, + ) def image_variations( self, @@ -245,4 +250,9 @@ class OpenAIImageVariationsHandler: error_response: Final[object] = getattr(e, "response", None) if error_headers is None and error_response: error_headers = getattr(error_response, "headers", None) - raise OpenAIError(status_code=status_code, message=error_text, headers=error_headers) + raise OpenAIError( + status_code=status_code, + message=error_text, + response=upstream_error_response(e), + headers=error_headers, + ) diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index af2e85c5f20..e0a23d6c99f 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -64,6 +64,7 @@ from .common_utils import ( is_openai_backed_api_base, is_output_token_limit_error, model_dump_provider_response, + upstream_error_response, ) from .workload_identity import ( build_async_openai_client, @@ -983,6 +984,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): raise OpenAIError( status_code=status_code, message=error_text, + response=upstream_error_response(e), headers=error_headers, body=error_body, ) @@ -1110,6 +1112,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): raise OpenAIError( status_code=status_code, message=message, + response=upstream_error_response(e), headers=error_headers, body=exception_body, ) @@ -1282,6 +1285,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): raise OpenAIError( status_code=getattr(e, "status_code", 500), message=str(e), + response=upstream_error_response(e), headers=error_headers, body=exception_body, ) @@ -1409,7 +1413,12 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): error_response: Final[object] = getattr(e, "response", None) if error_headers is None and error_response: error_headers = getattr(error_response, "headers", None) - raise OpenAIError(status_code=status_code, message=error_text, headers=error_headers) + raise OpenAIError( + status_code=status_code, + message=error_text, + response=upstream_error_response(e), + headers=error_headers, + ) def embedding( self, @@ -1499,7 +1508,12 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): error_response: Final[object] = getattr(e, "response", None) if error_headers is None and error_response: error_headers = getattr(error_response, "headers", None) - raise OpenAIError(status_code=status_code, message=error_text, headers=error_headers) + raise OpenAIError( + status_code=status_code, + message=error_text, + response=upstream_error_response(e), + headers=error_headers, + ) async def aimage_generation( self, @@ -1666,7 +1680,11 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): original_response=str(e), ) if hasattr(e, "status_code"): - raise OpenAIError(status_code=getattr(e, "status_code", 500), message=str(e)) + raise OpenAIError( + status_code=getattr(e, "status_code", 500), + message=str(e), + response=upstream_error_response(e), + ) else: raise OpenAIError(status_code=500, message=str(e)) diff --git a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py index 19e5387da9a..88501a23235 100644 --- a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py @@ -24,6 +24,7 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.bedrock.common_utils import BedrockError from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.openai.common_utils import OpenAIError +from litellm.llms.vertex_ai.common_utils import VertexAIError from litellm.proxy.common_utils.openai_error_payload import openai_error_type from litellm.types.utils import LlmProviders @@ -54,8 +55,8 @@ def test_openai_cyber_policy_uses_the_structured_error_code(shape: str, code: st assert caught.value.code == code assert caught.value.body == body assert caught.value.status_code == 400 - assert caught.value.response is response - assert caught.value.request is request + assert caught.value.response.status_code == 400 + assert caught.value.request.url == request.url assert caught.value.response.headers["x-request-id"] == "cyber-policy-request" @@ -2940,3 +2941,80 @@ def test_transport_timeout_maps_to_timeout( assert exc_info.value.status_code == 408 assert f"Request timed out. Error_str: {reason}" in str(exc_info.value) + + +def test_bedrock_server_error_keeps_the_aws_body_without_the_signed_request() -> None: + body: Final = '{"message":"Bedrock is unable to process your request."}' + signed_request: Final = httpx.Request( + "POST", + "https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-opus-5-5/converse", + headers={"authorization": "AWS4-HMAC-SHA256 Credential=AKIDSIGNED/20261029/us-east-1/bedrock/aws4_request"}, + content=b'{"messages":[{"role":"user","content":[{"text":"hi"}]}]}', + ) + upstream: Final = httpx.Response( + 500, content=body, headers={"x-amzn-requestid": "req-aws"}, request=signed_request + ) + + with pytest.raises(litellm.ServiceUnavailableError) as raised: + exception_type( + model="bedrock/anthropic.claude-opus-5-5", + original_exception=BedrockError(status_code=500, message=body, response=upstream), + custom_llm_provider="bedrock", + ) + + assert raised.value.response.text == body + assert raised.value.response.headers["x-amzn-requestid"] == "req-aws" + assert "AKIDSIGNED" not in str(raised.value.request.headers) + assert raised.value.request.content == b"" + + +@pytest.mark.parametrize("provider", ("gemini", "vertex_ai")) +def test_gemini_server_error_response_carries_the_upstream_error_body(provider: str) -> None: + body: Final = '{"error":{"code":500,"message":"Internal error encountered.","status":"INTERNAL"}}' + + with pytest.raises(litellm.InternalServerError) as raised: + exception_type( + model="gemini-3.1-pro", + original_exception=VertexAIError(status_code=500, message=body, headers=None), + custom_llm_provider=provider, + ) + + assert raised.value.response.status_code == 500 + assert raised.value.response.json() == json.loads(body) + + +@pytest.mark.parametrize( + ("status_code", "message", "mapped_error"), + ( + (500, "The server had an error processing your request. Sorry about that!", litellm.InternalServerError), + (520, "Web server is returning an unknown error", litellm.InternalServerError), + (504, "A timeout occurred", litellm.Timeout), + (500, "Mistral API raised a streaming error", litellm.APIError), + ), +) +def test_openai_error_matched_by_its_message_keeps_the_upstream_body( + status_code: int, message: str, mapped_error: type[Exception] +) -> None: + body: Final = json.dumps({"error": {"message": message, "type": "server_error", "param": None, "code": None}}) + sent_request: Final = httpx.Request( + "POST", + "https://api.openai.com/v1/chat/completions", + headers={"authorization": "Bearer sk-openai-secret"}, + content=b'{"messages":[{"role":"user","content":"hi"}]}', + ) + upstream: Final = httpx.Response( + status_code, content=body, headers={"x-request-id": "req-openai"}, request=sent_request + ) + + with pytest.raises(mapped_error) as raised: + exception_type( + model="gpt-5.6", + original_exception=openai.APIStatusError(message, response=upstream, body=json.loads(body)), + custom_llm_provider="openai", + ) + + error_response: Final = getattr(raised.value, "response") + assert error_response.text == body + assert error_response.headers["x-request-id"] == "req-openai" + assert "sk-openai-secret" not in str(error_response.request.headers) + assert error_response.request.content == b"" diff --git a/tests/unit/llms/azure/test_azure_exception_mapping.py b/tests/unit/llms/azure/test_azure_exception_mapping.py index cbbf70c691b..018b499f401 100644 --- a/tests/unit/llms/azure/test_azure_exception_mapping.py +++ b/tests/unit/llms/azure/test_azure_exception_mapping.py @@ -1,3 +1,4 @@ +import datetime import json from collections.abc import Awaitable, Callable, Mapping from typing import Final @@ -12,6 +13,7 @@ import respx import litellm from litellm.exceptions import BadRequestError, ContentPolicyViolationError, ContextWindowExceededError from litellm.litellm_core_utils.exception_mapping_utils import exception_type +from litellm.llms.azure.common_utils import AzureOpenAIError @respx.mock @@ -632,3 +634,202 @@ def test_azure_content_policy_wins_over_context_length_code(monkeypatch: pytest. error: Final = _azure_chat_error(monkeypatch, _azure_error_response(400, policy_error)) assert type(error) is ContentPolicyViolationError + + +_AZURE_REFUSAL_BODY: Final = '{"error":{"message":"azure refused","type":"upstream_refusal","code":"refused"}}' +_AZURE_DEPLOYMENT_NOT_FOUND_BODY: Final = '{"error":{"code":"DeploymentNotFound","message":"no such deployment"}}' +_AZURE_INTERNAL_ERROR_BODY: Final = '{"error":{"code":"InternalServerError","message":"Internal server error"}}' +_AZURE_CONTENT_FILTER_BODY: Final = json.dumps( + { + "error": { + "message": "The response was filtered due to the prompt triggering Azure OpenAI's content management policy.", + "type": None, + "param": "prompt", + "code": "content_filter", + "status": 400, + "innererror": { + "code": "ResponsibleAIPolicyViolation", + "content_filter_result": {"violence": {"filtered": True, "severity": "high"}}, + }, + } + } +) +_AZURE_KEY: Final = "azure-provider-secret" +_AZURE_BASE: Final = "https://azure-upstream.openai.azure.com" +_AZURE_ARGS: Final = {"api_base": _AZURE_BASE, "api_version": "2024-10-21", "api_key": _AZURE_KEY, "max_retries": 0} + + +async def _azure_completion() -> None: + litellm.completion(model="azure/gpt-4.1-mini", messages=[{"role": "user", "content": "hi"}], **_AZURE_ARGS) + + +async def _azure_completion_stream() -> None: + list( + litellm.completion( + model="azure/gpt-4.1-mini", messages=[{"role": "user", "content": "hi"}], stream=True, **_AZURE_ARGS + ) + ) + + +async def _azure_acompletion() -> None: + await litellm.acompletion(model="azure/gpt-4.1-mini", messages=[{"role": "user", "content": "hi"}], **_AZURE_ARGS) + + +async def _azure_acompletion_stream() -> None: + stream: Final = await litellm.acompletion( + model="azure/gpt-4.1-mini", messages=[{"role": "user", "content": "hi"}], stream=True, **_AZURE_ARGS + ) + async for _ in stream: + pass + + +async def _azure_embedding() -> None: + litellm.embedding(model="azure/text-embedding-3-large", input=["hi"], **_AZURE_ARGS) + + +async def _azure_aembedding() -> None: + await litellm.aembedding(model="azure/text-embedding-3-large", input=["hi"], **_AZURE_ARGS) + + +async def _azure_image_generation() -> None: + litellm.image_generation(model="azure/gpt-image-2", prompt="a cat", **_AZURE_ARGS) + + +async def _azure_text_completion() -> None: + litellm.text_completion(model="azure_text/gpt-35-turbo-instruct", prompt="hi", **_AZURE_ARGS) + + +async def _azure_text_completion_stream() -> None: + list(litellm.text_completion(model="azure_text/gpt-35-turbo-instruct", prompt="hi", stream=True, **_AZURE_ARGS)) + + +async def _azure_atext_completion() -> None: + await litellm.atext_completion(model="azure_text/gpt-35-turbo-instruct", prompt="hi", **_AZURE_ARGS) + + +async def _azure_atext_completion_stream() -> None: + stream: Final = await litellm.atext_completion( + model="azure_text/gpt-35-turbo-instruct", prompt="hi", stream=True, **_AZURE_ARGS + ) + async for _ in stream: + pass + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "route", + ( + _azure_completion, + _azure_completion_stream, + _azure_acompletion, + _azure_acompletion_stream, + _azure_embedding, + _azure_aembedding, + _azure_image_generation, + _azure_text_completion, + _azure_text_completion_stream, + _azure_atext_completion, + _azure_atext_completion_stream, + ), + ids=lambda route: route.__name__.removeprefix("_azure_"), +) +@pytest.mark.parametrize( + ("status_code", "body", "mapped_error"), + ( + (400, _AZURE_REFUSAL_BODY, BadRequestError), + (400, _AZURE_CONTENT_FILTER_BODY, BadRequestError), + (403, _AZURE_REFUSAL_BODY, litellm.APIError), + (404, _AZURE_DEPLOYMENT_NOT_FOUND_BODY, litellm.NotFoundError), + (500, _AZURE_INTERNAL_ERROR_BODY, openai.APIError), + (502, _AZURE_REFUSAL_BODY, litellm.BadGatewayError), + (504, _AZURE_REFUSAL_BODY, litellm.Timeout), + ), +) +@respx.mock +async def test_azure_error_keeps_the_upstream_body_without_the_api_key( + route: Callable[[], Awaitable[None]], + status_code: int, + body: str, + mapped_error: type[Exception], + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx.post(url__regex=r"https://azure-upstream\.openai\.azure\.com/.*").mock( + return_value=httpx.Response(status_code, content=body, headers={"content-type": "application/json"}) + ) + + with pytest.raises(mapped_error) as raised: + await route() + + error_response: Final = getattr(raised.value, "response") + assert error_response.status_code == status_code + assert error_response.text == body + assert error_response.json() == json.loads(body) + for exposed in (error_response.request, getattr(raised.value, "request")): + assert _AZURE_KEY not in str(exposed.headers) + assert exposed.content == b"" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "route", + ( + _azure_completion, + _azure_completion_stream, + _azure_acompletion, + _azure_acompletion_stream, + _azure_embedding, + _azure_aembedding, + _azure_text_completion, + _azure_atext_completion, + _azure_atext_completion_stream, + ), + ids=lambda route: route.__name__.removeprefix("_azure_"), +) +@respx.mock +async def test_azure_error_response_keeps_the_upstream_elapsed_time( + route: Callable[[], Awaitable[None]], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx.post(url__regex=r"https://azure-upstream\.openai\.azure\.com/.*").mock( + return_value=httpx.Response(400, content=_AZURE_REFUSAL_BODY, headers={"content-type": "application/json"}) + ) + + with pytest.raises(BadRequestError) as raised: + await route() + + assert raised.value.response.elapsed > datetime.timedelta(0) + + +def test_azure_client_side_missing_key_error_keeps_the_empty_placeholder_response() -> None: + missing_key: Final = openai.OpenAIError( + "The api_key client option must be set either by passing api_key to the client or by setting the " + "OPENAI_API_KEY environment variable" + ) + + with pytest.raises(litellm.AuthenticationError) as raised: + exception_type(model="azure/gpt-4.1-mini", original_exception=missing_key, custom_llm_provider="azure") + + assert raised.value.response.status_code == 401 + assert raised.value.response.text == "" + + +@pytest.mark.parametrize( + ("status_code", "message", "mapped_error"), + ( + (500, "azure_client is not an instance of AzureOpenAI or OpenAI", litellm.APIError), + (408, "the deployment did not answer in time", litellm.Timeout), + (504, "the deployment did not answer in time", litellm.Timeout), + ), +) +def test_azure_error_litellm_raises_itself_has_no_upstream_response( + status_code: int, message: str, mapped_error: type[Exception] +) -> None: + with pytest.raises(mapped_error) as raised: + exception_type( + model="azure/gpt-4.1-mini", + original_exception=AzureOpenAIError(status_code=status_code, message=message), + custom_llm_provider="azure", + ) + + assert getattr(raised.value, "response", None) is None diff --git a/tests/unit/llms/openai/completion/test_completion_handler.py b/tests/unit/llms/openai/completion/test_completion_handler.py index 9e2754eef68..628c9bbd783 100644 --- a/tests/unit/llms/openai/completion/test_completion_handler.py +++ b/tests/unit/llms/openai/completion/test_completion_handler.py @@ -93,24 +93,34 @@ async def test_acompletion_forwards_client_headers_to_provider( assert request_headers["x-mycorp-llmcall-id"] == "abc-123" +_REJECTION_BODY: Final = '{"error":{"message":"bad prompt","type":"invalid_request_error"}}' + + @pytest.fixture def rejected_completions_endpoint(): return respx.post("https://api.openai.com/v1/completions").mock( return_value=Response( 400, - json={"error": {"message": "bad prompt", "type": "invalid_request_error"}}, - headers={"x-request-id": "req-1"}, + content=_REJECTION_BODY, + headers={"x-request-id": "req-1", "content-type": "application/json"}, ) ) +def _assert_carries_the_upstream_rejection(rejected: litellm.BadRequestError) -> None: + assert rejected.status_code == 400 + assert rejected.litellm_response_headers["x-request-id"] == "req-1" + assert rejected.response.text == _REJECTION_BODY + assert rejected.response.json() == json.loads(_REJECTION_BODY) + assert rejected.response.headers["x-request-id"] == "req-1" + + @respx.mock def test_completion_surfaces_the_status_and_headers_of_a_rejected_request(rejected_completions_endpoint): with pytest.raises(litellm.BadRequestError) as rejected: text_completion(model="gpt-3.5-turbo-instruct", prompt="hello") - assert rejected.value.status_code == 400 - assert rejected.value.litellm_response_headers["x-request-id"] == "req-1" + _assert_carries_the_upstream_rejection(rejected.value) @respx.mock @@ -118,8 +128,7 @@ def test_streaming_completion_surfaces_the_status_and_headers_of_a_rejected_requ with pytest.raises(litellm.BadRequestError) as rejected: list(text_completion(model="gpt-3.5-turbo-instruct", prompt="hello", stream=True)) - assert rejected.value.status_code == 400 - assert rejected.value.litellm_response_headers["x-request-id"] == "req-1" + _assert_carries_the_upstream_rejection(rejected.value) @respx.mock @@ -130,8 +139,7 @@ async def test_acompletion_surfaces_the_status_and_headers_of_a_rejected_request with pytest.raises(litellm.BadRequestError) as rejected: await atext_completion(model="gpt-3.5-turbo-instruct", prompt="hello") - assert rejected.value.status_code == 400 - assert rejected.value.litellm_response_headers["x-request-id"] == "req-1" + _assert_carries_the_upstream_rejection(rejected.value) @respx.mock diff --git a/tests/unit/llms/openai/image_variations/__init__.py b/tests/unit/llms/openai/image_variations/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/openai/image_variations/test_openai_image_variations_handler.py b/tests/unit/llms/openai/image_variations/test_openai_image_variations_handler.py new file mode 100644 index 00000000000..83ddbec6038 --- /dev/null +++ b/tests/unit/llms/openai/image_variations/test_openai_image_variations_handler.py @@ -0,0 +1,51 @@ +import json +from io import BytesIO +from typing import Final + +import httpx +import pytest + +import litellm +from litellm.llms.openai.common_utils import OpenAIError + +_UPSTREAM_ERROR_BODY: Final = '{"error":{"message":"image is not a valid PNG","type":"invalid_request_error"}}' + + +def _upstream_rejection(_request: httpx.Request) -> httpx.Response: + return httpx.Response( + 400, + content=_UPSTREAM_ERROR_BODY, + headers={"x-request-id": "req_variation_400", "content-type": "application/json"}, + ) + + +_VARIATION_KWARGS: Final = { + "custom_llm_provider": "openai", + "api_key": "transport-only", + "api_base": "https://upstream.example/v1", + "max_retries": 0, +} + + +async def _request_variation(is_async: bool) -> None: + if is_async: + await litellm.aimage_variation(image=BytesIO(b"image-bytes"), **_VARIATION_KWARGS) + return + litellm.image_variation(image=BytesIO(b"image-bytes"), **_VARIATION_KWARGS) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("is_async", (False, True), ids=("sync", "async")) +async def test_rejected_image_variation_carries_the_upstream_error_response( + is_async: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + transport: Final = httpx.MockTransport(_upstream_rejection) + monkeypatch.setattr(litellm, "client_session", httpx.Client(transport=transport)) + monkeypatch.setattr(litellm, "aclient_session", httpx.AsyncClient(transport=transport)) + + with pytest.raises((litellm.BadRequestError, OpenAIError)) as raised: + await _request_variation(is_async) + + assert raised.value.response.text == _UPSTREAM_ERROR_BODY + assert raised.value.response.json() == json.loads(_UPSTREAM_ERROR_BODY) + assert raised.value.response.headers["x-request-id"] == "req_variation_400" diff --git a/tests/unit/llms/openai/test_openai.py b/tests/unit/llms/openai/test_openai.py index 5b27a446fb3..3e1cbad9308 100644 --- a/tests/unit/llms/openai/test_openai.py +++ b/tests/unit/llms/openai/test_openai.py @@ -1,8 +1,9 @@ import asyncio +import gzip import importlib import json import os -from collections.abc import Callable, Mapping +from collections.abc import Awaitable, Callable, Mapping from types import ModuleType from typing import Final, Literal, Protocol from unittest.mock import MagicMock, Mock, patch @@ -375,6 +376,277 @@ async def test_acompletion_preserves_public_errors_for_both_http_clients( assert not http_client.is_closed +_UPSTREAM_ERROR_BODY: Final = ( + '{"error":{"message":"upstream rejected the request","type":"invalid_request_error",' + '"param":"response_format.type","code":"invalid_value"}}' +) +_UPSTREAM_BASE: Final = "https://upstream.example/v1" +_UPSTREAM_TRANSPORT: Final = httpx.MockTransport( + lambda _request: httpx.Response( + 400, + content=_UPSTREAM_ERROR_BODY, + headers={"x-request-id": "req_upstream_400", "x-upstream-trace": "trace-7", "content-type": "application/json"}, + ) +) +_MESSAGES: Final = [{"role": "user", "content": "hi"}] + + +def _rejecting_client() -> OpenAI: + return OpenAI( + api_key="transport-only", + base_url=_UPSTREAM_BASE, + max_retries=0, + http_client=httpx.Client(transport=_UPSTREAM_TRANSPORT), + ) + + +def _rejecting_async_client() -> AsyncOpenAI: + return AsyncOpenAI( + api_key="transport-only", + base_url=_UPSTREAM_BASE, + max_retries=0, + http_client=httpx.AsyncClient(transport=_UPSTREAM_TRANSPORT), + ) + + +def _upstream(provider: str, model: str) -> dict[str, object]: + return {"model": f"{provider}/{model}", "api_base": _UPSTREAM_BASE, "api_key": "transport-only", "num_retries": 0} + + +async def _completion(provider: str) -> None: + litellm.completion(messages=_MESSAGES, client=_rejecting_client(), **_upstream(provider, "gpt-5.6")) + + +async def _completion_stream(provider: str) -> None: + stream: Final = litellm.completion( + messages=_MESSAGES, client=_rejecting_client(), stream=True, **_upstream(provider, "gpt-5.6") + ) + list(stream) + + +async def _acompletion(provider: str) -> None: + await litellm.acompletion(messages=_MESSAGES, client=_rejecting_async_client(), **_upstream(provider, "gpt-5.6")) + + +async def _acompletion_stream(provider: str) -> None: + stream: Final = await litellm.acompletion( + messages=_MESSAGES, client=_rejecting_async_client(), stream=True, **_upstream(provider, "gpt-5.6") + ) + async for _ in stream: + pass + + +async def _embedding(provider: str) -> None: + litellm.embedding(input=["hi"], client=_rejecting_client(), **_upstream(provider, "text-embedding-3-large")) + + +async def _aembedding(provider: str) -> None: + await litellm.aembedding( + input=["hi"], client=_rejecting_async_client(), **_upstream(provider, "text-embedding-3-large") + ) + + +async def _image_generation(provider: str) -> None: + litellm.image_generation(prompt="a cat", client=_rejecting_client(), **_upstream(provider, "gpt-image-2")) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("provider", ("openai", "litellm_proxy")) +@pytest.mark.parametrize( + "route", + (_completion, _completion_stream, _acompletion, _acompletion_stream, _embedding, _aembedding, _image_generation), + ids=lambda route: route.__name__.lstrip("_"), +) +async def test_mapped_error_carries_the_upstream_error_response( + provider: str, route: Callable[[str], Awaitable[None]] +) -> None: + with pytest.raises(litellm.BadRequestError) as raised: + await route(provider) + + assert raised.value.response.text == _UPSTREAM_ERROR_BODY + assert raised.value.response.json() == json.loads(_UPSTREAM_ERROR_BODY) + assert raised.value.response.headers["x-request-id"] == "req_upstream_400" + assert raised.value.response.headers["x-upstream-trace"] == "trace-7" + + +_UPSTREAM_SERVER_ERROR_BODY: Final = ( + '{"error":{"message":"The server had an error while processing your request.","type":"server_error",' + '"param":null,"code":null}}' +) +_CHUNKED_UPSTREAM_HEADERS: Final = { + "content-type": "application/json", + "transfer-encoding": "chunked", + "x-request-id": "req_upstream_5xx", +} + + +def _chunked_upstream_client(status_code: int, content_encoding: Literal["gzip"] | None) -> httpx.AsyncClient: + body: Final = _UPSTREAM_SERVER_ERROR_BODY.encode() + encoding_headers: Final = {"content-encoding": content_encoding} if content_encoding else {} + return httpx.AsyncClient( + transport=httpx.MockTransport( + lambda _request: httpx.Response( + status_code, + headers={**_CHUNKED_UPSTREAM_HEADERS, **encoding_headers}, + stream=httpx.ByteStream(gzip.compress(body) if content_encoding else body), + ) + ) + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("provider", ("openai", "litellm_proxy")) +@pytest.mark.parametrize("stream", (False, True)) +@pytest.mark.parametrize( + ("status_code", "mapped_error"), + ( + (429, litellm.RateLimitError), + (500, litellm.InternalServerError), + (502, litellm.BadGatewayError), + (503, litellm.ServiceUnavailableError), + ), +) +@pytest.mark.parametrize("content_encoding", (None, "gzip")) +async def test_retryable_error_keeps_the_decoded_upstream_body_and_its_headers( + provider: str, + stream: bool, + status_code: int, + mapped_error: type[Exception], + content_encoding: Literal["gzip"] | None, +) -> None: + client: Final = AsyncOpenAI( + api_key="transport-only", + base_url=_UPSTREAM_BASE, + max_retries=0, + http_client=_chunked_upstream_client(status_code, content_encoding), + ) + + with pytest.raises(mapped_error) as raised: + await litellm.acompletion(messages=_MESSAGES, client=client, stream=stream, **_upstream(provider, "gpt-5.6")) + + error_response: Final = getattr(raised.value, "response") + assert error_response.status_code == status_code + assert error_response.text == _UPSTREAM_SERVER_ERROR_BODY + assert error_response.json() == json.loads(_UPSTREAM_SERVER_ERROR_BODY) + assert dict(error_response.headers) == { + **_CHUNKED_UPSTREAM_HEADERS, + **({"content-encoding": content_encoding} if content_encoding else {}), + } + + +_REFUSAL_BODY: Final = '{"error":{"message":"upstream refused","type":"upstream_refusal","param":null,"code":null}}' +_GATEWAY_TIMEOUT_BODY: Final = ( + '{"error":{"message":"litellm.Timeout: APITimeoutError - Request timed out.","type":"invalid_request_error",' + '"param":null,"code":"408"}}' +) +_PROVIDER_KEY: Final = "sk-provider-secret" +_QUERY_SECRET: Final = "query-secret" + + +@pytest.mark.parametrize("provider", ("openai", "litellm_proxy")) +@pytest.mark.parametrize( + ("status_code", "mapped_error", "body"), + ( + (400, litellm.BadRequestError, _REFUSAL_BODY), + (403, litellm.APIError, _REFUSAL_BODY), + (408, litellm.Timeout, _REFUSAL_BODY), + (408, litellm.Timeout, _GATEWAY_TIMEOUT_BODY), + (409, litellm.APIError, _REFUSAL_BODY), + (413, litellm.APIError, _REFUSAL_BODY), + (429, litellm.RateLimitError, _REFUSAL_BODY), + (500, litellm.InternalServerError, _REFUSAL_BODY), + (504, litellm.Timeout, _REFUSAL_BODY), + ), +) +def test_mapped_error_keeps_the_upstream_body_without_the_provider_credentials( + provider: str, status_code: int, mapped_error: type[Exception], body: str +) -> None: + client: Final = OpenAI( + api_key=_PROVIDER_KEY, + base_url=_UPSTREAM_BASE, + default_query={"key": _QUERY_SECRET}, + max_retries=0, + http_client=httpx.Client( + transport=httpx.MockTransport( + lambda _request: httpx.Response( + status_code, content=body, headers={"content-type": "application/json"} + ) + ) + ), + ) + + with pytest.raises(mapped_error) as raised: + litellm.completion( + model=f"{provider}/gpt-5.6", + messages=_MESSAGES, + client=client, + api_base=_UPSTREAM_BASE, + api_key=_PROVIDER_KEY, + num_retries=0, + ) + + error_response: Final = getattr(raised.value, "response") + assert error_response.status_code == status_code + assert error_response.text == body + assert error_response.json() == json.loads(body) + assert str(error_response.request.url) == f"{_UPSTREAM_BASE}/chat/completions" + for exposed in (error_response.request, getattr(raised.value, "request")): + assert _QUERY_SECRET not in str(exposed.url) + assert _PROVIDER_KEY not in str(exposed.headers) + assert exposed.content == b"" + + +def _time_out(request: httpx.Request) -> httpx.Response: + raise httpx.ReadTimeout("no reply", request=request) + + +async def _sync_completion_timing_out(provider: str) -> None: + litellm.completion( + model=f"{provider}/gpt-5.6", + messages=_MESSAGES, + client=OpenAI( + api_key=_PROVIDER_KEY, + base_url=_UPSTREAM_BASE, + max_retries=0, + http_client=httpx.Client(transport=httpx.MockTransport(_time_out)), + ), + api_base=_UPSTREAM_BASE, + api_key=_PROVIDER_KEY, + num_retries=0, + ) + + +async def _async_stream_timing_out(provider: str) -> None: + stream: Final = await litellm.acompletion( + model=f"{provider}/gpt-5.6", + messages=_MESSAGES, + stream=True, + client=AsyncOpenAI( + api_key=_PROVIDER_KEY, + base_url=_UPSTREAM_BASE, + max_retries=0, + http_client=httpx.AsyncClient(transport=httpx.MockTransport(_time_out)), + ), + api_base=_UPSTREAM_BASE, + api_key=_PROVIDER_KEY, + num_retries=0, + ) + async for _ in stream: + pass + + +@pytest.mark.asyncio +@pytest.mark.parametrize("provider", ("openai", "litellm_proxy")) +@pytest.mark.parametrize("call", (_sync_completion_timing_out, _async_stream_timing_out)) +async def test_timeout_before_any_reply_has_no_upstream_response( + provider: str, call: Callable[[str], Awaitable[None]] +) -> None: + with pytest.raises(litellm.Timeout) as raised: + await call(provider) + + assert getattr(raised.value, "response", None) is None + + @pytest.mark.asyncio async def test_acompletion_streams_text_deltas_over_injected_transport( http_backend: ModuleType, sdk_provider: str, monkeypatch: pytest.MonkeyPatch diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index aeab7a3d348..c867db2fd02 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -2,6 +2,7 @@ import asyncio import contextvars import copy import datetime +import gzip import json from collections.abc import AsyncIterable, Mapping from types import MappingProxyType, SimpleNamespace @@ -13,6 +14,7 @@ from tests.unit.proxy.router_entry_point_stubs import route_entry_points_to_stub import anthropic import httpx +import openai import pytest import respx from fastapi import HTTPException, Request, Response, status @@ -11675,6 +11677,67 @@ async def test_handle_llm_api_exception_forwards_litellm_response_headers_when_r assert exc_info.value.headers["llm_provider-x-request-id"] == "req_openai_400" +_UPSTREAM_REJECTION_BODY: Final = b'{"error":{"message":"upstream said no","type":"upstream_rejection"}}' + + +def _upstream_rejection_wire(content_encoding: Literal["gzip"] | None) -> tuple[bytes, dict[str, str]]: + wire_body: Final = gzip.compress(_UPSTREAM_REJECTION_BODY) if content_encoding else _UPSTREAM_REJECTION_BODY + return wire_body, { + "x-request-id": "req_upstream", + "x-upstream-trace": "trace-7", + "content-type": "application/json", + "content-length": str(len(wire_body)), + **({"content-encoding": content_encoding} if content_encoding else {}), + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("status_code", "mapped_error"), + ( + (400, litellm.BadRequestError), + (429, litellm.RateLimitError), + (503, litellm.ServiceUnavailableError), + ), +) +@pytest.mark.parametrize("content_encoding", (None, "gzip")) +async def test_handle_llm_api_exception_forwards_exactly_the_upstream_headers_of_a_mapped_openai_error( + status_code: int, mapped_error: type[Exception], content_encoding: Literal["gzip"] | None +) -> None: + wire_body, upstream_headers = _upstream_rejection_wire(content_encoding) + transport: Final = httpx.MockTransport( + lambda _request: httpx.Response(status_code, content=wire_body, headers=upstream_headers) + ) + with pytest.raises(mapped_error) as mapped: + await litellm.acompletion( + model="openai/gpt-5.6", + messages=[{"role": "user", "content": "hi"}], + num_retries=0, + client=openai.AsyncOpenAI( + api_key="transport-only", + base_url="https://upstream.example/v1", + max_retries=0, + http_client=httpx.AsyncClient(transport=transport), + ), + ) + proxy_logging_obj: Final = MagicMock() + proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None) + proxy_logging_obj.post_call_response_headers_hook = AsyncMock(return_value={}) + + with pytest.raises(ProxyException) as forwarded: + await ProxyBaseLLMRequestProcessing(data={}).handle_llm_api_exception( + e=mapped.value, + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"), + proxy_logging_obj=proxy_logging_obj, + ) + + assert forwarded.value.code == str(status_code) + assert "upstream said no" in forwarded.value.message + assert {name: value for name, value in forwarded.value.headers.items() if name.startswith("llm_provider-")} == { + f"llm_provider-{name}": value for name, value in upstream_headers.items() + } + + @pytest.mark.asyncio async def test_handle_llm_api_exception_logs_bug_report_for_unmapped_error( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/test_exception_header_preservation.py b/tests/unit/test_exception_header_preservation.py index 7acacf05226..c5a0868f36d 100644 --- a/tests/unit/test_exception_header_preservation.py +++ b/tests/unit/test_exception_header_preservation.py @@ -13,6 +13,7 @@ from typing import Final import httpx import pytest +import litellm from litellm.exceptions import ( BadRequestError, ContentPolicyViolationError, @@ -418,3 +419,21 @@ class TestBedrockErrorHeaders: headers = get_response_headers(dict(_response_headers)) assert headers.get("llm_provider-x-amzn-requestid") == "req-proxy-500" + + +@pytest.mark.parametrize( + "error_class", + (RateLimitError, ServiceUnavailableError, litellm.BadGatewayError, litellm.InternalServerError), +) +def test_error_built_from_an_unread_streaming_response_keeps_its_headers(error_class: type[Exception]) -> None: + unread: Final = httpx.Response( + 500, + headers={"x-request-id": "req-unread"}, + stream=httpx.ByteStream(b'{"error":"never read"}'), + ) + + error: Final = error_class(message="upstream failed", llm_provider="openai", model="gpt-5.6", response=unread) + + error_response: Final = getattr(error, "response") + assert error_response.text == "" + assert dict(error_response.headers) == {"x-request-id": "req-unread"} From dbcacc7ab7880d5115376cd6c22d22898c50d08f Mon Sep 17 00:00:00 2001 From: zoveki <349028329+zoveki@users.noreply.github.com> Date: Thu, 29 Oct 2026 23:34:57 -0700 Subject: [PATCH 07/12] fix(health): probe only models a partial wildcard serves (#49060) * fix(health): probe only models a partial wildcard serves A partial wildcard health check now probes the cheapest cost-map chat models the wildcard matches, keeps provider route prefixes such as Bedrock converse/ on the probe, and falls back only within the wildcard. A wildcard with no matching cost-map chat model reports an error naming model_info.health_check_model * fix(health): keep the provider-wide probe for an unmatched wildcard, skip retired models, and ignore Bedrock ARNs A partial wildcard with no served cost-map chat model falls back to the provider's cheapest chat models instead of failing, and fails with a message naming model_info.health_check_model only when the provider has none. The cheapest-model picker skips models past their cost-map deprecation_date, and a Bedrock ARN segment is never read as a route prefix. * fix(health): fall back to the provider's cheapest probe when a wildcard's own models fail A partial wildcard now probes up to three of its own cheapest chat models first and then the provider-wide cheapest ones, all from one cost-map scan, so stale or unpriced rows the wildcard matches cannot turn a healthy deployment red. Retired models are read by their resolved cost-map key, both deprecation checks use the UTC date, Bedrock only treats converse, invoke, and region names as route segments, and equal-price ties break by model id * fix(health): cap wildcard health probes at three and rank retired cost-map rows last A partial wildcard now probes two of its own cheapest cost-map chat models and then one provider-wide model, so a failing deployment makes no more calls than before. Rows past their cost-map deprecation date go last instead of being dropped, since some still serve, and the shared cheapest-model picker returns what it returned before. * fix(health): probe a wildcard's own models first, then the same three models main probes A partial wildcard's health check tries its two cheapest own cost-map models, then the provider's three cheapest models main already probes, so a row main shows healthy stays healthy. The deprecation check uses the server's local date, like the Baseten listing * fix(health): keep main's provider-wide probe order so equal-price ties match main --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../health_check_helpers.py | 80 +++- .../litellm_core_utils/llm_request_utils.py | 58 ++- .../test_health_check_helpers.py | 343 +++++++++++++++++- .../test_llm_request_utils.py | 84 ++++- .../proxy/test_health_check_max_tokens.py | 7 +- 5 files changed, 545 insertions(+), 27 deletions(-) diff --git a/litellm/litellm_core_utils/health_check_helpers.py b/litellm/litellm_core_utils/health_check_helpers.py index 940730eeb9f..e40299a218f 100644 --- a/litellm/litellm_core_utils/health_check_helpers.py +++ b/litellm/litellm_core_utils/health_check_helpers.py @@ -3,6 +3,8 @@ Helper functions for health check calls. """ import base64 +import itertools +import re from collections.abc import Awaitable, Callable, Mapping from types import MappingProxyType from typing import TYPE_CHECKING, Final, Literal @@ -26,6 +28,9 @@ IMAGE_EDIT_HEALTH_CHECK_PROMPT: Final = ( ) ANTHROPIC_MESSAGES_HEALTH_CHECK_MAX_TOKENS: Final = 16 +WILDCARD_HEALTH_CHECK_SERVED_MODEL_COUNT: Final = 2 +WILDCARD_HEALTH_CHECK_PROVIDER_MODEL_COUNT: Final = 3 +BEDROCK_ANY_MODEL_ROUTES: Final = frozenset({"converse", "invoke"}) EXTRACTION_HEALTH_CHECK_FIELDS: Final[Mapping[str, Mapping[str, str]]] = MappingProxyType( {"reachable": MappingProxyType({"type": "text", "instructions": "Any word from the text"})} ) @@ -90,6 +95,62 @@ def _ocr_health_check_document(model: str, custom_llm_provider: str) -> Document return native(model, custom_llm_provider) +def _provider_base_model(custom_llm_provider: str, model: str) -> str | None: + from litellm.types.utils import LlmProviders + from litellm.utils import ProviderConfigManager + + try: + model_info: Final = ProviderConfigManager.get_provider_model_info( + model=None, provider=LlmProviders(custom_llm_provider) + ) + return None if model_info is None else model_info.get_base_model(model) + except (ValueError, NotImplementedError): + return None + + +def _is_route_segment(custom_llm_provider: str, segment: str, model_after_segment: str) -> bool: + if custom_llm_provider == "bedrock": + from litellm.llms.bedrock.common_utils import get_all_bedrock_regions + + return segment in BEDROCK_ANY_MODEL_ROUTES or segment in get_all_bedrock_regions() + base_model: Final = _provider_base_model(custom_llm_provider, f"{segment}/{model_after_segment}") + return base_model is not None and not base_model.startswith(f"{segment}/") + + +def wildcard_health_check_models(wildcard: str, custom_llm_provider: str) -> tuple[str, ...]: + from litellm.litellm_core_utils.llm_request_utils import rank_chat_models_from_llm_provider + from litellm.router_utils.pattern_match_deployments import PatternMatchRouter + + provider_prefix: Final = f"{custom_llm_provider}/" + segments: Final = wildcard.removeprefix(provider_prefix).split("/") + route_segments: Final = tuple( + itertools.takewhile( + lambda segment_index: _is_route_segment( + custom_llm_provider, segments[segment_index], "/".join(segments[segment_index + 1 :]) + ), + range(len(segments) - 1), + ) + ) + route_prefix: Final = "".join(f"{segments[segment_index]}/" for segment_index in route_segments) + model_regex: Final = re.compile(PatternMatchRouter().pattern_to_regex("/".join(segments[len(route_segments) :]))) + ranked_model_ids: Final = tuple( + (ranked_model.model.removeprefix(provider_prefix), ranked_model) + for ranked_model in rank_chat_models_from_llm_provider(custom_llm_provider) + ) + served_model_ids: Final = sorted( + (ranked for ranked in ranked_model_ids if model_regex.fullmatch(ranked[0])), + key=lambda ranked: (ranked[1].is_past_deprecation_date, ranked[1].route_rank, ranked[1].cost, ranked[0]), + ) + served_probes: Final = itertools.islice( + dict.fromkeys(f"{provider_prefix}{route_prefix}{model_id}" for model_id, _ in served_model_ids), + WILDCARD_HEALTH_CHECK_SERVED_MODEL_COUNT, + ) + provider_probes: Final = ( + f"{provider_prefix}{model_id}" for model_id, _ in ranked_model_ids[:WILDCARD_HEALTH_CHECK_PROVIDER_MODEL_COUNT] + ) + return tuple(dict.fromkeys((*served_probes, *provider_probes))) + + class HealthCheckHelpers: @staticmethod async def ahealth_check_wildcard_models( @@ -99,23 +160,16 @@ class HealthCheckHelpers: litellm_logging_obj: "Logging", ) -> dict: from litellm import acompletion - from litellm.litellm_core_utils.llm_request_utils import ( - pick_cheapest_chat_models_from_llm_provider, - ) - # this is a wildcard model, we need to pick a random model from the provider - cheapest_models = pick_cheapest_chat_models_from_llm_provider(custom_llm_provider=custom_llm_provider, n=3) - if len(cheapest_models) == 0: + probe_models: Final = wildcard_health_check_models(wildcard=model, custom_llm_provider=custom_llm_provider) + if len(probe_models) == 0: raise Exception( - f"Unable to health check wildcard model for provider {custom_llm_provider}. Add a model on your config.yaml or contribute here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json" + f"Unable to health check wildcard model {model} for provider {custom_llm_provider}: the cost map has " + "no chat model for this provider. Set model_info.health_check_model to a model this wildcard serves" ) - if len(cheapest_models) > 1: - fallback_models = cheapest_models[1:] # Pick the last 2 models from the shuffled list - else: - fallback_models = None - model_params["model"] = cheapest_models[0] + model_params["model"] = probe_models[0] model_params["litellm_logging_obj"] = litellm_logging_obj - model_params["fallbacks"] = fallback_models + model_params["fallbacks"] = list(probe_models[1:]) or None model_params["max_tokens"] = model_params.get("max_tokens", 16) # GPT-5 models require max_output_tokens >= 16 await acompletion(**model_params) return {} diff --git a/litellm/litellm_core_utils/llm_request_utils.py b/litellm/litellm_core_utils/llm_request_utils.py index db881aa16c6..9c2ec0b8984 100644 --- a/litellm/litellm_core_utils/llm_request_utils.py +++ b/litellm/litellm_core_utils/llm_request_utils.py @@ -1,6 +1,11 @@ +import datetime from collections.abc import Iterable, Iterator, Mapping +from dataclasses import dataclass from typing import Final, cast +from pydantic import TypeAdapter +from typing_extensions import NotRequired, ReadOnly, TypedDict + import litellm from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider @@ -151,19 +156,52 @@ def pick_cheapest_chat_models_from_llm_provider(custom_llm_provider: str, n=1): Returns: list[str]: A list of the n cheapest chat models. """ - if custom_llm_provider not in litellm.models_by_provider: - return [] + return [ranked_model.model for ranked_model in rank_chat_models_from_llm_provider(custom_llm_provider)[:n]] + +@dataclass(frozen=True, slots=True) +class RankedChatModel: + model: str + route_rank: int + cost: float + is_past_deprecation_date: bool + + +def rank_chat_models_from_llm_provider(custom_llm_provider: str) -> tuple[RankedChatModel, ...]: known_models: Final[Iterable[str]] = litellm.models_by_provider.get(custom_llm_provider, ()) - ranked_rows: Final = ( - (route_rank, cost, model) - for model, cost in _chat_row_costs(known_models, custom_llm_provider) + ranked_models: Final = ( + RankedChatModel( + model=model, route_rank=route_rank, cost=cost, is_past_deprecation_date=is_past_deprecation_date + ) + for model, cost, is_past_deprecation_date in _chat_row_costs(known_models, custom_llm_provider) if (route_rank := _route_rank(model, custom_llm_provider)) is not None ) - return [model for _, _, model in sorted(ranked_rows, key=lambda row: row[:2])[:n]] + return tuple(sorted(ranked_models, key=lambda ranked_model: (ranked_model.route_rank, ranked_model.cost))) -def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator[tuple[str, float]]: +class _CostMapDeprecation(TypedDict): + deprecation_date: NotRequired[ReadOnly[object]] + + +_COST_MAP_DEPRECATION: Final = TypeAdapter(_CostMapDeprecation) + + +def _is_past_deprecation_date(cost_map_key: str) -> bool: + deprecation_date: Final = _COST_MAP_DEPRECATION.validate_python( + litellm.model_cost.get(cost_map_key) or {} # pyright: ignore[reportUnknownMemberType] # untyped cost map JSON + ).get("deprecation_date") + if not isinstance(deprecation_date, str): + return False + try: + return ( + datetime.date.fromisoformat(deprecation_date) + <= datetime.datetime.now(datetime.timezone.utc).astimezone().date() + ) + except ValueError: + return False + + +def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator[tuple[str, float, bool]]: for model in models: try: model_info = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider) @@ -171,7 +209,11 @@ def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator continue if model_info.get("mode") != "chat": continue - yield model, (model_info.get("input_cost_per_token") or 0.0) + (model_info.get("output_cost_per_token") or 0.0) + yield ( + model, + (model_info.get("input_cost_per_token") or 0.0) + (model_info.get("output_cost_per_token") or 0.0), + _is_past_deprecation_date(model_info["key"]), + ) def _route_rank(model: str, custom_llm_provider: str) -> int | None: diff --git a/tests/unit/litellm_core_utils/test_health_check_helpers.py b/tests/unit/litellm_core_utils/test_health_check_helpers.py index 6979d72ff14..950a91beb81 100644 --- a/tests/unit/litellm_core_utils/test_health_check_helpers.py +++ b/tests/unit/litellm_core_utils/test_health_check_helpers.py @@ -8,6 +8,7 @@ from types import MappingProxyType from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest import responses import respx @@ -19,6 +20,7 @@ from litellm.litellm_core_utils.health_check_helpers import ( HealthCheckHelpers, default_health_check_mode, native_health_check_mode, + wildcard_health_check_models, ) from litellm.main import ahealth_check from litellm.proxy._types import UserAPIKeyAuth @@ -1449,6 +1451,339 @@ async def test_groq_wildcard_health_check_uses_a_concrete_model( assert response == {} +_WILDCARD_CATALOG: Final = MappingProxyType( + { + "databricks": ( + "databricks/databricks-claude-fixture-d", + "databricks/databricks-claude-fixture-c", + "databricks/databricks-claude-fixture-b", + "databricks/databricks-claude-fixture-a", + "databricks/databricks-cheap-chat", + "databricks/databricks-claude-fixture-retired", + ), + "bedrock": ("meta.llama-fixture", "anthropic.claude-fixture"), + "cloudflare": ("cloudflare/@cf/meta/fixture-chat",), + "vertex_ai": ("claude-fixture@default",), + } +) +_WILDCARD_CATALOG_COST: Final = MappingProxyType( + { + "databricks/databricks-claude-fixture-retired": 1e-10, + "databricks/databricks-cheap-chat": 1e-9, + "databricks/databricks-claude-fixture-a": 2e-9, + "databricks/databricks-claude-fixture-b": 3e-9, + "databricks/databricks-claude-fixture-c": 4e-9, + "databricks/databricks-claude-fixture-d": 5e-9, + "meta.llama-fixture": 1e-9, + "anthropic.claude-fixture": 2e-9, + "cloudflare/@cf/meta/fixture-chat": 1e-9, + "claude-fixture@default": 1e-9, + } +) +_DATABRICKS_FIXTURE_API_BASE: Final = "https://fixture.cloud.databricks.com/serving-endpoints" + + +@pytest.fixture +def wildcard_catalog(monkeypatch: pytest.MonkeyPatch) -> None: + providers: Final = {model: provider for provider, models in _WILDCARD_CATALOG.items() for model in models} + litellm.register_model( + { + model: { + "litellm_provider": providers[model], + "mode": "chat", + "max_tokens": 1024, + "input_cost_per_token": cost, + "output_cost_per_token": cost, + **({"deprecation_date": "2000-01-01"} if model.endswith("-retired") else {}), + } + for model, cost in _WILDCARD_CATALOG_COST.items() + } + ) + for provider, models in _WILDCARD_CATALOG.items(): + monkeypatch.setitem(litellm.models_by_provider, provider, list(models)) + + +_DATABRICKS_PROVIDER_PROBES: Final = ( + "databricks/databricks-claude-fixture-retired", + "databricks/databricks-cheap-chat", + "databricks/databricks-claude-fixture-a", +) +_BEDROCK_PROVIDER_PROBES: Final = ("bedrock/meta.llama-fixture", "bedrock/anthropic.claude-fixture") +_DATABRICKS_SERVED_CLAUDE_PROBES: Final = ( + "databricks/databricks-claude-fixture-a", + "databricks/databricks-claude-fixture-b", +) + + +@pytest.mark.parametrize( + ("custom_llm_provider", "wildcard", "expected"), + [ + ("databricks", "databricks-claude-*", (*_DATABRICKS_SERVED_CLAUDE_PROBES, *_DATABRICKS_PROVIDER_PROBES[:2])), + ( + "databricks", + "databricks/databricks-claude-*", + (*_DATABRICKS_SERVED_CLAUDE_PROBES, *_DATABRICKS_PROVIDER_PROBES[:2]), + ), + ("databricks", "*", (*_DATABRICKS_PROVIDER_PROBES[1:], _DATABRICKS_PROVIDER_PROBES[0])), + ("databricks", "databricks-claude-fixture-r*", _DATABRICKS_PROVIDER_PROBES), + ("databricks", "databricks-llama-*", _DATABRICKS_PROVIDER_PROBES), + ("databricks", "*-fixture", _DATABRICKS_PROVIDER_PROBES), + ("databricks", "team-endpoints/*", _DATABRICKS_PROVIDER_PROBES), + ("bedrock", "converse/anthropic.*", ("bedrock/converse/anthropic.claude-fixture", *_BEDROCK_PROVIDER_PROBES)), + ( + "bedrock", + "converse/*", + ( + "bedrock/converse/meta.llama-fixture", + "bedrock/converse/anthropic.claude-fixture", + *_BEDROCK_PROVIDER_PROBES, + ), + ), + ("bedrock", "invoke/anthropic.*", ("bedrock/invoke/anthropic.claude-fixture", *_BEDROCK_PROVIDER_PROBES)), + ("bedrock", "us-east-1/anthropic.*", ("bedrock/us-east-1/anthropic.claude-fixture", *_BEDROCK_PROVIDER_PROBES)), + ("bedrock", "agent/anthropic.*", _BEDROCK_PROVIDER_PROBES), + ("bedrock", "nova/*", _BEDROCK_PROVIDER_PROBES), + ("bedrock", "nova-2/*", _BEDROCK_PROVIDER_PROBES), + ("bedrock", "mantle/*", _BEDROCK_PROVIDER_PROBES), + ("bedrock", "openai/*", _BEDROCK_PROVIDER_PROBES), + ("bedrock", "chat_completions/*", _BEDROCK_PROVIDER_PROBES), + ("bedrock", "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/*", _BEDROCK_PROVIDER_PROBES), + ( + "bedrock", + "converse/arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/*", + _BEDROCK_PROVIDER_PROBES, + ), + ("cloudflare", "@cf/*", ("cloudflare/@cf/meta/fixture-chat",)), + ("vertex_ai", "claude-*", ("vertex_ai/claude-fixture@default",)), + ("vertex_ai", "meta/*", ("vertex_ai/claude-fixture@default",)), + ("fixture-gateway", "team/*", ()), + ], +) +@pytest.mark.usefixtures("wildcard_catalog") +def test_wildcard_health_check_probes_the_cheapest_served_chat_models_before_the_provider_cheapest( + custom_llm_provider: str, wildcard: str, expected: tuple[str, ...] +) -> None: + assert wildcard_health_check_models(wildcard=wildcard, custom_llm_provider=custom_llm_provider) == expected + + +def _fail_first_databricks_claude_probe(request: httpx.Request) -> httpx.Response: + model: Final = json.loads(request.content)["model"] + if model == "databricks-claude-fixture-a": + return httpx.Response(500, json={"error_code": "INTERNAL_ERROR", "message": "fixture outage"}) + return httpx.Response( + 200, + json={ + "id": "chatcmpl-health", + "object": "chat.completion", + "created": 1, + "model": model, + "choices": [{"index": 0, "message": {"role": "assistant", "content": "2"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("wildcard_catalog") +async def test_partial_wildcard_health_check_falls_back_within_the_wildcard_before_any_other_model( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post(f"{_DATABRICKS_FIXTURE_API_BASE}/chat/completions").mock( + side_effect=_fail_first_databricks_claude_probe + ) + + response: Final = await ahealth_check( + { + "model": "databricks/databricks-claude-*", + "api_key": "fake-key", + "api_base": _DATABRICKS_FIXTURE_API_BASE, + "messages": [{"role": "user", "content": "What is 1 + 1?"}], + } + ) + + probed_models: Final = [json.loads(call.request.content)["model"] for call in upstream.calls] + assert probed_models[0] == "databricks-claude-fixture-a" + assert probed_models[-1] == "databricks-claude-fixture-b" + assert all(model.startswith("databricks-claude-") for model in probed_models) + assert "databricks-claude-fixture-a" not in probed_models[1:] + assert response == {} + + +def _answer_only_the_retired_databricks_claude_model(request: httpx.Request) -> httpx.Response: + model: Final = json.loads(request.content)["model"] + if model != "databricks-claude-fixture-retired": + return httpx.Response(404, json={"error_code": "ENDPOINT_NOT_FOUND", "message": f"{model} does not exist"}) + return _fail_first_databricks_claude_probe(request) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("wildcard_catalog") +async def test_partial_wildcard_health_check_stays_healthy_while_any_model_the_provider_probes_answers( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post(f"{_DATABRICKS_FIXTURE_API_BASE}/chat/completions").mock( + side_effect=_answer_only_the_retired_databricks_claude_model + ) + + response: Final = await ahealth_check( + { + "model": "databricks/databricks-claude-*", + "api_key": "fake-key", + "api_base": _DATABRICKS_FIXTURE_API_BASE, + "messages": [{"role": "user", "content": "What is 1 + 1?"}], + } + ) + + assert [json.loads(call.request.content)["model"] for call in upstream.calls] == [ + "databricks-claude-fixture-a", + "databricks-claude-fixture-b", + "databricks-claude-fixture-retired", + ] + assert response == {} + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("wildcard_catalog") +async def test_partial_wildcard_health_check_that_fails_on_every_probe_tries_its_own_models_then_the_provider_probes( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post(f"{_DATABRICKS_FIXTURE_API_BASE}/chat/completions").respond( + 401, json={"error_code": "UNAUTHENTICATED", "message": "fixture bad token"} + ) + + response: Final = await ahealth_check( + { + "model": "databricks/databricks-claude-*", + "api_key": "fake-key", + "api_base": _DATABRICKS_FIXTURE_API_BASE, + "messages": [{"role": "user", "content": "What is 1 + 1?"}], + } + ) + + assert [json.loads(call.request.content)["model"] for call in upstream.calls] == [ + "databricks-claude-fixture-a", + "databricks-claude-fixture-b", + "databricks-claude-fixture-retired", + "databricks-cheap-chat", + ] + assert "error" in response + + +_DATABRICKS_EQUAL_PRICE_MODELS: Final = ( + "databricks/databricks-free-d", + "databricks/databricks-free-c", + "databricks/databricks-free-b", + "databricks/databricks-free-a", +) + + +def _register_equal_price_databricks_models(monkeypatch: pytest.MonkeyPatch, listed_order: tuple[str, ...]) -> None: + litellm.register_model( + { + model: { + "litellm_provider": "databricks", + "mode": "chat", + "max_tokens": 1024, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + } + for model in listed_order + } + ) + monkeypatch.setitem(litellm.models_by_provider, "databricks", list(listed_order)) + + +@pytest.mark.parametrize("listed_order", [_DATABRICKS_EQUAL_PRICE_MODELS, _DATABRICKS_EQUAL_PRICE_MODELS[::-1]]) +def test_partial_wildcard_health_check_orders_its_own_equal_price_models_by_id_whatever_the_listing_order( + monkeypatch: pytest.MonkeyPatch, listed_order: tuple[str, ...] +) -> None: + _register_equal_price_databricks_models(monkeypatch, listed_order) + + assert wildcard_health_check_models(wildcard="databricks/databricks-free-*", custom_llm_provider="databricks")[ + :2 + ] == ("databricks/databricks-free-a", "databricks/databricks-free-b") + + +def _answer_only_the_first_listed_databricks_free_model(request: httpx.Request) -> httpx.Response: + model: Final = json.loads(request.content)["model"] + if model != "databricks-free-d": + return httpx.Response(404, json={"error_code": "ENDPOINT_NOT_FOUND", "message": f"{model} does not exist"}) + return _fail_first_databricks_claude_probe(request) + + +@pytest.mark.asyncio +async def test_wildcard_health_check_falls_back_to_equal_price_provider_models_in_the_provider_listing_order( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + _register_equal_price_databricks_models(monkeypatch, _DATABRICKS_EQUAL_PRICE_MODELS) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post(f"{_DATABRICKS_FIXTURE_API_BASE}/chat/completions").mock( + side_effect=_answer_only_the_first_listed_databricks_free_model + ) + + response: Final = await ahealth_check( + { + "model": "databricks/databricks-claude-*", + "api_key": "fake-key", + "api_base": _DATABRICKS_FIXTURE_API_BASE, + "messages": [{"role": "user", "content": "What is 1 + 1?"}], + } + ) + + assert [json.loads(call.request.content)["model"] for call in upstream.calls] == ["databricks-free-d"] + assert response == {} + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("wildcard_catalog") +async def test_partial_wildcard_health_check_without_a_served_cost_map_model_probes_the_provider_cheapest_chat_model( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post(f"{_DATABRICKS_FIXTURE_API_BASE}/chat/completions").mock( + side_effect=_fail_first_databricks_claude_probe + ) + + response: Final = await ahealth_check( + { + "model": "databricks/databricks-llama-*", + "api_key": "fake-key", + "api_base": _DATABRICKS_FIXTURE_API_BASE, + "messages": [{"role": "user", "content": "What is 1 + 1?"}], + } + ) + + assert json.loads(upstream.calls[0].request.content)["model"] == "databricks-claude-fixture-retired" + assert response == {} + + +@pytest.mark.asyncio +async def test_partial_wildcard_health_check_on_a_provider_without_cost_map_chat_models_names_health_check_model( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setitem(litellm.models_by_provider, "databricks", []) + + response: Final = await ahealth_check( + { + "model": "databricks/databricks-llama-*", + "api_key": "fake-key", + "api_base": _DATABRICKS_FIXTURE_API_BASE, + "messages": [{"role": "user", "content": "What is 1 + 1?"}], + } + ) + + assert "model_info.health_check_model" in response["error"], response["error"] + assert respx_mock.calls.call_count == 0 + + @pytest.mark.asyncio async def test_cohere_rerank_health_check_returns_a_successful_response( monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter @@ -1632,7 +1967,12 @@ async def test_wildcard_health_check_with_pinned_provider_calls_a_model_without_ monkeypatch.setitem( litellm.model_cost, "fake-together-chat", - {"input_cost_per_token": 1e-9, "output_cost_per_token": 1e-9, "mode": "chat", "litellm_provider": "together_ai"}, + { + "input_cost_per_token": 1e-9, + "output_cost_per_token": 1e-9, + "mode": "chat", + "litellm_provider": "together_ai", + }, ) monkeypatch.setitem(litellm.models_by_provider, "together_ai", ("fake-together-chat",)) litellm.in_memory_llm_clients_cache.flush_cache() @@ -1647,4 +1987,3 @@ async def test_wildcard_health_check_with_pinned_provider_calls_a_model_without_ assert "error" not in result, result assert json.loads(together_route.calls.last.request.content)["model"] == "fake-together-chat" assert not other_hosts.called - diff --git a/tests/unit/litellm_core_utils/test_llm_request_utils.py b/tests/unit/litellm_core_utils/test_llm_request_utils.py index 0a2bd5cec1b..e7350fae395 100644 --- a/tests/unit/litellm_core_utils/test_llm_request_utils.py +++ b/tests/unit/litellm_core_utils/test_llm_request_utils.py @@ -8,6 +8,7 @@ from litellm.litellm_core_utils.llm_request_utils import ( ensure_extra_body_is_safe, flatten_form_field_values, pick_cheapest_chat_models_from_llm_provider, + rank_chat_models_from_llm_provider, serialize_multipart_form_fields, ) @@ -149,6 +150,24 @@ def deepseek_rows_whose_cheapest_ids_route_elsewhere(monkeypatch: pytest.MonkeyP "deepseek/fake-cheapest-embedding": {"input_cost_per_token": 1e-13, "mode": "embedding"}, "openai/fake-free-chat": {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "mode": "chat"}, "deepseek/fake-cheap-chat": {"input_cost_per_token": 1e-12, "output_cost_per_token": 1e-12, "mode": "chat"}, + "deepseek/fake-retired-chat": { + "input_cost_per_token": 1e-14, + "output_cost_per_token": 1e-14, + "mode": "chat", + "deprecation_date": "2000-01-01", + }, + "deepseek/fake-retiring-chat": { + "input_cost_per_token": 2e-12, + "output_cost_per_token": 2e-12, + "mode": "chat", + "deprecation_date": "2999-01-01", + }, + "deepseek/fake-misdated-chat": { + "input_cost_per_token": 2.5e-12, + "output_cost_per_token": 2.5e-12, + "mode": "chat", + "deprecation_date": "next spring", + }, "fake-free-chat-without-provider": {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "mode": "chat"}, } for name, row in rows.items(): @@ -156,12 +175,73 @@ def deepseek_rows_whose_cheapest_ids_route_elsewhere(monkeypatch: pytest.MonkeyP monkeypatch.setitem(litellm.models_by_provider, PICKER_PROVIDER, tuple(rows)) -def test_pick_cheapest_chat_models_prefers_provider_routed_ids_and_skips_ids_routed_elsewhere( +def test_pick_cheapest_chat_models_prefers_provider_routed_ids_skips_ids_routed_elsewhere_and_keeps_retired_ones( deepseek_rows_whose_cheapest_ids_route_elsewhere: None, ) -> None: assert pick_cheapest_chat_models_from_llm_provider(PICKER_PROVIDER, n=10) == [ + "deepseek/fake-retired-chat", "deepseek/fake-cheap-chat", + "deepseek/fake-retiring-chat", + "deepseek/fake-misdated-chat", "deepseek/fake-pricey-chat", "fake-free-chat-without-provider", ] - assert pick_cheapest_chat_models_from_llm_provider(PICKER_PROVIDER, n=1) == ["deepseek/fake-cheap-chat"] + assert pick_cheapest_chat_models_from_llm_provider(PICKER_PROVIDER, n=1) == ["deepseek/fake-retired-chat"] + + +def test_rank_chat_models_flags_only_the_rows_whose_cost_map_deprecation_date_has_passed( + deepseek_rows_whose_cheapest_ids_route_elsewhere: None, +) -> None: + assert { + ranked_model.model: ranked_model.is_past_deprecation_date + for ranked_model in rank_chat_models_from_llm_provider(PICKER_PROVIDER) + } == { + "deepseek/fake-retired-chat": True, + "deepseek/fake-cheap-chat": False, + "deepseek/fake-retiring-chat": False, + "deepseek/fake-misdated-chat": False, + "deepseek/fake-pricey-chat": False, + "fake-free-chat-without-provider": False, + } + + +@pytest.mark.parametrize( + "listed_order", + [("deepseek/fake-tie-b", "deepseek/fake-tie-a"), ("deepseek/fake-tie-a", "deepseek/fake-tie-b")], +) +def test_pick_cheapest_chat_models_keeps_the_provider_listing_order_for_equal_price_models( + monkeypatch: pytest.MonkeyPatch, listed_order: tuple[str, str] +) -> None: + for name in listed_order: + monkeypatch.setitem( + litellm.model_cost, + name, + { + "input_cost_per_token": 1e-12, + "output_cost_per_token": 1e-12, + "mode": "chat", + "litellm_provider": PICKER_PROVIDER, + }, + ) + monkeypatch.setitem(litellm.models_by_provider, PICKER_PROVIDER, listed_order) + + assert pick_cheapest_chat_models_from_llm_provider(PICKER_PROVIDER, n=2) == list(listed_order) + + +def test_rank_chat_models_flags_a_retired_model_the_provider_lists_without_its_cost_map_prefix( + monkeypatch: pytest.MonkeyPatch, +) -> None: + rows: Final = { + "vertex_ai/fake-retired-vertex-chat": {"input_cost_per_token": 1e-14, "deprecation_date": "2000-01-01"}, + "vertex_ai/fake-live-vertex-chat": {"input_cost_per_token": 1e-12, "deprecation_date": "2999-01-01"}, + } + for name, row in rows.items(): + monkeypatch.setitem(litellm.model_cost, name, {**row, "mode": "chat", "litellm_provider": "vertex_ai"}) + monkeypatch.setitem( + litellm.models_by_provider, "vertex_ai", tuple(name.removeprefix("vertex_ai/") for name in rows) + ) + + assert [ + (ranked_model.model, ranked_model.is_past_deprecation_date) + for ranked_model in rank_chat_models_from_llm_provider("vertex_ai") + ] == [("fake-retired-vertex-chat", True), ("fake-live-vertex-chat", False)] diff --git a/tests/unit/proxy/test_health_check_max_tokens.py b/tests/unit/proxy/test_health_check_max_tokens.py index e48bf4edc09..df0429316ed 100644 --- a/tests/unit/proxy/test_health_check_max_tokens.py +++ b/tests/unit/proxy/test_health_check_max_tokens.py @@ -11,6 +11,7 @@ import respx import litellm from litellm.litellm_core_utils.health_check_helpers import HealthCheckHelpers +from litellm.litellm_core_utils.llm_request_utils import RankedChatModel from litellm.proxy import health_check as hc_module from litellm.proxy.health_check import ( _is_strategy_router_deployment, @@ -85,8 +86,10 @@ async def test_ahealth_check_wildcard_models_respects_max_tokens(): """ with ( patch( - "litellm.litellm_core_utils.llm_request_utils.pick_cheapest_chat_models_from_llm_provider", - return_value=["gpt-4o-mini"], + "litellm.litellm_core_utils.llm_request_utils.rank_chat_models_from_llm_provider", + return_value=( + RankedChatModel(model="gpt-4o-mini", route_rank=0, cost=0.0, is_past_deprecation_date=False), + ), ), patch("litellm.acompletion", new_callable=AsyncMock), ): From 4a5c5665faec4efd3bd2dc31436c9f6f8b25cf2e Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 29 Oct 2026 23:49:55 -0700 Subject: [PATCH 08/12] fix(azure_ai)!: keep OpenAI prompt cache breakpoints on the Foundry chat completions path (#45299) * fix(azure_ai): keep OpenAI prompt cache breakpoints on the Foundry chat completions path * fix(azure_ai): keep only OpenAI content part types when preserving prompt cache breakpoints * fix(azure_ai): share the breakpoint check with the cache-mark step and keep only text parts A deployment's model_info opt-in is registered under its provider-prefixed cost-map key without a litellm_provider, so that prefix now names the provider the flag speaks for. * refactor(azure_ai): read text parts through a walrus and mark new test values Final * fix(azure_ai): read a router-registered breakpoint opt-in only for azure_ai and keep a message mark off dropped file parts * fix(azure_ai): read the breakpoint opt-in from each deployment's own model_info and base_model A router deployment without its own supports_prompt_cache_breakpoint flag now reads the cost map as loaded, so a sibling's opt-in on the same model string no longer turns its requests into 400s. The shared hook no longer reads router registered keys for azure_ai. * fix(azure_ai): send breakpoints in Foundry's shape under drop_params and fall back to the model name * test(integration): expect the injected breakpoint on the Foundry GPT-6 chat wire --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../anthropic_cache_control_hook.py | 33 +- litellm/llms/azure_ai/chat/transformation.py | 111 +++++- ...ect_prompt_cache_breakpoint_owned_proxy.py | 17 +- .../chat/test_azure_ai_transformation.py | 356 ++++++++++++++++++ 4 files changed, 499 insertions(+), 18 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index fc1f1830129..45c82589d2d 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -23,6 +23,7 @@ from litellm._logging import verbose_logger from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.custom_prompt_management import CustomPromptManagement from litellm.integrations.prompt_management_base import PromptManagementClient +from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.prompt_templates.common_utils import ( cache_control_as_prompt_cache_breakpoint, with_prompt_cache_breakpoint, @@ -175,7 +176,10 @@ def _hosted_entry_flag(entry: Mapping[str, object], resolve_provider: Callable[[ def _hosted_openai_dialect_flag( - model: str, custom_llm_provider: str | None, resolve_provider: Callable[[str], str | None] + model: str, + custom_llm_provider: str | None, + resolve_provider: Callable[[str], str | None], + model_cost: Mapping[str, Mapping[str, object]] | None = None, ) -> bool | None: """ Explicit opt-in for an OpenAI-shaped deployment served by another provider. @@ -194,14 +198,19 @@ def _hosted_openai_dialect_flag( """ import litellm - exact_entry: Final = litellm.model_cost.get(model) - exact_provider: Final = exact_entry.get("litellm_provider") if isinstance(exact_entry, dict) else None + cost_map: Final = ( + cast(Mapping[str, Mapping[str, object]], litellm.model_cost) # cast-ok: the live map holds catalog entries + if model_cost is None + else model_cost + ) + exact_entry: Final = cost_map.get(model) + exact_provider: Final = exact_entry.get("litellm_provider") if isinstance(exact_entry, Mapping) else None if custom_llm_provider is None and (exact_provider == "openai" or ("/" not in model and exact_provider is None)): return None provider: Final = custom_llm_provider or resolve_provider(model) if provider is None or provider == "openai": return None - if isinstance(exact_entry, dict) and exact_provider == provider: + if isinstance(exact_entry, Mapping) and exact_provider == provider: return _hosted_entry_flag(exact_entry, lambda: provider) from litellm.utils import get_potential_model_names @@ -214,8 +223,8 @@ def _hosted_openai_dialect_flag( names["stripped_model_name"], names["provider_prefixed_model_name"], ) - entries: Final = (litellm.model_cost.get(candidate) for candidate in candidates) - flags: Final = (_hosted_entry_flag(entry, lambda: provider) for entry in entries if isinstance(entry, dict)) + entries: Final = (cost_map.get(candidate) for candidate in candidates) + flags: Final = (_hosted_entry_flag(entry, lambda: provider) for entry in entries if isinstance(entry, Mapping)) return next((flag for flag in flags if flag is not None), None) @@ -537,6 +546,18 @@ class AnthropicCacheControlHook(CustomPromptManagement): return False return prompt_cache_options is not None or targets_openai_api(api_base) + @staticmethod + def catalog_targets_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str) -> bool: + return ( + _hosted_openai_dialect_flag( + model, + custom_llm_provider, + AnthropicCacheControlHook._resolve_provider, + GetModelCostMap.loaded_model_cost_map(), + ) + is True + ) + @staticmethod def _resolve_provider(model: str) -> str | None: return _served_provider_for_model(model) diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index d382d653a53..5381447f28f 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -6,15 +6,19 @@ from typing import TYPE_CHECKING, Final, cast import httpx from httpx import Response +from pydantic import TypeAdapter, ValidationError import litellm from litellm._logging import verbose_logger +from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook from litellm.litellm_core_utils.prompt_templates.common_utils import ( content_parts, convert_content_list_to_str, drop_tool_reference_parts_from_tool_messages, filter_value_from_dict, is_pdf_file_part, + prompt_cache_breakpoint_for_wire, + with_prompt_cache_breakpoint, ) from litellm.llms.azure.common_utils import BaseAzureLLM from litellm.llms.azure_ai.common_utils import ( @@ -27,7 +31,7 @@ from litellm.llms.openai.common_utils import drop_params_from_unprocessable_enti from litellm.llms.openai.openai import OpenAIChatCompletionResponseIterator, OpenAIConfig from litellm.llms.xai.chat.transformation import XAIChatConfig from litellm.secret_managers.main import get_secret_str -from litellm.types.llms.openai import AllMessageValues +from litellm.types.llms.openai import AllMessageValues, PromptCacheBreakpoint from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ModelResponse, ModelResponseStream, ProviderField from litellm.utils import add_path_to_api_base, supports_tool_choice @@ -62,6 +66,72 @@ def _has_list_only_content_part(message: Mapping[str, object]) -> bool: return any(_is_list_only_content_part(part) for part in content_parts(message)) +_CONTENT_PARTS_ADAPTER: Final = TypeAdapter(list[dict[str, object]]) + + +def _text_parts(message: Mapping[str, object]) -> tuple[Mapping[str, object], ...]: + content: Final = message.get("content") + if not isinstance(content, list): + return () + try: + parts: Final = _CONTENT_PARTS_ADAPTER.validate_python(content) + except ValidationError: + return () + return tuple(part for part in parts if part.get("type") == "text") + + +def _with_only_text_parts(message: AllMessageValues) -> AllMessageValues: + if _has_list_only_content_part(message) or not (text_parts := _text_parts(message)): + return message + text_only: Final = {**message, "content": list(text_parts)} + return cast(AllMessageValues, text_only) # cast-ok: the same message, text parts only + + +_FIELDS: Final = TypeAdapter(Mapping[str, object]) + + +def _foundry_breakpoint(marker: object) -> PromptCacheBreakpoint | None: + if prompt_cache_breakpoint_for_wire(marker, drop_params=True) is None: + return None + return PromptCacheBreakpoint(mode="explicit") + + +def _part_with_foundry_breakpoint(part: object) -> object: + if not isinstance(part, Mapping): + return part + fields: Final = _FIELDS.validate_python(part) + if "prompt_cache_breakpoint" not in fields: + return fields + unmarked: Final = {key: value for key, value in fields.items() if key != "prompt_cache_breakpoint"} + return with_prompt_cache_breakpoint(unmarked, _foundry_breakpoint(fields["prompt_cache_breakpoint"])) + + +def _with_breakpoints_for_foundry(message: AllMessageValues, drop_params: bool) -> AllMessageValues: + renamed: Final = AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(_with_only_text_parts(message)) + parts: Final = content_parts(renamed) if drop_params else () + if not parts: + return renamed + foundry_message: Final = {**renamed, "content": [_part_with_foundry_breakpoint(part) for part in parts]} + return cast(AllMessageValues, foundry_message) # cast-ok: the same message, its breakpoints in Foundry's shape + + +def _deployment_reads_breakpoints(model: str, litellm_params: Mapping[str, object]) -> bool: + base_model: Final = litellm_params.get("base_model") + named_models: Final = (base_model, model) if isinstance(base_model, str) and base_model else (model,) + model_info: Final = litellm_params.get("model_info") + if not isinstance(model_info, Mapping): + return any( + AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(name, "azure_ai") for name in named_models + ) + deployment_flag: Final = _FIELDS.validate_python(model_info).get("supports_prompt_cache_breakpoint") + if isinstance(deployment_flag, bool): + return deployment_flag + return any( + AnthropicCacheControlHook.catalog_targets_openai_prompt_cache_breakpoint(name, "azure_ai") + for name in named_models + ) + + class AzureAIChatCompletionResponseIterator(OpenAIChatCompletionResponseIterator): def chunk_parser(self, chunk: dict[str, object]) -> ModelResponseStream: # mutable-ok: mirrors override contract usage: Final = chunk.get("usage") @@ -251,6 +321,8 @@ class AzureAIStudioConfig(OpenAIConfig): self, messages: list[AllMessageValues], model: str, + reads_breakpoints: bool | None = None, + drop_params: bool = False, ) -> list: """ - Azure AI Studio doesn't support content as a list. This handles: @@ -260,13 +332,31 @@ class AzureAIStudioConfig(OpenAIConfig): Azure AI Foundry backends set additionalProperties=false and reject these with "Extra inputs are not permitted", which breaks multi-turn Anthropic-format clients that echo thinking blocks back as history. - 2. Transforms list content to a string. + 2. Transforms list content to a string, except on deployments that read OpenAI explicit + prompt cache breakpoints (the deployment's own model_info opt-in, else the cost-map entry + of its base_model or name, as loaded and without a sibling deployment's opt-in): only a + content part can carry the breakpoint (a message-level one is silently ignored), so those + keep their text parts, with each Anthropic `cache_control` mark renamed to a breakpoint the + way the openai provider renames it. Every other part is dropped, exactly as the string + conversion drops it, before the marks are renamed, so a message-level mark lands on the + last text part. Under drop_params each breakpoint goes out as `{"mode": "explicit"}`, the only + shape Foundry takes, and a malformed one is dropped. 3. If message contains an image, audio, or PDF file part, send as is (user-intended) Operates on a deep copy so the caller's messages keep their thinking blocks and provider metadata, which a fallback to another provider still needs. """ - stripped_messages: Final = copy.deepcopy(drop_tool_reference_parts_from_tool_messages(messages)) + deployment_reads_breakpoints: Final = ( + AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(model, "azure_ai") + if reads_breakpoints is None + else reads_breakpoints + ) + copied_messages: Final = copy.deepcopy(drop_tool_reference_parts_from_tool_messages(messages)) + stripped_messages: Final = ( + [_with_breakpoints_for_foundry(message, drop_params) for message in copied_messages] + if deployment_reads_breakpoints + else copied_messages + ) for message in stripped_messages: message_dict = cast(dict, message) # cast-ok: TypedDict is a runtime dict stripped on our copy for field in NON_OPENAI_SPEC_MESSAGE_FIELDS: @@ -275,6 +365,10 @@ class AzureAIStudioConfig(OpenAIConfig): if _has_list_only_content_part(message): continue + if deployment_reads_breakpoints and (text_parts := _text_parts(message)): + message_dict["content"] = list(text_parts) + continue + texts = convert_content_list_to_str(message=message) if texts: message["content"] = texts @@ -329,7 +423,16 @@ class AzureAIStudioConfig(OpenAIConfig): if extra_body and isinstance(extra_body, dict): optional_params.update(extra_body) optional_params.pop("max_retries", None) - return super().transform_request(model, messages, optional_params, litellm_params, headers) + return { + "model": model, + "messages": self._transform_messages( + messages, + model, + _deployment_reads_breakpoints(model, _FIELDS.validate_python(litellm_params)), + bool(litellm_params.get("drop_params") or litellm.drop_params), + ), + **optional_params, + } def transform_response( self, diff --git a/tests/integration/providers/test_openai_dialect_prompt_cache_breakpoint_owned_proxy.py b/tests/integration/providers/test_openai_dialect_prompt_cache_breakpoint_owned_proxy.py index 3df73d3b767..d470605b1e5 100644 --- a/tests/integration/providers/test_openai_dialect_prompt_cache_breakpoint_owned_proxy.py +++ b/tests/integration/providers/test_openai_dialect_prompt_cache_breakpoint_owned_proxy.py @@ -153,15 +153,16 @@ def _foundry_deployment(rig: _Rig, scenario: Scenario, *, bare: bool) -> str: return name -def _assert_foundry_chat_wire(received: Request, prompt: str, *, options: JsonValue | None) -> None: +def _assert_foundry_chat_wire(received: Request, user_content: JsonValue, *, options: JsonValue | None) -> None: body: Final = body_of(received) + marked_system: Final = [{"type": "text", "text": SYSTEM, "prompt_cache_breakpoint": EXPLICIT}] assert urlsplit(received.target).netloc == _FOUNDRY_HOST, received.target assert urlsplit(received.target).path == _FOUNDRY_CHAT_PATH, received.target - assert "prompt_cache_breakpoint" not in received.body.decode(), received.body assert body["model"] == _FOUNDRY_MODEL, received.body - assert body["messages"] == [{"role": "system", "content": SYSTEM}, {"role": "user", "content": prompt}], ( - received.body - ) + assert body["messages"] == [ + {"role": "system", "content": marked_system}, + {"role": "user", "content": user_content}, + ], received.body assert body.get("prompt_cache_options") == options, received.body @@ -185,7 +186,7 @@ def test_b9_b12_foundry_gpt6_responses_reach_the_openai_dialect_on_the_foundry_h @pytest.mark.timeout(_CELL_TIMEOUT) -def test_b10_foundry_gpt6_messages_bridge_to_chat_without_a_breakpoint(rig: _Rig) -> None: +def test_b10_foundry_gpt6_messages_bridge_to_chat_keeps_the_injected_breakpoint(rig: _Rig) -> None: marker: Final = fresh_marker() with rig.gateway.scenario() as scenario: name: Final = _foundry_deployment(rig, scenario, bare=False) @@ -193,12 +194,12 @@ def test_b10_foundry_gpt6_messages_bridge_to_chat_without_a_breakpoint(rig: _Rig rig.gateway, rig.wire, "messages", request_body("messages", name, prompt_text(marker)) ) assert_answered(outcome, marker) - _assert_foundry_chat_wire(received, prompt_text(marker), options=IMPLICIT) + _assert_foundry_chat_wire(received, [{"type": "text", "text": prompt_text(marker)}], options=IMPLICIT) success_row(name, marker) @pytest.mark.timeout(_CELL_TIMEOUT) -def test_b11_foundry_gpt6_chat_stays_on_the_plain_chat_wire(rig: _Rig) -> None: +def test_b11_foundry_gpt6_chat_stays_on_the_chat_wire_with_the_injected_breakpoint(rig: _Rig) -> None: marker: Final = fresh_marker() with rig.gateway.scenario() as scenario: name: Final = _foundry_deployment(rig, scenario, bare=False) diff --git a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py index 6269fd367fe..145d45317c3 100644 --- a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py +++ b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py @@ -1,6 +1,7 @@ import json import traceback from collections.abc import Callable +from types import MappingProxyType from typing import Final from unittest.mock import MagicMock, patch @@ -9,6 +10,7 @@ import respx from httpx import Response import litellm +from litellm import Router from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map from litellm.llms.azure_ai.azure_model_router.transformation import ( @@ -880,6 +882,360 @@ def test_azure_ai_stripping_does_not_mutate_caller_messages(): } +def test_azure_ai_keeps_prompt_cache_breakpoint_for_a_model_that_supports_it(_local_model_cost_map): + """ + Foundry GPT-6 deployments take OpenAI explicit prompt caching, and the breakpoint only + counts on a content part, so a system message carrying one must reach the request body as + the content-part list it arrived in while the Anthropic-only fields are still stripped. + """ + request: Final = AzureAIStudioConfig().transform_request( + model="gpt-6-astra", + messages=[ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "stable instructions", + "prompt_cache_breakpoint": {"mode": "explicit"}, + "cache_control": {"type": "ephemeral"}, + } + ], + }, + {"role": "user", "content": [{"type": "text", "text": "hi"}]}, + ], + optional_params={"prompt_cache_options": {"mode": "explicit"}}, + litellm_params={}, + headers={}, + ) + + assert request["messages"][0]["content"] == [ + {"type": "text", "text": "stable instructions", "prompt_cache_breakpoint": {"mode": "explicit"}} + ] + assert request["messages"][1]["content"] == [{"type": "text", "text": "hi"}] + assert request["prompt_cache_options"] == {"mode": "explicit"} + + +def test_azure_ai_still_flattens_list_content_for_a_model_without_the_breakpoint_flag(_local_model_cost_map): + request: Final = AzureAIStudioConfig().transform_request( + model="gpt-4o", + messages=[ + { + "role": "system", + "content": [ + {"type": "text", "text": "stable ", "prompt_cache_breakpoint": {"mode": "explicit"}}, + {"type": "text", "text": "instructions"}, + ], + }, + {"role": "user", "content": [{"type": "text", "text": "hi"}]}, + ], + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert request["messages"][0]["content"] == "stable instructions" + assert request["messages"][1]["content"] == "hi" + assert not _find_key_anywhere(request["messages"], "prompt_cache_breakpoint") + + +def test_azure_ai_drops_thinking_parts_from_history_while_keeping_the_breakpoint(_local_model_cost_map): + """ + A multi-turn client can echo an assistant turn back with thinking parts inside its content + list. Foundry rejects those parts, and the string conversion always dropped them, so a model + that keeps content parts must still drop them while the text parts and the breakpoint stay. + """ + request: Final = AzureAIStudioConfig().transform_request( + model="gpt-6-astra", + messages=[ + { + "role": "system", + "content": [ + {"type": "text", "text": "stable instructions", "prompt_cache_breakpoint": {"mode": "explicit"}} + ], + }, + {"role": "user", "content": "read the file"}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "The user wants me to read a file.", "signature": "sig"}, + {"type": "redacted_thinking", "data": "opaque"}, + {"type": "text", "text": "Reading it now."}, + ], + }, + {"role": "user", "content": [{"type": "text", "text": "thanks"}]}, + ], + optional_params={"prompt_cache_options": {"mode": "explicit"}}, + litellm_params={}, + headers={}, + ) + + assert request["messages"][0]["content"] == [ + {"type": "text", "text": "stable instructions", "prompt_cache_breakpoint": {"mode": "explicit"}} + ] + assert request["messages"][2]["content"] == [{"type": "text", "text": "Reading it now."}] + assert request["messages"][3]["content"] == [{"type": "text", "text": "thanks"}] + assert not _find_key_anywhere(request["messages"], "thinking") + assert not _find_key_anywhere(request["messages"], "data") + + + +def test_azure_ai_renames_a_cache_control_mark_to_a_breakpoint_for_a_model_that_reads_them(_local_model_cost_map): + request: Final = AzureAIStudioConfig().transform_request( + model="gpt-6-astra", + messages=[ + { + "role": "system", + "content": [{"type": "text", "text": "stable instructions", "cache_control": {"type": "ephemeral"}}], + }, + {"role": "user", "content": "hi", "cache_control": {"type": "ephemeral"}}, + ], + optional_params={"prompt_cache_options": {"mode": "explicit"}}, + litellm_params={}, + headers={}, + ) + + assert request["messages"][0]["content"] == [ + {"type": "text", "text": "stable instructions", "prompt_cache_breakpoint": {"mode": "explicit"}} + ] + assert request["messages"][1]["content"] == [ + {"type": "text", "text": "hi", "prompt_cache_breakpoint": {"mode": "explicit"}} + ] + assert not _find_key_anywhere(request["messages"], "cache_control") + + +def test_azure_ai_drops_a_file_part_next_to_the_breakpoint_like_the_string_conversion(_local_model_cost_map): + messages: Final = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "summarize", "prompt_cache_breakpoint": {"mode": "explicit"}}, + {"type": "file", "file": {"file_id": "file-held-by-another-provider"}}, + ], + } + ] + + kept: Final = AzureAIStudioConfig().transform_request( + model="gpt-6-astra", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + flattened: Final = AzureAIStudioConfig().transform_request( + model="gpt-4o", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + assert kept["messages"][0]["content"] == [ + {"type": "text", "text": "summarize", "prompt_cache_breakpoint": {"mode": "explicit"}} + ] + assert flattened["messages"][0]["content"] == "summarize" + + +def test_azure_ai_sends_a_pdf_part_next_to_the_breakpoint_as_is(_local_model_cost_map): + content: Final = [ + {"type": "file", "file": {"filename": "document.pdf", "file_data": "data:application/pdf;base64,JVBERi0xLjQK"}}, + {"type": "text", "text": "What is the codeword?", "prompt_cache_breakpoint": {"mode": "explicit"}}, + ] + + request: Final = AzureAIStudioConfig().transform_request( + model="gpt-6-astra", + messages=[{"role": "user", "content": content}], + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert request["messages"] == [{"role": "user", "content": content}] + + +def test_azure_ai_puts_a_message_level_mark_on_the_last_text_part_when_a_file_part_follows(_local_model_cost_map): + request: Final = AzureAIStudioConfig().transform_request( + model="gpt-6-astra", + messages=[ + { + "role": "user", + "cache_control": {"type": "ephemeral"}, + "content": [ + {"type": "text", "text": "summarize"}, + {"type": "file", "file": {"file_id": "file-held-by-another-provider"}}, + ], + } + ], + optional_params={"prompt_cache_options": {"mode": "explicit"}}, + litellm_params={}, + headers={}, + ) + + assert request["messages"][0]["content"] == [ + {"type": "text", "text": "summarize", "prompt_cache_breakpoint": {"mode": "explicit"}} + ] + + +@pytest.mark.parametrize( + ("registered", "keeps_parts"), + [ + ({"azure_ai/qa-foundry-alias": {"litellm_provider": "azure_ai", "supports_prompt_cache_breakpoint": True}}, True), + ({"qa-foundry-alias": {"litellm_provider": "openai", "supports_prompt_cache_breakpoint": True}}, False), + ], + ids=["azure_ai-entry-opts-in", "openai-entry-does-not-speak-for-azure_ai"], +) +def test_azure_ai_keeps_parts_only_when_the_azure_ai_entry_reads_breakpoints( + _local_model_cost_map, registered: dict[str, dict[str, object]], keeps_parts: bool +): + litellm.register_model(model_cost=registered, persist_across_reloads=False, add_to_provider_model_sets=False) + breakpoint_part: Final = {"type": "text", "text": "stable", "prompt_cache_breakpoint": {"mode": "explicit"}} + + request: Final = AzureAIStudioConfig().transform_request( + model="qa-foundry-alias", + messages=[{"role": "system", "content": [breakpoint_part]}], + optional_params={}, + litellm_params={"custom_llm_provider": "azure_ai", "drop_params": True}, + headers={}, + ) + + assert request["messages"][0]["content"] == ([breakpoint_part] if keeps_parts else "stable") + + +@pytest.mark.parametrize( + "registration_order", [("opted-in", "opted-out", "unflagged"), ("unflagged", "opted-out", "opted-in")] +) +def test_azure_ai_deployment_reads_only_its_own_breakpoint_flag_not_a_siblings_sharing_its_model_string( + _local_model_cost_map, respx_mock: respx.MockRouter, registration_order: tuple[str, ...] +) -> None: + breakpoint_flags: Final = MappingProxyType( + { + "opted-in": {"supports_prompt_cache_breakpoint": True}, + "opted-out": {"supports_prompt_cache_breakpoint": False}, + } + ) + def deployment(host: str, model_info: dict[str, object]) -> dict[str, object]: + return { + "model_name": host, + "litellm_params": { + "model": "azure_ai/qa-shared-deployment", + "api_base": f"https://{host}.services.ai.azure.com/models", + "api_key": "azure-ai-test-key", + }, + "model_info": model_info, + } + + routes: Final = { + host: respx_mock.post(host=f"{host}.services.ai.azure.com", path="/models/chat/completions").mock( + return_value=Response( + 200, + json={ + "id": "chatcmpl-azure-ai", + "object": "chat.completion", + "created": 1700000000, + "model": "qa-shared-deployment", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6}, + }, + ) + ) + for host in registration_order + } + router: Final = Router( + model_list=[ + deployment(host, {"id": f"{host}-id", **breakpoint_flags.get(host, {})}) for host in registration_order + ] + ) + marked_system: Final = { + "role": "system", + "content": [{"type": "text", "text": "stable", "cache_control": {"type": "ephemeral"}}], + } + + for host in registration_order: + router.completion(model=host, messages=[marked_system, {"role": "user", "content": "hi"}]) + + def sent_system_content(host: str) -> object: + return json.loads(routes[host].calls.last.request.content)["messages"][0]["content"] + + assert sent_system_content("opted-in") == [ + {"type": "text", "text": "stable", "prompt_cache_breakpoint": {"mode": "explicit"}} + ] + assert sent_system_content("opted-out") == "stable" + assert sent_system_content("unflagged") == "stable" + + +@pytest.mark.parametrize( + "litellm_params", + [ + {"base_model": "azure_ai/gpt-6-astra"}, + {"base_model": "azure_ai/gpt-6-astra", "model_info": {"id": "my-astra-deployment-id"}}, + ], +) +def test_azure_ai_keeps_parts_for_a_custom_deployment_name_whose_base_model_reads_breakpoints( + _local_model_cost_map, litellm_params: dict[str, object] +): + breakpoint_part: Final = {"type": "text", "text": "stable", "prompt_cache_breakpoint": {"mode": "explicit"}} + + request: Final = AzureAIStudioConfig().transform_request( + model="my-astra-deployment", + messages=[{"role": "system", "content": [breakpoint_part]}], + optional_params={}, + litellm_params=litellm_params, + headers={}, + ) + + assert request["messages"][0]["content"] == [breakpoint_part] + + +@pytest.mark.parametrize( + "litellm_params", + [ + {"base_model": "azure/cost-tracking-row"}, + {"base_model": "azure/cost-tracking-row", "model_info": {"id": "astra-deployment-id"}}, + ], +) +def test_azure_ai_falls_back_to_the_model_name_when_the_base_model_row_does_not_read_breakpoints( + _local_model_cost_map, litellm_params: dict[str, object] +): + breakpoint_part: Final = {"type": "text", "text": "stable", "prompt_cache_breakpoint": {"mode": "explicit"}} + + request: Final = AzureAIStudioConfig().transform_request( + model="gpt-6-astra", + messages=[{"role": "system", "content": [breakpoint_part]}], + optional_params={}, + litellm_params=litellm_params, + headers={}, + ) + + assert request["messages"][0]["content"] == [breakpoint_part] + + +@pytest.mark.parametrize( + "sent_marker, litellm_params, wire_part", + [ + ( + {"mode": "explicit", "ttl": "30m"}, + {"drop_params": True}, + {"type": "text", "text": "stable", "prompt_cache_breakpoint": {"mode": "explicit"}}, + ), + ({"type": "ephemeral"}, {"drop_params": True}, {"type": "text", "text": "stable"}), + ( + {"mode": "explicit", "ttl": "30m"}, + {}, + {"type": "text", "text": "stable", "prompt_cache_breakpoint": {"mode": "explicit", "ttl": "30m"}}, + ), + ], +) +def test_azure_ai_reshapes_breakpoints_for_foundry_only_under_drop_params( + _local_model_cost_map, + sent_marker: dict[str, object], + litellm_params: dict[str, object], + wire_part: dict[str, object], +): + request: Final = AzureAIStudioConfig().transform_request( + model="gpt-6-astra", + messages=[ + {"role": "system", "content": [{"type": "text", "text": "stable", "prompt_cache_breakpoint": sent_marker}]} + ], + optional_params={}, + litellm_params=litellm_params, + headers={}, + ) + + assert request["messages"][0]["content"] == [wire_part] + + def test_azure_ai_reference_only_tool_result_is_sent_as_empty_text(): messages: Final = [ {"role": "user", "content": "load the WebFetch tool"}, From 953750ed41c69ea909fa4064a42317b04fa8a39b Mon Sep 17 00:00:00 2001 From: Devon Krisman <844724+dkrisman@users.noreply.github.com> Date: Fri, 30 Oct 2026 03:06:57 -0400 Subject: [PATCH 09/12] fix(websearch-interception)!: stream native server-tool blocks to clients (#40967) * fix(websearch-interception): stream native server-tool blocks to clients Streaming native clients (Anthropic SDK, Claude Desktop, Claude Code) never received the server_tool_use and web_search_tool_result blocks the interception layer builds for them, so citation panels stayed empty and result counts read zero. Two gaps in the same flow: FakeAnthropicMessagesStreamIterator._create_content_block_chunks only knew text, thinking, redacted_thinking and tool_use; any other block type emitted a bare content_block_stop with no start and no payload, silently dropping the block from the fake stream. server_tool_use now streams as content_block_start plus input_json_delta and any other server-generated block rides whole in content_block_start, matching Anthropic's native streaming of these types. try_short_circuit_search only recognized a native client by finding a web_search_* tool in the tools list, but the pre-request and deployment hooks convert that tool to the LiteLLM standard shape before the short-circuit runs, recording the conversion in WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY. The flag now counts as native too, so the synthetic response carries the native blocks on the converted path instead of only when hook ordering happens to leave the tool untouched. Fixes #35333 * fix(anthropic): accept server_tool_use and web_search_tool_result blocks when costing a /v1/messages response The short-circuit now returns native web search blocks, and the cost path validated the served body against a content union that only allowed text, tool_use and thinking blocks, so non-streaming requests answered 500 * fix(anthropic): keep every server_tool_use field when re-streaming a /v1/messages reply * fix(anthropic): keep tool_use fields, redacted thinking data and text citations when re-streaming a /v1/messages reply * test(websearch-interception): build the cost test's logging object with a datetime start time * fix(websearch-interception): send search snippets once in the direct search answer * fix(guardrails): mask web search queries, titles and urls on /v1/messages and deliver stream rewrites past server_tool_use blocks Output guardrails now scan the native web search blocks the short-circuit emits, ahead of the reply text so guardrails that read the last text still scan the reply. A guardrail reply of a different length falls back to the reply-only write-back. Ended-stream tool call rewrites land on server_tool_use blocks too, the ended-stream scan key covers the result strings, and legacy-hook pipeline steps pass through the result strings their response view lacks. The wire test reads the streamed server_tool_use input from its input_json_delta. * fix(guardrails): deliver a legacy post-call hook's tool-call rewrites on streams instead of discarding the stream rewrite A legacy hook that masked a tool call's arguments on a buffered stream made the whole rewrite undeliverable, text included, so the original stream went out. With web search blocks in the stream that now hit any hook masking the search query. The adapter hands same-count tool-call rewrites to the translation's write-back, as a unified guardrail's already are, and still discards a rewrite that adds or drops a tool call. * fix(guardrails): show a legacy post-call hook the web search result blocks of a /v1/messages stream The legacy hook's view of a buffered Anthropic stream is rebuilt from the assembled ModelResponse, which keeps web_search_tool_result blocks only in provider-specific fields, so a hook masking result titles on a stream never saw them. The hook's view now carries those blocks, its rewrite of them goes back through the ended-stream write-back, and the adapter no longer needs to pass texts outside the hook's view through untouched. * test(guardrails): type the legacy hook stream test helpers * fix(guardrails): scan a streamed web search query as text like the non-streaming reply * test(guardrails): type the masking guardrail test helpers * fix(guardrails): keep the block id when scanning a streamed web search query --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../websearch_interception/handler.py | 15 +- .../websearch_interception/transformation.py | 10 +- .../chat/guardrail_translation/handler.py | 335 ++++++++++++++++-- .../messages/native/fake_stream_iterator.py | 56 +-- .../proxy/policy_engine/pipeline_executor.py | 36 +- litellm/types/llms/anthropic.py | 19 + .../test_websearch_interception_wire.py | 27 +- .../test_websearch_short_circuit.py | 122 +++++++ .../test_anthropic_guardrail_handler.py | 307 +++++++++++++++- .../native/test_fake_stream_iterator.py | 145 +++++++- .../policy_engine/test_pipeline_executor.py | 238 +++++++++++-- 11 files changed, 1191 insertions(+), 119 deletions(-) diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index 4739b3ee561..2bb89248be4 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -398,14 +398,17 @@ class WebSearchInterceptionLogger(CustomLogger): (t for t in tools if is_anthropic_native_web_search_tool(t)), None, ) + emit_native_blocks: Final = native_tool is not None or ( + kwargs is not None and bool(kwargs.get(WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY)) + ) outcome: Final = await self._short_circuit_search_outcome(query, kwargs=kwargs) search_result_text: Final = WebSearchTransformation.search_outcome_text(outcome) content: Final[list[dict[str, object]]] = [] - if native_tool is not None: + if emit_native_blocks: tool_use_id: Final = f"srvtoolu_{uuid.uuid4().hex}" - tool_name: Final = native_tool.get("name") or "web_search" + tool_name: Final = (native_tool.get("name") if native_tool is not None else None) or "web_search" content.append( { "type": "server_tool_use", @@ -415,10 +418,10 @@ class WebSearchInterceptionLogger(CustomLogger): } ) content.append( - WebSearchTransformation.build_web_search_outcome_block(tool_use_id=tool_use_id, outcome=outcome) + WebSearchTransformation.build_web_search_outcome_block( + tool_use_id=tool_use_id, outcome=outcome, with_snippets=False + ) ) - # Keep the text block so non-native short-circuit callers (Claude Code, - # github_copilot, etc.) see the same payload they always have. content.append({"type": "text", "text": search_result_text}) response: Final[dict[str, object]] = { @@ -435,7 +438,7 @@ class WebSearchInterceptionLogger(CustomLogger): verbose_logger.debug( "WebSearchInterception: Short-circuit search completed, returning synthetic response (%s chars, native_blocks=%s)", len(search_result_text), - native_tool is not None, + emit_native_blocks, ) return response diff --git a/litellm/integrations/websearch_interception/transformation.py b/litellm/integrations/websearch_interception/transformation.py index 58ae8bd6e48..55564dd1134 100644 --- a/litellm/integrations/websearch_interception/transformation.py +++ b/litellm/integrations/websearch_interception/transformation.py @@ -412,6 +412,7 @@ class WebSearchTransformation: def build_web_search_tool_result_block( tool_use_id: str, search_response: SearchResponse | None, + with_snippets: bool = True, ) -> dict[str, object]: """ Build an Anthropic-native ``web_search_tool_result`` content block. @@ -445,6 +446,8 @@ class WebSearchTransformation: ``litellm.asearch()``. If None or empty, the block is still emitted with an empty result list (signals "search ran, no results" rather than "search did not run"). + with_snippets: False when a text block in the same reply already + carries the snippets. """ items: Final[list[dict[str, object]]] = [] if search_response is not None: @@ -460,7 +463,7 @@ class WebSearchTransformation: "title": title, "page_age": page_age, "encrypted_content": "", - "snippet": getattr(r, "snippet", "") or "", + **({"snippet": getattr(r, "snippet", "") or ""} if with_snippets else {}), } ) return { @@ -481,12 +484,15 @@ class WebSearchTransformation: } @staticmethod - def build_web_search_outcome_block(tool_use_id: str, outcome: SearchOutcome) -> dict[str, object]: + def build_web_search_outcome_block( + tool_use_id: str, outcome: SearchOutcome, with_snippets: bool = True + ) -> dict[str, object]: match outcome: case SearchSucceeded(response=response): return WebSearchTransformation.build_web_search_tool_result_block( tool_use_id=tool_use_id, search_response=response, + with_snippets=with_snippets, ) case SearchFailed(error_code=error_code): return WebSearchTransformation.build_web_search_tool_result_error_block( diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index e351542da01..7f688ae733b 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -24,7 +24,7 @@ from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config from typing_extensions import ReadOnly, TypedDict, assert_never from litellm._logging import verbose_proxy_logger -from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.llms.anthropic.messages.adapters.transformation import ( OPENAI_MAX_TOOL_NAME_LENGTH, @@ -61,6 +61,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import ( unappliable_request_rewrite, ) from litellm.types.llms.anthropic import ( + ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX, AllAnthropicToolsValues, AnthropicMessagesRequest, ) @@ -153,6 +154,7 @@ class _TextAttrBlock(Protocol): text: str +@runtime_checkable class _WritableMessage(Protocol): @overload def get(self, key: str, /) -> object | None: ... @@ -195,6 +197,140 @@ class _ToolCallShape: arguments: str +@dataclass(frozen=True, slots=True) +class _WebSearchString: + block_idx: int + result_idx: int | None + field: str + value: str + + +def _streamed_web_search_result_blocks(response: ModelResponse) -> tuple[Mapping[str, object], ...]: + choice: Final = response.choices[0] if response.choices else None + fields: Final = as_str_mapping(choice.message.provider_specific_fields) if isinstance(choice, Choices) else None + blocks: Final = ( + (as_str_mapping(item) for item in iter_items(fields.get("web_search_results"))) + if fields is not None and isinstance(fields.get("web_search_results"), list) + else () + ) + return tuple( + deepcopy(block) for block in blocks if block is not None and block.get("type") == "web_search_tool_result" + ) + + +def _web_search_results(block: Mapping[str, object]) -> Iterable[object]: + return iter_items(block.get("content")) if isinstance(block.get("content"), list) else () + + +def _web_search_result_strings(block_idx: int, result_idx: int, result: object) -> tuple[_WebSearchString, ...]: + fields: Final = as_str_mapping(result) + if fields is None or fields.get("type") != "web_search_result": + return () + values: Final = ((field, fields.get(field)) for field in ("title", "url")) + return tuple( + _WebSearchString(block_idx, result_idx, field, value) + for field, value in values + if isinstance(value, str) and value + ) + + +def _is_server_tool_use(block: Mapping[str, object]) -> bool: + block_id: Final = block.get("id") + match block.get("type"): + case "server_tool_use": + return True + case "tool_use": + return isinstance(block_id, str) and block_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX) + case _: + return False + + +def _web_search_block_strings( + block_idx: int, block: Mapping[str, object], *, with_query: bool +) -> tuple[_WebSearchString, ...]: + """The strings a client reads off a native web search block: the ``server_tool_use`` query and each + ``web_search_tool_result`` result's title and url.""" + match block.get("type"): + case "server_tool_use" | "tool_use" if with_query and _is_server_tool_use(block): + tool_input: Final = as_str_mapping(block.get("input")) + query: Final = None if tool_input is None else tool_input.get("query") + return (_WebSearchString(block_idx, None, "query", query),) if isinstance(query, str) and query else () + case "web_search_tool_result": + return tuple( + chain.from_iterable( + _web_search_result_strings(block_idx, result_idx, result) + for result_idx, result in enumerate(_web_search_results(block)) + ) + ) + case _: + return () + + +def _rewritten_web_search_result( + result_idx: int, result: object, rewrites: Mapping[tuple[int | None, str], str] +) -> object: + fields: Final = as_str_mapping(result) + if fields is None: + return result + return { + **fields, + **{field: rewrites[(result_idx, field)] for field in ("title", "url") if (result_idx, field) in rewrites}, + } + + +def _rewritten_web_search_block( + block: Mapping[str, object], rewrites: Sequence[tuple[_WebSearchString, str]] +) -> Mapping[str, object]: + by_location: Final = MappingProxyType({(target.result_idx, target.field): value for target, value in rewrites}) + if _is_server_tool_use(block): + query: Final = by_location.get((None, "query")) + tool_input: Final = as_str_mapping(block.get("input")) or MappingProxyType({}) + return block if query is None else {**block, "input": {**tool_input, "query": query}} + if not isinstance(block.get("content"), list): + return block + return { + **block, + "content": [ + _rewritten_web_search_result(result_idx, result, by_location) + for result_idx, result in enumerate(_web_search_results(block)) + ], + } + + +def _write_back_web_search_block(content_block: object, rewrites: Sequence[tuple[_WebSearchString, str]]) -> None: + block: Final = as_str_mapping(content_block) + if block is None or not isinstance(content_block, _WritableMessage): + return + rewritten: Final = _rewritten_web_search_block(block, rewrites) + writable: Final = _as_writable(content_block) + for key in ("input", "content"): + if key in rewritten: + writable[key] = rewritten[key] + + +def _split_web_search_texts( + guardrailed: Sequence[str], web_search_count: int, reply_count: int +) -> tuple[Sequence[str], Sequence[str]]: + split_at: Final = web_search_count if len(guardrailed) == web_search_count + reply_count else 0 + return guardrailed[:split_at], guardrailed[split_at:] + + +def _web_search_rewrites_by_block( + strings: Sequence[_WebSearchString], guardrailed: Sequence[object] +) -> Mapping[int, tuple[tuple[_WebSearchString, str], ...]]: + rewrites: Final = tuple( + (target, value) + for target, value in zip(strings, guardrailed) + if isinstance(value, str) and value != target.value + ) + return MappingProxyType( + { + block_idx: tuple(rewrite for rewrite in rewrites if rewrite[0].block_idx == block_idx) + for block_idx in {target.block_idx for target, _ in rewrites} + } + ) + + def _is_client_tool_use(block: Mapping[str, object]) -> bool: return ( block.get("type") == "tool_use" @@ -289,6 +425,7 @@ def _write_back_message_text(message: _WritableMessage, target: MessageTextTarge _TOOL_USE_INPUT_ADAPTER: Final = TypeAdapter(dict[str, object]) _DECODED_SSE_DATA: Final = TypeAdapter(object) +_REBUILT_TOOL_CALL_BLOCK_TYPES: Final = frozenset({"tool_use", "server_tool_use"}) _RELEASED_TOOL_USE_STOP: Final = ( b"event: message_delta\n" b'data: {"type": "message_delta", "delta": {"stop_reason": "tool_use", "stop_sequence": null}, ' @@ -315,6 +452,51 @@ def _write_back_tool_use( block["name"] = shape.name +def _streamed_query_strings( + tool_call_blocks: Sequence[tuple[int, Mapping[str, object]]], tool_calls: Sequence[_ToolCallShape] +) -> tuple[_WebSearchString, ...]: + if len(tool_call_blocks) != len(tool_calls): + return () + return tuple( + chain.from_iterable( + _web_search_block_strings( + index, {**block, "input": _rewritten_tool_use_input(tool_call.arguments)}, with_query=True + ) + for (index, block), tool_call in zip(tool_call_blocks, tool_calls) + ) + ) + + +def _query_rewrites(strings: Sequence[_WebSearchString], guardrailed: Sequence[object]) -> Mapping[int, str]: + return MappingProxyType( + { + target.block_idx: value + for target, value in zip(strings, guardrailed) + if isinstance(value, str) and value != target.value + } + ) + + +def _with_query(tool_call: _ToolCallShape, query: str) -> _ToolCallShape: + arguments: Final = _rewritten_tool_use_input(tool_call.arguments) + if arguments is None: + return tool_call + return _ToolCallShape(name=tool_call.name, arguments=json.dumps({**arguments, "query": query})) + + +def _with_query_rewrites( + tool_calls: tuple[_ToolCallShape, ...], + tool_call_blocks: Sequence[tuple[int, Mapping[str, object]]], + query_rewrites: Mapping[int, str], +) -> tuple[_ToolCallShape, ...]: + if not query_rewrites or len(tool_call_blocks) != len(tool_calls): + return tool_calls + return tuple( + _with_query(tool_call, query_rewrites[index]) if index in query_rewrites else tool_call + for (index, _), tool_call in zip(tool_call_blocks, tool_calls) + ) + + @dataclass(frozen=True, slots=True) class _SSEFieldRewrite: """One field of one nested section of a buffered SSE event, rewritten.""" @@ -402,7 +584,12 @@ class AnthropicMessagesHandler(BaseTranslation): def post_call_hook_response(self, response: object) -> object: if not isinstance(response, ModelResponse): return response - return self.adapter.translate_openai_response_to_anthropic(response) + translated: Final = self.adapter.translate_openai_response_to_anthropic(response) + result_blocks: Final = _streamed_web_search_result_blocks(response) + if not result_blocks: + return translated + content: Final = translated.get("content") or () + return {**translated, "content": [*result_blocks, *content]} @staticmethod def _build_streaming_usage_response( @@ -1273,9 +1460,17 @@ class AnthropicMessagesHandler(BaseTranslation): task_mappings, tool_calls_to_check, ) + block_fields: Final = tuple(map(self._output_block_fields, response_content)) + web_search_strings: Final = tuple( + chain.from_iterable( + _web_search_block_strings(content_idx, fields[1], with_query=True) + for content_idx, fields in enumerate(block_fields) + if fields is not None + ) + ) # Step 2: Apply guardrail to all texts in batch - if texts_to_check or tool_calls_to_check: + if texts_to_check or tool_calls_to_check or web_search_strings: request_data = self._prepare_request_data( request_data, response, @@ -1284,7 +1479,7 @@ class AnthropicMessagesHandler(BaseTranslation): ) inputs: Final = self._build_guardrail_inputs( - texts_to_check, + [*(target.value for target in web_search_strings), *texts_to_check], images_to_check, tool_calls_to_check, response, @@ -1297,14 +1492,19 @@ class AnthropicMessagesHandler(BaseTranslation): logging_obj=litellm_logging_obj, ) - guardrailed_texts: Final = guardrailed_inputs.get("texts", []) + web_search_rewrites, reply_rewrites = _split_web_search_texts( + guardrailed_inputs.get("texts", []), len(web_search_strings), len(texts_to_check) + ) # Step 3: Map guardrail responses back to original response structure await self._apply_guardrail_responses_to_output( response=response, - responses=guardrailed_texts, + responses=list(reply_rewrites), task_mappings=task_mappings, ) + self._write_back_web_search_strings( + response_content, _web_search_rewrites_by_block(web_search_strings, web_search_rewrites) + ) verbose_proxy_logger.debug("Anthropic Messages: Processed output response: %s", response) @@ -1350,8 +1550,14 @@ class AnthropicMessagesHandler(BaseTranslation): string_so_far = first_choice.message.content pre_guardrail_tool_calls: Final = _tool_call_shapes(tool_calls_list or ()) guardrail_inputs: Final = GenericGuardrailAPIInputs() - if string_so_far: - guardrail_inputs["texts"] = [string_so_far] + leading_texts: Final = (string_so_far,) if isinstance(string_so_far, str) and string_so_far else () + tool_call_blocks: Final = self._streamed_tool_call_blocks(responses_so_far) + streamed_query_strings: Final = _streamed_query_strings(tool_call_blocks, pre_guardrail_tool_calls) + streamed_result_strings: Final = self._streamed_web_search_result_strings(responses_so_far) + streamed_web_search_strings: Final = (*streamed_result_strings, *streamed_query_strings) + texts_to_check: Final = [*(target.value for target in streamed_web_search_strings), *leading_texts] + if texts_to_check: + guardrail_inputs["texts"] = texts_to_check if tool_calls_list: guardrail_inputs["tool_calls"] = tool_calls_list @@ -1374,29 +1580,42 @@ class AnthropicMessagesHandler(BaseTranslation): responses_so_far, request_data ) raise - guardrailed_texts: Final = _guardrailed_inputs.get("texts") + web_search_rewrites, reply_rewrites = _split_web_search_texts( + _guardrailed_inputs.get("texts") or [], len(streamed_web_search_strings), len(leading_texts) + ) if ( deliver_ended_stream_rewrites and isinstance(string_so_far, str) and string_so_far - and guardrailed_texts - and guardrailed_texts[0] != string_so_far + and reply_rewrites + and reply_rewrites[0] != string_so_far ): self._write_ended_stream_text_rewrite( responses_so_far, - guardrailed_texts[0], + reply_rewrites[0], guardrail_name=guardrail_to_apply.guardrail_name or "unknown", ) + if deliver_ended_stream_rewrites: + self._write_ended_stream_web_search_rewrites( + responses_so_far, + _web_search_rewrites_by_block(streamed_result_strings, web_search_rewrites), + ) if deliver_ended_stream_rewrites: returned_tool_calls: Final = _guardrailed_inputs.get("tool_calls") self._write_ended_stream_tool_call_rewrites( responses_so_far, pre_guardrail_tool_calls=pre_guardrail_tool_calls, - post_guardrail_tool_calls=_tool_call_shapes( - returned_tool_calls - if isinstance(returned_tool_calls, list) - and len(returned_tool_calls) == len(pre_guardrail_tool_calls) - else tool_calls_list or () + post_guardrail_tool_calls=_with_query_rewrites( + _tool_call_shapes( + returned_tool_calls + if isinstance(returned_tool_calls, list) + and len(returned_tool_calls) == len(pre_guardrail_tool_calls) + else tool_calls_list or () + ), + tool_call_blocks, + _query_rewrites( + streamed_query_strings, web_search_rewrites[len(streamed_result_strings) :] + ), ), guardrail_name=guardrail_to_apply.guardrail_name or "unknown", ) @@ -1451,6 +1670,14 @@ class AnthropicMessagesHandler(BaseTranslation): request_data["litellm_metadata"] = user_metadata return request_data + @staticmethod + def _write_back_web_search_strings( + response_content: Sequence[object], + rewrites_by_block: Mapping[int, tuple[tuple[_WebSearchString, str], ...]], + ) -> None: + for block_idx, rewrites in rewrites_by_block.items(): + _write_back_web_search_block(_content_block_at(response_content, block_idx), rewrites) + @staticmethod def _get_response_content(response: object) -> Sequence[object]: """Extract content list from a dict or object response.""" @@ -1573,30 +1800,22 @@ class AnthropicMessagesHandler(BaseTranslation): ) -> None: """Deliver ended-stream guardrail tool-call rewrites by rewriting the buffered chunks in place: the rebuilt response lists tool calls in the - order of the stream's ``tool_use`` blocks, so the nth rewritten call lands - on the nth block, its first ``input_json_delta`` carrying the full rewritten + order of the stream's ``tool_use`` and ``server_tool_use`` blocks, so the nth + rewritten call lands on the nth block, its first ``input_json_delta`` carrying the full rewritten arguments, every later one blanked, and ``content_block_start`` carrying the rewritten name. Blocks that do not line up with the rebuilt tool calls make the rewrite undeliverable, so the pipeline executor discards it and releases the original chunks.""" if post_guardrail_tool_calls == pre_guardrail_tool_calls: return - block_indices: Final = tuple( - index - for item in responses_so_far - for event in cls._iter_sse_events(item) - if event.get("type") == "content_block_start" - and isinstance(block := event.get("content_block"), Mapping) - and block.get("type") == "tool_use" - and isinstance(index := event.get("index"), int) - ) + block_indices: Final = tuple(index for index, _ in cls._streamed_tool_call_blocks(responses_so_far)) if len(block_indices) != len(post_guardrail_tool_calls): from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite raise UndeliverableStreamRewrite( guardrail_name, f"the guardrail returned {len(post_guardrail_tool_calls)} tool calls for a stream that carried " - f"{len(block_indices)} tool_use blocks", + f"{len(block_indices)} tool_use or server_tool_use blocks", ) rewrites_by_block: Final = MappingProxyType( { @@ -1636,6 +1855,60 @@ class AnthropicMessagesHandler(BaseTranslation): ), ) + @classmethod + def _streamed_web_search_result_strings(cls, responses_so_far: Sequence[object]) -> tuple[_WebSearchString, ...]: + events: Final = chain.from_iterable(cls._iter_sse_events(item) for item in responses_so_far) + result_blocks: Final = ( + (index, block) + for event in events + if event.get("type") == "content_block_start" + and isinstance(index := event.get("index"), int) + and (block := as_str_mapping(event.get("content_block"))) is not None + ) + return tuple( + chain.from_iterable( + _web_search_block_strings(index, block, with_query=False) for index, block in result_blocks + ) + ) + + @classmethod + def _write_ended_stream_web_search_rewrites( + cls, + responses_so_far: MutableSequence[object], # mutable-ok: rewrites the caller's buffered chunks in place + rewrites_by_block: Mapping[int, tuple[tuple[_WebSearchString, str], ...]], + ) -> None: + """Deliver ended-stream guardrail rewrites of web search result titles and urls into the + ``content_block_start`` that carries each whole ``web_search_tool_result`` block.""" + if not rewrites_by_block: + return + + def rewrite_result_block(event: Mapping[str, object]) -> _SSEFieldRewrite | None: + index: Final = event.get("index") + block: Final = as_str_mapping(event.get("content_block")) + rewrites: Final = rewrites_by_block.get(index) if isinstance(index, int) else None + if event.get("type") != "content_block_start" or block is None or rewrites is None: + return None + return _SSEFieldRewrite( + "content_block", "content", _rewritten_web_search_block(block, rewrites).get("content") + ) + + cls._rewrite_ended_stream_events(responses_so_far, rewrite_result_block) + + @classmethod + def _streamed_tool_call_blocks( + cls, responses_so_far: Sequence[object] + ) -> tuple[tuple[int, Mapping[str, object]], ...]: + events: Final = chain.from_iterable(cls._iter_sse_events(item) for item in responses_so_far) + return tuple( + (index, block) + for event in events + if event.get("type") == "content_block_start" + and (block := as_str_mapping(event.get("content_block"))) is not None + and isinstance(block_type := block.get("type"), str) + and block_type in _REBUILT_TOOL_CALL_BLOCK_TYPES + and isinstance(index := event.get("index"), int) + ) + @classmethod def _streamed_tool_use_ids(cls, responses_so_far: Sequence[object]) -> Mapping[int, str]: """The id of each buffered ``tool_use`` block, by its content block index.""" @@ -1738,8 +2011,10 @@ class AnthropicMessagesHandler(BaseTranslation): def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None: stream_ended: Final = self._check_streaming_has_ended(responses_so_far) tool_use_fingerprints: Final = self._streamed_tool_use_fingerprints(responses_so_far) + streamed_text: Final = self.get_streaming_string_so_far(responses_so_far) + result_strings: Final = self._streamed_web_search_result_strings(responses_so_far) if stream_ended else () return StreamingScanKey( - texts=(self.get_streaming_string_so_far(responses_so_far),), + texts=(*(target.value for target in result_strings), streamed_text), tool_calls=tool_use_fingerprints if stream_ended else (), stream_ended=stream_ended, tool_calls_in_flight=bool(tool_use_fingerprints) and not stream_ended, diff --git a/litellm/llms/anthropic/messages/native/fake_stream_iterator.py b/litellm/llms/anthropic/messages/native/fake_stream_iterator.py index 39bdfbbc53a..3a26737a8ac 100644 --- a/litellm/llms/anthropic/messages/native/fake_stream_iterator.py +++ b/litellm/llms/anthropic/messages/native/fake_stream_iterator.py @@ -12,11 +12,20 @@ import json from collections.abc import Mapping, Sequence from typing import Final, cast +from pydantic import TypeAdapter + from litellm.llms.anthropic.messages.native.utils import anthropic_message_delta from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) +_CITATIONS: Final = TypeAdapter(tuple[object, ...]) + + +def _citations_of(block: Mapping[str, object]) -> tuple[object, ...]: + citations: Final = block.get("citations") + return _CITATIONS.validate_python(citations) if isinstance(citations, list) else () + class FakeAnthropicMessagesStreamIterator: """ @@ -47,16 +56,30 @@ class FakeAnthropicMessagesStreamIterator: def _create_content_block_chunks(self, block_dict: Mapping[str, object], index: int) -> list[bytes]: """Build SSE chunks for a single content block.""" - chunks: Final = [] + chunks: Final = list[bytes]() block_type: Final = block_dict.get("type") if block_type == "text": + citations: Final = _citations_of(block_dict) content_block_start = { "type": "content_block_start", "index": index, - "content_block": {"type": "text", "text": ""}, + "content_block": {"type": "text", "text": "", "citations": ()} + if citations + else {"type": "text", "text": ""}, } chunks.append(f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode()) + citation_deltas: Final = ( + { + "type": "content_block_delta", + "index": index, + "delta": {"type": "citations_delta", "citation": citation}, + } + for citation in citations + ) + chunks.extend( + f"event: content_block_delta\ndata: {json.dumps(delta)}\n\n".encode() for delta in citation_deltas + ) text: Final = block_dict.get("text", "") content_block_delta = { "type": "content_block_delta", @@ -89,36 +112,23 @@ class FakeAnthropicMessagesStreamIterator: } chunks.append(f"event: content_block_delta\ndata: {json.dumps(signature_delta)}\n\n".encode()) - elif block_type == "redacted_thinking": - content_block_start = { + elif block_type in ("tool_use", "server_tool_use"): + empty_input: Final[Mapping[str, object]] = {} + tool_use_start: Final = { "type": "content_block_start", "index": index, - "content_block": {"type": "redacted_thinking"}, + "content_block": {**block_dict, "input": empty_input}, } - chunks.append(f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode()) - - elif block_type == "tool_use": - content_block_start = { - "type": "content_block_start", - "index": index, - "content_block": { - "type": "tool_use", - "id": block_dict.get("id"), - "name": block_dict.get("name"), - "input": {}, - }, - } - chunks.append(f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode()) - input_data: Final = block_dict.get("input", {}) - content_block_delta = { + chunks.append(f"event: content_block_start\ndata: {json.dumps(tool_use_start)}\n\n".encode()) + tool_use_delta: Final = { "type": "content_block_delta", "index": index, "delta": { "type": "input_json_delta", - "partial_json": json.dumps(input_data), + "partial_json": json.dumps(block_dict.get("input", {})), }, } - chunks.append(f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode()) + chunks.append(f"event: content_block_delta\ndata: {json.dumps(tool_use_delta)}\n\n".encode()) else: passthrough_start: Final = { diff --git a/litellm/proxy/policy_engine/pipeline_executor.py b/litellm/proxy/policy_engine/pipeline_executor.py index 4ad29bd3478..47572dd7ceb 100644 --- a/litellm/proxy/policy_engine/pipeline_executor.py +++ b/litellm/proxy/policy_engine/pipeline_executor.py @@ -98,15 +98,11 @@ def _count_change(sent: tuple[object, ...] | None, returned: tuple[object, ...] return (len(sent), len(returned)) -def _tool_call_mismatch_reason( - sent: tuple[tuple[object, object], ...] | None, returned: tuple[tuple[object, object], ...] | None -) -> str | None: - if sent == returned: - return None +def _tool_call_count_mismatch_reason(sent: Sequence[object] | None, returned: Sequence[object] | None) -> str | None: sent_count: Final = len(sent or ()) returned_count: Final = len(returned or ()) if sent_count == returned_count: - return "the legacy hook changed a tool call's name or arguments, which this path cannot write back" + return None return f"the legacy hook returned {returned_count} tool calls for a stream that carried {sent_count}" @@ -205,11 +201,10 @@ class _LegacyHookStreamAdapter(CustomGuardrail): non-streaming hooks, an exception it raises ends the stream through the executor's fail/error classification, and the response it hands back, or the one it changed in place and returned ``None`` for, is re-scanned by the same translation so its texts reach the - client through the translation's ended-stream write-back. A - replacement whose scanned texts do not line up with the originals, or whose tool calls - differ from them, is undeliverable, so the executor releases the original chunks. A stream - that carried no text to scan, such as a tool-only Anthropic message, stays deliverable as - long as the hook left the tool calls alone.""" + client through the translation's ended-stream write-back. Tool calls the hook rewrote go to the translation's + tool-call write-back like a unified guardrail's. A replacement whose scanned texts or tool + calls do not line up with the originals by count is undeliverable, so the executor releases + the original chunks.""" def __init__( self, @@ -258,14 +253,19 @@ class _LegacyHookStreamAdapter(CustomGuardrail): guardrail_name, f"the legacy hook returned {returned_text_count} texts for a stream that carried {sent_text_count}", ) - tool_call_mismatch: Final = _tool_call_mismatch_reason( - _tool_call_shapes(inputs.get("tool_calls")), _tool_call_shapes(rescanned.get("tool_calls")) + sent_tool_calls: Final = inputs.get("tool_calls") + rewritten_tool_calls: Final = rescanned.get("tool_calls") + tool_call_count_mismatch: Final = _tool_call_count_mismatch_reason(sent_tool_calls, rewritten_tool_calls) + if tool_call_count_mismatch is not None: + raise UndeliverableStreamRewrite(guardrail_name, tool_call_count_mismatch) + texts_update: Final[GenericGuardrailAPIInputs] = {"texts": rewritten} if rewritten else {} + tool_calls_update: Final[GenericGuardrailAPIInputs] = ( + {"tool_calls": rewritten_tool_calls} + if rewritten_tool_calls is not None + and _rewrote(_tool_call_shapes(sent_tool_calls), _tool_call_shapes(rewritten_tool_calls)) + else {} ) - if tool_call_mismatch is not None: - raise UndeliverableStreamRewrite(guardrail_name, tool_call_mismatch) - if not rewritten: - return inputs - rewritten_inputs: Final[GenericGuardrailAPIInputs] = {**inputs, "texts": rewritten} + rewritten_inputs: Final[GenericGuardrailAPIInputs] = {**inputs, **texts_update, **tool_calls_update} return rewritten_inputs async def _rescan( diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index 50c4a2340a3..a648376c542 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -713,6 +713,23 @@ class AnthropicResponseContentBlockRedactedThinking(LiteLLMBaseModel): data: str +class AnthropicResponseContentBlockServerToolUse(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + type: Literal["server_tool_use"] + id: str + name: str + input: Mapping[str, object] + + +class AnthropicResponseContentBlockWebSearchToolResult(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + type: Literal["web_search_tool_result"] + tool_use_id: str + content: Sequence[Mapping[str, object]] | Mapping[str, object] + + class AnthropicResponseUsageBlock(LiteLLMBaseModel): model_config = ConfigDict(extra="allow") @@ -744,6 +761,8 @@ class AnthropicResponse(LiteLLMBaseModel): | AnthropicResponseContentBlockToolUse | AnthropicResponseContentBlockThinking | AnthropicResponseContentBlockRedactedThinking + | AnthropicResponseContentBlockServerToolUse + | AnthropicResponseContentBlockWebSearchToolResult ] """Content generated by the model.""" diff --git a/tests/integration/messages_endpoint/providers/anthropic/test_websearch_interception_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_websearch_interception_wire.py index 5ec860910e5..1bd9f18253e 100644 --- a/tests/integration/messages_endpoint/providers/anthropic/test_websearch_interception_wire.py +++ b/tests/integration/messages_endpoint/providers/anthropic/test_websearch_interception_wire.py @@ -29,6 +29,24 @@ def sse_events(text: str) -> tuple[tuple[str, dict[str, object]], ...]: ) +def streamed_blocks(events: tuple[tuple[str, dict[str, object]], ...]) -> tuple[dict[str, object], ...]: + input_json: Final = tuple( + (event["index"], event["delta"]["partial_json"]) + for name, event in events + if name == "content_block_delta" and event["delta"]["type"] == "input_json_delta" + ) + return tuple( + { + **event["content_block"], + "input": json.loads("".join(part for index, part in input_json if index == event["index"])), + } + if any(index == event["index"] for index, _ in input_json) + else event["content_block"] + for name, event in events + if name == "content_block_start" + ) + + @pytest.mark.covers("other.provider_wire.bedrock.websearch_interception_streamed_capped_turn_ends_with_native_results") def test_streamed_web_search_turn_capped_by_max_agentic_loops_ends_turn_with_snippets_and_ordered_blocks( gateway: Gateway, tmp_path: Path @@ -136,7 +154,7 @@ def test_streamed_web_search_turn_capped_by_max_agentic_loops_ends_turn_with_sni for earlier, earlier_event in events[:position] if earlier == "content_block_start" }, response.text - started: Final = tuple(event["content_block"] for name, event in events if name == "content_block_start") + started: Final = streamed_blocks(events) search_ids: Final = tuple(block["id"] for block in started if block["type"] == "server_tool_use") assert search_ids and all(search_id.startswith("srvtoolu_") for search_id in search_ids), response.text assert started[-1] == {"type": "text", "text": ""}, response.text @@ -162,7 +180,12 @@ def test_streamed_web_search_turn_capped_by_max_agentic_loops_ends_turn_with_sni ) ), response.text assert ( - "".join(event["delta"]["text"] for name, event in events if name == "content_block_delta") == "turn-1" + "".join( + event["delta"]["text"] + for name, event in events + if name == "content_block_delta" and event["delta"]["type"] == "text_delta" + ) + == "turn-1" ), response.text assert [event["delta"]["stop_reason"] for name, event in events if name == "message_delta"] == [ "end_turn" diff --git a/tests/unit/integrations/websearch_interception/test_websearch_short_circuit.py b/tests/unit/integrations/websearch_interception/test_websearch_short_circuit.py index ef050d96dc5..10cb6ff12e9 100644 --- a/tests/unit/integrations/websearch_interception/test_websearch_short_circuit.py +++ b/tests/unit/integrations/websearch_interception/test_websearch_short_circuit.py @@ -5,7 +5,9 @@ Tests the short-circuit path that detects web-search-only /v1/messages requests and executes the search directly without routing through the backend LLM. """ +import datetime import json +from collections.abc import Iterator, Mapping, Sequence from typing import Final from unittest.mock import AsyncMock, patch @@ -16,6 +18,7 @@ import litellm from litellm.integrations.websearch_interception.handler import ( WebSearchInterceptionLogger, ) +from litellm.messages.dispatch import MessagesResult # --------------------------------------------------------------------------- # Detection tests @@ -73,6 +76,42 @@ class TestTryShortCircuitSearch: assert "Result" in text_block["text"] mock_search.assert_called_once_with("Search for Claude Code releases") + @pytest.mark.asyncio + async def test_emits_native_blocks_when_tools_already_converted(self, perplexity_search: respx.Route): + """Pre-request/deployment hooks convert the native tool to the LiteLLM + standard shape before the short-circuit runs and record that in + WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY; the flag alone must be enough for + native blocks to be emitted.""" + from litellm.integrations.websearch_interception.handler import ( + WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY, + ) + + logger: Final = WebSearchInterceptionLogger(enabled_providers=["hosted_vllm"]) + + result: Final = await logger.try_short_circuit_search( + model="hosted_vllm/some-model", + messages=[{"role": "user", "content": "Search for something"}], + tools=[ + { + "name": "litellm_web_search", + "description": "Search the web for information.", + "input_schema": { + "type": "object", + "properties": {"query": {"type": "string"}}, + "required": ["query"], + }, + } + ], + custom_llm_provider="hosted_vllm", + kwargs={WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY: True}, + ) + + assert perplexity_search.call_count == 1 + assert result is not None + assert [b["type"] for b in result["content"]] == ["server_tool_use", "web_search_tool_result", "text"] + assert result["content"][0]["name"] == "web_search" + assert [r["url"] for r in result["content"][1]["content"]] == ["https://example.com/releases"] + @pytest.mark.asyncio async def test_does_not_short_circuit_mixed_tools(self): """Mix of web_search and other tools → NOT short-circuited""" @@ -468,6 +507,89 @@ class TestShortCircuitEntryPoint: assert text_block["text"] == "results" +class TestShortCircuitThroughMessagesEntryPoint: + """The pre-request hook renames the native tool before the short-circuit + runs, so only a request through anthropic_messages shows whether native + clients still get their server_tool_use and web_search_tool_result blocks.""" + + NATIVE_TOOL: Final = {"type": "web_search_20250305", "name": "web_search"} + + @pytest.fixture(autouse=True) + def _intercept_web_search(self, perplexity_search: respx.Route) -> Iterator[respx.Route]: + with patch("litellm.callbacks", [WebSearchInterceptionLogger(enabled_providers=["bedrock"])]): + yield perplexity_search + + @staticmethod + async def _send(*, tools: Sequence[Mapping[str, object]], stream: bool, **kwargs: object) -> MessagesResult: + return await litellm.anthropic_messages( + model="bedrock/converse/us.anthropic.claude-sonnet-5-5", + messages=[{"role": "user", "content": "latest python release"}], + tools=tools, + max_tokens=64, + stream=stream, + **kwargs, + ) + + @pytest.mark.asyncio + async def test_native_tool_gets_native_blocks(self, perplexity_search: respx.Route): + response: Final = await self._send(tools=[self.NATIVE_TOOL], stream=False) + + assert perplexity_search.call_count == 1 + assert [block["type"] for block in response["content"]] == [ + "server_tool_use", + "web_search_tool_result", + "text", + ] + server_tool_use, tool_result, text = response["content"] + assert server_tool_use["name"] == "web_search" + assert server_tool_use["input"] == {"query": "latest python release"} + assert tool_result["tool_use_id"] == server_tool_use["id"] + assert [result["url"] for result in tool_result["content"]] == ["https://example.com/releases"] + assert "Release notes" in text["text"] + assert "Release notes" not in json.dumps(tool_result) + + @pytest.mark.asyncio + async def test_native_blocks_reply_can_be_costed(self): + from litellm.litellm_core_utils.litellm_logging import Logging + + logging_obj: Final = Logging( + model="bedrock/converse/us.anthropic.claude-sonnet-5-5", + messages=[{"role": "user", "content": "latest python release"}], + stream=False, + call_type="anthropic_messages", + start_time=datetime.datetime(2026, 1, 1), + litellm_call_id="short-circuit-cost", + function_id="short-circuit-cost", + ) + logging_obj.update_environment_variables(litellm_params={}, optional_params={}) + response: Final = await self._send(tools=[self.NATIVE_TOOL], stream=False, litellm_logging_obj=logging_obj) + + assert logging_obj.response_cost_calculator(result=response) == 0.0 + + @pytest.mark.asyncio + async def test_native_tool_streams_native_blocks(self): + stream: Final = await self._send(tools=[self.NATIVE_TOOL], stream=True) + body: Final = b"".join([chunk async for chunk in stream]) + + starts: Final = [ + json.loads(line.removeprefix(b"data: "))["content_block"] + for line in body.splitlines() + if line.startswith(b"data: ") and b'"content_block_start"' in line + ] + assert [block["type"] for block in starts] == ["server_tool_use", "web_search_tool_result", "text"] + assert starts[1]["tool_use_id"] == starts[0]["id"] + + @pytest.mark.asyncio + async def test_litellm_standard_tool_stays_text_only(self): + from litellm.integrations.websearch_interception.tools import ( + get_litellm_web_search_tool, + ) + + response: Final = await self._send(tools=[get_litellm_web_search_tool()], stream=False) + + assert [block["type"] for block in response["content"]] == ["text"] + + class TestShortCircuitSafeguardResults: """A short-circuited reply never reaches the backend, so the proxy answers Claude Code's safeguards itself.""" diff --git a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index 141ad8a0481..d815e1eedd3 100644 --- a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -6,6 +6,7 @@ with guardrail transformations, specifically testing edge cases with empty choic """ import json +import re from typing import Any, Literal, Optional, Final from unittest.mock import MagicMock, patch @@ -552,13 +553,25 @@ class TestAnthropicMessagesHandlerStreamingOutputProcessing: assert chunks == original @pytest.mark.asyncio - async def test_deliver_ended_stream_tool_use_rewrite_with_server_tool_use_block_fails_closed(self): - from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite - + async def test_deliver_ended_stream_tool_use_rewrite_lands_on_server_tool_use_and_tool_use_blocks(self): handler = AnthropicMessagesHandler() server_tool_use = [ - ("content_block_start", {"type": "content_block_start", "index": 0, "content_block": {"type": "server_tool_use", "id": "srvtoolu_1", "name": "web_search", "input": {}}}), - ("content_block_delta", {"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": '{"query": "fruit"}'}}), + ( + "content_block_start", + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "server_tool_use", "id": "srvtoolu_1", "name": "web_search", "input": {}}, + }, + ), + ( + "content_block_delta", + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "input_json_delta", "partial_json": '{"query": "fruit"}'}, + }, + ), ("content_block_stop", {"type": "content_block_stop", "index": 0}), ] tool_use = self._ended_tool_use_sse_chunks() @@ -568,6 +581,26 @@ class TestAnthropicMessagesHandlerStreamingOutputProcessing: + [chunk.replace(b'"index": 0', b'"index": 1') for chunk in tool_use[1:]] ) + await handler.process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=self._argument_masking_guardrail(), + litellm_logging_obj=MagicMock(), + deliver_ended_stream_rewrites=True, + ) + + assert self._partial_jsons(chunks) == ['{"fruit": "[MASKED]"}', '{"fruit": "[MASKED]"}', "", ""] + raw = b"".join(chunks).decode() + assert '"id": "srvtoolu_1"' in raw and '"id": "toolu_1"' in raw + assert "persim" not in raw and '"query"' not in raw + + @pytest.mark.asyncio + async def test_deliver_ended_stream_tool_use_rewrite_on_an_unindexed_tool_block_fails_closed(self): + from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite + + handler = AnthropicMessagesHandler() + tool_use = self._ended_tool_use_sse_chunks() + chunks = tool_use[:1] + [tool_use[1].replace(b'"index": 0', b'"index": "0"')] + tool_use[2:] + with pytest.raises(UndeliverableStreamRewrite): await handler.process_output_streaming_response( responses_so_far=chunks, @@ -576,6 +609,270 @@ class TestAnthropicMessagesHandlerStreamingOutputProcessing: deliver_ended_stream_rewrites=True, ) + @staticmethod + def _python_masking_guardrail() -> CustomGuardrail: + class MaskPython(CustomGuardrail): + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict[str, object], + input_type: Literal["request", "response"], + logging_obj: Optional[LiteLLMLoggingObj] = None, + ) -> GenericGuardrailAPIInputs: + for tool_call in inputs.get("tool_calls", []): + if isinstance(tool_call, ChatCompletionMessageToolCall): + tool_call.function.arguments = re.sub( + "python", "[MASKED]", tool_call.function.arguments, flags=re.I + ) + return { + **inputs, + "texts": [re.sub("python", "[MASKED]", text, flags=re.I) for text in inputs.get("texts", [])], + } + + return MaskPython(guardrail_name="test") + + @staticmethod + def _python_texts_masking_guardrail() -> CustomGuardrail: + class MaskPythonTexts(CustomGuardrail): + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict[str, object], + input_type: Literal["request", "response"], + logging_obj: Optional[LiteLLMLoggingObj] = None, + ) -> GenericGuardrailAPIInputs: + return { + **inputs, + "texts": [re.sub("python", "[MASKED]", text, flags=re.I) for text in inputs.get("texts", [])], + } + + return MaskPythonTexts(guardrail_name="test") + + @staticmethod + def _web_search_result_block() -> dict[str, object]: + return { + "type": "web_search_tool_result", + "tool_use_id": "srvtoolu_1", + "content": [ + { + "type": "web_search_result", + "url": "https://docs.python.org/3.14/", + "title": "Python 3.14 docs", + "encrypted_content": "", + "page_age": None, + }, + { + "type": "web_search_result", + "url": "https://peps.example/745", + "title": "PEP 745", + "encrypted_content": "", + "page_age": None, + }, + ], + } + + @classmethod + def _web_search_stream_chunks(cls, server_tool_block_type: str = "server_tool_use") -> list[bytes]: + events = [ + ( + "message_start", + { + "type": "message_start", + "message": { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [], + "stop_reason": None, + "usage": {"input_tokens": 1, "output_tokens": 0}, + }, + }, + ), + ( + "content_block_start", + { + "type": "content_block_start", + "index": 0, + "content_block": { + "type": server_tool_block_type, + "id": "srvtoolu_1", + "name": "web_search", + "input": {}, + }, + }, + ), + ( + "content_block_delta", + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "input_json_delta", "partial_json": '{"query": "python 3.14 release date"}'}, + }, + ), + ("content_block_stop", {"type": "content_block_stop", "index": 0}), + ( + "content_block_start", + {"type": "content_block_start", "index": 1, "content_block": cls._web_search_result_block()}, + ), + ("content_block_stop", {"type": "content_block_stop", "index": 1}), + ( + "content_block_start", + {"type": "content_block_start", "index": 2, "content_block": {"type": "text", "text": ""}}, + ), + ( + "content_block_delta", + { + "type": "content_block_delta", + "index": 2, + "delta": {"type": "text_delta", "text": "Python 3.14 shipped"}, + }, + ), + ("content_block_stop", {"type": "content_block_stop", "index": 2}), + ( + "message_delta", + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": 2}, + }, + ), + ("message_stop", {"type": "message_stop"}), + ] + return [f"event: {name}\ndata: {json.dumps(payload)}\n\n".encode() for name, payload in events] + + @pytest.mark.parametrize("server_tool_block_type", ["server_tool_use", "tool_use"]) + @pytest.mark.parametrize( + "guardrail_factory", + [_python_masking_guardrail, _python_texts_masking_guardrail], + ids=["texts_and_tool_calls", "texts_only"], + ) + @pytest.mark.asyncio + async def test_deliver_ended_stream_rewrites_mask_the_web_search_query_results_and_text( + self, guardrail_factory, server_tool_block_type + ): + chunks = self._web_search_stream_chunks(server_tool_block_type) + + await AnthropicMessagesHandler().process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=guardrail_factory(), + litellm_logging_obj=MagicMock(), + deliver_ended_stream_rewrites=True, + ) + + raw = b"".join(chunks).decode() + assert "python" not in raw.lower() + result_block = next( + json.loads(line[len("data:") :])["content_block"] + for line in raw.split("\n") + if line.startswith("data:") and '"web_search_tool_result"' in line + ) + assert [(result["title"], result["url"]) for result in result_block["content"]] == [ + ("[MASKED] 3.14 docs", "https://docs.[MASKED].org/3.14/"), + ("PEP 745", "https://peps.example/745"), + ] + assert '"id": "srvtoolu_1"' in raw and '"tool_use_id": "srvtoolu_1"' in raw + + @staticmethod + def _last_text_redacting_guardrail() -> CustomGuardrail: + class RedactLastText(CustomGuardrail): + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + return {**inputs, "texts": ["[REDACTED] " + inputs["texts"][-1]]} + + return RedactLastText(guardrail_name="test") + + @pytest.mark.asyncio + async def test_output_response_keeps_the_reply_as_the_last_text_a_guardrail_scans(self): + response = { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [ + { + "type": "server_tool_use", + "id": "srvtoolu_1", + "name": "web_search", + "input": {"query": "python 3.14 release date"}, + }, + self._web_search_result_block(), + {"type": "text", "text": "Python 3.14 shipped"}, + ], + "stop_reason": "end_turn", + } + + await AnthropicMessagesHandler().process_output_response( + response=response, + guardrail_to_apply=self._last_text_redacting_guardrail(), + litellm_logging_obj=MagicMock(), + ) + + assert response["content"][0]["input"] == {"query": "python 3.14 release date"} + assert response["content"][1] == self._web_search_result_block() + assert response["content"][2] == {"type": "text", "text": "[REDACTED] Python 3.14 shipped"} + + @pytest.mark.asyncio + async def test_deliver_ended_stream_keeps_the_reply_as_the_last_text_a_guardrail_scans(self): + chunks = self._web_search_stream_chunks() + + await AnthropicMessagesHandler().process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=self._last_text_redacting_guardrail(), + litellm_logging_obj=MagicMock(), + deliver_ended_stream_rewrites=True, + ) + + raw = b"".join(chunks).decode() + assert "[REDACTED] Python 3.14 shipped" in raw + assert raw.count("[REDACTED]") == 1 + assert json.dumps(self._web_search_result_block()) in raw + + def test_ended_stream_scan_key_differs_from_a_mid_stream_key_by_the_web_search_results(self): + handler = AnthropicMessagesHandler() + chunks = self._web_search_stream_chunks() + + mid_stream_key = handler.get_streaming_scan_key(chunks[:-2]) + ended_key = handler.get_streaming_scan_key(chunks) + + assert mid_stream_key.texts == ("Python 3.14 shipped",) + assert ended_key != mid_stream_key + assert ended_key.texts[-1] == "Python 3.14 shipped" + assert {"Python 3.14 docs", "https://docs.python.org/3.14/"} <= set(ended_key.texts) + + @pytest.mark.asyncio + async def test_output_response_masks_the_web_search_query_results_and_text(self): + response = { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [ + { + "type": "server_tool_use", + "id": "srvtoolu_1", + "name": "web_search", + "input": {"query": "python 3.14 release date"}, + }, + self._web_search_result_block(), + {"type": "text", "text": "Python 3.14 shipped"}, + ], + "stop_reason": "end_turn", + } + + await AnthropicMessagesHandler().process_output_response( + response=response, + guardrail_to_apply=self._python_masking_guardrail(), + litellm_logging_obj=MagicMock(), + ) + + assert "python" not in json.dumps(response).lower() + assert response["content"][0]["input"] == {"query": "[MASKED] 3.14 release date"} + assert [(result["title"], result["url"]) for result in response["content"][1]["content"]] == [ + ("[MASKED] 3.14 docs", "https://docs.[MASKED].org/3.14/"), + ("PEP 745", "https://peps.example/745"), + ] + assert response["content"][2] == {"type": "text", "text": "[MASKED] 3.14 shipped"} + @pytest.mark.asyncio async def test_ended_stream_rewrite_leaves_chunks_untouched_by_default(self): handler = AnthropicMessagesHandler() diff --git a/tests/unit/llms/anthropic/messages/native/test_fake_stream_iterator.py b/tests/unit/llms/anthropic/messages/native/test_fake_stream_iterator.py index 233145f2706..ece5bb39225 100644 --- a/tests/unit/llms/anthropic/messages/native/test_fake_stream_iterator.py +++ b/tests/unit/llms/anthropic/messages/native/test_fake_stream_iterator.py @@ -1,6 +1,10 @@ import json -from collections.abc import Iterable, Mapping -from typing import Final +from collections.abc import Iterable, Mapping, Sequence +from typing import Final, cast + +import anthropic +import httpx +import pytest from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, @@ -91,3 +95,140 @@ def test_message_delta_omits_absent_optional_fields(): "usage": {"input_tokens": 10, "output_tokens": 2}, } assert _message_delta(response) == {"stop_reason": "end_turn", "stop_sequence": None} + + +_WEB_SEARCH_RESULT_BLOCK: Final = { + "type": "web_search_tool_result", + "tool_use_id": "srvtoolu_01", + "content": [ + { + "type": "web_search_result", + "url": "https://example.com/a", + "title": "Example A", + "encrypted_content": "abc123", + } + ], +} + + +def _content_events(content: Sequence[Mapping[str, object]]) -> tuple[tuple[str, dict[str, object]], ...]: + response: Final = { + "id": "msg_ws", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-6", + "content": list(content), + "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 5}, + } + return tuple( + event for event in _decoded_events(FakeAnthropicMessagesStreamIterator(response=response)) + if event[0].startswith("content_block_") + ) + + +def test_server_tool_use_streams_empty_start_then_input_json_delta(): + """A real Anthropic stream opens server_tool_use with an empty input and + sends the query as an input_json_delta, which SDK accumulators read.""" + caller: Final = {"type": "code_execution_20260120", "tool_id": "srvtoolu_00"} + events: Final = _content_events( + [ + { + "type": "server_tool_use", + "id": "srvtoolu_01", + "name": "web_search", + "input": {"query": "example search"}, + "caller": caller, + } + ] + ) + assert [event_type for event_type, _ in events] == [ + "content_block_start", + "content_block_delta", + "content_block_stop", + ] + assert events[0][1]["content_block"] == { + "type": "server_tool_use", + "id": "srvtoolu_01", + "name": "web_search", + "input": {}, + "caller": caller, + } + delta: Final = events[1][1]["delta"] + assert isinstance(delta, dict) + assert delta["type"] == "input_json_delta" + assert json.loads(delta["partial_json"]) == {"query": "example search"} + + +def test_web_search_tool_result_rides_whole_in_content_block_start(): + events: Final = _content_events([_WEB_SEARCH_RESULT_BLOCK]) + assert [event_type for event_type, _ in events] == ["content_block_start", "content_block_stop"] + assert events[0][1]["content_block"] == _WEB_SEARCH_RESULT_BLOCK + + +def test_native_search_blocks_keep_indices_aligned_with_text(): + events: Final = _content_events( + [ + {"type": "server_tool_use", "id": "srvtoolu_01", "name": "web_search", "input": {"query": "q"}}, + _WEB_SEARCH_RESULT_BLOCK, + {"type": "text", "text": "Here is what I found."}, + ] + ) + starts: Final = tuple(payload for event_type, payload in events if event_type == "content_block_start") + stops: Final = tuple(payload["index"] for event_type, payload in events if event_type == "content_block_stop") + assert [(start["index"], cast(dict[str, object], start["content_block"])["type"]) for start in starts] == [ + (0, "server_tool_use"), + (1, "web_search_tool_result"), + (2, "text"), + ] + assert stops == (0, 1, 2) + + +_CALLER: Final = {"type": "code_execution_20260120", "tool_id": "srvtoolu_00"} + +_REPLAYED_BLOCKS: Final = ( + {"type": "thinking", "thinking": "Look up the release page.", "signature": "sig_01"}, + {"type": "redacted_thinking", "data": "EncryptedThinking=="}, + {"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": {"city": "Paris"}, "caller": _CALLER}, + {"type": "server_tool_use", "id": "srvtoolu_01", "name": "web_fetch", "input": {"url": "https://example.com/a"}, "caller": _CALLER}, + _WEB_SEARCH_RESULT_BLOCK, + { + "type": "text", + "text": "Example A is the newest release.", + "citations": [ + { + "type": "web_search_result_location", + "cited_text": "Example A was released today.", + "url": "https://example.com/a", + "title": "Example A", + "encrypted_index": "idx_01", + } + ], + }, +) + + +def _sdk_rebuilt_content(block: Mapping[str, object]) -> list[dict[str, object]]: + response: Final = { + "id": "msg_replay", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [block], + "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 5}, + } + body: Final = b"".join(FakeAnthropicMessagesStreamIterator(response=response)) + transport: Final = httpx.MockTransport( + lambda _request: httpx.Response(200, content=body, headers={"content-type": "text/event-stream"}) + ) + client: Final = anthropic.Anthropic(api_key="sk-ant-test", http_client=httpx.Client(transport=transport)) + with client.messages.stream( + model="claude-sonnet-5-5", max_tokens=64, messages=[{"role": "user", "content": "hi"}] + ) as stream: + return [content_block.model_dump(exclude_none=True) for content_block in stream.get_final_message().content] + + +@pytest.mark.parametrize("block", _REPLAYED_BLOCKS, ids=lambda block: str(block["type"])) +def test_anthropic_sdk_rebuilds_every_block_field_from_the_stream(block: Mapping[str, object]): + assert _sdk_rebuilt_content(block) == [block] diff --git a/tests/unit/proxy/policy_engine/test_pipeline_executor.py b/tests/unit/proxy/policy_engine/test_pipeline_executor.py index c42878bd4f7..da855753b38 100644 --- a/tests/unit/proxy/policy_engine/test_pipeline_executor.py +++ b/tests/unit/proxy/policy_engine/test_pipeline_executor.py @@ -5,9 +5,12 @@ Uses mock guardrails to validate pipeline execution without external services. """ import copy +import json import logging import pickle -from typing import Literal +from collections.abc import Callable, Iterable, Mapping, Sequence +from itertools import chain +from typing import Final, Literal from unittest.mock import MagicMock import pytest @@ -15,6 +18,7 @@ import pytest import litellm from litellm.caching.dual_cache import DualCache from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.llms.anthropic.chat.guardrail_translation.handler import AnthropicMessagesHandler from litellm.llms.base_llm.guardrail_translation.utils import UnappliableRequestRewrite from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.guardrails.guardrail_hooks.custom_code.custom_code_guardrail import ( @@ -1726,21 +1730,6 @@ async def test_streaming_step_discards_legacy_rewrite_whose_texts_do_not_line_up assert chunks == [_chunk()] -@pytest.mark.asyncio -async def test_streaming_step_discards_legacy_rewrite_that_changes_a_tool_call(monkeypatch, caplog): - masked_tool_call = {"function": {"name": "lookup", "arguments": '{"ssn": "[MASKED]"}'}} - guardrail = _LegacyHookGuardrail( - replacement=_legacy_replacement("[REWRITTEN] hello world", tool_calls=[masked_tool_call]) - ) - chunks = [_chunk()] - - with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): - result = await _run_legacy_streaming_step(monkeypatch, guardrail, chunks) - - _assert_passed_with_discard_warning(result, caplog, "the legacy hook changed a tool call's name or arguments") - assert chunks == [_chunk()] - - @pytest.mark.asyncio async def test_streaming_step_discards_legacy_rewrite_that_drops_the_tool_calls(monkeypatch, caplog): guardrail = _LegacyHookGuardrail(replacement=_legacy_replacement("[REWRITTEN] hello world", tool_calls=[])) @@ -1772,21 +1761,6 @@ async def test_streaming_step_passes_a_tool_only_stream_the_legacy_hook_left_alo assert not any("discarded" in record.getMessage() for record in caplog.records) -@pytest.mark.asyncio -async def test_streaming_step_discards_a_legacy_tool_call_rewrite_on_a_tool_only_stream(monkeypatch, caplog): - masked_tool_call = {"function": {"name": "lookup", "arguments": '{"ssn": "[MASKED]"}'}} - guardrail = _LegacyHookGuardrail(replacement=_legacy_replacement(tool_calls=[masked_tool_call])) - chunks = [_tool_only_chunk()] - - with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): - result = await _run_legacy_streaming_step( - monkeypatch, guardrail, chunks, translation=_ToolOnlyLegacyScanningTranslation() - ) - - _assert_passed_with_discard_warning(result, caplog, "the legacy hook changed a tool call's name or arguments") - assert chunks == [_tool_only_chunk()] - - class _NoHooksGuardrail(CustomGuardrail): pass @@ -1825,6 +1799,208 @@ async def test_streaming_step_discards_a_legacy_rewrite_the_translation_cannot_r assert chunks == [_chunk()] +_AnthropicHookResponse = dict[str, list[dict[str, object]]] + + +class _AnthropicTextRewritingLegacyGuardrail(_LegacyHookGuardrail): + async def async_post_call_success_hook( + self, data: dict[str, object], user_api_key_dict: UserAPIKeyAuth, response: _AnthropicHookResponse + ) -> None: + self.calls.append({"response": copy.deepcopy(response)}) + for block in response["content"]: + if block["type"] == "text" and isinstance(text := block.get("text"), str): + block["text"] = "[REWRITTEN] " + text + + +def _anthropic_web_search_stream() -> list[bytes]: + result_block: Final = { + "type": "web_search_tool_result", + "tool_use_id": "srvtoolu_1", + "content": [ + {"type": "web_search_result", "url": "https://example.test/a", "title": "A", "encrypted_content": ""} + ], + } + events: Final = ( + { + "type": "message_start", + "message": { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "m", + "content": [], + "stop_reason": None, + "usage": {"input_tokens": 1, "output_tokens": 0}, + }, + }, + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "server_tool_use", "id": "srvtoolu_1", "name": "web_search", "input": {}}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "input_json_delta", "partial_json": '{"query": "a"}'}, + }, + {"type": "content_block_stop", "index": 0}, + {"type": "content_block_start", "index": 1, "content_block": result_block}, + {"type": "content_block_stop", "index": 1}, + {"type": "content_block_start", "index": 2, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 2, "delta": {"type": "text_delta", "text": "hello world"}}, + {"type": "content_block_stop", "index": 2}, + {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 2}}, + {"type": "message_stop"}, + ) + return _anthropic_sse(events) + + +def _anthropic_sse(events: Iterable[Mapping[str, object]]) -> list[bytes]: + return [f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() for event in events] + + +def _anthropic_tool_only_stream() -> list[bytes]: + return _anthropic_sse( + ( + { + "type": "message_start", + "message": { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "m", + "content": [], + "stop_reason": None, + "usage": {"input_tokens": 1, "output_tokens": 0}, + }, + }, + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "tool_use", "id": "toolu_1", "name": "lookup", "input": {}}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "input_json_delta", "partial_json": '{"ssn"'}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "input_json_delta", "partial_json": ': "123"}'}, + }, + {"type": "content_block_stop", "index": 0}, + {"type": "message_delta", "delta": {"stop_reason": "tool_use"}, "usage": {"output_tokens": 2}}, + {"type": "message_stop"}, + ) + ) + + +def _streamed_events(chunks: Sequence[bytes]) -> list[dict[str, object]]: + return [ + json.loads(line[len("data: ") :]) + for line in b"".join(chunks).decode().splitlines() + if line.startswith("data: ") + ] + + +def _streamed_block_deltas(chunks: Sequence[bytes], index: int, field: str) -> str: + deltas: Final = ( + event["delta"] + for event in _streamed_events(chunks) + if event.get("index") == index and event["type"] == "content_block_delta" + ) + return "".join(str(delta[field]) for delta in deltas if isinstance(delta, dict)) + + +def _streamed_web_search_results(chunks: Sequence[bytes]) -> list[dict[str, str]]: + blocks: Final = ( + event["content_block"] for event in _streamed_events(chunks) if event["type"] == "content_block_start" + ) + results: Final = ( + block["content"] for block in blocks if isinstance(block, dict) and block["type"] == "web_search_tool_result" + ) + return list(chain.from_iterable(results)) + + +@pytest.mark.asyncio +async def test_streaming_step_delivers_a_legacy_hook_mask_of_web_search_result_titles( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + chunks: Final = _anthropic_web_search_stream() + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + result: Final = await _run_legacy_streaming_step( + monkeypatch, _AnthropicMaskingLegacyGuardrail(), chunks, translation=AnthropicMessagesHandler() + ) + + assert [step.outcome for step in result.step_results] == ["pass"] + assert [(item["title"], item["url"]) for item in _streamed_web_search_results(chunks)] == [ + ("[MASKED]", "https://example.test/a") + ] + assert _streamed_block_deltas(chunks, 2, "text") == "[MASKED] world" + assert not any("discarded" in record.getMessage() for record in caplog.records) + + +class _AnthropicMaskingLegacyGuardrail(_LegacyHookGuardrail): + async def async_post_call_success_hook( + self, data: dict[str, object], user_api_key_dict: UserAPIKeyAuth, response: _AnthropicHookResponse + ) -> None: + for block in response["content"]: + if block["type"] == "text" and isinstance(text := block.get("text"), str): + block["text"] = text.replace("hello", "[MASKED]") + if block["type"] == "tool_use" and isinstance(tool_input := block.get("input"), dict): + block["input"] = {key: "[MASKED]" for key in tool_input} + if block["type"] == "web_search_tool_result" and isinstance(results := block.get("content"), list): + block["content"] = [{**result, "title": "[MASKED]"} for result in results] + + +@pytest.mark.parametrize( + "stream, masked_key", + [(_anthropic_web_search_stream, "query"), (_anthropic_tool_only_stream, "ssn")], + ids=["web_search_query", "tool_only"], +) +@pytest.mark.asyncio +async def test_streaming_step_delivers_a_legacy_hook_mask_of_a_streamed_tool_input( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + stream: Callable[[], list[bytes]], + masked_key: str, +) -> None: + chunks: Final = stream() + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + result: Final = await _run_legacy_streaming_step( + monkeypatch, _AnthropicMaskingLegacyGuardrail(), chunks, translation=AnthropicMessagesHandler() + ) + + assert result.terminal_action == "allow" + assert [step.outcome for step in result.step_results] == ["pass"] + assert json.loads(_streamed_block_deltas(chunks, 0, "partial_json")) == {masked_key: "[MASKED]"} + assert "hello" not in b"".join(chunks).decode() + assert not any("discarded" in record.getMessage() for record in caplog.records) + + +@pytest.mark.asyncio +async def test_streaming_step_delivers_a_legacy_text_rewrite_on_a_web_search_stream( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + guardrail: Final = _AnthropicTextRewritingLegacyGuardrail() + chunks: Final = _anthropic_web_search_stream() + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + result: Final = await _run_legacy_streaming_step( + monkeypatch, guardrail, chunks, translation=AnthropicMessagesHandler() + ) + + assert result.terminal_action == "allow" + assert [step.outcome for step in result.step_results] == ["pass"] + raw: Final = b"".join(chunks).decode() + assert "[REWRITTEN] hello world" in raw + assert '"title": "A"' in raw and '"url": "https://example.test/a"' in raw + assert not any("discarded" in record.getMessage() for record in caplog.records) + + @pytest.mark.asyncio async def test_later_legacy_step_sees_the_stream_as_the_earlier_step_left_it(monkeypatch): masker = _LegacyHookGuardrail(replacement=_legacy_replacement("[REWRITTEN] hello world")) From 56388fd13de12f13e7f49a43e0050ad590baa94d Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 00:11:55 -0700 Subject: [PATCH 10/12] fix(count_tokens): count a history that carries a compaction block (#44784) * fix(count_tokens): count a compaction block instead of answering 500 A message history that replays Anthropic's signed compaction block made /v1/messages/count_tokens answer HTTP 500: neither the Anthropic nor the Vertex AI partner count-tokens request carried the compact-2026-09-04 beta, the provider rejected the block with a 400, the proxy fell back to the local tokenizer, and that raised on the unknown block type. Both provider count paths now add the compaction beta whenever the history carries a compaction block, and the local tokenizer counts the block's summary text like a text block, skipping the signature. * fix(token_counter): skip the tokenizer for empty thinking and compaction text * fix(count_tokens): forward context_management on Anthropic counts and count chat compaction blocks locally Vertex AI compaction support for inference and counting moves to its own PR, so the Vertex count handler is back to main here * fix(anthropic): map list-form context_management to edits for count_tokens and inference betas Count requests forwarded the OpenAI list form of context_management unchanged, while inference maps it to Anthropic edits, and the inference beta helper read the list as no edits. Both now map it once. A replayed compaction block with an empty summary no longer reaches the tokenizer. * fix(anthropic): forward count context_management only with edits, map Vertex list-form betas, and return provider context_management * fix(anthropic): drop an unmappable context_management list under drop_params --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/litellm_core_utils/token_counter.py | 43 ++++-- litellm/llms/anthropic/common_utils.py | 74 ++++++++-- .../llms/anthropic/count_tokens/handler.py | 4 +- .../anthropic/count_tokens/token_counter.py | 1 + .../anthropic/count_tokens/transformation.py | 31 ++++- .../messages/native/transformation.py | 26 +--- .../anthropic/messages/transformation.py | 25 +--- .../proxy/anthropic_endpoints/endpoints.py | 18 ++- .../litellm_core_utils/test_token_counter.py | 91 ++++++++++++ .../test_anthropic_messages_transformation.py | 62 ++++++++- ...t_anthropic_count_tokens_transformation.py | 131 +++++++++++++++++- .../llms/anthropic/test_count_tokens_oauth.py | 2 +- .../vertex_ai/count_tokens/test_context.py | 3 +- ...artner_models_anthropic_messages_config.py | 23 +++ .../anthropic_endpoints/test_endpoints.py | 68 ++++++++- 15 files changed, 520 insertions(+), 82 deletions(-) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index ea3e645ffdf..800a0939cec 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -17,6 +17,7 @@ import anyio import anyio.lowlevel import httpx import tiktoken +from pydantic import TypeAdapter from tiktoken.model import MODEL_TO_ENCODING from typing_extensions import ParamSpec, TypeVar @@ -69,6 +70,7 @@ from litellm.types.llms.anthropic import ( AnthropicMessagesToolResultParam, AnthropicMessagesToolUseParam, AnthropicSearchResultParam, + CompactionBlock, ) from litellm.types.llms.openai import ( AllMessageValues, @@ -937,6 +939,8 @@ def _count_messages( default_token_count, params.pdf_pricing, ) + elif key == "provider_specific_fields": + num_tokens += sum(map(params.count_function, _compaction_summaries(value))) elif key == "search_results" and isinstance(value, list): from litellm.litellm_core_utils.prompt_templates.common_utils import ( extract_search_results_text, @@ -951,6 +955,25 @@ def _count_messages( return num_tokens +_OBJECT_LIST: Final = TypeAdapter(list[object]) + + +def _compaction_summaries(provider_specific_fields: object) -> tuple[str, ...]: + fields: Final = as_str_mapping(provider_specific_fields) + blocks: Final = fields.get("compaction_blocks") if fields is not None else None + block_fields: Final = ( + tuple(map(as_str_mapping, _OBJECT_LIST.validate_python(blocks))) if isinstance(blocks, list) else () + ) + return tuple( + content + for block in block_fields + if block is not None + and block.get("type") == "compaction" + and isinstance(content := block.get("content"), str) + and content + ) + + def _count_extra( count_function: TokenCounterFunction, tools: list[ChatCompletionToolParam] | None, @@ -1648,6 +1671,11 @@ def _count_anthropic_content( return tokens +_TEXT_FIELD_BY_BLOCK_TYPE: Final = MappingProxyType( + {"text": "text", "thinking": "thinking", "redacted_thinking": "thinking", "compaction": "content"} +) + + def _count_content_list( count_function: TokenCounterFunction, content_list: str @@ -1659,6 +1687,7 @@ def _count_content_list( | AnthropicMessagesImageParam | AnthropicMessagesDocumentParam | AnthropicSearchResultParam + | CompactionBlock ], use_default_image_token_count: bool, default_token_count: int | None, @@ -1670,8 +1699,10 @@ def _count_content_list( for c in content_list: if isinstance(c, str): num_tokens += count_function(c) - elif c["type"] == "text": - num_tokens += count_function(str(c.get("text", ""))) + elif c["type"] in _TEXT_FIELD_BY_BLOCK_TYPE: + block_text = str(c.get(_TEXT_FIELD_BY_BLOCK_TYPE[c["type"]]) or "") + if block_text: + num_tokens += count_function(block_text) elif c["type"] == "image_url": image_url = c.get("image_url") num_tokens += _count_image_tokens(image_url, use_default_image_token_count) @@ -1704,12 +1735,6 @@ def _count_content_list( default_token_count, pdf_pricing, ) - elif c["type"] in ("thinking", "redacted_thinking"): - # Claude extended thinking content block - # Count the thinking text and skip the opaque blobs (signature, redacted data) - thinking_text = str(c.get("thinking", "")) - if thinking_text: - num_tokens += count_function(thinking_text) elif c["type"] == "tool_reference": # Anthropic tool-search reference block: a lightweight pointer to # a deferred tool, e.g. {"type": "tool_reference", "tool_name": ...}. @@ -1727,7 +1752,7 @@ def _count_content_list( f"Invalid content item type: {content_type}. " f"Expected str or dict with 'type' field " f"(text, image_url, image, document, file, tool_use, tool_result, thinking, redacted_thinking, " - f"tool_reference, search_result)." + f"compaction, tool_reference, search_result)." ) return num_tokens except Exception as e: diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 0bc519187f4..82977f97854 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -8,12 +8,13 @@ import re from collections.abc import Container, Mapping, MutableMapping, Sequence from dataclasses import dataclass from datetime import datetime, timezone +from itertools import chain from types import MappingProxyType from typing import Annotated, Any, ClassVar, Final, Literal, TypeVar from urllib.parse import quote, urlparse import httpx -from pydantic import BeforeValidator, ConfigDict, Field, StrictBool, TypeAdapter, ValidationError +from pydantic import BeforeValidator, ConfigDict, Field, JsonValue, StrictBool, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict import litellm @@ -24,6 +25,7 @@ from litellm.constants import ( DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET, ) from litellm.exceptions import UnsupportedParamsError +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_file_ids_from_messages, is_encrypted_reasoning_block, @@ -46,6 +48,7 @@ from litellm.llms.anthropic.wif import ( from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.llms.anthropic import ( + ANTHROPIC_BETA_HEADER_VALUES, ANTHROPIC_HOSTED_TOOLS, ANTHROPIC_MID_CONVERSATION_OUTPUT_CONFIG_BETA_HEADER, ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER, @@ -118,6 +121,7 @@ _CLAUDE_CODE_BILLING_HEADER_PREFIX: Final = "x-anthropic-billing-header:" OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object]) OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object]) _DECODED_JSON: Final = TypeAdapter(object) +_JSON_VALUE: Final = TypeAdapter(JsonValue) _CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/") @@ -177,24 +181,68 @@ def is_anthropic_messages_url(url: str) -> bool: return parsed_url.hostname == "api.anthropic.com" or parsed_url.path.removesuffix("/").endswith("/v1/messages") +def _message_blocks(message: object) -> Sequence[object]: + fields: Final = as_str_mapping(message) + blocks: Final = validated_object_list(fields.get("content")) if fields is not None else None + return blocks or () + + +def _is_signed_compaction_block(block: object) -> bool: + fields: Final = as_str_mapping(block) + if fields is None or fields.get("type") != "compaction": + return False + signature: Final = fields.get("signature") + return isinstance(signature, str) and bool(signature) + + +def carries_signed_compaction_block(messages: Sequence[object]) -> bool: + return any(map(_is_signed_compaction_block, chain.from_iterable(map(_message_blocks, messages)))) + + def requires_native_compaction_beta( custom_llm_provider: str, optional_params: Mapping[str, object], messages: Sequence[object], ) -> bool: return custom_llm_provider == "anthropic" and ( - optional_params.get("compaction") is not None - or any( - isinstance(block, Mapping) - and block.get("type") == "compaction" - and isinstance(block.get("signature"), str) - and bool(block.get("signature")) - for message in messages - if isinstance(message, Mapping) - for content in (message.get("content"),) - if isinstance(content, (list, tuple)) - for block in content + optional_params.get("compaction") is not None or carries_signed_compaction_block(messages) + ) + + +def anthropic_context_management(context_management: JsonValue) -> JsonValue: + if not isinstance(context_management, list): + return context_management + from litellm.llms.anthropic.chat.transformation import AnthropicConfig + + edits: Final = AnthropicConfig.map_openai_context_management_to_anthropic( + [entry for entry in context_management if isinstance(entry, dict)] + ) + return context_management if edits is None else _JSON_VALUE.validate_python(edits) + + +def _context_management_edits(spec: JsonValue) -> Sequence[object]: + mapping: Final = as_str_mapping(spec) + return (validated_object_list(mapping.get("edits")) if mapping is not None else None) or () + + +def context_management_with_edits(context_management: JsonValue) -> JsonValue: + spec: Final = anthropic_context_management(context_management) + return spec if _context_management_edits(spec) else None + + +def context_management_betas(context_management: JsonValue) -> tuple[str, ...]: + edits: Final = _context_management_edits(anthropic_context_management(context_management)) + edit_types: Final = tuple(edit.get("type") for edit in map(as_str_mapping, edits) if edit is not None) + return tuple( + beta + for beta, used in ( + (ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value, "compact_20260112" in edit_types), + ( + ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value, + any(edit_type != "compact_20260112" for edit_type in edit_types), + ), ) + if used ) @@ -1367,8 +1415,6 @@ class AnthropicModelInfo(BaseLLMModelInfo): if is_vertex_request is True: # Vertex AI requires web search beta header for web search to work if web_search_tool_used: - from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES - headers["anthropic-beta"] = ANTHROPIC_BETA_HEADER_VALUES.WEB_SEARCH_2025_03_05.value elif len(all_betas) > 0: headers["anthropic-beta"] = ",".join(all_betas) diff --git a/litellm/llms/anthropic/count_tokens/handler.py b/litellm/llms/anthropic/count_tokens/handler.py index f8bf7232d9c..7a7d1d776d3 100644 --- a/litellm/llms/anthropic/count_tokens/handler.py +++ b/litellm/llms/anthropic/count_tokens/handler.py @@ -39,6 +39,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig): tools: list[dict[str, JsonValue]] | None = None, system: JsonValue = None, optional_params: Mapping[str, JsonValue] | None = None, + context_management: JsonValue = None, drop_params: bool = False, ) -> dict[str, JsonValue]: """ @@ -70,6 +71,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig): tools=tools, system=system, optional_params=optional_params, + context_management=context_management, drop_params=drop_params, ) @@ -81,7 +83,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig): verbose_logger.debug("Making request to: %s", endpoint_url) # Get required headers - headers: Final = self.get_count_tokens_headers(auth_header) + headers: Final = self.get_count_tokens_headers(auth_header, messages, context_management) # Use LiteLLM's async httpx client async_client: Final = get_async_httpx_client(llm_provider=litellm.LlmProviders.ANTHROPIC) diff --git a/litellm/llms/anthropic/count_tokens/token_counter.py b/litellm/llms/anthropic/count_tokens/token_counter.py index 91fc77773d1..99b1cd7f241 100644 --- a/litellm/llms/anthropic/count_tokens/token_counter.py +++ b/litellm/llms/anthropic/count_tokens/token_counter.py @@ -132,6 +132,7 @@ class AnthropicTokenCounter(BaseTokenCounter): tools=tools, system=system, optional_params=count_options, + context_management=context_management.spec if context_management is not None else None, drop_params=drops_unsupported_params(litellm_params.get("drop_params")), ) diff --git a/litellm/llms/anthropic/count_tokens/transformation.py b/litellm/llms/anthropic/count_tokens/transformation.py index e60ffe67c8d..1c8f0c206cd 100644 --- a/litellm/llms/anthropic/count_tokens/transformation.py +++ b/litellm/llms/anthropic/count_tokens/transformation.py @@ -27,11 +27,15 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import ) from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.anthropic.common_utils import ( + carries_signed_compaction_block, + context_management_betas, + context_management_with_edits, is_claude_model_target, merge_anthropic_beta_headers, tools_without_custom_field, ) from litellm.llms.anthropic.wif import resolve_anthropic_base +from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES from litellm.types.llms.openai import AllMessageValues, ChatCompletionImageObject, ValidUserMessageContentTypes from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse @@ -376,6 +380,7 @@ class AnthropicCountTokensConfig: tools: list[dict[str, JsonValue]] | None = None, system: JsonValue = None, optional_params: Mapping[str, JsonValue] | None = None, + context_management: JsonValue = None, drop_params: bool = False, ) -> dict[str, JsonValue]: # mutable-ok: provider transport requires JSON dictionaries """ @@ -408,6 +413,7 @@ class AnthropicCountTokensConfig: drop_custom=drop_params, ), ), + ("context_management", context_management_with_edits(context_management)), ) if value is not None } @@ -419,18 +425,33 @@ class AnthropicCountTokensConfig: ) ) - def get_count_tokens_headers(self, auth_header: Mapping[str, str]) -> dict[str, str]: + def get_count_tokens_headers( + self, + auth_header: Mapping[str, str], + messages: Sequence[Mapping[str, JsonValue]], + context_management: JsonValue = None, + ) -> dict[str, str]: """The count-tokens headers around a resolved Anthropic auth header (``AnthropicModelInfo.get_auth_header``): x-api-key for a static key, an Authorization bearer for ``ANTHROPIC_AUTH_TOKEN`` and for sk-ant-oat tokens, whose mandatory oauth beta - merges with the token-counting beta instead of replacing it.""" + merges with the token-counting beta instead of replacing it. A history that carries a + signed compaction block also needs the compaction beta, or Anthropic rejects the block + as an unknown content type, and ``context_management`` edits need the betas inference sends.""" + compaction_betas: Final = ( + (ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value,) + if carries_signed_compaction_block(messages) + else () + ) + request_betas: Final = ( + ANTHROPIC_TOKEN_COUNTING_BETA_VERSION, + *compaction_betas, + *context_management_betas(context_management), + ) return { "Content-Type": "application/json", "anthropic-version": "2023-06-01", **auth_header, - "anthropic-beta": merge_anthropic_beta_headers( - auth_header.get("anthropic-beta"), ANTHROPIC_TOKEN_COUNTING_BETA_VERSION - ), + "anthropic-beta": merge_anthropic_beta_headers(auth_header.get("anthropic-beta"), request_betas), } def validate_request( diff --git a/litellm/llms/anthropic/messages/native/transformation.py b/litellm/llms/anthropic/messages/native/transformation.py index c086618e829..ef855eff180 100644 --- a/litellm/llms/anthropic/messages/native/transformation.py +++ b/litellm/llms/anthropic/messages/native/transformation.py @@ -43,6 +43,7 @@ from ...common_utils import ( OBJECT_LIST_ADAPTER, AnthropicError, AnthropicModelInfo, + context_management_betas, is_claude_model_target, merge_anthropic_beta_headers, optionally_handle_anthropic_oauth, @@ -745,6 +746,8 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): ) if transformed_context_management is not None: anthropic_messages_optional_request_params["context_management"] = transformed_context_management + elif drops_unsupported_params(litellm_params.drop_params): + anthropic_messages_optional_request_params.pop("context_management") ####### get required params for all anthropic messages requests ###### # Lazy %s: the f-string previously stringified the entire messages @@ -873,28 +876,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): if requires_native_compaction_beta(custom_llm_provider, optional_params, messages): beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value) - # Check for context management - context_management_param: Final = optional_params.get("context_management") - if context_management_param is not None: - # Check edits array for compact_20260112 type - edits: Final = context_management_param.get("edits", ()) - has_compact = False - has_other = False - - for edit in edits: - edit_type = edit.get("type", "") - if edit_type == "compact_20260112": - has_compact = True - else: - has_other = True - - # Add compact header if any compact edits exist - if has_compact: - beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value) - - # Add context management header if any other edits exist - if has_other: - beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value) + beta_values.update(context_management_betas(optional_params.get("context_management"))) # Check for structured outputs. Anthropic's newer request shape nests # the schema under output_config.format; the older top-level diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py index bfa94396ea7..da82766c2b9 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py @@ -3,7 +3,7 @@ from typing import Final from litellm.exceptions import UnsupportedParamsError from litellm.litellm_core_utils.core_helpers import drops_unsupported_params -from litellm.llms.anthropic.common_utils import AnthropicModelInfo +from litellm.llms.anthropic.common_utils import AnthropicModelInfo, context_management_betas from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, messages_carry_output_config, @@ -171,28 +171,7 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert if existing_beta: beta_values.update(b.strip() for b in existing_beta.split(",")) - # Check for context management - context_management_param: Final = optional_params.get("context_management") - if context_management_param is not None: - # Check edits array for compact_20260112 type - edits: Final = context_management_param.get("edits", []) - has_compact = False - has_other = False - - for edit in edits: - edit_type = edit.get("type", "") - if edit_type == "compact_20260112": - has_compact = True - else: - has_other = True - - # Add compact header if any compact edits exist - if has_compact: - beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value) - - # Add context management header if any other edits exist - if has_other: - beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value) + beta_values.update(context_management_betas(optional_params.get("context_management"))) # Check for web search tool for tool in tools: diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 1a2e610d5c4..0e204cf9211 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -8,7 +8,7 @@ from typing import Final from fastapi import APIRouter, Depends, HTTPException, Request, Response, status from fastapi.responses import JSONResponse -from pydantic import JsonValue, ValidationError +from pydantic import JsonValue, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict import litellm @@ -54,6 +54,7 @@ from litellm.proxy.litellm_pre_call_utils import ( should_auto_drop_params_for_agentic_cli, ) from litellm.router import Router +from litellm.types.utils import TokenCountResponse router: Final = APIRouter() @@ -334,6 +335,15 @@ def _count_tokens_body_options(data: Mapping[str, JsonValue], auto_drop_params: } +_PROVIDER_COUNT_REPLY: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) + + +def _provider_context_management(response: TokenCountResponse) -> Mapping[str, object]: + original: Final = _PROVIDER_COUNT_REPLY.validate_python(response.original_response) + context_management: Final = original.get("context_management") if original is not None else None + return {} if context_management is None else {"context_management": context_management} + + @router.post( "/v1/messages/count_tokens", tags=["[beta] Anthropic Messages Token Counting"], @@ -413,7 +423,11 @@ async def count_tokens( log_llm_api_exception(e, litellm_call_id) return _anthropic_error_json_response(proxy_exception_from_route_error(e), request) else: - return {"input_tokens": token_response.total_tokens, **count_source_fields(token_response)} + return { + "input_tokens": token_response.total_tokens, + **_provider_context_management(token_response), + **count_source_fields(token_response), + } @router.post( diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index c2f8e12a726..697ac76b4e5 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -1962,6 +1962,97 @@ def test_token_counter_with_redacted_thinking_content(): assert token_counter(model=model, messages=with_block) == token_counter(model=model, messages=without_block) + +def test_token_counter_with_compaction_block(): + """ + A replayed on-demand compaction block (Anthropic's signed summary of the earlier turns) counts its + summary text like a text block and nothing for its signature. It used to raise, which turned every + /v1/messages/count_tokens call on a compacted history into a 500 once the provider count fell back here. + """ + model = "anthropic/claude-sonnet-5-5" + summary = "The user is building a recipe app in Python and asked for one-sentence class descriptions." + follow_up = {"role": "user", "content": [{"type": "text", "text": "Now do the same for Ingredient."}]} + signed_block = {"type": "compaction", "content": summary, "signature": "EqQBCkYIBRgCKkBjZ2xhc3M" * 40} + + as_compaction = [{"role": "assistant", "content": [signed_block]}, follow_up] + unsigned = [{"role": "assistant", "content": [{"type": "compaction", "content": summary}]}, follow_up] + as_text = [{"role": "assistant", "content": [{"type": "text", "text": summary}]}, follow_up] + + assert token_counter(model=model, messages=as_compaction) == token_counter(model=model, messages=as_text) + assert token_counter(model=model, messages=unsigned) == token_counter(model=model, messages=as_text) + + +def test_token_counter_counts_chat_compaction_blocks_like_their_summary_text(): + """ + Chat clients replay a compaction block in the assistant message's provider_specific_fields, and inference + sends it to Anthropic as a content block, so the local count includes its summary text and skips its signature. + """ + model = "anthropic/claude-sonnet-5-5" + summary = "The user is building a recipe app in Python and asked for one-sentence class descriptions." + follow_up = {"role": "user", "content": "Now do the same for Ingredient."} + signed_block = {"type": "compaction", "content": summary, "signature": "EqQBCkYIBRgCKkBjZ2xhc3M" * 40} + + replayed = [ + {"role": "assistant", "content": None, "provider_specific_fields": {"compaction_blocks": [signed_block]}}, + follow_up, + ] + as_text = [{"role": "assistant", "content": summary}, follow_up] + + assert token_counter(model=model, messages=replayed) == token_counter(model=model, messages=as_text) + + +class _BosEncoder: + def encode_batch_fast(self, texts: list[str]) -> list[list[int]]: + return [[0] * (len(text.split()) + 1) for text in texts] + + +def test_token_counter_charges_nothing_for_a_chat_compaction_block_with_no_summary(): + """ + A tokenizer that prepends a BOS token charges one token for the empty string, so a replayed + compaction block with an empty summary must not reach it, like an empty thinking block. + """ + model = "anthropic/claude-sonnet-5-5" + tokenizer = {"type": "huggingface_tokenizer", "tokenizer": _BosEncoder()} + follow_up = {"role": "user", "content": "Now do the same for Ingredient."} + replayed = [ + { + "role": "assistant", + "content": None, + "provider_specific_fields": {"compaction_blocks": [{"type": "compaction", "content": ""}]}, + }, + follow_up, + ] + bare = [{"role": "assistant", "content": None}, follow_up] + + assert token_counter(model=model, messages=replayed, custom_tokenizer=tokenizer) == token_counter( + model=model, messages=bare, custom_tokenizer=tokenizer + ) + + +def test_count_content_list_skips_the_tokenizer_for_empty_block_text(): + """ + An empty thinking block, a redacted_thinking block (no thinking text at all) and a compaction + block with no summary carry no tokens, even under a tokenizer that charges for the empty string + (the llama-2 tokenizer prepends its BOS token to every encode call). Counting them through the + tokenizer added one token per block on such models. + """ + from litellm.litellm_core_utils.token_counter import _count_content_list + + def charges_for_empty_text(text: str) -> int: + return len(text.split()) + 1 + + empty_blocks = [ + {"type": "thinking", "thinking": "", "signature": "EqcLCkYICxgCKkCrqu6lP..."}, + {"type": "thinking", "thinking": None, "signature": "EqcLCkYICxgCKkCrqu6lP..."}, + {"type": "redacted_thinking", "data": "EmwKAhgBEgy3va3pzix/LafPsn4aDFIT2Xlxh0L5L8rLVyIw"}, + {"type": "compaction", "content": ""}, + {"type": "text", "text": ""}, + ] + + assert _count_content_list(charges_for_empty_text, empty_blocks, False, None) == 0 + assert _count_content_list(charges_for_empty_text, [{"type": "thinking", "thinking": "two words"}], False, None) == 3 + + def test_token_counter_with_tool_reference_block(): """ Regression test: a message containing an Anthropic tool-search diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py index 1c004d8c36e..8fa9d534a8c 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py @@ -24,7 +24,7 @@ from litellm.llms.tencent.messages.transformation import TencentAnthropicMessage from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation import ( VertexAIPartnerModelsAnthropicMessagesConfig, ) -from litellm.types.llms.anthropic import AnthropicFinishReason +from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES, AnthropicFinishReason from litellm.types.router import GenericLiteLLMParams from tests.unit.llms.anthropic.test_anthropic_wif import ( ScriptedPoster, @@ -321,6 +321,66 @@ def _derived_beta_header(config: AnthropicMessagesConfig) -> dict: ) +_COMPACT_EDIT_BETA: Final = ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value +_CLEARING_EDIT_BETA: Final = ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value + + +@pytest.mark.parametrize( + ("context_management", "betas"), + ( + pytest.param({"edits": [{"type": "compact_20260112"}]}, {_COMPACT_EDIT_BETA}, id="compact"), + pytest.param({"edits": [{"type": "clear_tool_uses_20250919"}]}, {_CLEARING_EDIT_BETA}, id="clearing"), + pytest.param( + {"edits": [{"type": "compact_20260112"}, {"type": "clear_thinking_20251015"}]}, + {_COMPACT_EDIT_BETA, _CLEARING_EDIT_BETA}, + id="both", + ), + pytest.param( + [{"type": "compaction", "compact_threshold": 150000}], {_COMPACT_EDIT_BETA}, id="openai_list_form" + ), + ), +) +def test_context_management_edits_add_the_beta_each_edit_needs( + context_management: JsonValue, betas: set[str], local_beta_headers_config: None +) -> None: + headers: Final = AnthropicMessagesConfig()._update_headers_with_anthropic_beta( + headers={}, optional_params={"context_management": context_management}, messages=[] + ) + + assert set(headers["anthropic-beta"].split(",")) == betas + + +_UNMAPPABLE_LIST_FORM: Final = [{"type": "clear_tool_uses_20250919"}] + + +@pytest.mark.parametrize("config_class", [AnthropicMessagesConfig, VertexAIPartnerModelsAnthropicMessagesConfig]) +@pytest.mark.parametrize( + ("context_management", "drop_params", "forwarded"), + ( + pytest.param(_UNMAPPABLE_LIST_FORM, True, None, id="unmappable_dropped_under_drop_params"), + pytest.param(_UNMAPPABLE_LIST_FORM, False, _UNMAPPABLE_LIST_FORM, id="unmappable_kept_for_the_provider_400"), + pytest.param( + [{"type": "compaction"}], + True, + {"edits": [{"type": "compact_20260112"}]}, + id="mappable_still_mapped_under_drop_params", + ), + ), +) +def test_context_management_the_list_form_cannot_map_is_dropped_only_under_drop_params( + config_class: type[AnthropicMessagesConfig], context_management: JsonValue, drop_params: bool, forwarded: JsonValue +) -> None: + request: Final = config_class().transform_anthropic_messages_request( + model="claude-haiku-5-5", + messages=[{"role": "user", "content": "Hi"}], + anthropic_messages_optional_request_params={"max_tokens": 50, "context_management": context_management}, + litellm_params=GenericLiteLLMParams(drop_params=drop_params), + headers={}, + ) + + assert request.get("context_management") == forwarded + + @pytest.mark.parametrize("config", _classifier_configs(), ids=type) def test_routes_whose_platform_runs_the_classifier_keep_safeguards_and_the_beta( config: AnthropicMessagesConfig, local_beta_headers_config: None diff --git a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py index a3e723eaca2..aa7fa2727b4 100644 --- a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py +++ b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py @@ -26,7 +26,24 @@ from litellm.llms.azure_ai.anthropic.count_tokens.token_counter import AzureAIAn from litellm.llms.azure_ai.anthropic.count_tokens.transformation import ( AzureAIAnthropicCountTokensConfig, ) -from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse +from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse +from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION +from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES + +COMPACTION_BETA = ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value +COMPACTED_HISTORY = [ + { + "role": "assistant", + "content": [ + { + "type": "compaction", + "content": "The user is building a recipe app and asked for one-sentence class descriptions.", + "signature": "EqQBCkYIBRgCKkBjZ2xhc3M" * 40, + } + ], + }, + {"role": "user", "content": "Now do the same for Ingredient."}, +] @pytest.mark.parametrize( @@ -890,3 +907,115 @@ async def test_counter_sends_the_structured_output_tool_choice_and_thinking_chat inference["tool_choice"], inference["tools"], ) + + +def test_headers_add_the_compaction_beta_only_for_a_compacted_history(): + config = AnthropicCountTokensConfig() + auth_header = {"x-api-key": "sk-ant-api03-test-key"} + + with_block = config.get_count_tokens_headers(auth_header, COMPACTED_HISTORY)["anthropic-beta"].split(",") + plain = config.get_count_tokens_headers(auth_header, [{"role": "user", "content": "hi"}])["anthropic-beta"] + + assert COMPACTION_BETA in with_block + assert ANTHROPIC_TOKEN_COUNTING_BETA_VERSION in with_block + assert plain == ANTHROPIC_TOKEN_COUNTING_BETA_VERSION + + +@pytest.mark.asyncio +async def test_handler_sends_the_compaction_beta_with_a_compacted_history(httpx_transport_clients): + with respx.mock: + route = respx.post("https://gateway.example/v1/messages/count_tokens").mock( + return_value=httpx.Response(200, json={"input_tokens": 404}) + ) + result = await AnthropicCountTokensHandler().handle_count_tokens_request( + model="claude-sonnet-5-5", + messages=COMPACTED_HISTORY, + auth_header={"x-api-key": "sk-ant-api03-test-key"}, + api_base="https://gateway.example", + ) + + assert result == {"input_tokens": 404} + betas = route.calls.last.request.headers["anthropic-beta"].split(",") + assert COMPACTION_BETA in betas + assert ANTHROPIC_TOKEN_COUNTING_BETA_VERSION in betas + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("httpx_transport_clients") +@pytest.mark.parametrize( + ("context_management", "forwarded", "sent_beta", "unsent_beta"), + ( + pytest.param( + {"edits": [{"type": "compact_20260112"}]}, + {"edits": [{"type": "compact_20260112"}]}, + ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value, + ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value, + id="compact", + ), + pytest.param( + {"edits": [{"type": "clear_tool_uses_20250919"}]}, + {"edits": [{"type": "clear_tool_uses_20250919"}]}, + ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value, + ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value, + id="clearing", + ), + pytest.param( + [{"type": "compaction", "compact_threshold": 150000}], + {"edits": [{"type": "compact_20260112", "trigger": {"type": "input_tokens", "value": 150000}}]}, + ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value, + ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value, + id="openai_list_form", + ), + ), +) +async def test_counter_forwards_context_management_with_the_beta_inference_sends( + context_management: JsonValue, forwarded: JsonValue, sent_beta: str, unsent_beta: str +) -> None: + """Anthropic counts an unsigned compaction block only under the compact_20260112 edit that inference sends, + and a clearing edit changes the count, so the count carries the request's context_management as inference + sends it, the OpenAI list form mapped to edits, and its beta.""" + with respx.mock: + route: Final = respx.post("https://gateway.example/v1/messages/count_tokens").mock( + return_value=httpx.Response(200, json={"input_tokens": 259}) + ) + result: Final = await AnthropicTokenCounter().count_tokens( + model_to_use="claude-opus-5-5", + messages=[{"role": "assistant", "content": [{"type": "compaction", "content": "Recipe app summary."}]}], + contents=None, + deployment={"litellm_params": {"api_key": "sk-ant-api03-test-key", "api_base": "https://gateway.example"}}, + context_management=CountContextManagement(spec=context_management, drop_invalid_edits=False), + ) + + sent: Final = route.calls.last.request + betas: Final = sent.headers["anthropic-beta"].split(",") + assert result is not None and result.total_tokens == 259 + assert TypeAdapter(dict[str, JsonValue]).validate_json(sent.content)["context_management"] == forwarded + assert (sent_beta in betas, unsent_beta in betas) == (True, False) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("httpx_transport_clients") +@pytest.mark.parametrize("context_management", ({}, {"edits": []}, []), ids=("empty_object", "empty_edits", "empty_list")) +async def test_counter_leaves_out_context_management_without_edits(context_management: JsonValue) -> None: + """Anthropic's count endpoint rejects a context_management with no edits, which inference accepts, so the count + leaves it out and adds no context-management beta.""" + with respx.mock: + route: Final = respx.post("https://gateway.example/v1/messages/count_tokens").mock( + return_value=httpx.Response(200, json={"input_tokens": 11}) + ) + result: Final = await AnthropicTokenCounter().count_tokens( + model_to_use="claude-opus-5-5", + messages=[{"role": "user", "content": "Hello"}], + contents=None, + deployment={"litellm_params": {"api_key": "sk-ant-api03-test-key", "api_base": "https://gateway.example"}}, + context_management=CountContextManagement(spec=context_management, drop_invalid_edits=False), + ) + + sent: Final = route.calls.last.request + betas: Final = frozenset(sent.headers.get("anthropic-beta", "").split(",")) + assert result is not None and result.total_tokens == 11 + assert "context_management" not in TypeAdapter(dict[str, JsonValue]).validate_json(sent.content) + assert not betas & { + ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value, + ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value, + } diff --git a/tests/unit/llms/anthropic/test_count_tokens_oauth.py b/tests/unit/llms/anthropic/test_count_tokens_oauth.py index b8a7c001b37..3e3b408d825 100644 --- a/tests/unit/llms/anthropic/test_count_tokens_oauth.py +++ b/tests/unit/llms/anthropic/test_count_tokens_oauth.py @@ -48,7 +48,7 @@ FEDERATED_DEPLOYMENT = { def count_tokens_headers_for(api_key: str) -> dict[str, str]: auth_header = AnthropicModelInfo.get_auth_header(api_key=api_key) assert auth_header is not None - return AnthropicCountTokensConfig().get_count_tokens_headers(auth_header) + return AnthropicCountTokensConfig().get_count_tokens_headers(auth_header, [{"role": "user", "content": "hi"}]) @pytest.fixture diff --git a/tests/unit/llms/vertex_ai/count_tokens/test_context.py b/tests/unit/llms/vertex_ai/count_tokens/test_context.py index 38eb87debfd..67733884e12 100644 --- a/tests/unit/llms/vertex_ai/count_tokens/test_context.py +++ b/tests/unit/llms/vertex_ai/count_tokens/test_context.py @@ -666,7 +666,8 @@ async def test_anthropic_view_clears_only_the_turns_after_a_compaction_block_whe *_agent_turns(2, first_round=3), ) - view: Final = await inference_request_view(turns, None, "anthropic", _edits(clear_before_compact)) + with patch.object(litellm, "token_counter", side_effect=RuntimeError("tokenizer unavailable")): + view: Final = await inference_request_view(turns, None, "anthropic", _edits(clear_before_compact)) assert isinstance(view, InferenceRequestView) assert _tool_outputs_kept(view) == (True, True) diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py index 1e7c3d52814..7f16957ddc4 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py @@ -526,6 +526,29 @@ def test_both_compact_and_context_management_headers_added(): ) +def test_openai_list_form_context_management_sends_the_compact_beta(): + config: Final = VertexAIPartnerModelsAnthropicMessagesConfig() + + with ( + patch.object(config, "_ensure_access_token", return_value=("token", "test-project")), + patch.object(config, "get_complete_vertex_url", return_value="https://mock-url"), + ): + updated_headers, _ = config.validate_anthropic_messages_environment( + headers={}, + model="claude-vertex-ai-opus-4-6", + messages=[], + optional_params={"context_management": [{"type": "compaction", "compact_threshold": 150000}]}, + litellm_params={ + "vertex_ai_project": "test-project", + "vertex_ai_location": "us-central1", + "vertex_credentials": "{}", + }, + api_base=None, + ) + + assert updated_headers["anthropic-beta"].split(",") == [ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value] + + def test_validate_environment_always_refreshes_token_ignoring_stale_bearer(): """Regression: stale Authorization in shared deployment extra_headers must not skip token refresh on /v1/messages — _ensure_access_token is always called.""" diff --git a/tests/unit/proxy/anthropic_endpoints/test_endpoints.py b/tests/unit/proxy/anthropic_endpoints/test_endpoints.py index f0317af915f..604fc84a096 100644 --- a/tests/unit/proxy/anthropic_endpoints/test_endpoints.py +++ b/tests/unit/proxy/anthropic_endpoints/test_endpoints.py @@ -6,6 +6,7 @@ import json import logging import unittest from collections.abc import AsyncGenerator, AsyncIterator, Callable, Mapping +from types import MappingProxyType from typing import Final from unittest.mock import AsyncMock, MagicMock, patch @@ -587,6 +588,48 @@ class TestErrorLogCarriesCallId: assert call_id in record.getMessage() + +class TestCountTokensCompactedHistory: + """A history carrying Anthropic's signed compaction block used to fail /v1/messages/count_tokens once the count + ran on the local tokenizer, which rejected the block type.""" + + @pytest.mark.asyncio + async def test_count_tokens_counts_a_compaction_block_like_its_summary_text(self): + import litellm.proxy.anthropic_endpoints.endpoints as ep + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy._types import UserAPIKeyAuth + + summary = "The user is building a recipe app and asked for one-sentence class descriptions." + follow_up = {"role": "user", "content": "Now do the same for Ingredient."} + compacted = { + "model": "claude-sonnet-5-5", + "messages": [ + { + "role": "assistant", + "content": [{"type": "compaction", "content": summary, "signature": "EqQBCkYIBRgCKkBjZ2xhc3M" * 40}], + }, + follow_up, + ], + } + as_text = { + "model": "claude-sonnet-5-5", + "messages": [{"role": "assistant", "content": [{"type": "text", "text": summary}]}, follow_up], + } + request = MagicMock() + request.headers = {} + + async def count(body): + with ( + patch.object(ep, "read_request_body", new=AsyncMock(return_value=body)), # test-quality-ok: endpoint reads the body via a module function; no injection seam + patch.object(proxy_server, "llm_router", None), # test-quality-ok: module global; with no router the count runs on the local tokenizer, the path that raised + ): + return await ep.count_tokens(request=request, user_api_key_dict=UserAPIKeyAuth()) + + counted = await count(compacted) + + assert counted["input_tokens"] > 0 + assert counted == await count(as_text) + def _claude_router() -> Router: return Router( model_list=[ @@ -759,7 +802,10 @@ async def test_count_tokens_drops_malformed_tools_the_counted_deployment_drops_e def _count_on_claude( - monkeypatch: pytest.MonkeyPatch, peer: Callable[[httpx.Request], httpx.Response], headers: Mapping[str, str] + monkeypatch: pytest.MonkeyPatch, + peer: Callable[[httpx.Request], httpx.Response], + headers: Mapping[str, str], + extra_body: Mapping[str, object] = MappingProxyType({}), ) -> httpx.Response: from fastapi import FastAPI @@ -799,7 +845,7 @@ def _count_on_claude( app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth() return TestClient(app).post( "/v1/messages/count_tokens", - json={"model": "count-log-claude", "messages": [{"role": "user", "content": "Hello"}]}, + json={"model": "count-log-claude", "messages": [{"role": "user", "content": "Hello"}], **extra_body}, headers=dict(headers), ) @@ -827,6 +873,24 @@ def test_count_tokens_logs_one_line_with_the_callers_call_id_model_and_count( ) +def test_count_tokens_returns_the_context_management_the_provider_applied(monkeypatch: pytest.MonkeyPatch) -> None: + applied: Final = {"original_input_tokens": 640} + response: Final = _count_on_claude( + monkeypatch, + lambda request: httpx.Response( + 200, json={"input_tokens": 576, "context_management": applied}, request=request + ), + {}, + {"context_management": {"edits": [{"type": "clear_tool_uses_20250919"}]}}, + ) + + assert response.json() == { + "input_tokens": 576, + "context_management": applied, + "litellm_count_source": "provider", + }, response.text + + def test_count_tokens_provider_rejection_logs_its_status_and_answers_the_same_error( monkeypatch: pytest.MonkeyPatch, token_count_log_lines: Callable[[], tuple[dict[str, object], ...]] ) -> None: From cfb77ecc3ad72d9f3602dbda22d78d7e84739289 Mon Sep 17 00:00:00 2001 From: mika-lt <349045450+mika-lt@users.noreply.github.com> Date: Fri, 30 Oct 2026 00:14:49 -0700 Subject: [PATCH 11/12] fix(proxy): list only routable models for a wildcard alias over a provider org (#49152) * fix(proxy): list only routable models for a wildcard alias over a provider org A wildcard alias over an org-scoped deployment (together_ai/* over together_ai/nosuchorg/*) listed the provider's whole catalog on /v1/models and /model/info, and every listed id routed to a model the provider 404s. A skipped deployment path segment now counts as a routing prefix only when the routed model resolves to a catalog id through the get_model_info lookup order, so responses/, Bedrock converse/, and region paths list as before. Together's model list now also carries the cost map's together_ai rows, skipping the together-ai-* pricing tiers, so an alias over a real org lists the models Together serves today. * fix(proxy): keep flat catalog ids under routes the cost map can't parse, drop retired Together rows * fix(proxy): check skipped path segments only on Together so Hugging Face provider routes keep listing --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/__init__.py | 5 ++ litellm/constants.py | 2 +- litellm/proxy/auth/model_checks.py | 34 ++++++++- litellm/responses/main.py | 8 +- litellm/utils.py | 46 ++++++------ tests/unit/proxy/auth/test_model_checks.py | 87 +++++++++++++++++++++- 6 files changed, 150 insertions(+), 32 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index d59edcaff89..5f14c605cd0 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -872,6 +872,11 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje # ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params. if "-to-" not in key and "fireworks-ai-default" not in key: fireworks_ai_models.add(key) + elif value.get("litellm_provider") == "together_ai": + if _is_past_deprecation_date(value): + together_ai_models.discard(key) + elif not key.startswith("together-ai-"): + together_ai_models.add(key) elif value.get("litellm_provider") == "baseten": if _is_past_deprecation_date(value): baseten_models.discard(key) diff --git a/litellm/constants.py b/litellm/constants.py index cfee777f36a..e7c9c5dcbfb 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1274,7 +1274,7 @@ empower_models: Final = set( ] ) -together_ai_models: Final[set] = set( +together_ai_models: Final[set[str]] = set( [ # llama llms - chat "togethercomputer/llama-2-70b-chat", diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index c40ee190e12..b21315529eb 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -14,14 +14,16 @@ from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping from litellm.proxy._types import SpecialModelNames, UserAPIKeyAuth from litellm.repositories.object_permission_repository import ObjectPermissionRepository +from litellm.responses.main import strip_responses_routing_prefix from litellm.router import Router from litellm.router_utils.fallback_event_handlers import get_fallback_model_group from litellm.router_utils.pattern_match_deployments import PatternMatchRouter from litellm.types.router import CredentialLiteLLMParams, LiteLLM_Params from litellm.types.utils import LlmProviders -from litellm.utils import ProviderConfigManager, get_valid_models +from litellm.utils import ProviderConfigManager, get_valid_models, potential_model_names_in_lookup_order LLM_PROVIDER_PREFIXES: Final = frozenset(llm_provider.value for llm_provider in LlmProviders) +_PROVIDERS_SENDING_THE_DEPLOYMENT_PATH_AS_THE_MODEL_ID: Final = frozenset({LlmProviders.TOGETHER_AI.value}) _CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields) @@ -331,6 +333,25 @@ def _routes_back_to( ) +def _resolves_to_catalog_model(model: str, provider: str, provider_model_ids: frozenset[str]) -> bool: + provider_id_prefix: Final = f"{provider}/" + resolved_model: Final = ( + f"{provider_id_prefix}{strip_responses_routing_prefix(model.removeprefix(provider_id_prefix))}" + ) + return any( + name.removeprefix(provider_id_prefix) in provider_model_ids + for name in potential_model_names_in_lookup_order(resolved_model, provider) + ) + + +def _routes_to_catalog_model( + routes_to_model: Callable[[str, str], bool], provider: str, provider_model_ids: frozenset[str] +) -> Callable[[str, str], bool]: + return lambda listed_id, model: ( + routes_to_model(listed_id, model) and _resolves_to_catalog_model(model, provider, provider_model_ids) + ) + + def _ids_routing_to_catalog_models( name_pattern: str, deployment_pattern: str, @@ -344,8 +365,15 @@ def _ids_routing_to_catalog_models( return () provider_id_prefix: Final = f"{provider}/" provider_model_ids: Final = tuple(catalog_id.removeprefix(provider_id_prefix) for catalog_id in catalog_ids) + deployment_model_pattern_without_provider: Final = deployment_pattern.removeprefix(provider_id_prefix) catalog_pattern: Final = _without_routing_segments( - deployment_pattern.removeprefix(provider_id_prefix), provider, name_pattern, provider_model_ids + deployment_model_pattern_without_provider, provider, name_pattern, provider_model_ids + ) + routing_prefix: Final = deployment_model_pattern_without_provider.removesuffix(catalog_pattern) + reaches_catalog_model: Final = ( + _routes_to_catalog_model(routes_to_model, provider, frozenset(provider_model_ids)) + if routing_prefix and provider in _PROVIDERS_SENDING_THE_DEPLOYMENT_PATH_AS_THE_MODEL_ID + else routes_to_model ) deployment_model_pattern: Final = f"{provider_id_prefix}{catalog_pattern}" deployment_regex: Final = re.compile(PatternMatchRouter().pattern_to_regex(deployment_model_pattern)) @@ -369,7 +397,7 @@ def _ids_routing_to_catalog_models( for listed_id, catalog_id in listed_ids if listed_id and _routes_back_to( - listed_id, name_regex, (deployment_model_pattern, catalog_id), deployment_pattern, routes_to_model + listed_id, name_regex, (deployment_model_pattern, catalog_id), deployment_pattern, reaches_catalog_model ) ) ) diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 98356e9a5d4..063ef9d91ea 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -759,7 +759,7 @@ def _dispatched_model_name( from litellm.litellm_core_utils.get_llm_provider_logic import resolve_model_and_provider_for_metadata provider_model, _ = resolve_model_and_provider_for_metadata(model=model, custom_llm_provider=custom_llm_provider) - return _strip_responses_routing_prefix(provider_model) + return strip_responses_routing_prefix(provider_model) def _will_bridge_to_chat_completions( @@ -1245,7 +1245,7 @@ def _pop_use_chat_completions_api_kw(kwargs: dict[str, object]) -> bool: _RESPONSES_ROUTING_PREFIX: Final = "responses/" -def _strip_responses_routing_prefix(model: str) -> str: +def strip_responses_routing_prefix(model: str) -> str: if not model.startswith(_RESPONSES_ROUTING_PREFIX): return model return model[len(_RESPONSES_ROUTING_PREFIX) :] @@ -1273,7 +1273,7 @@ def _resolve_model_provider_for_responses( litellm_params.api_key = dynamic_api_key if dynamic_api_base is not None: litellm_params.api_base = dynamic_api_base - return _strip_responses_routing_prefix(provider_model), resolved_provider + return strip_responses_routing_prefix(provider_model), resolved_provider def _apply_managed_file_id_mapping( @@ -2893,7 +2893,7 @@ async def _aresponses_websocket( api_key=api_key, litellm_params=litellm_params, ) - resolved_model: Final = _strip_responses_routing_prefix(provider_model) + resolved_model: Final = strip_responses_routing_prefix(provider_model) litellm_params_dict["data_residency"] = infer_openai_data_residency( _custom_llm_provider, diff --git a/litellm/utils.py b/litellm/utils.py index 41a3fddf3a3..e85e089289a 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6708,6 +6708,26 @@ def _first_registered_match( return next(matches, (None, None)) +def _model_cost_lookup_order( + model: str, potential_model_names: PotentialModelNamesAndCustomLLMProvider +) -> tuple[str, ...]: + return ( + potential_model_names["combined_model_name"], + model, + potential_model_names["region_free_combined_model_name"], + potential_model_names["split_model"], + potential_model_names["combined_stripped_model_name"], + potential_model_names["stripped_model_name"], + potential_model_names["provider_prefixed_model_name"], + ) + + +def potential_model_names_in_lookup_order(model: str, custom_llm_provider: str) -> tuple[str, ...]: + return _model_cost_lookup_order( + model, get_potential_model_names(model=model, custom_llm_provider=custom_llm_provider) + ) + + def _get_model_info_from_generalization( model: str, potential_model_names: PotentialModelNamesAndCustomLLMProvider, @@ -6726,15 +6746,7 @@ def _get_model_info_from_generalization( still had a priced exact name to try. O(number of rules); only call after the exact lookups have missed. """ - candidates: Final = ( - potential_model_names["combined_model_name"], - model, - potential_model_names["region_free_combined_model_name"], - potential_model_names["split_model"], - potential_model_names["combined_stripped_model_name"], - potential_model_names["stripped_model_name"], - potential_model_names["provider_prefixed_model_name"], - ) + candidates: Final = _model_cost_lookup_order(model, potential_model_names) if any(get_model_cost_key(candidate) is not None for candidate in candidates): return None for candidate in candidates: @@ -6973,11 +6985,6 @@ def get_model_info_helper( model=model, custom_llm_provider=custom_llm_provider or declared_authenticating_provider(model) ) - combined_model_name: Final = potential_model_names["combined_model_name"] - stripped_model_name: Final = potential_model_names["stripped_model_name"] - combined_stripped_model_name: Final = potential_model_names["combined_stripped_model_name"] - provider_prefixed_model_name: Final = potential_model_names["provider_prefixed_model_name"] - split_model: Final = potential_model_names["split_model"] custom_llm_provider = potential_model_names["custom_llm_provider"] model_cost_custom_llm_provider: Final = custom_llm_provider ######################### @@ -7044,16 +7051,9 @@ def get_model_info_helper( and strip it. Tried last so no model that already resolves through 1-6 can change. """ - lookup_order: Final = ( - combined_model_name, - model, - potential_model_names["region_free_combined_model_name"], - split_model, - combined_stripped_model_name, - stripped_model_name, - provider_prefixed_model_name, + lookup: Final = _first_registered_match( + _model_cost_lookup_order(model, potential_model_names), model_cost_custom_llm_provider ) - lookup: Final = _first_registered_match(lookup_order, model_cost_custom_llm_provider) key: str | None = lookup[0] _model_info: dict[str, Any] | None = lookup[1] diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index 3f0f97e63b0..c67610c24a1 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -1057,7 +1057,9 @@ def test_wildcard_lists_each_model_once_under_its_full_owner_id(provider: str): import litellm from litellm.proxy.auth.model_checks import get_known_models_from_wildcard - full_ids: Final = {f"{provider}/{model.removeprefix(f'{provider}/')}" for model in litellm.models_by_provider[provider]} + full_ids: Final = { + f"{provider}/{model.removeprefix(f'{provider}/')}" for model in litellm.models_by_provider[provider] + } assert any(model.count("/") > 1 for model in full_ids) assert sorted(get_known_models_from_wildcard(f"{provider}/*")) == sorted(full_ids) @@ -1372,12 +1374,57 @@ def test_partial_wildcard_lists_catalog_ids_stored_with_the_provider_prefix( assert sorted(listed) == expected +@pytest.mark.parametrize( + ("provider", "routing_segment"), [("bedrock", "converse"), ("bedrock", "us-east-1"), ("openai", "responses")] +) +def test_routing_segment_lists_the_same_catalog_ids_as_the_bare_provider_wildcard( + provider: str, routing_segment: str +) -> None: + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + bare_listing: Final = get_known_models_from_wildcard(f"{provider}/*", LiteLLM_Params(model=f"{provider}/*")) + routed_listing: Final = get_known_models_from_wildcard( + f"{provider}/*", LiteLLM_Params(model=f"{provider}/{routing_segment}/*") + ) + + assert bare_listing + assert sorted(routed_listing) == sorted(bare_listing) + + +def test_provider_wildcard_lists_cost_map_models_but_not_pricing_tiers() -> None: + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + + model_id: Final = "together_ai/partial-wildcard-org/model-t" + pricing_tier: Final = "together-ai-partial-wildcard-tier" + with _registered_chat_models({model_id: "together_ai", pricing_tier: "together_ai"}): + listed: Final = get_known_models_from_wildcard("together_ai/*") + + assert model_id in listed + assert [listed_id for listed_id in listed if pricing_tier in listed_id] == [] + + +def test_provider_wildcard_stops_listing_a_cost_map_model_once_it_is_retired() -> None: + import litellm + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + + model_id: Final = "together_ai/partial-wildcard-org/model-retired" + with _registered_chat_models({model_id: "together_ai"}): + listed_while_served: Final = get_known_models_from_wildcard("together_ai/*") + litellm.add_known_models({model_id: {"litellm_provider": "together_ai", "deprecation_date": "2000-01-01"}}) + listed_once_retired: Final = get_known_models_from_wildcard("together_ai/*") + + assert model_id in listed_while_served + assert model_id not in listed_once_retired + + _ROUTE_CF_ID: Final = "cloudflare/@cf/partial-route-org/model-a-instruct" _ROUTE_CF_SUFFIXED_ID: Final = "cloudflare/@cf/partial-route-org/model-b-instruct-v2" _ROUTE_HF_ID: Final = "cloudflare/@hf/partial-route-org/model-c" _ROUTE_GEMINI_ID: Final = "gemini-partial-route-9-flash" _ROUTE_OPENAI_ID: Final = "gpt-partial-route-9-mini" _ROUTE_BEDROCK_ID: Final = "anthropic.partial-route-9-v1:0" +_ROUTE_TOGETHER_ID: Final = "together_ai/partial-route-org/model-t" _ROUTE_CF_IDS: Final = {_ROUTE_CF_ID, _ROUTE_CF_SUFFIXED_ID} @@ -1456,6 +1503,29 @@ _ROUTE_CF_IDS: Final = {_ROUTE_CF_ID, _ROUTE_CF_SUFFIXED_ID} set(), id="same-name-namespace-the-catalog-lacks", ), + pytest.param( + "together/*", "together_ai/partial-route-missing-org/*", None, set(), id="alias-over-org-the-catalog-lacks" + ), + pytest.param( + "openai/*", + "openai/chat_completions/*", + None, + {f"openai/chat_completions/{_ROUTE_OPENAI_ID}"}, + id="chat-completions-bridge-route", + ), + pytest.param( + "bedrock/*", "bedrock/llama/*", None, {f"bedrock/llama/{_ROUTE_BEDROCK_ID}"}, id="invoke-format-route" + ), + pytest.param( + "together/*", + "together_ai/partial-route-org/*", + None, + {_ROUTE_TOGETHER_ID}, + id="alias-over-org-a-cost-map-catalog-carries", + ), + pytest.param( + "bedrock/*", "bedrock/us-east-1/*", None, {f"bedrock/us-east-1/{_ROUTE_BEDROCK_ID}"}, id="region-route" + ), pytest.param( "*", "openai/*", "openai/gpt-partial-route-9*", set(), id="provider-prefixed-key-filter-over-catch-all" ), @@ -1501,6 +1571,7 @@ def test_every_listed_partial_wildcard_id_routes_to_the_catalog_model_it_names( _ROUTE_GEMINI_ID: "gemini", _ROUTE_OPENAI_ID: "openai", _ROUTE_BEDROCK_ID: "bedrock", + _ROUTE_TOGETHER_ID: "together_ai", } ): v1_models: Final = get_complete_model_list( @@ -1535,6 +1606,20 @@ def test_every_listed_partial_wildcard_id_routes_to_the_catalog_model_it_names( assert listed_model_info_names == sorted(reachable) +def test_inference_provider_route_lists_every_id_the_plain_provider_wildcard_lists() -> None: + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + plain: Final = get_known_models_from_wildcard("huggingface/*", LiteLLM_Params(model="huggingface/*")) + routed: Final = get_known_models_from_wildcard( + "huggingface/*", LiteLLM_Params(model="huggingface/featherless-ai/*") + ) + + assert plain + assert [model for model in plain if "/" not in model.removeprefix("huggingface/")] == [] + assert sorted(routed) == sorted(plain) + + def test_catch_all_lists_no_id_a_more_specific_sibling_wildcard_routes_elsewhere() -> None: from litellm import Router from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info, get_complete_model_list From e81dd348a425c180c7a632c1eecea7a37682ad96 Mon Sep 17 00:00:00 2001 From: Filip Sinitsin Date: Fri, 30 Oct 2026 07:18:15 +0000 Subject: [PATCH 12/12] fix(proxy)!: log and bill Responses-style bodies sent to /cursor/chat/completions (#30277) * fix(proxy): restore spend logging for /cursor/chat/completions (#30126) The /cursor/chat/completions route processes requests through the Responses API pipeline, but _get_metadata_variable_name did not treat it as a litellm_metadata route. Proxy auth metadata therefore landed in the user-facing metadata param, so the spend tracking callback saw user_api_key=None and silently skipped the SpendLogs write. Add the route to LITELLM_METADATA_ROUTES and cover it with regression tests. * test(proxy): pin provider metadata and key billing on /cursor/chat/completions A Responses-format body on the Cursor route now sends OpenAI only the metadata the client sent and writes a spend row that bills the key. The chat-body branch, which this fix also moves onto litellm_metadata, still bills the key with its team and request tags. The scripted chat reply gets a unique id per test, since a shared id let only the first spend row per database land * fix(proxy): pick the Cursor route's metadata field from the body shape A Responses-style body on /cursor/chat/completions keeps proxy metadata in litellm_metadata, the same as /v1/responses, while a Chat Completions body keeps metadata, the same as /v1/chat/completions. Adding the whole route to the internal-metadata list let a client's chat-body tags replace the key's tags in spend logs and skip tag budgets. * fix(proxy): keep get_metadata_variable_name callable with only the request * fix(proxy): check client tags against tag budgets and read agent limits from litellm_metadata on Responses bodies A Responses-style body on /cursor/chat/completions now keeps its proxy fields in litellm_metadata, so the pre-call tag budget check and the v3 agent rate limiter stopped seeing the client's metadata.tags and the x-litellm-agent-id value. The pre-auth tag check now merges the caller's metadata.tags into the selected metadata field, sharing one helper with request setup, and the limiter reads agent_id from either field * fix(proxy): check client tags sent in a JSON-string metadata against tag budgets on Responses bodies * fix(proxy): answer 422, not 500, when a budget rejects a Responses body whose metadata is a JSON string * fix(proxy): keep key guardrails, metadata type errors and failure-row client details on Responses-style bodies --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/integrations/custom_guardrail.py | 32 ++-- litellm/proxy/auth/auth_checks.py | 10 +- litellm/proxy/common_request_processing.py | 16 +- .../hooks/parallel_request_limiter_v3.py | 3 +- .../proxy/hooks/proxy_track_cost_callback.py | 37 +++- litellm/proxy/litellm_pre_call_utils.py | 88 ++++++++-- .../proxy/response_api_endpoints/endpoints.py | 12 +- litellm/proxy/utils.py | 10 +- .../providers/test_cursor_responses_wire.py | 163 +++++++++++++++--- .../integrations/test_custom_guardrail.py | 14 ++ .../hooks/test_parallel_request_limiter_v3.py | 5 +- .../hooks/test_proxy_track_cost_callback.py | 50 ++++++ .../response_api_endpoints/test_endpoints.py | 2 +- .../proxy/test_common_request_processing.py | 34 ++++ .../unit/proxy/test_litellm_pre_call_utils.py | 68 ++++++++ .../test_post_call_failure_hook.py | 29 ++++ 16 files changed, 482 insertions(+), 91 deletions(-) diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 9e2dc902d55..a9fb5c29bad 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -9,6 +9,7 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, cast, get_args import httpx +from pydantic import TypeAdapter from litellm._internal_context import with_service_target from litellm._logging import verbose_logger @@ -218,6 +219,15 @@ def _request_is_streaming(data: object, event_type: GuardrailEventHooks | None = ) +_REQUEST_FIELDS: Final = TypeAdapter(Mapping[str, object]) + + +def _guardrails_listed_in(metadata: object) -> list[str | dict[str, DynamicGuardrailParams]]: + if not isinstance(metadata, dict): + return [] + return metadata.get("guardrails") or [] + + class CustomGuardrail(CustomLogger): # If True, during_call runs async_moderation_hook instead of the unified apply_guardrail path. use_native_during_call_hook: ClassVar[bool] = False @@ -791,28 +801,22 @@ class CustomGuardrail(CustomLogger): return True raise - def get_guardrail_from_metadata(self, data: dict) -> list[str] | list[dict[str, DynamicGuardrailParams]]: + def get_guardrail_from_metadata(self, data: dict) -> list[str | dict[str, DynamicGuardrailParams]]: """ - Returns the guardrail(s) to be run from the metadata or root + Returns the guardrail(s) to be run from the root, else from both metadata fields together """ if "guardrails" in data: return data["guardrails"] - # Check both metadata locations. For regular endpoints move_guardrails_to_metadata - # writes to "metadata"; for thread/assistant endpoints it writes to - # "litellm_metadata". We check the one that actually contains the "guardrails" - # key so that a non-empty litellm_metadata without guardrails does not shadow - # the merged list stored in metadata (which would cause team guardrails to be - # silently skipped while default_on=True policy guardrails still fire). - for meta_key in ("metadata", "litellm_metadata"): - meta = data.get(meta_key) or {} - if isinstance(meta, dict) and "guardrails" in meta: - return meta.get("guardrails") or [] - return [] + request_fields: Final = _REQUEST_FIELDS.validate_python(data) + return [ + *_guardrails_listed_in(request_fields.get("metadata")), + *_guardrails_listed_in(request_fields.get("litellm_metadata")), + ] def _guardrail_is_in_requested_guardrails( self, - requested_guardrails: list[str] | list[dict[str, DynamicGuardrailParams]], + requested_guardrails: Sequence[str | dict[str, DynamicGuardrailParams]], ) -> bool: for _guardrail in requested_guardrails: if isinstance(_guardrail, dict): diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 1ef5b1072ed..b8d9530d2eb 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -446,7 +446,7 @@ def _request_has_trace_id(request_body: Mapping[str, object], request: Request, from litellm.proxy.litellm_pre_call_utils import ( _trace_id_from_traceparent, # pyright: ignore[reportPrivateUsage] # reuse W3C traceparent parser get_chain_id_from_headers, - metadata_variable_name_for_route, + metadata_variable_name_for_request, ) if get_chain_id_from_headers(dict(request.headers)): @@ -460,7 +460,7 @@ def _request_has_trace_id(request_body: Mapping[str, object], request: Request, traceparent: Final[str | None] = request.headers.get("traceparent") if isinstance(traceparent, str) and _trace_id_from_traceparent(traceparent): return True - metadata_variable_name: Final = metadata_variable_name_for_route(get_request_route(request)) + metadata_variable_name: Final = metadata_variable_name_for_request(get_request_route(request), request_body) selected_metadata: Final[object] = request_body.get(metadata_variable_name) if isinstance(selected_metadata, dict): selected_metadata_dict: Final = _TRACE_ID_METADATA_ADAPTER.validate_python(selected_metadata) @@ -1367,11 +1367,15 @@ async def common_checks( if not skip_all_budget_checks: # Key metadata.tags are injected into request_body here so the tag budget # check can read them; this mutation must run before the gathered checks. - from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, metadata_variable_name_for_request if valid_token is not None: for tag_target in LiteLLMProxyRequestSetup.pre_auth_tag_merge_targets(request, request_body): LiteLLMProxyRequestSetup.pre_seed_litellm_metadata_for_route(request_data=tag_target, route=route) + LiteLLMProxyRequestSetup.merge_caller_body_tags( + request_data=tag_target, + metadata_variable_name=metadata_variable_name_for_request(route, tag_target), + ) LiteLLMProxyRequestSetup.apply_key_tags_pre_auth(request_data=tag_target, user_api_key_dict=valid_token) LiteLLMProxyRequestSetup.apply_authenticated_baggage_tags(request, request_body) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 7cd752c2297..a79d8fe29db 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -2401,9 +2401,10 @@ class ProxyBaseLLMRequestProcessing: version=version, proxy_config=proxy_config, ) + request_fields: Final = _REQUEST_FIELDS.validate_python(self.data) + metadata_variable_name: Final = get_metadata_variable_name(request, request_fields) if logging_auth is not None: - metadata_variable_name: Final = get_metadata_variable_name(request) - request_metadata: Final = self.data.get(metadata_variable_name) + request_metadata: Final = request_fields.get(metadata_variable_name) if isinstance(request_metadata, dict): self.data[metadata_variable_name] = { **TypeAdapter(dict[str, object]).validate_python(request_metadata), @@ -2431,12 +2432,11 @@ class ProxyBaseLLMRequestProcessing: # Store queue time in metadata after add_litellm_data_to_request to ensure it's preserved if queue_time_seconds is not None: - _metadata_variable_name: Final = get_metadata_variable_name(request) - if _metadata_variable_name not in self.data: - self.data[_metadata_variable_name] = {} - if not isinstance(self.data[_metadata_variable_name], dict): - self.data[_metadata_variable_name] = {} - self.data[_metadata_variable_name]["queue_time_seconds"] = queue_time_seconds + if metadata_variable_name not in self.data: + self.data[metadata_variable_name] = {} + if not isinstance(self.data[metadata_variable_name], dict): + self.data[metadata_variable_name] = {} + self.data[metadata_variable_name]["queue_time_seconds"] = queue_time_seconds if isinstance(model, str): reject_url_valued_destination("model", model) diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 14b08a09116..15282679622 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -3312,7 +3312,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): if key_agent_id: return key_agent_id metadata: Final = data.get("metadata") or {} - return metadata.get("agent_id") + litellm_metadata: Final = data.get("litellm_metadata") or {} + return metadata.get("agent_id") or litellm_metadata.get("agent_id") def _get_session_id_from_data(self, data: dict) -> str | None: """Extract session_id from request metadata or litellm_session_id.""" diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 7d8b2e8329c..9c4a4bc775b 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -37,7 +37,7 @@ from litellm.proxy.db.db_spend_update_writer import ( get_llm_router, ) from litellm.proxy.hooks.search_free_tier import apply_search_free_tier -from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, metadata_variable_name_for_route +from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, metadata_variable_name_for_request from litellm.proxy.route_llm_request import ProxyModelNotFoundError from litellm.proxy.spend_tracking.spend_counter_batch import post_call_counter_keys, spend_counter_batch_scope from litellm.proxy.spend_tracking.spend_event import ( @@ -137,19 +137,34 @@ _CAPTURED_IDENTITY_CALL_TYPES: Final[frozenset[str]] = frozenset( _OBJECT_MAPPING: Final = TypeAdapter(ObjectMapping) -def _proxy_stamped_used_client_oauth_token( - request_data: Mapping[str, object], request_route: str | None -) -> bool | None: +def _proxy_stamped_metadata(request_data: Mapping[str, object], request_route: str | None) -> Mapping[str, object]: proxy_bucket: Final = ( get_metadata_variable_name_from_kwargs(request_data) if request_route is None - else metadata_variable_name_for_route(request_route) + else metadata_variable_name_for_request(request_route, request_data) ) - proxy_metadata: Final = request_data.get(proxy_bucket) - stamped: Final = proxy_metadata.get("used_client_oauth_token") if isinstance(proxy_metadata, dict) else None + return _str_keyed_fields(request_data.get(proxy_bucket)) + + +def _proxy_stamped_used_client_oauth_token( + request_data: Mapping[str, object], request_route: str | None +) -> bool | None: + stamped: Final = _proxy_stamped_metadata(request_data, request_route).get("used_client_oauth_token") return stamped if isinstance(stamped, bool) else None +_PROXY_STAMPED_REQUEST_FIELDS: Final = ("requester_ip_address", "user_agent") + + +def _proxy_stamped_request_fields( + request_data: Mapping[str, object], request_route: str | None +) -> Mapping[str, object]: + if request_route is None: + return {} + proxy_metadata: Final = _proxy_stamped_metadata(request_data, request_route) + return {field: proxy_metadata[field] for field in _PROXY_STAMPED_REQUEST_FIELDS if field in proxy_metadata} + + def _str_keyed_fields(value: object) -> Mapping[str, object]: return value if is_str_keyed_mapping(value) else {} @@ -455,7 +470,13 @@ class _ProxyDBLogger(CustomLogger): _metadata["used_client_oauth_token"] = _proxy_stamped_used_client_oauth_token(request_fields, request_route) existing_metadata: Final[dict] = request_data.get("metadata", None) or {} - existing_metadata.update({**_routed_metadata_fallback(request_fields), **_metadata}) + existing_metadata.update( + { + **_routed_metadata_fallback(request_fields), + **_proxy_stamped_request_fields(request_fields, request_route), + **_metadata, + } + ) litellm_metadata_bucket: Final = request_data.get("litellm_metadata") litellm_metadata: Final[ObjectMapping] = ( diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index a87fc815402..c1e541d20a9 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -379,6 +379,8 @@ LITELLM_METADATA_ROUTES: Final = ( "interactions", ) +CURSOR_CHAT_COMPLETIONS_ROUTE: Final = "/cursor/chat/completions" + LITELLM_TRACE_CONTROL_METADATA_FIELDS: Final = frozenset( { "mask_input", @@ -636,7 +638,7 @@ def _has_tag_budget(tag: "LiteLLM_TagTable | None") -> bool: return tag.litellm_budget_table.max_budget is not None -def _normalized_metadata_slot( +def normalized_metadata_slot( request_data: MutableMapping[str, object], metadata_variable_name: str ) -> dict[str, object]: """Return the request's metadata slot as a dict, normalising it in place first. @@ -653,6 +655,12 @@ def _normalized_metadata_slot( return normalized +def _holds_metadata_object(value: object) -> bool: + if value is None or isinstance(value, dict): + return True + return isinstance(value, str) and isinstance(_DECODED_JSON.validate_python(safe_json_loads(value)), dict) + + def _metadata_object_or_error(field: str, value: object) -> Mapping[str, object] | ProxyException: if value is None: return {} @@ -863,7 +871,7 @@ def _strip_router_reserved_metadata( ) -def get_metadata_variable_name(request: Request) -> str: +def get_metadata_variable_name(request: Request, data: Mapping[str, object] | None = None) -> str: """ Helper to return what the "metadata" field should be called in the request data @@ -874,7 +882,10 @@ def get_metadata_variable_name(request: Request) -> str: # Inline imports — auth_utils/route_checks participate in a proxy import cycle. from litellm.proxy.auth.auth_utils import get_request_route # noqa: PLC0415 - return metadata_variable_name_for_route(get_request_route(request)) + route: Final = get_request_route(request) + if data is None: + return metadata_variable_name_for_route(route) + return metadata_variable_name_for_request(route, data) _get_metadata_variable_name: Final = get_metadata_variable_name @@ -890,6 +901,25 @@ def metadata_variable_name_for_route(route: str) -> Literal["metadata", "litellm return "metadata" +def is_chat_completions_body(data: Mapping[str, object]) -> bool: + messages: Final = data.get("messages") + if isinstance(messages, list) and messages: + return True + return "messages" in data and "input" not in data + + +def is_cursor_responses_request(route: str, body: Mapping[str, object]) -> bool: + return CURSOR_CHAT_COMPLETIONS_ROUTE in route and not is_chat_completions_body(body) + + +def metadata_variable_name_for_request( + route: str, body: Mapping[str, object] +) -> Literal["metadata", "litellm_metadata"]: + if is_cursor_responses_request(route, body): + return "litellm_metadata" + return metadata_variable_name_for_route(route) + + def _promoted_trace_control_fields( requester_metadata: Mapping[str, object], litellm_metadata: Mapping[str, object], @@ -2309,7 +2339,9 @@ class LiteLLMProxyRequestSetup: ``_tag_max_budget_check`` resolve to ``litellm_metadata``, leaving header tags invisible to per-tag budget enforcement. """ - if any(metadata_route in route for metadata_route in LITELLM_METADATA_ROUTES): + if any(metadata_route in route for metadata_route in LITELLM_METADATA_ROUTES) or is_cursor_responses_request( + route, request_data + ): request_data.setdefault("litellm_metadata", {}) @staticmethod @@ -2380,7 +2412,7 @@ class LiteLLMProxyRequestSetup: return tags: Final = _STRING_TAGS_ADAPTER.validate_python(raw_tags) slot: Final = metadata_key if metadata_key is not None else get_metadata_variable_name_from_kwargs(request_data) - metadata: Final = _normalized_metadata_slot(request_data, slot) + metadata: Final = normalized_metadata_slot(request_data, slot) existing: Final = metadata.get("tags") if existing and not isinstance(existing, list): return @@ -2393,6 +2425,34 @@ class LiteLLMProxyRequestSetup: tags=tuple(tag for tag in tags if tag not in sent_tags) ) + @staticmethod + def merge_caller_body_tags( + request_data: MutableMapping[str, object], # mutable-ok: the request body the tag merges write into + metadata_variable_name: str | None = None, + ) -> None: + slot: Final = ( + metadata_variable_name + if metadata_variable_name is not None + else get_metadata_variable_name_from_kwargs(request_data) + ) + if slot == "metadata" or not _holds_metadata_object(request_data.get(slot)): + return + raw_caller_metadata: Final = request_data.get("metadata") + caller_metadata: Final = ( + _DECODED_JSON.validate_python(safe_json_loads(raw_caller_metadata)) + if isinstance(raw_caller_metadata, str) + else raw_caller_metadata + ) + caller_tags: Final = caller_metadata.get("tags") if isinstance(caller_metadata, dict) else None + if not isinstance(caller_tags, list) or not caller_tags: + return + metadata: Final = normalized_metadata_slot(request_data, slot) + existing_tags: Final = metadata.get("tags") + metadata["tags"] = LiteLLMProxyRequestSetup.merge_tags( + request_tags=existing_tags if isinstance(existing_tags, list) else None, + tags_to_add=caller_tags, + ) + @staticmethod def apply_key_tags_pre_auth( request_data: dict, @@ -2408,7 +2468,7 @@ class LiteLLMProxyRequestSetup: return _metadata_variable_name: Final = get_metadata_variable_name_from_kwargs(request_data) - metadata: Final = _normalized_metadata_slot(request_data, _metadata_variable_name) + metadata: Final = normalized_metadata_slot(request_data, _metadata_variable_name) existing_tags: Final = metadata.get("tags") metadata["tags"] = LiteLLMProxyRequestSetup.merge_tags( @@ -2460,7 +2520,7 @@ class LiteLLMProxyRequestSetup: # from (litellm_metadata vs metadata) so the merged tags are visible # to _tag_max_budget_check. _metadata_variable_name: Final = get_metadata_variable_name_from_kwargs(request_data) - metadata: Final = _normalized_metadata_slot(request_data, _metadata_variable_name) + metadata: Final = normalized_metadata_slot(request_data, _metadata_variable_name) existing_tags: Final = metadata.get("tags") metadata["tags"] = LiteLLMProxyRequestSetup.merge_tags( @@ -2637,7 +2697,7 @@ async def add_litellm_data_to_request( } safe_add_api_version_from_query_params(data, request) - _metadata_variable_name: Final = get_metadata_variable_name(request) + _metadata_variable_name: Final = get_metadata_variable_name(request, _REQUEST_FIELDS.validate_python(data)) if data.get(_metadata_variable_name, None) is None: data[_metadata_variable_name] = {} @@ -2974,17 +3034,7 @@ async def add_litellm_data_to_request( tags_to_add=tags, ) - _caller_body_metadata: Final = data.get("metadata") if _metadata_variable_name != "metadata" else None - _caller_body_tags: Final = ( - _caller_body_metadata.get("tags") - if isinstance(_caller_body_metadata, dict) and isinstance(_caller_body_metadata.get("tags"), list) - else None - ) - if _caller_body_tags: - data[_metadata_variable_name]["tags"] = LiteLLMProxyRequestSetup.merge_tags( # rebind-ok: matches file idiom - request_tags=data[_metadata_variable_name].get("tags"), - tags_to_add=_caller_body_tags, - ) + LiteLLMProxyRequestSetup.merge_caller_body_tags(request_data=data, metadata_variable_name=_metadata_variable_name) # caller_tags: the tags the request's headers and body carry (x-litellm-tags, body "tags", and body # "metadata.tags" / "litellm_metadata.tags" read before the mapped tags are added below), diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 9d0d89f681e..ef59deb2e5e 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -79,6 +79,7 @@ from litellm.proxy.hooks.responses_id_security import ResponsesIDSecurity, decry from litellm.proxy.litellm_pre_call_utils import ( _CLIENT_PRICING_METADATA_FIELDS, # pyright: ignore[reportPrivateUsage] # the HTTP path's own strip list _UNTRUSTED_METADATA_CONTROL_FIELDS, # pyright: ignore[reportPrivateUsage] # the HTTP path's own strip list + is_chat_completions_body, ) from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attribution import ( request_tags_from_request_data, @@ -213,13 +214,6 @@ def _normalize_tool_dialect( return {**data, **{key: value for key, value in replaceable if key in data}} -def _is_chat_completions_body(data: Mapping[str, object]) -> bool: - messages: Final = data.get("messages") - if isinstance(messages, list) and messages: - return True - return "messages" in data and "input" not in data - - _CURSOR_THINKING_SEPARATOR: Final = "-thinking-" _CURSOR_FAST_SUFFIX: Final = "-fast" _CURSOR_THINKING_LEVELS: Final[frozenset[str]] = frozenset(get_args(REASONING_EFFORT)) @@ -261,7 +255,7 @@ def _resolve_cursor_model_variant( resolved: Final = {**data, "model": variant.base_model} if variant.reasoning_effort is None: return resolved - if _is_chat_completions_body(data): + if is_chat_completions_body(data): if "reasoning_effort" in data: return resolved return {**resolved, "reasoning_effort": variant.reasoning_effort} @@ -646,7 +640,7 @@ async def cursor_chat_completions( raw_body: Final = await read_request_body(request=request) - if _is_chat_completions_body(raw_body): + if is_chat_completions_body(raw_body): # Genuine chat completions body (Cursor sends these for models whose BYOK it # already fixed); delegate so behavior matches /chat/completions exactly. # Keyed on messages CONTENT, not key presence: Cursor can send a null or diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 9e125b95a68..b1055bb9124 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -250,7 +250,11 @@ from litellm.proxy.hooks.sensitive_data_routing import ( # noqa: F401, RUF100 PROXY_SensitiveDataRoutingHandler, _PROXY_SensitiveDataRoutingHandler, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export ) -from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, add_guardrails_from_auth_metadata +from litellm.proxy.litellm_pre_call_utils import ( + LiteLLMProxyRequestSetup, + add_guardrails_from_auth_metadata, + normalized_metadata_slot, +) from litellm.proxy.management_helpers.key_settings_audit import with_settings_updated_at from litellm.proxy.policy_engine.pipeline_executor import ( PipelineExecutor, @@ -3973,9 +3977,7 @@ class ProxyLogging: ) request_data["litellm_logging_obj"] = litellm_logging_obj # rebind-ok: lifted then popped by the caller request_data.update({REJECTED_BEFORE_REQUEST_SETUP_KEY: REJECTED_BEFORE_REQUEST_SETUP}) - if "metadata" not in request_data: - request_data["metadata"] = {} - request_data["metadata"].update(user_api_key_logged_metadata) + normalized_metadata_slot(request_data, "metadata").update(user_api_key_logged_metadata) if litellm_logging_obj is not None: ## UPDATE LOGGING INPUT diff --git a/tests/integration/providers/test_cursor_responses_wire.py b/tests/integration/providers/test_cursor_responses_wire.py index d77d35f2ff8..1bd79b5b575 100644 --- a/tests/integration/providers/test_cursor_responses_wire.py +++ b/tests/integration/providers/test_cursor_responses_wire.py @@ -3,11 +3,14 @@ from __future__ import annotations import json import os import uuid +from collections.abc import Mapping +from hashlib import sha256 from importlib.resources import files from typing import Final +import httpx import pytest -from integration._support.client import Gateway, Scenario +from integration._support.client import Gateway, Scenario, eventually, list_value, object_value from integration._support.responses_vendor import same_response from integration._support.wire import Reply, Request, Wire, wire_server from integration._support.worker_readiness import wait_for_worker_readiness @@ -38,6 +41,7 @@ _CHAT_SUFFIXES: Final[tuple[tuple[str, str], ...]] = (("", ""),) + tuple( (f"-thinking-{level}", level) for level in _CHAT_THINKING_LEVELS ) _PROMPT: Final = "Inspect the requested change." +_REQUEST_TAG: Final = "cursor-integration" _INPUT: Final = [{"role": "user", "content": [{"type": "input_text", "text": _PROMPT}]}] _CUSTOM_TOOL: Final = { "type": "custom", @@ -107,16 +111,30 @@ def _stream(identity: str) -> tuple[bytes, ...]: ) + (b"data: [DONE]\n\n",) -def _chat_completion(stream: bool) -> bytes | tuple[bytes, ...]: +def _chat_completion(stream: bool, identity: str) -> bytes | tuple[bytes, ...]: if stream: - return ( - b'data: {"id":"chat_cursor_1","object":"chat.completion.chunk","created":1,"model":"gpt-5","choices":[{"index":0,"delta":{"content":"Patch inspected."},"finish_reason":null}]}\n\n', - b'data: {"id":"chat_cursor_1","object":"chat.completion.chunk","created":1,"model":"gpt-5","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n', - b"data: [DONE]\n\n", - ) + return tuple( + f"data: {json.dumps(chunk)}\n\n".encode() + for chunk in ( + { + "id": identity, + "object": "chat.completion.chunk", + "created": 1, + "model": _MODEL, + "choices": [{"index": 0, "delta": {"content": "Patch inspected."}, "finish_reason": None}], + }, + { + "id": identity, + "object": "chat.completion.chunk", + "created": 1, + "model": _MODEL, + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + }, + ) + ) + (b"data: [DONE]\n\n",) return json.dumps( { - "id": "chat_cursor_1", + "id": identity, "object": "chat.completion", "created": 1, "model": _MODEL, @@ -133,11 +151,12 @@ def _chat_completion(stream: bool) -> bytes | tuple[bytes, ...]: ).encode() -def _deployment(scenario: Scenario, wire: Wire) -> str: +def _deployment(scenario: Scenario, wire: Wire, **pricing: JsonValue) -> str: model: Final = scenario.model( model=f"openai/{_MODEL}", api_key=_API_KEY, api_base=f"{wire.url}/v1", + **pricing, ) wait_for_worker_readiness(scenario.gateway, model, 1, _PROXY_WORKERS) return model @@ -167,7 +186,7 @@ def test_cursor_responses_input_uses_responses_wire_and_chat_output( assert request.method == "POST" assert request.headers["authorization"] == f"Bearer {_API_KEY}", dict(request.headers) if request.target == "/v1/chat/completions": - completion: Final = _chat_completion(stream) + completion: Final = _chat_completion(stream, raw_id) return Reply( content_type="text/event-stream" if stream else "application/json", chunks=completion if isinstance(completion, tuple) else None, @@ -175,7 +194,7 @@ def test_cursor_responses_input_uses_responses_wire_and_chat_output( ) assert request.target == "/v1/responses", request.target body: Final = _JSON_OBJECT.validate_json(request.body) - assert {key: value for key, value in body.items() if key != "metadata"} == { + assert body == { "model": _MODEL, "input": _INPUT, **reasoning, @@ -259,41 +278,69 @@ def test_cursor_responses_input_uses_responses_wire_and_chat_output( assert [request.target for request in drain_contract_requests(wire)] == ["/v1/responses"] -def test_cursor_responses_body_carries_no_proxy_metadata(gateway: Gateway) -> None: - pytest.skip( - "BUG: /cursor/chat/completions forwards proxy-internal metadata (key hash, team, user, client ip, user agent) " - "to the provider in the Responses body" - ) +@pytest.mark.parametrize( + "client_metadata", ({}, {"cursor_conversation": "integration"}), ids=("no_client_metadata", "client_metadata") +) +@pytest.mark.parametrize("stream", (False, True)) +def test_cursor_responses_body_sends_only_client_metadata_and_bills_the_key( + gateway: Gateway, + stream: bool, + client_metadata: Mapping[str, JsonValue], +) -> None: raw_id: Final = f"resp_cursor_metadata_{uuid.uuid4().hex}" + alias: Final = f"integration-cursor-{uuid.uuid4().hex}" + sent_metadata: Final = {"metadata": client_metadata} if client_metadata else {} def respond(request: Request) -> Reply: if request.method == "GET" and request.target == "/v1/models": return model_discovery_reply(_MODEL) assert request.method == "POST" and request.target == "/v1/responses", request.target + if stream: + return Reply(content_type="text/event-stream", chunks=_stream(raw_id)) return Reply(body=json.dumps(_response(raw_id)).encode()) with wire_server(respond) as wire, gateway.scenario() as scenario: model: Final = _deployment(scenario, wire) + key: Final = scenario.key(key_alias=alias) + hashed_key: Final = sha256(key.encode()).hexdigest() response: Final = gateway.request( "POST", "/cursor/chat/completions", { - "model": f"{model}-thinking-high", + "model": model, "input": _INPUT, "messages": [], "tools": [_CUSTOM_TOOL], + "stream": stream, + **sent_metadata, }, + key=key, ) assert response.status_code == 200, response.text requests: Final = drain_contract_requests(wire) assert len(requests) == 1, requests - body: Final = _JSON_OBJECT.validate_json(requests[0].body) - assert body == { + assert _JSON_OBJECT.validate_json(requests[0].body) == { "model": _MODEL, "input": _INPUT, - "reasoning": {"effort": "high"}, "tools": [_CUSTOM_TOOL], + "stream": stream, + **sent_metadata, }, requests[0].body + (row,) = eventually( + lambda: gateway.rows( + 'SELECT call_type, spend, metadata FROM "LiteLLM_SpendLogs" WHERE api_key = %s', (hashed_key,) + ), + lambda rows: len(rows) == 1, + seconds=70, + ) + assert row["call_type"] == "aresponses", row + assert object_value(row["metadata"])["user_api_key_alias"] == alias, row + assert isinstance(row["spend"], float) and row["spend"] > 0, row + eventually( + lambda: gateway.rows('SELECT spend FROM "LiteLLM_VerificationToken" WHERE token = %s', (hashed_key,)), + lambda tokens: len(tokens) == 1 and tokens[0]["spend"] == pytest.approx(row["spend"]), + seconds=70, + ) def test_cursor_stream_reports_usage_when_the_client_asks_for_it(gateway: Gateway) -> None: @@ -341,10 +388,12 @@ def test_cursor_messages_body_uses_chat_completions_wire( suffix: str, reasoning_effort: str, ) -> None: + chat_id: Final = f"chat_cursor_{uuid.uuid4().hex}" + def respond(request: Request) -> Reply: if request.method == "GET" and request.target == "/v1/models": return model_discovery_reply(_MODEL) - completion: Final = _chat_completion(stream) + completion: Final = _chat_completion(stream, chat_id) return Reply( content_type="text/event-stream" if stream else "application/json", chunks=completion if isinstance(completion, tuple) else None, @@ -367,6 +416,7 @@ def test_cursor_messages_body_uses_chat_completions_wire( "stream": stream, }, key=key, + headers={"x-litellm-tags": _REQUEST_TAG}, ) assert response.status_code == 200, response.text if stream: @@ -393,7 +443,7 @@ def test_cursor_messages_body_uses_chat_completions_wire( payload: Final = _JSON_OBJECT.validate_json(response.content) identity: Final = payload.get("id") created: Final = payload.get("created") - assert isinstance(identity, str) and same_response(identity, "chat_cursor_1"), response.text + assert isinstance(identity, str) and same_response(identity, chat_id), response.text assert isinstance(created, int) and created > 0, response.text assert payload == { "id": identity, @@ -456,3 +506,72 @@ def test_cursor_messages_body_uses_chat_completions_wire( **({"stream_options": {"include_usage": True}} if stream else {}), **({"reasoning_effort": reasoning_effort} if reasoning_effort else {}), }, requests[0].body + (row,) = eventually( + lambda: gateway.rows( + 'SELECT call_type, team_id, request_tags FROM "LiteLLM_SpendLogs" WHERE api_key = %s', + (sha256(key.encode()).hexdigest(),), + ), + lambda rows: len(rows) == 1, + seconds=70, + ) + assert (row["call_type"], row["team_id"]) == ("acompletion", team), row + assert _REQUEST_TAG in list_value(row["request_tags"]), row + + +@pytest.mark.parametrize( + ("body_shape", "metadata_as_json_string"), (("messages", False), ("input", False), ("input", True)) +) +def test_cursor_body_keeps_client_and_key_tags_and_enforces_the_client_tag_budget( + gateway: Gateway, body_shape: str, metadata_as_json_string: bool +) -> None: + chat_id: Final = f"chat_cursor_{uuid.uuid4().hex}" + response_id: Final = f"resp_cursor_tags_{uuid.uuid4().hex}" + budget_tag: Final = f"cursor-tag-budget-{uuid.uuid4().hex}" + key_tag: Final = f"cursor-key-tag-{uuid.uuid4().hex}" + client_metadata: Final = {"tags": [budget_tag]} + + def respond(request: Request) -> Reply: + if request.method == "GET" and request.target == "/v1/models": + return model_discovery_reply(_MODEL) + if request.target == "/v1/responses": + return Reply(body=json.dumps(_response(response_id)).encode()) + completion: Final = _chat_completion(False, chat_id) + return Reply(body=completion if isinstance(completion, bytes) else b"") + + def delete_tag() -> None: + gateway.post("/tag/delete", {"name": budget_tag}) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire, input_cost_per_token=0.01, output_cost_per_token=0.01) + key: Final = scenario.key(models=[model], metadata={"tags": [key_tag]}) + gateway.post("/tag/new", {"name": budget_tag, "max_budget": 0.0001}) + scenario.cleanups.callback(delete_tag) + + def tagged_chat(text: str) -> httpx.Response: + return gateway.request( + "POST", + "/cursor/chat/completions", + { + "model": model, + body_shape: [{"role": "user", "content": text}], + "metadata": json.dumps(client_metadata) if metadata_as_json_string else client_metadata, + }, + key=key, + ) + + first: Final = tagged_chat("first tagged chat") + assert first.status_code == 200, first.text + (row,) = eventually( + lambda: gateway.rows( + 'SELECT request_tags FROM "LiteLLM_SpendLogs" WHERE api_key = %s', + (sha256(key.encode()).hexdigest(),), + ), + lambda rows: len(rows) == 1, + seconds=70, + ) + assert {budget_tag, key_tag} <= set(list_value(row["request_tags"])), row + blocked: Final = eventually( + lambda: tagged_chat("tag budget probe"), lambda response: response.status_code != 200, seconds=70 + ) + assert blocked.status_code == 422, blocked.text + assert "budget" in blocked.text.lower(), blocked.text diff --git a/tests/unit/integrations/test_custom_guardrail.py b/tests/unit/integrations/test_custom_guardrail.py index 614ff97553a..0483888533e 100644 --- a/tests/unit/integrations/test_custom_guardrail.py +++ b/tests/unit/integrations/test_custom_guardrail.py @@ -3699,6 +3699,20 @@ def test_get_guardrail_from_metadata() -> None: ) == [{"test-guardrail": {"extra_body": {"key": "value"}}}] + +@pytest.mark.parametrize( + "caller_metadata", + [{"guardrails": []}, {"guardrails": ["not-a-guardrail"]}, {"guardrails": None}], +) +def test_caller_metadata_guardrails_never_hide_the_proxy_guardrails_in_litellm_metadata(caller_metadata: dict) -> None: + guardrail: Final = CustomGuardrail( + guardrail_name="key-guardrail", event_hook=GuardrailEventHooks.pre_call, default_on=False + ) + request_data: Final = {"metadata": caller_metadata, "litellm_metadata": {"guardrails": ["key-guardrail"]}} + + assert guardrail.should_run_guardrail(request_data, GuardrailEventHooks.pre_call) is True + + def test_guardrail_is_in_requested_guardrails() -> None: guardrail: Final = CustomGuardrail(guardrail_name="test-guardrail") diff --git a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py index b66e7f3d091..3aad2bda9bf 100644 --- a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py @@ -2591,7 +2591,8 @@ async def test_agent_session_rate_limit_skipped_without_session_id(): @pytest.mark.asyncio -async def test_agent_rate_limit_from_metadata_agent_id(): +@pytest.mark.parametrize("metadata_field", ("metadata", "litellm_metadata")) +async def test_agent_rate_limit_from_metadata_agent_id(metadata_field): """ Test that agent rate limits work when agent_id comes from request metadata (header) rather than from the API key. @@ -2639,7 +2640,7 @@ async def test_agent_rate_limit_from_metadata_agent_id(): cache=local_cache, data={ "model": "gpt-4", - "metadata": {"agent_id": _agent_id}, + metadata_field: {"agent_id": _agent_id}, }, call_type="", ) diff --git a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py index dc46b5faf4e..8d4f8cd87ad 100644 --- a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py @@ -623,6 +623,56 @@ async def test_async_post_call_failure_hook_reads_used_client_oauth_token_from_t assert json.loads(payload["metadata"])["used_client_oauth_token"] is expected + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "request_route, body", + [ + ( + "/cursor/chat/completions", + { + "input": "Hello", + "metadata": {"requester_ip_address": "203.0.113.9", "user_agent": "spoofed"}, + "litellm_metadata": {"requester_ip_address": "127.0.0.1", "user_agent": "Cursor/1.0"}, + }, + ), + ( + "/v1/chat/completions", + { + "messages": [{"role": "user", "content": "Hello"}], + "metadata": {"requester_ip_address": "127.0.0.1", "user_agent": "Cursor/1.0"}, + "litellm_metadata": {"requester_ip_address": "203.0.113.9", "user_agent": "spoofed"}, + }, + ), + ], +) +async def test_async_post_call_failure_hook_logs_the_proxy_stamped_ip_and_user_agent(request_route: str, body: dict): + request_data = { + "model": "gpt-4o", + "proxy_server_request": {"request_id": "test_request_id"}, + **body, + } + + with patch( + "litellm.proxy.db.db_spend_update_writer.DBSpendUpdateWriter.update_database", + new_callable=AsyncMock, + ) as mock_update_database: + await ProxyDBLogger().async_post_call_failure_hook( + request_data=request_data, + original_exception=Exception("invalid api key"), + user_api_key_dict=UserAPIKeyAuth(api_key="test_api_key", request_route=request_route), + ) + + payload = get_logging_payload( + kwargs=mock_update_database.call_args[1]["kwargs"], + response_obj={}, + start_time=datetime.now(), + end_time=datetime.now(), + ) + assert payload["requester_ip_address"] == "127.0.0.1" + assert json.loads(payload["metadata"])["user_agent"] == "Cursor/1.0" + + @pytest.mark.asyncio async def test_async_post_call_failure_hook_bills_guardrail_cost_on_blocked_request(): """LIT-5651: a request blocked by a guardrail never reaches the LLM, but the diff --git a/tests/unit/proxy/response_api_endpoints/test_endpoints.py b/tests/unit/proxy/response_api_endpoints/test_endpoints.py index a783ac70881..1e4dded801e 100644 --- a/tests/unit/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/unit/proxy/response_api_endpoints/test_endpoints.py @@ -3938,7 +3938,7 @@ class TestCursorInputArmFlattening: class TestChatCompletionsBodyDetection: def test_routing_matrix(self): - from litellm.proxy.response_api_endpoints.endpoints import _is_chat_completions_body + from litellm.proxy.litellm_pre_call_utils import is_chat_completions_body as _is_chat_completions_body assert _is_chat_completions_body({"messages": [{"role": "user", "content": "hi"}]}) is True assert _is_chat_completions_body({"messages": [{"role": "user", "content": "hi"}], "input": []}) is True diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index c867db2fd02..2d08609c93b 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -9711,6 +9711,40 @@ async def test_saved_weights_override_caller_input_and_preserve_key_precedence( assert logging.pre_call_hook.call_args.kwargs["data"].get("_router_weights") == expected +def _hook_data(**kwargs: object) -> object: + return kwargs["data"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("prompt, proxy_field, client_field", [ + ({"input": "hi"}, "litellm_metadata", "metadata"), + ({"messages": [{"role": "user", "content": "hi"}]}, "metadata", "litellm_metadata"), +]) +async def test_cursor_route_writes_proxy_fields_only_to_the_field_its_body_shape_picks( + prompt: dict[str, str | list[dict[str, str]]], + proxy_field: str, + client_field: str, +) -> None: + from litellm.proxy import proxy_server + + logging: Final = MagicMock(spec=ProxyLogging) + logging.pre_call_hook = AsyncMock(side_effect=_hook_data) + processor: Final = ProxyBaseLLMRequestProcessing(data={"model": "gpt-x", "metadata": {"client": "cursor"}, **prompt}) + data, _ = await processor.common_processing_pre_call_logic( + request=Request({"type": "http", "method": "POST", "path": "/cursor/chat/completions", "headers": []}), + general_settings={}, + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="hash"), + proxy_logging_obj=logging, + proxy_config=proxy_server.ProxyConfig(), + route_type="acompletion", + logging_auth=ProxyUserAPIKeyAuth(api_key="logging-hash", key_alias="logging-alias"), + ) + assert data[proxy_field]["user_api_key_alias"] == "logging-alias" + assert "queue_time_seconds" in data[proxy_field] + assert "user_api_key_alias" not in data.get(client_field, {}) + assert "queue_time_seconds" not in data.get(client_field, {}) + assert data["metadata"]["client"] == "cursor" + class TestPerRequestModelGroupAlias: """``router_settings.model_group_alias`` on a key or team has to be resolved by the proxy: the Router resolves aliases from its own shared instance diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 1ce61a8e9b3..2edb032b048 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -121,6 +121,23 @@ class TestGetMetadataVariableName: request = self._make_request("/chat/completions") assert get_metadata_variable_name(request) == "metadata" + @pytest.mark.parametrize( + "body, expected", + [ + ({"messages": [{"role": "user", "content": "hi"}]}, "metadata"), + ({"messages": []}, "metadata"), + ({"input": [{"role": "user", "content": "hi"}]}, "litellm_metadata"), + ({"input": "hi", "messages": []}, "litellm_metadata"), + ], + ) + def test_cursor_route_picks_the_metadata_field_by_body_shape(self, body, expected): + request = self._make_request("/cursor/chat/completions") + assert get_metadata_variable_name(request, body) == expected + + def test_cursor_route_without_a_body_keeps_the_route_answer(self): + request = self._make_request("/cursor/chat/completions") + assert get_metadata_variable_name(request) == "metadata" + def test_returns_metadata_for_completions(self): request = self._make_request("/v1/completions") assert get_metadata_variable_name(request) == "metadata" @@ -681,6 +698,45 @@ async def test_add_litellm_data_to_request_rejects_non_object_metadata(field, va assert field not in data +@pytest.mark.asyncio +@pytest.mark.parametrize("route", ["/v1/chat/completions", "/v1/responses", "/cursor/chat/completions"]) +@pytest.mark.parametrize("value, received_type", [("junk", "a string"), (5, "an integer"), ([1], "an array")]) +async def test_invalid_litellm_metadata_answers_400_when_metadata_carries_tags(route, value, received_type): + from litellm.proxy.auth.auth_checks import common_checks + from litellm.proxy.utils import ProxyLogging + + data = {"model": "gpt-4o", "input": "hi", "metadata": {"tags": ["client-tag"]}, "litellm_metadata": value} + request = _batches_request_mock() + await common_checks( + request_body=data, + team_object=None, + user_object=None, + end_user_object=None, + global_proxy_spend=None, + general_settings={}, + route=route, + llm_router=None, + proxy_logging_obj=ProxyLogging(user_api_key_cache=None), + valid_token=UserAPIKeyAuth(api_key="hashed-key"), + request=request, + ) + + with pytest.raises(ProxyException) as exc_info: + await add_litellm_data_to_request( + data=data, + request=request, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + assert exc_info.value.code == "400" + assert exc_info.value.message == ( + f"Invalid type for 'litellm_metadata': expected an object, but got {received_type} instead." + ) + + @pytest.mark.asyncio async def test_add_litellm_data_to_request_removes_every_invalid_metadata_field_before_raising(): """When both fields are invalid, the raise for the first must not leave the @@ -11098,3 +11154,15 @@ def test_mapped_baggage_tags_keep_non_string_client_tags_as_sent(): data: Final[dict[str, object]] = {"metadata": {"tags": [1, {"team": "x"}, "client-tag", "client-tag"]}} LiteLLMProxyRequestSetup.apply_authenticated_baggage_tags(request, data) assert data["metadata"]["tags"] == [1, {"team": "x"}, "client-tag", "client-tag", "tenant:a"] + + +@pytest.mark.parametrize("caller_metadata", ({"tags": ["client-tag"]}, json.dumps({"tags": ["client-tag"]}))) +def test_caller_tags_reach_litellm_metadata_whether_metadata_is_a_dict_or_a_json_string(caller_metadata): + data: Final[dict[str, object]] = { + "input": "hi", + "metadata": caller_metadata, + "litellm_metadata": {"tags": ["key-tag"]}, + } + LiteLLMProxyRequestSetup.merge_caller_body_tags(request_data=data) + assert set(data["litellm_metadata"]["tags"]) == {"key-tag", "client-tag"} + assert data["metadata"] == caller_metadata diff --git a/tests/unit/proxy/utils/proxy_logging/test_post_call_failure_hook.py b/tests/unit/proxy/utils/proxy_logging/test_post_call_failure_hook.py index f880d3548f7..4386d5aec94 100644 --- a/tests/unit/proxy/utils/proxy_logging/test_post_call_failure_hook.py +++ b/tests/unit/proxy/utils/proxy_logging/test_post_call_failure_hook.py @@ -4,6 +4,7 @@ and ``_handle_logging_proxy_only_error``.""" from __future__ import annotations import asyncio +import json from collections.abc import Callable from datetime import datetime from types import MappingProxyType @@ -1192,3 +1193,31 @@ async def test_handle_logging_proxy_only_path_flags_requests_that_died_before_re blocked_by_guardrail.get(REJECTED_BEFORE_REQUEST_SETUP_KEY) is REJECTED_BEFORE_REQUEST_SETUP, blocked_by_guardrail["litellm_call_id"], ) == (True, "client-supplied", False, "call-1") + + +@pytest.mark.asyncio +async def test_auth_failure_on_a_responses_body_with_a_json_string_metadata_is_logged_against_the_key( + proxy_logging, monkeypatch +): + from litellm.proxy import proxy_server + + recorded: list[dict] = [] + + class _RecordingLogger(CustomLogger): + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + recorded.append(kwargs) + + monkeypatch.setattr(proxy_server, "llm_router", None) + monkeypatch.setattr(litellm, "callbacks", [_RecordingLogger()]) + proxy_logging.alert_types = [] + + await proxy_logging.post_call_failure_hook( + request_data={"model": "internal-model", "input": "hi", "metadata": json.dumps({"tags": ["checkout"]})}, + original_exception=litellm.BudgetExceededError(current_cost=1.0, max_budget=0.5), + user_api_key_dict=UserAPIKeyAuth(api_key="sk-hashed", key_alias="checkout-key", request_route="/v1/responses"), + error_type=ProxyErrorTypes.auth_error, + route="/v1/responses", + ) + + assert len(recorded) == 1 + assert recorded[0]["standard_logging_object"]["metadata"]["user_api_key_alias"] == "checkout-key"