From b71491c6c910c15bc1580e9469b6d3ecb3b376b1 Mon Sep 17 00:00:00 2001 From: Siddhant Badola Date: Fri, 30 Oct 2026 09:27:59 -0400 Subject: [PATCH 01/74] fix(bedrock): route a Bedrock model id with no Invoke provider to Converse (#43290) * fix(bedrock): route unmapped converse-only family model ids to converse #42713 stopped Bedrock-native OpenAI ids that are not in the cost map yet from falling to the invoke route. The same gap exists for the other families that have no invoke transformation at all: minimax, nvidia, writer, zai, google and xai. An id from one of them that is newer than the cost map hits get_bedrock_route -> "invoke" -> get_bedrock_invoke_provider() -> None -> BedrockError, when Converse would have served it Add a second family regex for those six and default them to converse, the same way the openai family is handled. _OPENAI_FAMILY_MODEL_RE is left alone because the prompt-caching rule is openai-specific Every shipped id of these families already resolves to a converse entry via its base model (checked against the cost map), so this only changes behaviour for ids that are not in the map yet Tests: six unmapped ids route to converse, and an unmapped meta.* id still routes to invoke since meta has an invoke handler * fix(bedrock): route a model id with no Invoke provider to Converse A bedrock/ id missing from the cost map fell to the Invoke route, where an id that names no Invoke provider (a new vendor's model, a bare provisioned-model, imported-model, or custom-model-deployment ARN) fails locally with "Unknown provider=None" before any request is sent. Such ids now go to Converse, which replaces the per-family regex with the general rule. Ids that name an Invoke provider and explicit route prefixes are unchanged. * test(bedrock): expect Converse for ids with no Invoke provider in the nova ARN and not-found tests * fix(bedrock): accept tools on Converse for a model with no cost-map row or Invoke provider, and keep a 404 for an invalid id * fix(bedrock): answer 404 for an invalid model id instead of AWS's 400 AWS answers a mistyped Bedrock model id with a 400 ValidationException. The mapping already raised NotFoundError, but passing AWS's 400 response through made the openai base class report 400, so the proxy answered 400 and the router skipped its cooldown * fix(bedrock): drop lookaround regexes from tool schemas for a model the cost map does not list * fix(bedrock): apply the unknown-model defaults only to bare ids LiteLLM knows nothing about A bare Bedrock id with no cost-map row and no Invoke vendor gets Converse's open params, its reasoning_effort as sent, lookaround-free tool schemas, and a 404 for AWS's invalid-identifier reply, which keeps AWS's response headers. An explicit route like converse/, an ARN, or an id LiteLLM recognizes keeps main's params, regexes and error status, so drop_params still drops tools there * fix(bedrock): judge lookaround support by model id on every route, ignore price-only rows The regex check now strips route and region prefixes from the model and its base_model before asking whether LiteLLM knows the model, so bedrock/converse/ gets the same answer in a direct transform call as on the completion path. A cost-map row counts only when it names a provider, so a price-only register_model row leaves an unlisted id unknown, as on the proxy. AmazonInvokeConfig now inherits the Invoke provider lookup instead of keeping an identical copy. * fix(bedrock): treat a region path like the bare id when the cost map does not list it A deployment like bedrock/us-east-1/ now gets the same tools, reasoning_effort and typo 404 as bedrock/, so Claude Code works on it and a mistyped one still answers 404. * fix(bedrock): keep today's 404 for ids that reach Converse only by default, and keep an empty id local * fix(bedrock): answer 404 for an invalid id only when Converse rejected it, and keep a region path with no id local * fix(bedrock): keep a path-shaped id local, and map /v1/completions errors with the model's provider An id with a slash that is not an ARN, like bedrock/../../converse or an unknown region path, keeps Invoke and its local 404 instead of reaching AWS Converse. atext_completion mapped every error as OpenAI, so a Bedrock typo on /v1/completions answered 400 while chat answered 404. It now resolves the provider from the request's model, as the other async entry points do. * fix(text completion): keep /v1/completions error mapping as it is on main Mapping async /v1/completions errors with each provider's rules is wider than this Bedrock routing change. It moves classes, statuses, cooldown and fallbacks for most providers, and some lose context-window fallbacks. So this restores main's mapping here and leaves that change to its own PR. A Bedrock typo on /v1/completions answers AWS's 400, while chat keeps the 404. --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../exception_mapping_utils.py | 32 +- .../bedrock/chat/converse_transformation.py | 8 +- .../base_invoke_transformation.py | 62 +--- litellm/llms/bedrock/common_utils.py | 68 +++- .../test_bedrock_kimi_k3_cache_point_wire.py | 2 +- .../test_exception_mapping_utils.py | 70 +++- .../chat/test_converse_transformation.py | 312 +++++++++++++++++- .../llms/bedrock/test_bedrock_common_utils.py | 107 +++++- .../llms/bedrock/test_nova_imported_models.py | 6 +- 9 files changed, 559 insertions(+), 108 deletions(-) diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 5a27f5446ad..33b6d4e1857 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -18,6 +18,7 @@ from litellm.litellm_core_utils.bug_report import ( ) from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none from litellm.litellm_core_utils.secret_redaction import redact_string as redact_secret_string +from litellm.llms.bedrock.common_utils import bedrock_model_reaches_converse_by_default from litellm.llms.bigmodel.common_utils import raise_bigmodel_business_error from litellm.types.utils import LlmProviders @@ -938,13 +939,22 @@ _BEDROCK_MANTLE_CONTEXT_WINDOW_GENERIC_MESSAGE: Final = ( ) -def _bedrock_mantle_context_window_response(original_exception: _ProviderHTTPException) -> httpx.Response: +def _bedrock_response_with_status(original_exception: _ProviderHTTPException, status_code: int) -> httpx.Response: original_response: Final = getattr(original_exception, "response", None) if not isinstance(original_response, httpx.Response) or getattr(original_response, "_request", None) is None: - return httpx.Response(status_code=400, request=httpx.Request(method="POST", url="https://api.openai.com/v1/")) - if original_response.status_code == 400: + return httpx.Response( + status_code=status_code, request=httpx.Request(method="POST", url="https://api.openai.com/v1/") + ) + if original_response.status_code == status_code: return original_response - return httpx.Response(status_code=400, headers=original_response.headers, request=original_response.request) + return httpx.Response(status_code=status_code, headers=original_response.headers, request=original_response.request) + + +def _came_from_bedrock_converse(original_exception: _ProviderHTTPException) -> bool: + original_response: Final = getattr(original_exception, "response", None) + if not isinstance(original_response, httpx.Response) or getattr(original_response, "_request", None) is None: + return False + return original_response.request.url.path.endswith(("/converse", "/converse-stream")) def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None: @@ -980,7 +990,7 @@ def _map_bedrock_exception( message=mantle_context_window_message, model=model, llm_provider=custom_llm_provider, - response=_bedrock_mantle_context_window_response(original_exception), + response=_bedrock_response_with_status(original_exception, 400), ) if ( "too many tokens" in error_str @@ -1003,6 +1013,18 @@ def _map_bedrock_exception( llm_provider="bedrock", response=getattr(original_exception, "response", None), ) + elif ( + "The provided model identifier is invalid" in error_str + and bool(model) + and _came_from_bedrock_converse(original_exception) + and bedrock_model_reaches_converse_by_default(model) + ): + raise NotFoundError( + message=f"BedrockException - {error_str}", + model=model, + llm_provider="bedrock", + response=_bedrock_response_with_status(original_exception, 404), + ) elif "Malformed input request" in error_str: raise BadRequestError( message=f"BedrockException - {error_str}", diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 443e722168d..5e621a5f322 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -63,6 +63,7 @@ from litellm.llms.bedrock.chat.tool_result_images import place_tool_result_image from litellm.llms.bedrock.common_utils import ( bedrock_chat_uses_native_structured_output, bedrock_cleaned_output_schema, + bedrock_model_is_unknown, bedrock_model_supports_regex_lookaround, bedrock_native_output_format, bedrock_native_output_is_opt_in, @@ -529,13 +530,14 @@ class AmazonConverseConfig(BaseConfig): """ Handle the reasoning_effort parameter based on the model type. - - GPT-OSS and DeepSeek V3 models: passed through unchanged via additionalModelRequestFields. + - GPT-OSS, DeepSeek V3 and unknown models (``bedrock_model_is_unknown``): passed through unchanged + via additionalModelRequestFields. - OpenAI GPT-5.x and GPT-6 models: mapped to ``reasoning.effort`` via additionalModelRequestFields. - Nova 2 models: transformed to reasoningConfig. - Anthropic models: mapped to ``thinking`` (and ``output_config.effort`` on adaptive Claude 4.6 / 4.7). """ - if "gpt-oss" in model or "deepseek" in model: + if "gpt-oss" in model or "deepseek" in model or bedrock_model_is_unknown(model): optional_params["reasoning_effort"] = reasoning_effort elif self._is_openai_gpt_reasoning_model(model): reasoning: Final[BedrockConverseGptReasoningEffortBlock] = {"effort": reasoning_effort} @@ -681,7 +683,7 @@ class AmazonConverseConfig(BaseConfig): "parallel_tool_calls", ] - if "arn" in model: # we can't infer the model from the arn, so just add all params + if "arn" in model or bedrock_model_is_unknown(model): supported_params.append("tools") supported_params.append("tool_choice") supported_params.append("thinking") diff --git a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py index 852179e8caa..877fb4ca692 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py @@ -2,7 +2,7 @@ import copy import json import time from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final, cast, get_args +from typing import TYPE_CHECKING, Any, Final import httpx from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config @@ -74,10 +74,6 @@ _GUARDRAIL_CONFIG_EXPECTED_FORMAT: Final = ( "{'guardrailIdentifier': str, 'guardrailVersion': str, 'trace': 'enabled'|'disabled'|'enabled_full'}" ) -_INVOKE_PROVIDERS: Final = TypeAdapter(tuple[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, ...]).validate_python( - get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL) -) - def _response_header(headers: Mapping[str, str], name: str) -> str | None: return headers.get(name, None) @@ -562,62 +558,6 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): """ return False - @staticmethod - def get_bedrock_invoke_provider( - model: str, - ) -> litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None: - """ - Helper function to get the bedrock provider from the model - - handles 4 scenarios: - 1. model=invoke/anthropic.claude-3-5-sonnet-20240620-v1:0 -> Returns `anthropic` - 2. model=anthropic.claude-3-5-sonnet-20240620-v1:0 -> Returns `anthropic` - 3. model=llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n -> Returns `llama` - 4. model=us.amazon.nova-pro-v1:0 -> Returns `nova` - """ - if model.startswith("invoke/"): - model = model.replace("invoke/", "", 1) - - # Special case: Check for "nova" in model name first (before "amazon") - # This handles amazon.nova-* models which would otherwise match "amazon" (Titan) - if "nova" in model.lower(): - if "nova" in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL): - return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, "nova") - - _split_model: Final = model.split(".")[0] - if _split_model in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL): - return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, _split_model) - - # If not a known provider, check for pattern with two slashes - provider = AmazonInvokeConfig._get_provider_from_model_path(model) - if provider is not None: - return provider - - for provider in _INVOKE_PROVIDERS: - if provider in model: - return provider - return None - - @staticmethod - def _get_provider_from_model_path( - model_path: str, - ) -> litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None: - """ - Helper function to get the provider from a model path with format: provider/model-name - - Args: - model_path (str): The model path (e.g., 'llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n' or 'anthropic/model-name') - - Returns: - Optional[str]: The provider name, or None if no valid provider found - """ - parts: Final = model_path.split("/") - if len(parts) >= 1: - provider: Final = parts[0] - if provider in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL): - return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, provider) - return None - def convert_messages_to_prompt(self, model: str, messages, provider, custom_prompt_dict) -> tuple[str, list | None]: # handle anthropic prompts and amazon titan prompts prompt = "" diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index d88d7e6280c..d6f5d5b6bd7 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -1098,6 +1098,11 @@ def _model_cost_entry(key: str) -> Mapping[str, object] | None: return None if raw is None else _MODEL_COST_ENTRY_ADAPTER.validate_python(raw) +def _cost_map_row_names_a_provider(key: str) -> bool: + entry: Final = _model_cost_entry(key) + return entry is not None and entry.get("litellm_provider") is not None + + def _bedrock_price_map_entries(model: str) -> tuple[Mapping[str, object] | None, ...]: return tuple( _model_cost_entry(key) @@ -1567,7 +1572,8 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[ The deployment's ``model_info.supports_regex_lookaround`` wins, then the ``model_prices_and_context_window.json`` entry of its ``base_model``, then the - entry of ``model`` itself. A model nobody flagged keeps its schema as sent. + entry of ``model`` itself. A model nobody flagged keeps its schema as sent, unless + no candidate names a model LiteLLM knows, whatever route prefix it carries. """ params: Final = litellm_params or {} deployment_flag: Final = deployment_model_info(params).get("supports_regex_lookaround") @@ -1576,7 +1582,41 @@ def bedrock_model_supports_regex_lookaround(model: str, litellm_params: Mapping[ base_model: Final = params.get("base_model") candidates: Final = (*((base_model,) if isinstance(base_model, str) else ()), model) flags: Final = (_bedrock_converse_model_flag(candidate, "supports_regex_lookaround") for candidate in candidates) - return next((flag for flag in flags if flag is not None), True) + flag: Final = next((flag for flag in flags if flag is not None), None) + if flag is not None: + return flag + return not all(bedrock_model_is_unknown(split_bedrock_region_path(candidate)[1]) for candidate in candidates) + + +def _bedrock_model_id_without_region(model: str) -> str: + routed_id: Final = model.removeprefix("bedrock/") + region, separator, regional_id = routed_id.partition("/") + return regional_id if separator and region in get_all_bedrock_regions() else routed_id + + +def bedrock_model_is_unknown(model: str) -> bool: + model_id: Final = _bedrock_model_id_without_region(model) + if "/" in model_id: + return False + base: Final = get_bedrock_base_model(model_id) + regional_ids: Final = (f"{region}.{base}" for region in get_bedrock_cross_region_inference_regions()) + if any(_cost_map_row_names_a_provider(key) for key in (*_bedrock_model_info_candidates(model_id), *regional_ids)): + return False + from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM + + return BaseAWSLLM.get_bedrock_invoke_provider(model_id) is None + + +def _names_no_invoke_provider(model: str) -> bool: + from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM + + model_id: Final = _bedrock_model_id_without_region(model) + names_one_model: Final = bool(model_id) and ("/" not in model_id or model_id.startswith("arn:")) + return names_one_model and BaseAWSLLM.get_bedrock_invoke_provider(model) is None + + +def bedrock_model_reaches_converse_by_default(model: str) -> bool: + return BedrockModelInfo.named_bedrock_route(model, None) is None and _names_no_invoke_provider(model) _BedrockConverseModelFlag: TypeAlias = Literal[ @@ -1585,10 +1625,13 @@ _BedrockConverseModelFlag: TypeAlias = Literal[ ] -def _bedrock_converse_model_flag(model: str, key: _BedrockConverseModelFlag) -> bool | None: +def _bedrock_model_info_candidates(model: str) -> tuple[str, ...]: base: Final = get_bedrock_base_model(model) - candidates: Final = dict.fromkeys((model, base, _BEDROCK_MODEL_VERSION_SUFFIX_RE.sub("", base))) - for candidate in candidates: + return tuple(dict.fromkeys((model, base, _BEDROCK_MODEL_VERSION_SUFFIX_RE.sub("", base)))) + + +def _bedrock_converse_model_flag(model: str, key: _BedrockConverseModelFlag) -> bool | None: + for candidate in _bedrock_model_info_candidates(model): with contextlib.suppress(Exception): model_info = get_cached_model_info()( model=candidate, @@ -1766,8 +1809,19 @@ class BedrockModelInfo(BaseLLMModelInfo): default (``bedrock_runtime_chat_completions_is_default``) and ``chat_completions/`` opts any other model in; ``request_params`` (the caller's chat params) sends such a request to Converse when it needs a feature only Converse serves, and ``converse/`` pins a model to Converse. Every other OpenAI-family - model stays on Converse without the prefix. + model stays on Converse without the prefix. A model the cost map does not list as Converse goes to + Invoke only when its id names an Invoke provider, and to Converse otherwise. """ + named_route: Final = BedrockModelInfo.named_bedrock_route(model, request_params) + if named_route is not None: + return named_route + return "converse" if _names_no_invoke_provider(model) else "invoke" + + @staticmethod + def named_bedrock_route( + model: str, + request_params: Mapping[str, object] | None, + ) -> BedrockRoute | None: route_mappings: dict[ str, Literal[ @@ -1822,7 +1876,7 @@ class BedrockModelInfo(BaseLLMModelInfo): return "converse" if _OPENAI_FAMILY_MODEL_RE.search(base_model): return "converse" - return "invoke" + return None @staticmethod def _explicit_converse_route(model: str) -> bool: diff --git a/tests/integration/providers/test_bedrock_kimi_k3_cache_point_wire.py b/tests/integration/providers/test_bedrock_kimi_k3_cache_point_wire.py index 854da6fce0b..0772e928b35 100644 --- a/tests/integration/providers/test_bedrock_kimi_k3_cache_point_wire.py +++ b/tests/integration/providers/test_bedrock_kimi_k3_cache_point_wire.py @@ -1037,7 +1037,7 @@ def test_s07_a_5kb_caller_model_under_the_wildcard_is_a_bounded_4xx_and_liveline error: Final = _JSON.validate_python(_JSON.validate_python(json.loads(outcome.raw))["error"]) assert "a" * 5120 in str(error["message"]), outcome.raw[:300] assert_within_factor_of_control(oversized.seconds, control=control.seconds, factor=20) - assert rig.wire.drain() == () + assert len(rig.wire.drain()) == 1 _failure_row(outcome.call_id) diff --git a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py index 88501a23235..efb9648cf13 100644 --- a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py @@ -60,7 +60,6 @@ def test_openai_cyber_policy_uses_the_structured_error_code(shape: str, code: st assert caught.value.response.headers["x-request-id"] == "cyber-policy-request" - def _debug_info_for_api_base(configured_api_base: str) -> str: request: Final = httpx.Request("POST", "https://gateway.example/v1/chat/completions") original: Final = OpenAIError( @@ -1602,6 +1601,75 @@ def test_bedrock_mantle_400_maps_to_bad_request(): assert type(excinfo.value) is litellm.BadRequestError +class _ErrorWithoutResponse(Exception): + def __init__(self, message: str, status_code: int) -> None: + super().__init__(message) + self.message = message + self.status_code = status_code + + +def test_bedrock_mantle_context_overflow_without_a_provider_response_answers_400(): + with pytest.raises(litellm.ContextWindowExceededError) as excinfo: + exception_type( + model="openai.gpt-5.6-sol", + original_exception=_ErrorWithoutResponse( + '{"error":{"code":"context_length_exceeded","message":"too long"}}', 400 + ), + custom_llm_provider="bedrock_mantle", + ) + + assert excinfo.value.response.status_code == 400 + + +def _invalid_model_identifier_from(url: str) -> BedrockError: + return BedrockError( + status_code=400, + message='{"message":"The provided model identifier is invalid."}', + response=httpx.Response(status_code=400, request=httpx.Request(method="POST", url=url)), + ) + + +@pytest.mark.parametrize( + "path", + ["/model/us.newvendor.typo-1/converse", "/model/us.newvendor.typo-1/converse-stream"], +) +def test_bedrock_converse_rejecting_an_id_with_no_invoke_provider_answers_404(path): + with pytest.raises(litellm.NotFoundError) as excinfo: + exception_type( + model="us.newvendor.typo-1", + original_exception=_invalid_model_identifier_from(f"https://bedrock-runtime.us-east-1.amazonaws.com{path}"), + custom_llm_provider="bedrock", + ) + + assert excinfo.value.response.status_code == 404 + + +@pytest.mark.parametrize( + "original_exception", + [ + _invalid_model_identifier_from("https://bedrock-runtime.us-east-1.amazonaws.com/model/claude-typo-9/invoke"), + _ErrorWithoutResponse("The provided model identifier is invalid.", 400), + ], + ids=["invoke", "no-response"], +) +def test_bedrock_invalid_model_identifier_not_from_converse_stays_a_bad_request(original_exception): + with pytest.raises(litellm.BadRequestError) as excinfo: + exception_type(model="claude-typo-9", original_exception=original_exception, custom_llm_provider="bedrock") + + assert excinfo.value.status_code == 400 + + +def test_bedrock_invalid_model_identifier_without_a_model_stays_a_bad_request(): + with pytest.raises(litellm.BadRequestError) as excinfo: + exception_type( + model=None, + original_exception=_ErrorWithoutResponse("The provided model identifier is invalid.", 400), + custom_llm_provider="bedrock", + ) + + assert excinfo.value.status_code == 400 + + def test_bedrock_mantle_context_overflow_maps_to_context_window_exceeded(): from litellm.llms.base_llm.chat.transformation import BaseLLMException diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index d82ba192b1f..743e7e74431 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -1221,6 +1221,97 @@ def test_transform_request_drops_lookaround_regex_when_the_deployment_model_info assert properties == _LOOKAROUND_FREE_PROPERTIES +@pytest.mark.parametrize( + ("model", "litellm_params"), + [ + ("us.newvendor.model-1", None), + ("converse/us.newvendor.model-1", None), + ("us-east-1/us.newvendor.model-1", None), + ("us.newvendor.model-1", {"base_model": "bedrock/converse/us.newvendor.model-1"}), + ], +) +def test_transform_request_drops_lookaround_regex_for_a_model_the_cost_map_does_not_list(model, litellm_params): + assert "us.newvendor.model-1" not in litellm.model_cost + + properties: Final = _tool_schema_properties(model, _ARTIFACT_DATA_ANTHROPIC_TOOL, litellm_params) + + assert properties == _LOOKAROUND_FREE_PROPERTIES + + +@pytest.mark.parametrize( + "model", + [ + "arn:aws:bedrock:us-east-1:123456789012:custom-model-deployment/abc123", + "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123", + "us.anthropic.claude-unreleased-v1:0", + "eu.openai.gpt-unreleased", + "apac.newvendor.listed-1", + ], +) +def test_transform_request_keeps_lookaround_regex_for_an_unlisted_model_litellm_still_recognizes(model, monkeypatch): + monkeypatch.setitem(litellm.model_cost, "us.newvendor.listed-1", {"litellm_provider": "bedrock_converse"}) + + properties = _tool_schema_properties(model, _ARTIFACT_DATA_ANTHROPIC_TOOL) + + assert properties == _ARTIFACT_DATA_INPUT_SCHEMA["properties"] + + +def test_transform_request_keeps_lookaround_regex_for_an_unlisted_model_whose_deployment_opts_out(): + properties = _tool_schema_properties( + "us.newvendor.model-1", _ARTIFACT_DATA_ANTHROPIC_TOOL, {"model_info": {"supports_regex_lookaround": True}} + ) + + assert properties == _ARTIFACT_DATA_INPUT_SCHEMA["properties"] + + +@pytest.mark.parametrize( + "deployment_params", + [ + {"model": "bedrock/us.newvendor.model-1"}, + {"model": "bedrock/converse/us.newvendor.model-1", "allowed_openai_params": ["tools"]}, + ], +) +def test_a_router_deployment_of_a_model_the_cost_map_does_not_list_drops_lookaround_regex( + respx_mock, deployment_params +): + route: Final = respx_mock.post( + "https://bedrock-runtime.us-east-1.amazonaws.com/model/us.newvendor.model-1/converse" + ).mock( + return_value=httpx.Response( + 200, + json={ + "output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}, + }, + ) + ) + router: Final = Router( + model_list=[ + { + "model_name": "unlisted", + "litellm_params": { + **deployment_params, + "aws_access_key_id": "fake-access-key", + "aws_secret_access_key": "fake-secret-key", + "aws_region_name": "us-east-1", + }, + "model_info": {"input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6}, + } + ] + ) + + router.completion( + model="unlisted", + messages=[{"role": "user", "content": "hi"}], + tools=[_ARTIFACT_DATA_OPENAI_TOOL], + client=HTTPHandler(), + ) + + tool: Final = json.loads(route.calls[0].request.content)["toolConfig"]["tools"][0] + assert tool["toolSpec"]["inputSchema"]["json"]["properties"] == _LOOKAROUND_FREE_PROPERTIES + + def test_transform_request_keeps_lookaround_regex_when_the_deployment_model_info_opts_out(): properties = _tool_schema_properties( "global.moonshotai.kimi-k3", _ARTIFACT_DATA_ANTHROPIC_TOOL, {"model_info": {"supports_regex_lookaround": True}} @@ -9189,18 +9280,223 @@ def test_bedrock_tools_transformation_valid_params(): assert "test" in result[0]["toolSpec"]["inputSchema"]["json"]["required"] -def test_not_found_error(): +def _invalid_model_identifier_reply(sent: list[httpx.Request]): + def respond(request: httpx.Request) -> httpx.Response: + sent.append(request) + return httpx.Response( + 400, + json={"message": "The provided model identifier is invalid."}, + headers={"x-amzn-ErrorType": "ValidationException", "x-amzn-RequestId": "req-123"}, + ) + + return respond + + +@pytest.mark.parametrize( + "model, path", + [ + ("bedrock/bad_model", b"/model/bad_model/converse"), + ("bedrock/us-east-1/bad_model", b"/model/bad_model/converse"), + ("bedrock/newvendor.listed-v1:0", b"/model/newvendor.listed-v1%3A0/converse"), + ("bedrock/us-east-1/newvendor.listed-v1:0", b"/model/newvendor.listed-v1%3A0/converse"), + ], +) +def test_not_found_error(model, path, monkeypatch): + monkeypatch.setitem(litellm.model_cost, "newvendor.listed-v1:0", {"litellm_provider": "bedrock", "mode": "chat"}) + sent: Final[list[httpx.Request]] = [] + + with pytest.raises(litellm.NotFoundError) as exc_info: + completion( + model=model, + messages=[{"role": "user", "content": "What is the meaning of life"}], + client=HTTPHandler( + client=httpx.Client(transport=httpx.MockTransport(_invalid_model_identifier_reply(sent))) + ), + aws_access_key_id="testing", + aws_secret_access_key="testing", + aws_region_name="us-west-2", + ) + + assert exc_info.value.status_code == 404 + assert exc_info.value.response.status_code == 404 + assert exc_info.value.response.headers["x-amzn-requestid"] == "req-123" + assert [request.url.raw_path for request in sent] == [path] + + +@pytest.mark.parametrize( + "model", ["bedrock/", "bedrock/us-east-1/", "bedrock/../../converse", "bedrock/xx-fake-1/us.xai.grok-4.7"] +) +def test_a_model_id_that_is_empty_or_a_path_fails_locally_without_calling_aws(model): + sent: Final[list[httpx.Request]] = [] + with pytest.raises(litellm.NotFoundError): completion( - model="bedrock/bad_model", - messages=[ - { - "role": "user", - "content": "What is the meaning of life", - } - ], + model=model, + messages=[{"role": "user", "content": "What is the meaning of life"}], + client=HTTPHandler( + client=httpx.Client(transport=httpx.MockTransport(_invalid_model_identifier_reply(sent))) + ), + aws_access_key_id="testing", + aws_secret_access_key="testing", + aws_region_name="us-west-2", ) + assert sent == [] + + +@pytest.mark.parametrize( + "model, path", + [ + ("bedrock/converse/bad_model", b"/model/bad_model/converse"), + ("bedrock/anthropic.claude-typo-v1:0", b"/model/anthropic.claude-typo-v1:0/invoke"), + ], +) +def test_invalid_model_identifier_stays_a_bad_request_for_a_model_litellm_recognizes(model, path): + sent: Final[list[httpx.Request]] = [] + + with pytest.raises(litellm.BadRequestError) as exc_info: + completion( + model=model, + messages=[{"role": "user", "content": "What is the meaning of life"}], + client=HTTPHandler( + client=httpx.Client(transport=httpx.MockTransport(_invalid_model_identifier_reply(sent))) + ), + aws_access_key_id="testing", + aws_secret_access_key="testing", + aws_region_name="us-west-2", + ) + + assert exc_info.value.status_code == 400 + assert [request.url.raw_path for request in sent] == [path] + + +_WEATHER_TOOL: Final = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the weather for a city", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + }, +} + + +def _converse_tool_call_reply(sent: list[httpx.Request]): + def respond(request: httpx.Request) -> httpx.Response: + sent.append(request) + return httpx.Response( + 200, + json={ + "output": { + "message": { + "role": "assistant", + "content": [ + {"toolUse": {"toolUseId": "t1", "name": "get_weather", "input": {"city": "Paris"}}} + ], + } + }, + "stopReason": "tool_use", + "usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}, + }, + ) + + return respond + + +@pytest.mark.parametrize("model", ["bedrock/us.newvendor.model-1", "bedrock/us-east-1/us.newvendor.model-1"]) +def test_model_missing_from_cost_map_sends_tools_to_converse(model): + assert "us.newvendor.model-1" not in litellm.model_cost + sent: Final[list[httpx.Request]] = [] + + response: Final = completion( + model=model, + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + tools=[_WEATHER_TOOL], + tool_choice="auto", + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_converse_tool_call_reply(sent)))), + aws_access_key_id="testing", + aws_secret_access_key="testing", + aws_region_name="us-west-2", + ) + + assert response.choices[0].message.tool_calls[0].function.name == "get_weather" + assert [request.url.raw_path for request in sent] == [b"/model/us.newvendor.model-1/converse"] + tool_config: Final = json.loads(sent[0].content)["toolConfig"] + assert [tool["toolSpec"]["name"] for tool in tool_config["tools"]] == ["get_weather"] + assert tool_config["toolChoice"] == {"auto": {}} + + +def test_a_price_only_row_for_an_unlisted_model_still_sends_tools_to_converse(monkeypatch): + monkeypatch.setitem( + litellm.model_cost, "us.newvendor.priced-1", {"input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6} + ) + sent: Final[list[httpx.Request]] = [] + + completion( + model="bedrock/us.newvendor.priced-1", + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + tools=[_WEATHER_TOOL], + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_converse_tool_call_reply(sent)))), + aws_access_key_id="testing", + aws_secret_access_key="testing", + aws_region_name="us-west-2", + ) + + tool_config: Final = json.loads(sent[0].content)["toolConfig"] + assert [tool["toolSpec"]["name"] for tool in tool_config["tools"]] == ["get_weather"] + + +def test_explicit_converse_route_to_an_unlisted_model_still_drops_tools_under_drop_params(): + sent: Final[list[httpx.Request]] = [] + + completion( + model="bedrock/converse/us.newvendor.model-1", + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + tools=[_WEATHER_TOOL], + drop_params=True, + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_converse_tool_call_reply(sent)))), + aws_access_key_id="testing", + aws_secret_access_key="testing", + aws_region_name="us-west-2", + ) + + assert "toolConfig" not in json.loads(sent[0].content) + + +def test_unknown_model_gets_reasoning_effort_as_sent(): + sent: Final[list[httpx.Request]] = [] + + completion( + model="bedrock/us.newvendor.model-1", + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + reasoning_effort="high", + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_converse_tool_call_reply(sent)))), + aws_access_key_id="testing", + aws_secret_access_key="testing", + aws_region_name="us-west-2", + ) + + assert json.loads(sent[0].content)["additionalModelRequestFields"] == {"reasoning_effort": "high"} + + +def test_mapped_model_without_function_calling_still_rejects_tools(): + model: Final = "bedrock/converse/meta.llama3-8b-instruct-v1:0" + assert "meta.llama3-8b-instruct-v1:0" in litellm.model_cost + assert not litellm.supports_function_calling(model=model) + sent: Final[list[httpx.Request]] = [] + + with pytest.raises(litellm.UnsupportedParamsError): + completion( + model=model, + messages=[{"role": "user", "content": "What is the weather in Paris?"}], + tools=[_WEATHER_TOOL], + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_converse_tool_call_reply(sent)))), + aws_access_key_id="testing", + aws_secret_access_key="testing", + aws_region_name="us-west-2", + ) + + assert sent == [] + @pytest.mark.parametrize( "model, expected_base_model", diff --git a/tests/unit/llms/bedrock/test_bedrock_common_utils.py b/tests/unit/llms/bedrock/test_bedrock_common_utils.py index dd89d9c46b6..20c0e10cb81 100644 --- a/tests/unit/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/unit/llms/bedrock/test_bedrock_common_utils.py @@ -214,24 +214,17 @@ def test_system_defined_inference_profile_arn_still_routes_to_converse(): assert route == "converse" -def test_other_opaque_arn_types_still_route_to_invoke(): - """ - Only application-inference-profile ARNs are auto-routed to converse. Other - opaque ARNs (provisioned-model, imported-model, custom-model-deployment) - also yield no invoke provider, but they are frequently invoke-only with - provider-specific body formats, so routing them to converse could break - them. Guards the deliberate scope against an over-broad "any opaque ARN -> - converse" generalization. - """ - for arn_segment in ( - "provisioned-model/abcdefgh1234", - "imported-model/abcdefgh1234", - "custom-model-deployment/abcdefgh1234", - ): - route = BedrockModelInfo.get_bedrock_route( - model=f"bedrock/arn:aws:bedrock:us-east-1:123412341234:{arn_segment}" - ) - assert route == "invoke", f"{arn_segment} should stay on invoke route" +@pytest.mark.parametrize( + "arn_segment", + ["provisioned-model/abcdefgh1234", "imported-model/abcdefgh1234", "custom-model-deployment/abcdefgh1234"], +) +def test_bare_opaque_arn_with_no_invoke_provider_routes_to_converse(arn_segment: str): + from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM + + model: Final = f"bedrock/arn:aws:bedrock:us-east-1:123412341234:{arn_segment}" + assert BaseAWSLLM.get_bedrock_invoke_provider(model) is None + assert BedrockModelInfo.get_bedrock_route(model) == "converse" + assert BedrockModelInfo.get_bedrock_route(f"bedrock/llama/arn:aws:bedrock:us-east-1:123412341234:{arn_segment}") == "invoke" def test_govcloud_cross_region_inference_prefix(): @@ -1098,6 +1091,84 @@ def test_unmapped_openai_family_model_routes_to_converse(): assert BedrockModelInfo.get_bedrock_route(imported) == "openai" +@pytest.mark.parametrize( + "unmapped", + [ + "bedrock/minimax.minimax-m99-unmapped", + "bedrock/us.nvidia.nemotron-99b-unmapped", + "bedrock/writer.palmyra-x99-unmapped", + "bedrock/zai.glm-99-unmapped", + "bedrock/google.gemma-99-unmapped", + "bedrock/global.xai.grok-99-unmapped", + ], +) +def test_unmapped_converse_only_family_model_routes_to_converse(unmapped: str): + import litellm + + assert BedrockModelInfo.get_base_model(unmapped) not in litellm.bedrock_converse_models + assert BedrockModelInfo.get_bedrock_route(unmapped) == "converse" + + +def test_unmapped_model_of_family_with_invoke_handler_still_routes_to_invoke(): + from typing import Final + + import litellm + + unmapped: Final = "bedrock/meta.llama99-unmapped-v1:0" + assert BedrockModelInfo.get_base_model(unmapped) not in litellm.bedrock_converse_models + assert BedrockModelInfo.get_bedrock_route(unmapped) == "invoke" + + +@pytest.mark.parametrize( + ("model", "model_path", "response_json"), + [ + ( + "bedrock/us.newvendor.model-1", + b"/model/us.newvendor.model-1/converse", + { + "output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}, + }, + ), + ( + "bedrock/mistral.mistral-99-unmapped-v1:0", + b"/model/mistral.mistral-99-unmapped-v1:0/invoke", + {"outputs": [{"text": "ok", "stop_reason": "stop"}]}, + ), + ], +) +def test_completion_on_unmapped_model_posts_converse_unless_its_id_names_an_invoke_provider( + model: str, model_path: bytes, response_json: dict[str, object] +): + import httpx + + from litellm.llms.custom_httpx.http_handler import HTTPHandler + + assert BedrockModelInfo.get_base_model(model) not in litellm.bedrock_converse_models + sent: Final[list[httpx.Request]] = [] + + def respond(request: httpx.Request) -> httpx.Response: + sent.append(request) + return httpx.Response( + 200, + json=response_json, + headers={"x-amzn-bedrock-input-token-count": "1", "x-amzn-bedrock-output-token-count": "1"}, + ) + + response: Final = litellm.completion( + model=model, + messages=[{"role": "user", "content": "hello"}], + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))), + aws_access_key_id="testing", + aws_secret_access_key="testing", + aws_region_name="us-west-2", + ) + + assert response.choices[0].message.content == "ok" + assert [request.url.raw_path for request in sent] == [model_path] + + @pytest.mark.parametrize( ("model", "expected"), [ diff --git a/tests/unit/llms/bedrock/test_nova_imported_models.py b/tests/unit/llms/bedrock/test_nova_imported_models.py index e3677aaf9e6..231f3097dcd 100644 --- a/tests/unit/llms/bedrock/test_nova_imported_models.py +++ b/tests/unit/llms/bedrock/test_nova_imported_models.py @@ -24,10 +24,8 @@ class TestGetBedrockRoute: def test_nova2_prefix_routes_to_converse(self): assert BedrockModelInfo.get_bedrock_route(NOVA2_MODEL) == "converse" - def test_plain_arn_routes_to_invoke(self): - # Without spec prefix, ARN doesn't match converse models - result = BedrockModelInfo.get_bedrock_route(f"bedrock/{NOVA_ARN}") - assert result == "invoke" + def test_plain_arn_without_an_invoke_provider_routes_to_converse(self): + assert BedrockModelInfo.get_bedrock_route(f"bedrock/{NOVA_ARN}") == "converse" class TestGetBedrockBaseModel: From cbfb1abdb3fe86a139431360ea7d561579dbfb7f Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 06:58:05 -0700 Subject: [PATCH 02/74] fix(proxy)!: label shared-handler errors the way the route catch-alls do (#39555) * fix(proxy): name shared-handler errors by the status they answer handle_exception_on_proxy pinned type=internal_server_error on every exception it wrapped, so a management route rejecting a caller's own argument answered {"type":"internal_server_error","code":"400"} and an SDK branching on the type read its own bad request as a gateway outage. Derive the type from the status the response actually carries, keeping a type the exception already names. * test(proxy): pin a 500 for exceptions whose status is None * fix(proxy): keep internal_server_error for timeouts and 5xx, redact hosts in shared handler errors Provider 5xx and timeouts keep internal_server_error so retry loops still retry them, the shared handler redacts internal hosts from raw exception messages, and /key/list routes its HTTPException through the shared handler instead of its own copy of the old label. * fix(proxy): label shared-handler errors like the chat route, pick the status before the type, redact only gateway failures * fix(proxy): pass shared-handler error messages through as on main Redacting timeout and 5xx messages changed what every shared-handler route returns, which relabeling the error type does not need, so messages go back to the verbatim text main sends. * fix(proxy): redact only the status-less failure main never sent * fix(proxy): answer a generic 500 for a status-less failure and expect auth_error on the /user/new 403 A failure with no status now answers "Internal server error" instead of its own text, since that text can name a guardrail's private host and pattern redaction misses cluster DNS names and IPv6 addresses. The /user/new integration test now expects the auth_error type main's route catch-alls give a 403 the proxy raises * fix(proxy): answer only a 4xx or 5xx carried status and fall back to the route default otherwise A numeric-string status outside 400 to 599 made /v1/messages, images, rerank and the OpenAI pass-through answer HTTP 200 with an error body or drop the connection, and non-ASCII digits or a 5000-digit string slipped through isdecimal. error_status_code now accepts an int or a three-digit ASCII string in 400 to 599, the range the chat route already enforces * refactor(proxy): parse a three-digit string status without recursing --- .../common_utils/openai_error_payload.py | 48 +++-- .../proxy/guardrails/guardrail_endpoints.py | 3 +- litellm/proxy/utils.py | 35 ++-- .../test_guardrail_submissions.py | 4 +- .../test_user_self_update_guard.py | 2 +- .../test_user_delete_revokes_keys.py | 4 +- .../test_apply_guardrail_endpoint.py | 2 +- .../common_utils/test_openai_error_payload.py | 31 ++- .../credential_endpoints/test_endpoints.py | 17 ++ .../guardrails/test_guardrail_endpoints.py | 35 +++- .../test_key_management_endpoints.py | 2 +- .../proxy/utils/helpers/test_error_helpers.py | 181 +++++++++++++++++- 12 files changed, 311 insertions(+), 53 deletions(-) diff --git a/litellm/proxy/common_utils/openai_error_payload.py b/litellm/proxy/common_utils/openai_error_payload.py index 53b2a05d40f..a948c390375 100644 --- a/litellm/proxy/common_utils/openai_error_payload.py +++ b/litellm/proxy/common_utils/openai_error_payload.py @@ -28,6 +28,8 @@ _OPENAI_ERROR_TYPE_BY_STATUS: Final[Mapping[int, str]] = MappingProxyType( _AUTH_FAILURE_STATUS_CODES: Final = frozenset({status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN}) +_HTTP_ERROR_STATUSES: Final = range(status.HTTP_400_BAD_REQUEST, 600) + _ERROR_BODY: Final = TypeAdapter(dict[str, object]) _FAILOVER_SQLSTATES: Final = frozenset({"25006", "57P01", "57P02", "57P03"}) @@ -47,13 +49,26 @@ def attribute_of(value: object, name: str, default: object = None) -> object: def error_status_code(exc: object, default: int) -> int: - """The HTTP status an exception carries as ``status_code`` or, the way ``ProxyException`` - stores it, as a stringified ``code``; ``default`` when it carries neither.""" - carried: Final = attribute_of(exc, "status_code") - if isinstance(carried, int) and not isinstance(carried, bool): + """The 4xx or 5xx HTTP status an exception carries as ``status_code`` (an int or a decimal string) or, + the way ``ProxyException`` stores it, as a stringified ``code``; ``default`` when it carries neither.""" + carried: Final = _http_error_status(attribute_of(exc, "status_code")) + if carried is not None: return carried stringified: Final = attribute_of(exc, "code") - return int(stringified) if isinstance(stringified, str) and stringified.isdecimal() else default + from_code: Final = _http_error_status(stringified) if isinstance(stringified, str) else None + return default if from_code is None else from_code + + +def _http_error_status(value: object) -> int | None: + match value: + case bool(): + return None + case int() if value in _HTTP_ERROR_STATUSES: + return value + case str() if len(value) == 3 and value.isascii() and value.isdigit() and int(value) in _HTTP_ERROR_STATUSES: + return int(value) + case _: + return None def openai_error_type(exc: object, status_code: int) -> str: @@ -105,9 +120,9 @@ def proxy_exception_for(exc: Exception, default_status_code: int) -> ProxyExcept code=status.HTTP_503_SERVICE_UNAVAILABLE, ) case HTTPException(status_code=status_code, detail=detail): - return _openai_shaped_proxy_exception(exc, str(detail), status_code) + return openai_shaped_proxy_exception(exc, str(detail), status_code) case _: - return _openai_shaped_proxy_exception( + return openai_shaped_proxy_exception( exc, str(exc) or type(exc).__name__, error_status_code(exc, default_status_code) ) @@ -176,14 +191,19 @@ def _route_failure_error_type(exc: Exception, raised_status: int, answered_statu return openai_error_type(exc, answered_status) -def _openai_shaped_proxy_exception(exc: Exception, message: str, status_code: int) -> ProxyException: +def openai_shaped_proxy_exception(exc: Exception, message: str, status_code: int) -> ProxyException: answered_status: Final = answered_status_code(message, status_code) - return ProxyException( - message=message, - type=_catch_all_error_type(exc, answered_status), - param=openai_error_param(exc), - code=answered_status, - ) + return _proxy_exception(exc, message, answered_status, _catch_all_error_type(exc, answered_status)) + + +def status_typed_proxy_exception(exc: HTTPException) -> ProxyException: + message: Final = str(exc.detail) + answered_status: Final = answered_status_code(message, exc.status_code) + return _proxy_exception(exc, message, answered_status, openai_error_type(exc, answered_status)) + + +def _proxy_exception(exc: Exception, message: str, answered_status: int, error_type: str) -> ProxyException: + return ProxyException(message=message, type=error_type, param=openai_error_param(exc), code=answered_status) def _catch_all_error_type(exc: Exception, status_code: int) -> str: diff --git a/litellm/proxy/guardrails/guardrail_endpoints.py b/litellm/proxy/guardrails/guardrail_endpoints.py index a5822bfb0e3..363b9162e00 100644 --- a/litellm/proxy/guardrails/guardrail_endpoints.py +++ b/litellm/proxy/guardrails/guardrail_endpoints.py @@ -22,6 +22,7 @@ from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.callback_utils import CALLBACK_VAR_ENCRYPTED_PREFIX +from litellm.proxy.common_utils.openai_error_payload import status_typed_proxy_exception from litellm.proxy.common_utils.path_utils import is_within, safe_join from litellm.proxy.guardrails.content_filter_data import CATEGORIES_DIR, DATA_ROOTS, category_dirs, find_category_file from litellm.proxy.guardrails.guardrail_hooks.custom_code.bounded_execution import ( @@ -2533,7 +2534,7 @@ async def apply_guardrail( e = transformed_exception except Exception: verbose_proxy_logger.exception("apply_guardrail: post_call_failure_hook failed") - raise handle_exception_on_proxy(e) + raise handle_exception_on_proxy(status_typed_proxy_exception(e) if isinstance(e, HTTPException) else e) # Success logging outside except so a hook error never triggers failure handlers. response = await _emit_guardrail_success_logs( diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index b1055bb9124..74e16628bc8 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -85,8 +85,8 @@ from litellm.proxy._types import ( from litellm.proxy.bug_report_config import build_proxy_bug_report from litellm.proxy.common_utils.openai_error_payload import ( attribute_of, - litellm_call_id_headers, - openai_error_param, + error_status_code, + openai_shaped_proxy_exception, with_litellm_call_id, ) from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error @@ -9186,39 +9186,30 @@ def _recreate_writer_on_read_only_transaction(prisma_client: "PrismaClient | Non asyncio.create_task(prisma_client.recreate_read_only_writer(reason="postgres_read_only_transaction")) +def _client_message(e: Exception) -> str: + status_is_none: Final = hasattr(e, "status_code") and attribute_of(e, "status_code") is None + return "Internal server error" if status_is_none else strip_bug_report_notice(str(e)) + + def handle_exception_on_proxy(e: Exception, litellm_call_id: str | None = None) -> ProxyException: """ Returns an Exception as ProxyException, this ensures all exceptions are OpenAI API compatible """ - from fastapi import status - verbose_proxy_logger.exception("Exception: %s", e) if PrismaDBExceptionHandler.is_read_only_transaction_error(e): from litellm.proxy.proxy_server import prisma_client _recreate_writer_on_read_only_transaction(prisma_client) - headers: Final = litellm_call_id_headers(litellm_call_id) - if isinstance(e, HTTPException): - return ProxyException( - message=getattr(e, "detail", f"error({e})"), - type=ProxyErrorTypes.internal_server_error, - param=openai_error_param(e), - headers=headers, - code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), - ) - elif isinstance(e, ProxyException): + if isinstance(e, ProxyException): return with_litellm_call_id(e, litellm_call_id) - _status_code: Final = getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR) + if isinstance(e, HTTPException): + return with_litellm_call_id(openai_shaped_proxy_exception(e, str(e.detail), e.status_code), litellm_call_id) if should_report_bug(e): verbose_proxy_logger.error(bug_report_notice(build_proxy_bug_report(e))) - return ProxyException( - message=strip_bug_report_notice(str(e)), - type=ProxyErrorTypes.internal_server_error, - param=openai_error_param(e), - headers=headers, - code=_status_code, - ) + message: Final = _client_message(e) + status_code: Final = error_status_code(e, status.HTTP_500_INTERNAL_SERVER_ERROR) + return with_litellm_call_id(openai_shaped_proxy_exception(e, message, status_code), litellm_call_id) def require_enterprise_license(feature: str | None = None) -> None: diff --git a/tests/integration/authorization/test_guardrail_submissions.py b/tests/integration/authorization/test_guardrail_submissions.py index 16b9cbf31dc..fb095010eb2 100644 --- a/tests/integration/authorization/test_guardrail_submissions.py +++ b/tests/integration/authorization/test_guardrail_submissions.py @@ -432,7 +432,7 @@ def test_team_guardrail_submissions_require_admin_approval( f"Guardrail '{first_name}' not found. " "Please ensure the guardrail is configured in your LiteLLM proxy." ), - "type": "internal_server_error", + "type": "invalid_request_error", "param": None, "code": "404", } @@ -481,7 +481,7 @@ def test_team_guardrail_submissions_require_admin_approval( f"Guardrail '{second_name}' not found. " "Please ensure the guardrail is configured in your LiteLLM proxy." ), - "type": "internal_server_error", + "type": "invalid_request_error", "param": None, "code": "404", } diff --git a/tests/integration/authorization/test_user_self_update_guard.py b/tests/integration/authorization/test_user_self_update_guard.py index 2c495beb77e..4294e8a088d 100644 --- a/tests/integration/authorization/test_user_self_update_guard.py +++ b/tests/integration/authorization/test_user_self_update_guard.py @@ -279,7 +279,7 @@ def test_new_user_with_admin_role_requires_proxy_admin(gateway: Gateway, role: s assert refused.json() == { "error": { "message": f"Only proxy admins can create administrative users (proxy_admin, proxy_admin_viewer). Attempted to create user with role: {role}. Your role: internal_user", - "type": "internal_server_error", + "type": "auth_error", "param": None, "code": "403", } diff --git a/tests/integration/management/test_user_delete_revokes_keys.py b/tests/integration/management/test_user_delete_revokes_keys.py index 513a80b7319..2a1bd210b49 100644 --- a/tests/integration/management/test_user_delete_revokes_keys.py +++ b/tests/integration/management/test_user_delete_revokes_keys.py @@ -170,7 +170,7 @@ def test_user_delete_revokes_keys_and_memberships(gateway: Gateway, peer: Gatewa assert info.json() == { "error": { "message": f"User {user_id} not found", - "type": "internal_server_error", + "type": "invalid_request_error", "param": None, "code": "404", } @@ -337,7 +337,7 @@ def test_scim_user_delete_revokes_keys(gateway: Gateway, peer: Gateway, monkeypa expected_info: Final = { "error": { "message": f"User {user_id} not found", - "type": "internal_server_error", + "type": "invalid_request_error", "param": None, "code": "404", } diff --git a/tests/integration/observability/test_apply_guardrail_endpoint.py b/tests/integration/observability/test_apply_guardrail_endpoint.py index 9fa7d1a74db..f50679bd54e 100644 --- a/tests/integration/observability/test_apply_guardrail_endpoint.py +++ b/tests/integration/observability/test_apply_guardrail_endpoint.py @@ -156,7 +156,7 @@ def test_apply_guardrail_returns_a_block_error_for_generic_guardrails(gateway: G assert error.model_dump() == { "error": { "message": _BLOCK_REASON, - "type": "internal_server_error", + "type": "invalid_request_error", "param": None, "code": 400, } diff --git a/tests/unit/proxy/common_utils/test_openai_error_payload.py b/tests/unit/proxy/common_utils/test_openai_error_payload.py index acefc4c1e2c..d4ab811d505 100644 --- a/tests/unit/proxy/common_utils/test_openai_error_payload.py +++ b/tests/unit/proxy/common_utils/test_openai_error_payload.py @@ -111,10 +111,13 @@ def test_the_default_status_stands_when_the_exception_carries_none(default: int) assert error_status_code(ValueError("boom"), default) == default -@pytest.mark.parametrize("carried_status", [True, False, "429", None, 429.0]) -def test_a_non_int_carried_status_falls_back_to_the_default(carried_status: object): - """True is an int in Python but not an HTTP status, and a stringified one would break - every caller that compares the code numerically.""" +@pytest.mark.parametrize( + "carried_status", + [True, False, "slow down", "", None, 429.0, 200, 1301, 0, "200", "1301", "0", "0429", " 429", "\u0664\u0662\u0669", "4" * 5000], +) +def test_a_carried_status_that_is_not_an_error_status_falls_back_to_the_default(carried_status: object): + """True is an int in Python but not an HTTP status, and answering a 200 or a 1301 either reads + as success or makes uvicorn drop the connection.""" class _Carrier(Exception): status_code = carried_status @@ -122,6 +125,22 @@ def test_a_non_int_carried_status_falls_back_to_the_default(carried_status: obje assert error_status_code(_Carrier("boom"), 500) == 500 +def test_a_status_carried_as_a_decimal_string_is_answered_as_that_status(): + """Provider stream-error chunks pass the vendor's code through as a string ``status_code``, + and the shared handler used to answer it as is, so dropping it turned a rate limit into a 500.""" + + class _Carrier(Exception): + status_code = "429" + + assert error_status_code(_Carrier("slow down"), 500) == 429 + assert proxy_exception_for(_Carrier("slow down"), default_status_code=500).to_dict() == { + "message": "slow down", + "type": "rate_limit_error", + "param": None, + "code": "429", + } + + def test_a_proxy_exception_keeps_the_status_it_was_raised_with(): """ProxyException stores its status as the string ``code`` rather than ``status_code``, so a route tail that rewraps one used to answer a 4xx rejection as a 500.""" @@ -130,8 +149,8 @@ def test_a_proxy_exception_keeps_the_status_it_was_raised_with(): assert error_status_code(rejection, 500) == 400 -@pytest.mark.parametrize("carried_code", [None, "None", "", "rate_limited", "4xx", 404]) -def test_a_code_that_is_not_a_decimal_string_falls_back_to_the_default(carried_code: object): +@pytest.mark.parametrize("carried_code", [None, "None", "", "rate_limited", "4xx", 404, "200", "1301"]) +def test_a_code_that_is_not_a_stringified_error_status_falls_back_to_the_default(carried_code: object): """Only ProxyException's stringified status is a status; ``code`` on anything else (OpenAI's ``invalid_api_key``, a stray int) says nothing about the HTTP answer.""" diff --git a/tests/unit/proxy/credential_endpoints/test_endpoints.py b/tests/unit/proxy/credential_endpoints/test_endpoints.py index 47dd6d561b3..3d8e92988d9 100644 --- a/tests/unit/proxy/credential_endpoints/test_endpoints.py +++ b/tests/unit/proxy/credential_endpoints/test_endpoints.py @@ -3277,6 +3277,23 @@ class TestCredentialDisplayName: assert listed["openai-prod"]["credential_values"]["api_key"] != "sk-old" +def test_looking_up_a_missing_credential_is_labelled_a_client_error_not_a_gateway_failure( + restore_credential_list: None, +): + """Naming a credential that does not exist is the caller's own mistake, and the shared + handler used to label every one of these internal_server_error, so an SDK branching on + the type read a 404 it can never retry past as a gateway outage and backed off forever.""" + response: Final = _call_as("GET", "/credentials/by_name/definitely-not-there") + + assert response.status_code == 404, f"lookup answered {response.status_code}: {response.text}" + assert response.json()["error"] == { + "message": "Credential not found. Got credential name: definitely-not-there", + "type": "invalid_request_error", + "param": None, + "code": "404", + } + + def test_get_credential_by_model_masks_the_deployment_credentials(monkeypatch): import asyncio diff --git a/tests/unit/proxy/guardrails/test_guardrail_endpoints.py b/tests/unit/proxy/guardrails/test_guardrail_endpoints.py index 6807a4fb569..7d912478877 100644 --- a/tests/unit/proxy/guardrails/test_guardrail_endpoints.py +++ b/tests/unit/proxy/guardrails/test_guardrail_endpoints.py @@ -13,7 +13,7 @@ import yaml from fastapi import HTTPException from litellm.integrations.custom_guardrail import CustomGuardrail -from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy._types import LitellmUserRoles, ProxyException, UserAPIKeyAuth from litellm.proxy.guardrails.guardrail_endpoints import ( CreateGuardrailRequest, ListGuardrailsResponse, @@ -2198,6 +2198,39 @@ def _patch_apply_guardrail_env(mocker, guardrail_result): return mock_guardrail +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("vendor_status", "expected_type"), + [(401, "authentication_error"), (403, "permission_error")], +) +async def test_a_guardrail_vendors_rejection_is_labelled_the_way_chat_labels_it( + mocker, vendor_status: int, expected_type: str +): + """A vendor refusing the guardrail's own credentials is not the caller's key being rejected, + so it must not read as the proxy's auth_error the way a 401 the proxy raises does.""" + mock_guardrail: Final = _patch_apply_guardrail_env(mocker, None) + mock_guardrail.apply_guardrail.side_effect = HTTPException( + status_code=vendor_status, detail="vendor refused the request" + ) + mocker.patch( + "litellm.proxy.proxy_server.proxy_logging_obj.post_call_failure_hook", AsyncMock(return_value=None) + ) + + with pytest.raises(ProxyException) as raised: + await apply_guardrail( + fastapi_request=mocker.Mock(), + request=ApplyGuardrailRequest(guardrail_name="test-guardrail", text="hello"), + user_api_key_dict=UserAPIKeyAuth(), + ) + + assert raised.value.to_dict() == { + "message": "vendor refused the request", + "type": expected_type, + "param": None, + "code": str(vendor_status), + } + + @pytest.mark.asyncio async def test_apply_guardrail_forwards_metadata_to_guardrail(mocker): """Client-supplied metadata must reach apply_guardrail via request_data so diff --git a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py index a0c32f2235f..62dd367b8e5 100644 --- a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py @@ -18750,7 +18750,7 @@ async def test_list_keys_rejects_invalid_expires(): expires="expred", ) - assert exc_info.value.code == "400" + assert (exc_info.value.code, exc_info.value.type) == ("400", "invalid_request_error") assert "Invalid expires value" in str(exc_info.value.message) diff --git a/tests/unit/proxy/utils/helpers/test_error_helpers.py b/tests/unit/proxy/utils/helpers/test_error_helpers.py index df399c8b1d2..b20857dbdaf 100644 --- a/tests/unit/proxy/utils/helpers/test_error_helpers.py +++ b/tests/unit/proxy/utils/helpers/test_error_helpers.py @@ -1,13 +1,20 @@ import asyncio import json +from typing import Final from unittest.mock import AsyncMock, MagicMock +import httpx +import openai import pytest from fastapi import HTTPException from prisma.errors import DataError +import litellm from litellm.proxy._types import ProxyErrorTypes, ProxyException +from litellm.proxy.common_request_processing import proxy_exception_from_route_error +from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for from litellm.proxy.utils import get_error_message_str, handle_exception_on_proxy +from litellm.types.router import RouterErrors def normalize(value): @@ -96,7 +103,7 @@ def test_handle_exception_on_proxy_happy_path_http_exception(): assert snapshot == { "is_proxy_exception": True, "message": "forbidden", - "type": ProxyErrorTypes.internal_server_error.value, + "type": "auth_error", "code": "403", } @@ -156,7 +163,7 @@ def test_handle_exception_on_proxy_uses_attached_status_code_when_present(): assert snapshot == { "code": "418", "message": "teapot", - "type": ProxyErrorTypes.internal_server_error.value, + "type": "invalid_request_error", } @@ -176,6 +183,176 @@ def test_handle_exception_on_proxy_error_path_none_input_wraps_as_500(): } +@pytest.mark.parametrize( + "status_code, expected_type", + [ + (400, "invalid_request_error"), + (401, "auth_error"), + (403, "auth_error"), + (404, "invalid_request_error"), + (408, "invalid_request_error"), + (409, "invalid_request_error"), + (422, "invalid_request_error"), + (429, "rate_limit_error"), + (500, ProxyErrorTypes.internal_server_error.value), + (503, ProxyErrorTypes.internal_server_error.value), + ], +) +def test_handle_exception_on_proxy_labels_an_http_exception_by_the_status_it_answers( + status_code: int, expected_type: str +): + """A management route that rejects a caller's own argument answers 400, and calling that + internal_server_error tells the caller's retry loop the gateway is down when nothing but + their request is wrong.""" + result: Final = handle_exception_on_proxy(HTTPException(status_code=status_code, detail="boom")) + + assert (result.code, result.type) == (str(status_code), expected_type) + + +def _exception_carrying_status(carried: int | None, message: str = "boom") -> Exception: + class _CarriedStatus(Exception): + status_code = carried + + return _CarriedStatus(message) + + +@pytest.mark.parametrize( + "status_code, expected_type", + [ + (400, "invalid_request_error"), + (404, "invalid_request_error"), + (408, "invalid_request_error"), + (429, "rate_limit_error"), + (500, ProxyErrorTypes.internal_server_error.value), + ], +) +def test_handle_exception_on_proxy_labels_a_carried_status_the_same_way(status_code: int, expected_type: str): + """Routes also reach this handler with plain exceptions carrying their own status, and + those must be named by that status rather than all landing on the 500 label.""" + result: Final = handle_exception_on_proxy(_exception_carrying_status(status_code)) + + assert (result.code, result.type) == (str(status_code), expected_type) + + +def test_handle_exception_on_proxy_keeps_the_type_the_exception_already_names(): + """A rate limit litellm raised already names itself, and overwriting that with the status + map's guess would drop what the client branches on.""" + result: Final = handle_exception_on_proxy( + litellm.RateLimitError(message="slow down", llm_provider="openai", model="gpt-4o") + ) + + assert (result.code, result.type) == ("429", "throttling_error") + + +@pytest.mark.parametrize( + "exc", + [ + openai.InternalServerError( + message="upstream down", + response=httpx.Response(500, request=httpx.Request("GET", "https://api.openai.com/v1/batches/b")), + body={"type": "server_error"}, + ), + litellm.Timeout(message="took too long", model="gpt-6.1-sol", llm_provider="openai"), + litellm.RateLimitError(message="slow down", llm_provider="openai", model="gpt-6.1-sol"), + litellm.AuthenticationError(message="bad key", llm_provider="openai", model="gpt-6.1-sol"), + litellm.BadRequestError(message="bad input", model="gpt-6.1-sol", llm_provider="openai"), + HTTPException(status_code=404, detail="no such batch"), + ], + ids=["provider_5xx_naming_its_own_type", "timeout", "rate_limit", "bad_key", "bad_request", "http_404"], +) +def test_handle_exception_on_proxy_labels_a_failure_the_way_the_chat_route_does(exc: Exception): + """Batch, file, and management routes answer through this handler while chat answers through the route + error path, and one client calling both must read the same type for the same failure.""" + shared: Final = handle_exception_on_proxy(exc) + chat: Final = proxy_exception_from_route_error(exc) + + assert (shared.code, shared.type) == (chat.code, chat.type) + + +@pytest.mark.parametrize( + "exc", + [ + HTTPException(status_code=400, detail="bad budget_duration"), + HTTPException(status_code=401, detail="not allowed"), + HTTPException(status_code=403, detail="admin only"), + HTTPException(status_code=404, detail="no such team"), + litellm.AuthenticationError(message="bad key", llm_provider="openai", model="gpt-6.1-sol"), + ValueError(f"{RouterErrors.no_deployments_with_tag_routing.value}=['paid']"), + _exception_carrying_status(409, "duplicate credential"), + ], + ids=["http_400", "http_401", "http_403", "http_404", "provider_401", "tag_denial", "carried_409"], +) +def test_handle_exception_on_proxy_labels_a_failure_the_way_route_catch_alls_do(exc: Exception): + """Management routes answer some failures through their own catch-all and the rest through this handler, + so a client reading /key/list and /key/generate must see the same type and status for the same failure.""" + shared: Final = handle_exception_on_proxy(exc) + catch_all: Final = proxy_exception_for(exc, default_status_code=500) + + assert (shared.code, shared.type, shared.message) == (catch_all.code, catch_all.type, catch_all.message) + + +@pytest.mark.parametrize( + "message, expected", + [ + (f"{RouterErrors.no_deployments_with_provider_budget_routing.value}, try later", ("429", "rate_limit_error")), + (f"{RouterErrors.no_deployments_with_tag_routing.value}=['paid']", ("401", "auth_error")), + ], + ids=["no_deployment_left", "no_deployment_for_tags"], +) +def test_handle_exception_on_proxy_names_a_router_failure_by_the_status_it_answers( + message: str, expected: tuple[str, str] +): + """A router failure that carries no status answers the 429 or 401 its message maps to, and a 429 labelled + internal_server_error reads as an outage to a client that branches on the type.""" + result: Final = handle_exception_on_proxy(ValueError(message)) + + assert (result.code, result.type) == expected + + +def test_handle_exception_on_proxy_keeps_the_path_a_rejected_request_names(): + """A provider rejecting a batch input names the caller's own bucket path, and redacting the /tmp part of it + leaves the caller nothing to fix their request with.""" + result: Final = handle_exception_on_proxy( + litellm.BadRequestError( + message="Invalid input_file_id gs://my-bucket/tmp/batch_input.jsonl", + model="gemini-3.1-pro-preview", + llm_provider="vertex_ai", + ) + ) + + assert result.code == "400" + assert "gs://my-bucket/tmp/batch_input.jsonl" in result.message + + +def test_handle_exception_on_proxy_keeps_the_message_an_exception_with_a_status_carries(): + """Main sends a status-bearing failure's message verbatim, and relabeling its type must not change that text.""" + result: Final = handle_exception_on_proxy(_exception_carrying_status(503, "Cannot connect to host 10.12.4.7:8443")) + + assert (result.code, result.message) == ("503", "Cannot connect to host 10.12.4.7:8443") + + +@pytest.mark.parametrize( + "message", + [ + "boom", + "Cannot connect to host 10.12.4.7:8443", + "Cannot connect to host guardrail.internal.svc.cluster.local:443", + "Cannot connect to host [fd00::12]:8443", + ], + ids=["plain", "ipv4", "cluster_dns", "ipv6"], +) +def test_handle_exception_on_proxy_answers_a_generic_500_for_an_exception_whose_status_is_none(message: str): + """A guardrail whose upstream refuses the connection raises with status_code=None, which crashed the handler + into a 503 blaming the auth database, and its text names private hosts no virtual key was ever sent.""" + result: Final = handle_exception_on_proxy(_exception_carrying_status(None, message)) + + assert (result.code, result.type, result.message) == ( + "500", + ProxyErrorTypes.internal_server_error.value, + "Internal server error", + ) + + @pytest.mark.parametrize( "exc", [ From 160a0b72cff4dbbc41bd5de1130966a207f0c17d Mon Sep 17 00:00:00 2001 From: simonp9 <349053457+simonp9@users.noreply.github.com> Date: Fri, 30 Oct 2026 07:34:10 -0700 Subject: [PATCH 03/74] fix(mcp): bill both calls of a streamed chat completions MCP auto-execute (#48899) * fix(mcp): bill both rounds of a streamed chat completions MCP auto-execute A streamed /v1/chat/completions request with an MCP tool set to require_approval "never" runs two model calls on one logging object, and the final-stream dedupe kept only the first round's spend row. The final usage chunk also showed only the second round. The MCP stream iterator now holds the first round's assembled response, prices it, and logs one success row that carries the usage and cost of both rounds, with the fixed margin charged once. The final usage chunk shows the same summed usage. A single-round stream still logs its one row when the answer finishes. When the follow-up round fails, the first round is logged from a copy of the logging object taken before the follow-up opened, and the failure row carries only the follow-up's partial usage with no second fixed margin. With proxy post-call guardrails armed, only (assembled, cache_hit) tuples are parked for the proxy, and the deferred log is armed only while the iterator reads, so the proxy's orphaned-callback cleanup never sees it after the Router's first-chunk read * fix(mcp): send the usage chunk of a streamed chat MCP request that ends in one round When a streamed chat completions MCP request ends in its first round (the model answers without a tool call, or no tool results come back), the iterator drained the inner stream to fire its spend log and dropped the usage chunk that stream produced, so a client that asked for include_usage got no usage chunk at all. The drained usage chunk is now held and sent to the client after the final content chunk and before [DONE], the same place a plain stream sends it, so the last usage chunk matches the billed row. A two-round request still drops the first round's usage chunk and sends the merged one from the follow-up, and a stream without include_usage still sends none * fix(mcp): hold round one's log for the guardrail, bill the chunk cost once, and keep round one's id When the follow-up call of a streamed chat MCP auto-execute fails, round one's success log now goes through the proxy's deferred guardrail release instead of straight to the handlers, so a post-call guardrail runs on it and the row carries it. Without a proxy release it logs directly as before The last usage chunk now takes its cost from the combined row's billed total, so a fixed cost margin is counted once in the chunk, as it is on the row. The follow-up's usage chunk is held until the follow-up ends to make that possible The combined row now keeps round one's id, the stream's first chunk id, which matches how a single-round stream is logged * fix(mcp): bill every chat MCP stream round once on failure, hang-up, and sync drop When the follow-up call fails, round 1's success row is now logged on a copy of the logging object that does not share the request's guardrail results, so a post-call guardrail that reports on the failed request counts once instead of on both rows A client hang-up during the follow-up now bills round 1 plus the follow-up's partial answer on one success row with round 1's id and the fixed margin counted once, the same way a plain stream's hang-up bills its partial answer The last usage chunk now carries round 1's usage and cost when only round 1 is billed, and a synchronous reader that drops the stream mid-answer bills round 1 on the loop it read with instead of discarding it The deferred release's logged_on argument is typed and every release closure accepts it, and the parked dispatch constants and protocols both MCP billing paths use now live in one module. The unit tests wait on a logging signal instead of a fixed delay and check the billed row through the proxy stream hook instead of private callback state * test(mcp): drive the unassembled follow-up through an upstream audio delta that is not base64 The test replaced litellm.stream_chunk_builder with a stub that raises. Round 2 now streams an audio delta whose data is not valid base64, so the real builder raises, the stream parks only best-effort usage, and the client's last usage chunk must carry round 1's tokens and cost * fix(mcp): count the fixed margin once in the combined stream row's usage cost * fix(mcp): bill round 2's partial answer when a client hangs up on a routed stream On the proxy the Router wraps the MCP stream, and when a client hangs up during round 2 the router's own cleanup closes that stream before the proxy's disconnect cleanup runs. The close logged round 1 alone, so round 2's partial answer was never billed. The MCP stream now bills round 1 plus round 2's partial answer as soon as the hang-up's cancellation reaches its read, before anything closes it * fix(mcp): count the follow-up's own messages in round 2's partial prompt The follow-up stream shares the request's logger, so it took the client's original messages as its prompt. When round 2 ended early, from a hang-up or a broken upstream, its partial usage counted only those messages and left out the assistant tool call and the tool results the follow-up actually sent. The follow-up stream now counts the messages the follow-up call sent * fix(mcp): bill round 2's partial when a routed stream's client hangs up between reads On a proxy the stream the disconnect cleanup holds is the router's fallback wrapper, so the cleanup never found the MCP stream's in-flight billing and released round 1 alone when the client hung up while the proxy sat at a yield. The round billing now registers its in-flight biller on the request's logging object, next to the parked release, and the cleanup looks it up there, so any wrapper chain bills round 1 plus round 2's partial once * fix(mcp): send the MCP stream usage chunk under the billed row id and count the partial off the loop A streamed chat completion that runs a LiteLLM-hosted MCP tool bills both model calls on one row under round 1's chatcmpl id, but the usage chunk it sent the client was round 2's held chunk and kept round 2's id, so a caller could not join the usage chunk to its row. The usage chunk now carries round 1's id in every path that sends it, with or without a deferred post-call guardrail, while its usage and cost and every content chunk's id stay as they were When the client hangs up during round 2, assembling round 2's partial answer counts the follow-up messages, tool result included, with the local tokenizer. That ran on the event loop and stalled every other request on the worker for about a second on a large tool result. The assembly now runs on a worker thread through asyncify from both the reader's cancellation and the proxy's disconnect cleanup, shielded from cancellation, and round 1 is checked again after the await so a release that happened meanwhile is never billed twice * test(mcp): wait on the loop with a hang guard instead of a 300 ms deadline * fix(mcp): give a proxy-rejected follow-up's failure row round 1's own cost breakdown * refactor(streaming): fold the partial response's empty guards into one check --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/litellm_core_utils/litellm_logging.py | 22 + .../litellm_core_utils/streaming_handler.py | 35 +- litellm/proxy/common_request_processing.py | 45 +- litellm/proxy/utils.py | 3 + .../responses/mcp/chat_completions_handler.py | 101 +- .../mcp/chat_stream_round_billing.py | 281 +++++ .../responses/mcp/parked_stream_dispatch.py | 72 ++ litellm/responses/mcp/stream_round_billing.py | 58 +- .../integration/_support/hook_count_probe.py | 18 + .../integration/mcp/test_mcp_llm_endpoints.py | 261 +++- .../proxy/test_common_request_processing.py | 28 + .../mcp/test_chat_completions_handler.py | 1050 ++++++++++++++++- .../mcp/test_parked_stream_dispatch.py | 17 + 13 files changed, 1894 insertions(+), 97 deletions(-) create mode 100644 litellm/responses/mcp/chat_stream_round_billing.py create mode 100644 litellm/responses/mcp/parked_stream_dispatch.py create mode 100644 tests/unit/responses/mcp/test_parked_stream_dispatch.py diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 4dd33fa5565..8bfb0b60693 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -568,6 +568,11 @@ def _resolve_mantle_region_for_cost( _PER_CALL_LITELLM_PARAM_DICTS: Final = frozenset({"metadata", "litellm_metadata"}) _LITELLM_PARAMS: Final = TypeAdapter(dict[str, object]) _REDACTED_RESULT: Final = TypeAdapter(object) +_GUARDRAIL_INFORMATION_KEY: Final = "standard_logging_guardrail_information" + + +def _without_guardrail_information(metadata: Mapping[str, object]) -> Mapping[str, object]: + return {key: value for key, value in metadata.items() if key != _GUARDRAIL_INFORMATION_KEY} def _provider_response_id(source: object) -> str | None: @@ -2566,6 +2571,23 @@ class Logging(LiteLLMLoggingBaseClass): attempt.model_call_details = dict(self.model_call_details) return attempt + def guardrail_scoped_copy(self) -> "Logging": + request_params: Final = _LITELLM_PARAMS.validate_python( + self.litellm_params or {} # pyright: ignore[reportUnknownMemberType] # Logging.litellm_params is untyped + ) + litellm_params: Final = { + **request_params, + **{ + key: _without_guardrail_information(_METADATA_MAPPING.validate_python(value)) + for key, value in request_params.items() + if key in _PER_CALL_LITELLM_PARAM_DICTS and isinstance(value, dict) + }, + } + scoped: Final = copy.copy(self) + scoped.litellm_params = litellm_params + scoped.model_call_details = {**self.model_call_details, "litellm_params": litellm_params} + return scoped + def dispatch_attempt_failure_handlers( self, exception: Exception, traceback_exception: str ) -> Coroutine[None, None, None]: diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 682b2a2f538..098a404ea14 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -2486,6 +2486,25 @@ class CustomStreamWrapper: def bill_failure_after_earlier_rounds(self, earlier: "BilledRoundsSpend") -> None: self._earlier_rounds_spend = earlier + def partial_response(self) -> "ModelResponse | TextCompletionResponse | None": + partial: Final = ( + litellm.stream_chunk_builder( + chunks=self.chunks, + messages=self.messages if isinstance(self.messages, list) else None, + logging_obj=self.logging_obj, + count_prompt_tokens=self.count_prompt_tokens, + ) + if self.chunks + else None + ) + usage: Final[object] = getattr(partial, "usage", None) + if partial is None or not isinstance(usage, Usage): + return None + if self.model: + partial.model = self.model + backfill_missing_cache_usage_fields(usage) + return partial + def _record_partial_usage_for_failure(self) -> None: """ A stream that breaks mid-flight still billed the provider for the chunks @@ -2495,23 +2514,15 @@ class CustomStreamWrapper: moves it into terminal failure usage before opening the next attempt, so a recovered attempt logs its own usage. """ - if self.logging_obj is None or not self.chunks: + if self.logging_obj is None: return try: - partial_response: Final = litellm.stream_chunk_builder( - chunks=self.chunks, - messages=self.messages if isinstance(self.messages, list) else None, - logging_obj=self.logging_obj, - count_prompt_tokens=self.count_prompt_tokens, - ) + partial_response: Final = self.partial_response() if partial_response is None: return - usage: Final = cast(Usage | None, getattr(partial_response, "usage", None)) - if usage is None: + usage: Final[object] = getattr(partial_response, "usage", None) + if not isinstance(usage, Usage): return - if self.model: - partial_response.model = self.model - backfill_missing_cache_usage_fields(usage) partial_cost: Final = self.logging_obj.response_cost_calculator(result=partial_response) or 0.0 if self._earlier_rounds_spend is None: self.logging_obj.model_call_details["combined_usage_object"] = usage diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index a79d8fe29db..48eb58adaf2 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -140,6 +140,7 @@ from litellm.proxy.utils import ( # noqa: F401 # legacy module exports _check_and_merge_model_level_guardrails, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export check_and_merge_model_level_guardrails, ) +from litellm.responses.mcp.parked_stream_dispatch import in_flight_round_biller, parked_success from litellm.responses.streaming_iterator import partial_responses_incomplete_event_on_disconnect, spawn_logging_task from litellm.router import Router from litellm.router_api import RoutingContext @@ -535,6 +536,16 @@ async def _bill_partial_streamed_spend_on_disconnect(request_data: dict, respons return await _dispatch_disconnect_billing_event(logging_obj, partial_response) +async def _bill_in_flight_round_on_disconnect(request_data: Mapping[str, object]) -> None: + bill_in_flight_round: Final = in_flight_round_biller(request_data.get("litellm_logging_obj")) + if bill_in_flight_round is None: + return + try: + await bill_in_flight_round() + except Exception as e: # noqa: BLE001 # disconnect billing is best-effort; never break stream teardown + verbose_proxy_logger.debug("Failed to bill the in-flight round after client disconnect: %s", e) + + async def _dispatch_disconnect_billing_event(logging_obj: LiteLLMLoggingObj, partial_response: object) -> bool: try: await logging_obj.dispatch_success_handlers( @@ -2083,6 +2094,13 @@ class _ClientStreamFailureReport: ) +def _metadata_logged_on(logged_on: LiteLLMLoggingObj) -> Mapping[str, object]: + logged_params: Final = object_mapping(logged_on.model_call_details.get("litellm_params")) or MappingProxyType({}) + return MappingProxyType( + {key: logged_params[key] for key in ("metadata", "litellm_metadata") if key in logged_params} + ) + + class ProxyBaseLLMRequestProcessing: def __init__(self, data: dict, served_model: str | None = None): self.data = data @@ -4151,11 +4169,21 @@ class ProxyBaseLLMRequestProcessing: _captured_user_api_key_dict: Final = user_api_key_dict _captured_logging_obj: Final = logging_obj - async def _on_deferred_stream_complete(assembled_response: object, cache_hit: object) -> None: + async def _on_deferred_stream_complete( + assembled_response: object, cache_hit: object, logged_on: LiteLLMLoggingObj | None = None + ) -> None: await ProxyBaseLLMRequestProcessing._run_deferred_stream_guardrails( - captured_data=_captured_data, + captured_data=_captured_data + if logged_on is None + else { + **_captured_data, + **{ + key: value for key, value in _metadata_logged_on(logged_on).items() if key in _captured_data + }, + "litellm_logging_obj": logged_on, + }, captured_user_api_key_dict=_captured_user_api_key_dict, - captured_logging_obj=_captured_logging_obj, + captured_logging_obj=_captured_logging_obj if logged_on is None else logged_on, assembled_response=assembled_response, cache_hit=cache_hit, ) @@ -4180,14 +4208,8 @@ class ProxyBaseLLMRequestProcessing: GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(async_coroutine=logging_coroutine) case (logging_coroutine,) if asyncio.iscoroutine(logging_coroutine): GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(async_coroutine=logging_coroutine) - case (assembled_response, cache_hit): - await _as_success_dispatcher(_captured_native_logging_obj).dispatch_success_handlers( - assembled_response, - cache_hit=cache_hit, - start_time=None, - end_time=None, - prefer_async_handlers=True, - ) + case _ if (success := parked_success(args)) is not None: + await success.dispatch(_captured_native_logging_obj) case _: verbose_proxy_logger.error( "Deferred stream logging dropped: unexpected stored args shape %s", @@ -4524,6 +4546,7 @@ class ProxyBaseLLMRequestProcessing: client_disconnected, ) if recorded_client_disconnect: + await _bill_in_flight_round_on_disconnect(_REQUEST_FIELDS.validate_python(request_data)) deferred_stream_logging_armed: Final = _deferred_stream_logging_is_armed(request_data) ProxyLogging.fire_deferred_stream_logging(request_data) # A disconnect-time success event (the deferred-guardrail flush diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 74e16628bc8..ddaa8d2e69d 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -91,6 +91,7 @@ from litellm.proxy.common_utils.openai_error_payload import ( ) from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error from litellm.responses.litellm_completion_transformation.compaction_items import messages_since_newest_compaction +from litellm.responses.mcp.parked_stream_dispatch import held_round_spend from litellm.types.guardrails import GuardrailEventHooks from litellm.types.proxy.model_listing import ModelInfoResponse from litellm.types.utils import MCP_GUARDRAIL_CALL_TYPES, CallTypes, CallTypesLiteral, ModelInfo, Usage @@ -4943,6 +4944,8 @@ class ProxyLogging: if not isinstance(error, _PROXY_ONLY_LLM_API_ERRORS): return False match _args: + case (ModelResponse(), *_) if (held := held_round_spend(logging_obj)) is not None: + logging_obj.record_billed_rounds_for_failure(held) case (ModelResponse() as assembled, *_): logging_obj.record_assembled_response_for_failure(assembled) case (BilledRoundsDispatch() as billed,): diff --git a/litellm/responses/mcp/chat_completions_handler.py b/litellm/responses/mcp/chat_completions_handler.py index cd072825ef5..1e07b768f62 100644 --- a/litellm/responses/mcp/chat_completions_handler.py +++ b/litellm/responses/mcp/chat_completions_handler.py @@ -1,15 +1,18 @@ """Helpers for handling MCP-aware `/chat/completions` requests.""" +import asyncio import logging from types import MappingProxyType from typing import TYPE_CHECKING, Final, cast +import anyio from pydantic import TypeAdapter from typing_extensions import TypedDict, Unpack from litellm._internal_context import internal_sub_call from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.responses.mcp.chat_stream_round_billing import ChatStreamRoundBilling, engaged_round_billing from litellm.responses.mcp.litellm_proxy_mcp_handler import ( LiteLLM_Proxy_MCP_Handler, ) @@ -19,7 +22,7 @@ from litellm.responses.mcp.round_billing import ( first_round_billed_on_failure, summed_cost_breakdown, ) -from litellm.types.utils import Message, ModelResponse +from litellm.types.utils import Message, ModelResponse, Usage from litellm.utils import CustomStreamWrapper if TYPE_CHECKING: @@ -258,10 +261,20 @@ async def acompletion_with_mcp( self.follow_up_stream = None self.follow_up_iterator = None self.follow_up_exhausted = False + self._round_billing: ChatStreamRoundBilling | None = engaged_round_billing(initial_stream.logging_obj) + self._unsent_usage_chunk: ModelResponseStream | None = None def __aiter__(self): return self + def __del__(self) -> None: + self.release_abandoned() + + def release_abandoned(self) -> None: + round_billing: Final[object] = getattr(self, "_round_billing", None) + if isinstance(round_billing, ChatStreamRoundBilling): + round_billing.release_abandoned() + def _add_mcp_list_tools_to_chunk(self, chunk: ModelResponseStream) -> ModelResponseStream: """Add mcp_list_tools to the first chunk.""" from litellm.types.utils import ( @@ -321,18 +334,54 @@ async def acompletion_with_mcp( return chunk - async def _drain_inner_stream(self): + async def _drain_inner_stream(self) -> tuple[ModelResponseStream, ...]: try: - while True: - await self._stream_iterator.__anext__() - except StopAsyncIteration: - pass + return tuple([chunk async for chunk in initial_stream]) except Exception: logging.getLogger("LiteLLM").exception( "Error draining inner MCP stream after final chunk; spend logging may be incomplete" ) + return () - async def __anext__(self): + async def _complete_first_round(self) -> ModelResponseStream | None: + drained: Final = await self._drain_inner_stream() + if self._round_billing is not None: + self._round_billing.complete_first_round() + usage_chunks: Final = tuple( + chunk for chunk in drained if isinstance(getattr(chunk, "usage", None), Usage) + ) + return usage_chunks[-1] if usage_chunks else None + + async def _follow_up_or_release_first_round(self, usage_chunk: ModelResponseStream | None) -> None: + if self.tool_results and self.complete_response: + await self._prepare_follow_up_call() + if self.follow_up_stream is not None: + return + self._unsent_usage_chunk = usage_chunk + if self._round_billing is not None: + self._round_billing.finish() + + async def __anext__(self) -> ModelResponseStream: + round_billing: Final = self._round_billing + if round_billing is None: + return await self._next_chunk() + with round_billing.armed_while_reading(): + try: + return await self._next_chunk() + except StopAsyncIteration: + usage_chunk: Final = round_billing.finish() + if usage_chunk is None: + raise + return usage_chunk + except asyncio.CancelledError: + with anyio.CancelScope(shield=True): + await round_billing.bill_in_flight_round() + raise + except Exception: + round_billing.fail() + raise + + async def _next_chunk(self) -> ModelResponseStream: # Phase 1: Collect and yield initial stream chunks if not self.stream_exhausted: # Get the iterator from the stream wrapper @@ -362,29 +411,22 @@ async def acompletion_with_mcp( if is_final: self.stream_exhausted = True + usage_chunk: Final = await self._complete_first_round() await self._process_tool_calls() chunk = self._add_mcp_tool_metadata_to_final_chunk(chunk) - if self.tool_results and self.complete_response: - await self._prepare_follow_up_call() - # Drain inner stream so CustomStreamWrapper fires its - # end-of-stream handler (dispatch_success_handlers → - # _ProxyDBLogger → LiteLLM_SpendLogs). The CSW may - # yield one usage chunk before raising StopAsyncIteration. - await self._drain_inner_stream() + await self._follow_up_or_release_first_round(usage_chunk) return chunk except StopAsyncIteration: self.stream_exhausted = True + drained_usage_chunk: Final = await self._complete_first_round() # Process tool calls after stream is exhausted await self._process_tool_calls() # If we have chunks, yield the final one with metadata if self.collected_chunks: final_chunk = self.collected_chunks[-1] final_chunk = self._add_mcp_tool_metadata_to_final_chunk(final_chunk) - # If we have tool results, prepare follow-up call - if self.tool_results and self.complete_response: - await self._prepare_follow_up_call() - await self._drain_inner_stream() + await self._follow_up_or_release_first_round(drained_usage_chunk) return final_chunk # Phase 2: Yield follow-up stream chunks if available @@ -400,7 +442,6 @@ async def acompletion_with_mcp( from litellm._logging import verbose_logger verbose_logger.debug("Follow-up chunk yielded: %s", chunk) - return chunk except StopAsyncIteration: self.follow_up_exhausted = True from litellm._logging import verbose_logger @@ -408,6 +449,10 @@ async def acompletion_with_mcp( verbose_logger.debug("Follow-up stream exhausted") # After follow-up stream is exhausted, check if we need to raise StopAsyncIteration raise StopAsyncIteration + client_chunk: Final = ( + chunk if self._round_billing is None else self._round_billing.client_chunk(chunk) + ) + return await self._next_chunk() if client_chunk is None else client_chunk # If we're here and follow_up_stream is None but we expected it, log a warning if ( @@ -420,7 +465,11 @@ async def acompletion_with_mcp( verbose_logger.warning("Follow-up stream was not created despite having tool results") - raise StopAsyncIteration + unsent_usage_chunk: Final = self._unsent_usage_chunk + if unsent_usage_chunk is None: + raise StopAsyncIteration + self._unsent_usage_chunk = None + return unsent_usage_chunk async def _process_tool_calls(self): """Process tool calls after streaming completes.""" @@ -488,10 +537,16 @@ async def acompletion_with_mcp( # This ensures the patch works correctly in tests import litellm + earlier_rounds: Final = None if self._round_billing is None else self._round_billing.bank_first_round() follow_up_response: Final = await litellm.acompletion(**follow_up_call_args) # Ensure follow-up response is a CustomStreamWrapper if isinstance(follow_up_response, CustomStreamWrapper): + if earlier_rounds is not None: + follow_up_response.bill_failure_after_earlier_rounds(earlier_rounds) + follow_up_response.messages = follow_up_messages + if self._round_billing is not None: + self._round_billing.follow_up_started(follow_up_response.partial_response) self.follow_up_stream = follow_up_response from litellm._logging import verbose_logger @@ -548,11 +603,13 @@ async def acompletion_with_mcp( def __aiter__(self): return self._custom_iterator + async def aclose(self) -> None: + self._custom_iterator.release_abandoned() + await super().aclose() + def __iter__(self): # For synchronous iteration, create a sync wrapper if self._sync_iterator is None: - import asyncio - try: self._sync_loop = asyncio.get_event_loop() except RuntimeError: diff --git a/litellm/responses/mcp/chat_stream_round_billing.py b/litellm/responses/mcp/chat_stream_round_billing.py new file mode 100644 index 00000000000..370de663606 --- /dev/null +++ b/litellm/responses/mcp/chat_stream_round_billing.py @@ -0,0 +1,281 @@ +import asyncio +from collections.abc import Callable, Generator +from contextlib import contextmanager +from dataclasses import dataclass, replace +from types import MappingProxyType +from typing import Final, TypeAlias + +from litellm.litellm_core_utils.asyncify import asyncify, is_event_loop_running +from litellm.litellm_core_utils.hidden_params import get_hidden_params, served_from_cache, set_hidden_params +from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend, UncachedPrice +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.responses.mcp.parked_stream_dispatch import ( + DISPATCH_RELEASE, + HELD_ROUND_SPEND, + IN_FLIGHT_ROUND, + PARKED_DISPATCH, + SERVED_FROM_CACHE, + DeferredStreamRelease, + parked_success, +) +from litellm.responses.mcp.round_billing import billed_for_every_round, merged_chat_usage, summed_cost_breakdown +from litellm.responses.streaming_iterator import spawn_logging_task +from litellm.types.utils import ModelResponse, ModelResponseStream, Usage + +Parked: TypeAlias = tuple[object, ...] +PartialAnswer: TypeAlias = Callable[[], object] + + +@dataclass(frozen=True, slots=True) +class _AssembledRound: + response: ModelResponse + usage: Usage + cache_hit: bool + parked: Parked + + +@dataclass(frozen=True, slots=True) +class _PricedRound: + response: ModelResponse + usage: Usage + price: UncachedPrice + cache_hit: bool + + @property + def spend(self) -> BilledRoundsSpend: + return BilledRoundsSpend(self.usage, self.price.cost or 0.0, self.price.cost_breakdown) + + +@dataclass(frozen=True, slots=True) +class _HeldUsageChunk: + chunk: ModelResponseStream + usage: Usage + + +@dataclass(frozen=True, slots=True) +class _FirstRound: + priced: _PricedRound + parked: Parked + logged_alone: Parked + banked: LiteLLMLoggingObj | None = None + usage_chunk: _HeldUsageChunk | None = None + + def client_usage_chunk(self, billed: ModelResponse | None) -> ModelResponseStream | None: + if self.usage_chunk is None: + return None + usage: Final = ( + self.priced.usage.model_copy(update=MappingProxyType({"cost": self.priced.price.cost})) + if billed is None + else merged_chat_usage(self.priced.usage, self.usage_chunk.usage).model_copy( + update=MappingProxyType({"cost": _cost(billed)}) + ) + ) + return self.usage_chunk.chunk.model_copy( + update=MappingProxyType({"id": self.priced.response.id, "usage": usage}) + ) + + +def _cost(response: ModelResponse) -> object: + hidden_params: Final = get_hidden_params(response) + return None if hidden_params is None else hidden_params.get("response_cost") + + +def _assembled_round(parked: Parked | None) -> _AssembledRound | None: + match parked: + case (ModelResponse() as response, cache_hit) if isinstance(usage := getattr(response, "usage", None), Usage): + return _AssembledRound(response, usage, cache_hit is True, parked) + case _: + return None + + +def engaged_round_billing(logging_obj: object) -> "ChatStreamRoundBilling | None": + return ChatStreamRoundBilling(logging_obj) if isinstance(logging_obj, LiteLLMLoggingObj) else None + + +class ChatStreamRoundBilling: + def __init__(self, logging_obj: LiteLLMLoggingObj) -> None: + self._logging_obj: Final = logging_obj + self._first: _FirstRound | None = None + self._partial_follow_up: PartialAnswer | None = None + self._reading_loop: asyncio.AbstractEventLoop | None = None + + @contextmanager + def armed_while_reading(self) -> Generator[None]: + armed: Final = getattr(self._logging_obj, DISPATCH_RELEASE, None) is None + if armed: + setattr(self._logging_obj, DISPATCH_RELEASE, self._log_success) + self._reading_loop = asyncio.get_running_loop() + try: + yield + finally: + if armed and self._peek_parked() is None: + self._disarm() + + def complete_first_round(self) -> None: + assembled: Final = _assembled_round(self._peek_parked()) + if assembled is not None: + self._first = _FirstRound(self._priced(assembled), assembled.parked, assembled.parked) + setattr(self._logging_obj, HELD_ROUND_SPEND, self._held_round_one_spend) + + def bank_first_round(self) -> BilledRoundsSpend | None: + self._bank_apart_from_the_follow_up() + return None if self._first is None else self._first.priced.spend + + def client_chunk(self, chunk: ModelResponseStream) -> ModelResponseStream | None: + usage: Final[object] = getattr(chunk, "usage", None) + if self._first is None or not isinstance(usage, Usage): + return chunk + self._first = replace(self._first, usage_chunk=_HeldUsageChunk(chunk, usage)) + return None + + def finish(self) -> ModelResponseStream | None: + first: Final = self._first + billed: Final = self._release_rounds() + return None if first is None else first.client_usage_chunk(billed) + + def fail(self) -> None: + self._bank_apart_from_the_follow_up() + first: Final = self._first + if first is None or first.banked is None or not self._holds_round_one_alone(first): + self.finish() + return + self._first = None + setattr(self._logging_obj, PARKED_DISPATCH, None) + self._log_through_the_release((*first.parked, first.banked.guardrail_scoped_copy())) + + def follow_up_started(self, partial_follow_up: PartialAnswer) -> None: + self._partial_follow_up = partial_follow_up + setattr(self._logging_obj, IN_FLIGHT_ROUND, self.bill_in_flight_round) + + async def bill_in_flight_round(self) -> None: + partial_follow_up: Final = self._partial_follow_up + if partial_follow_up is None or self._first_round_held_alone() is None: + return + final_round: Final = _assembled_round((await asyncify(partial_follow_up)(), False)) + first: Final = self._first_round_held_alone() + if final_round is None or first is None: + return + self._first = None + setattr( + self._logging_obj, PARKED_DISPATCH, self._billed_for_both_rounds(first.priced, self._priced(final_round)) + ) + + def release_abandoned(self) -> None: + if not self._owns_release(): + return + if is_event_loop_running() or self._idle_reading_loop() is not None: + self.finish() + return + setattr(self._logging_obj, PARKED_DISPATCH, None) + self._first = None + self._disarm() + + def _bank_apart_from_the_follow_up(self) -> None: + first: Final = self._first + if first is None or first.banked is not None or self._peek_parked() is not first.parked: + return + banked: Final = self._logging_obj.attempt_scoped_copy() + if not self._owns_release(): + self._first = replace(first, banked=banked) + return + logged_alone: Final = (*first.parked, banked) + self._first = replace(first, logged_alone=logged_alone, banked=banked) + setattr(self._logging_obj, PARKED_DISPATCH, logged_alone) + + def _release_rounds(self) -> ModelResponse | None: + parked: Final = self._take_parked() + first: Final = self._first + self._first = None + if parked is None: + self._disarm() + return None + if first is None or parked is first.logged_alone: + self._release(parked) + return None + final_round: Final = _assembled_round(parked) + if final_round is None: + self._release(first.logged_alone) + return None + billed, cache_hit = self._billed_for_both_rounds(first.priced, self._priced(final_round)) + self._release((billed, cache_hit)) + return billed + + def _billed_for_both_rounds(self, first: _PricedRound, final: _PricedRound) -> tuple[ModelResponse, bool]: + first_breakdown: Final = first.price.cost_breakdown + final_breakdown: Final = final.price.cost_breakdown + rounds: Final = billed_for_every_round(first.response, first_breakdown, final.response, final_breakdown) + usage: Final[object] = getattr(rounds, "usage", None) + charged_usage: Final = ( + usage.model_copy(update=MappingProxyType({"cost": _cost(rounds)})) if isinstance(usage, Usage) else usage + ) + billed: Final = rounds.model_copy(update=MappingProxyType({"id": first.response.id, "usage": charged_usage})) + cache_hit: Final = first.cache_hit and final.cache_hit + set_hidden_params(billed, {**(get_hidden_params(rounds) or {}), "cache_hit": cache_hit}) + self._logging_obj.cost_breakdown = summed_cost_breakdown( + first.response, first_breakdown, final.response, final_breakdown + ) + return billed, cache_hit + + def _priced(self, assembled: _AssembledRound) -> _PricedRound: + served: Final = assembled.cache_hit or served_from_cache(assembled.response) + price: Final = SERVED_FROM_CACHE if served else self._logging_obj.uncached_price(assembled.response) + priced: Final = assembled.response.model_copy() + set_hidden_params( + priced, + {**(get_hidden_params(assembled.response) or {}), "response_cost": price.cost, "cache_hit": served}, + ) + return _PricedRound(priced, assembled.usage, price, served) + + def _peek_parked(self) -> Parked | None: + parked: Final[Parked | None] = getattr(self._logging_obj, PARKED_DISPATCH, None) + return parked if isinstance(parked, tuple) else None + + def _take_parked(self) -> Parked | None: + parked: Final = self._peek_parked() + setattr(self._logging_obj, PARKED_DISPATCH, None) + return parked + + def _first_round_held_alone(self) -> _FirstRound | None: + first: Final = self._first + return first if first is not None and self._holds_round_one_alone(first) else None + + def _held_round_one_spend(self) -> BilledRoundsSpend | None: + first: Final = self._first_round_held_alone() + return None if first is None else first.priced.spend + + def _holds_round_one_alone(self, first: _FirstRound) -> bool: + parked: Final = self._peek_parked() + return parked is first.parked or parked is first.logged_alone + + def _owns_release(self) -> bool: + release: Final[object] = getattr(self._logging_obj, DISPATCH_RELEASE, None) + return release == self._log_success + + def _release(self, parked: Parked) -> None: + if getattr(self._logging_obj, DISPATCH_RELEASE, None) is not None and not self._owns_release(): + setattr(self._logging_obj, PARKED_DISPATCH, parked) + return + self._log_through_the_release(parked) + + def _log_through_the_release(self, parked: Parked) -> None: + armed: Final[DeferredStreamRelease | None] = getattr(self._logging_obj, DISPATCH_RELEASE, None) + setattr(self._logging_obj, DISPATCH_RELEASE, None) + logged: Final = (self._log_success if armed is None else armed)(*parked) + idle_reading_loop: Final = None if is_event_loop_running() else self._idle_reading_loop() + if idle_reading_loop is not None: + idle_reading_loop.run_until_complete(logged) + return + spawn_logging_task(asyncio.get_running_loop(), logged, task_name="Chat completions MCP stream success log") + + def _idle_reading_loop(self) -> asyncio.AbstractEventLoop | None: + loop: Final = self._reading_loop + return loop if loop is not None and not loop.is_closed() and not loop.is_running() else None + + def _disarm(self) -> None: + if self._owns_release(): + setattr(self._logging_obj, DISPATCH_RELEASE, None) + + async def _log_success(self, *parked: object) -> None: + success: Final = parked_success(parked) + if success is not None: + await success.dispatch(self._logging_obj) diff --git a/litellm/responses/mcp/parked_stream_dispatch.py b/litellm/responses/mcp/parked_stream_dispatch.py new file mode 100644 index 00000000000..792363f0bab --- /dev/null +++ b/litellm/responses/mcp/parked_stream_dispatch.py @@ -0,0 +1,72 @@ +from collections.abc import Coroutine +from dataclasses import dataclass +from typing import Final, Protocol + +from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend, UncachedPrice +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + +PARKED_DISPATCH: Final = "_deferred_stream_complete_args" +DISPATCH_RELEASE: Final = "_on_deferred_stream_complete" +IN_FLIGHT_ROUND: Final = "_bill_in_flight_round_on_disconnect" +HELD_ROUND_SPEND: Final = "_held_round_spend_for_failure" +SERVED_FROM_CACHE: Final = UncachedPrice(cost=0.0, cost_breakdown=None) + + +class DispatchesSuccessHandlers(Protocol): + async def dispatch_success_handlers( + self, + result: object = None, + start_time: object = None, + end_time: object = None, + cache_hit: object = None, + prefer_async_handlers: bool = False, + ) -> None: ... + + +class DeferredStreamRelease(Protocol): + def __call__(self, *parked: object) -> Coroutine[object, object, None]: ... + + +class InFlightRoundBiller(Protocol): + def __call__(self) -> Coroutine[object, object, None]: ... + + +class HeldRoundSpend(Protocol): + def __call__(self) -> BilledRoundsSpend | None: ... + + +def success_dispatcher(logger: LiteLLMLoggingObj) -> DispatchesSuccessHandlers: + return logger + + +@dataclass(frozen=True, slots=True) +class ParkedSuccess: + assembled: object + cache_hit: object + logged_on: LiteLLMLoggingObj | None = None + + async def dispatch(self, request_logger: LiteLLMLoggingObj) -> None: + logger: Final = request_logger if self.logged_on is None else self.logged_on + await success_dispatcher(logger).dispatch_success_handlers( + self.assembled, cache_hit=self.cache_hit, start_time=None, end_time=None, prefer_async_handlers=True + ) + + +def in_flight_round_biller(logging_obj: object) -> InFlightRoundBiller | None: + biller: Final[InFlightRoundBiller | None] = getattr(logging_obj, IN_FLIGHT_ROUND, None) + return biller if isinstance(logging_obj, LiteLLMLoggingObj) and callable(biller) else None + + +def held_round_spend(logging_obj: object) -> BilledRoundsSpend | None: + held: Final[HeldRoundSpend | None] = getattr(logging_obj, HELD_ROUND_SPEND, None) + return held() if isinstance(logging_obj, LiteLLMLoggingObj) and callable(held) else None + + +def parked_success(parked: tuple[object, ...]) -> ParkedSuccess | None: + match parked: + case (assembled, cache_hit): + return ParkedSuccess(assembled, cache_hit) + case (assembled, cache_hit, LiteLLMLoggingObj() as logged_on): + return ParkedSuccess(assembled, cache_hit, logged_on) + case _: + return None diff --git a/litellm/responses/mcp/stream_round_billing.py b/litellm/responses/mcp/stream_round_billing.py index 443147230bf..5cb256a8ac0 100644 --- a/litellm/responses/mcp/stream_round_billing.py +++ b/litellm/responses/mcp/stream_round_billing.py @@ -2,11 +2,18 @@ import asyncio from collections.abc import Callable, Coroutine, Sequence from dataclasses import dataclass from datetime import datetime -from typing import Final, Protocol, TypeAlias +from typing import Final, TypeAlias from litellm.litellm_core_utils.hidden_params import get_hidden_params, served_from_cache from litellm.litellm_core_utils.litellm_logging import BilledRoundsDispatch, BilledRoundsSpend, UncachedPrice from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.responses.mcp.parked_stream_dispatch import ( + DISPATCH_RELEASE, + PARKED_DISPATCH, + SERVED_FROM_CACHE, + parked_success, + success_dispatcher, +) from litellm.responses.mcp.round_billing import billed_for_every_round, merged_chat_usage, summed_cost_breakdown from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator, spawn_logging_task from litellm.responses.utils import ResponseAPILoggingUtils @@ -16,25 +23,6 @@ from litellm.types.utils import CostBreakdown, ModelResponse, Usage Parked: TypeAlias = tuple[object, ...] RestoresHeaders: TypeAlias = Callable[[ResponsesAPIStreamingResponse], None] -_PARKED_DISPATCH: Final = "_deferred_stream_complete_args" -_DISPATCH_RELEASE: Final = "_on_deferred_stream_complete" -_SERVED_FROM_CACHE: Final = UncachedPrice(cost=0.0, cost_breakdown=None) - - -class _DispatchesSuccessHandlers(Protocol): - async def dispatch_success_handlers( - self, - result: object = None, - start_time: object = None, - end_time: object = None, - cache_hit: object = None, - prefer_async_handlers: bool = False, - ) -> None: ... - - -def _success_dispatcher(logger: LiteLLMLoggingObj) -> _DispatchesSuccessHandlers: - return logger - def _event_loop_running() -> bool: try: @@ -198,8 +186,8 @@ class StreamedRoundBilling: self._billed: _BilledRounds | None = None self._banked: Parked | None = None self._banked_row: _RoundRow | None = None - if getattr(logging_obj, _DISPATCH_RELEASE, None) is None: - setattr(logging_obj, _DISPATCH_RELEASE, self._run_parked) + if getattr(logging_obj, DISPATCH_RELEASE, None) is None: + setattr(logging_obj, DISPATCH_RELEASE, self._run_parked) def complete_round( self, @@ -286,7 +274,7 @@ class StreamedRoundBilling: def _price(self, response: ResponsesAPIResponse, round_parked: Parked | None) -> UncachedPrice: if served_from_cache(response): - return _SERVED_FROM_CACHE + return SERVED_FROM_CACHE match round_parked: case (ModelResponse() as assembled, bool()): return self._logging_obj.uncached_price(assembled) @@ -297,7 +285,7 @@ class StreamedRoundBilling: self._banked = parked self._banked_row = row if parked is not None: - setattr(self._logging_obj, _PARKED_DISPATCH, parked) + setattr(self._logging_obj, PARKED_DISPATCH, parked) def _bank_apart_from_later_rounds(self, billed: _BilledRounds) -> None: row: Final = self._banked_row @@ -320,7 +308,7 @@ class StreamedRoundBilling: @staticmethod async def _log_success(success: _SuccessLog) -> None: success.carry_bridged_usage() - await _success_dispatcher(success.logger).dispatch_success_handlers( + await success_dispatcher(success.logger).dispatch_success_handlers( success.row.logged, start_time=success.logger.start_time, end_time=success.row.end_time, @@ -329,10 +317,10 @@ class StreamedRoundBilling: ) def _take_parked(self) -> Parked | None: - parked: Final[Parked | None] = getattr(self._logging_obj, _PARKED_DISPATCH, None) + parked: Final[Parked | None] = getattr(self._logging_obj, PARKED_DISPATCH, None) if not isinstance(parked, tuple): return None - setattr(self._logging_obj, _PARKED_DISPATCH, None) + setattr(self._logging_obj, PARKED_DISPATCH, None) return parked async def _run_parked(self, *parked: object) -> None: @@ -341,29 +329,27 @@ class StreamedRoundBilling: await dispatch case (dispatch,) if asyncio.iscoroutine(dispatch): await dispatch - case (assembled, cache_hit): - await _success_dispatcher(self._logging_obj).dispatch_success_handlers( - assembled, cache_hit=cache_hit, start_time=None, end_time=None, prefer_async_handlers=True - ) + case _ if (success := parked_success(parked)) is not None: + await success.dispatch(self._logging_obj) case _: return def _owns_release(self) -> bool: - release: Final[object] = getattr(self._logging_obj, _DISPATCH_RELEASE, None) + release: Final[object] = getattr(self._logging_obj, DISPATCH_RELEASE, None) return release == self._run_parked def _release(self, parked: Parked) -> None: - if getattr(self._logging_obj, _DISPATCH_RELEASE, None) is not None and not self._owns_release(): - setattr(self._logging_obj, _PARKED_DISPATCH, parked) + if getattr(self._logging_obj, DISPATCH_RELEASE, None) is not None and not self._owns_release(): + setattr(self._logging_obj, PARKED_DISPATCH, parked) return - setattr(self._logging_obj, _DISPATCH_RELEASE, None) + setattr(self._logging_obj, DISPATCH_RELEASE, None) spawn_logging_task( asyncio.get_running_loop(), self._run_parked(*parked), task_name="Responses MCP stream success logging" ) def _disarm(self) -> None: if self._owns_release(): - setattr(self._logging_obj, _DISPATCH_RELEASE, None) + setattr(self._logging_obj, DISPATCH_RELEASE, None) @staticmethod def _close(parked: Parked | None) -> None: diff --git a/tests/integration/_support/hook_count_probe.py b/tests/integration/_support/hook_count_probe.py index 0d4056e8fb9..3cd72f1a6ba 100644 --- a/tests/integration/_support/hook_count_probe.py +++ b/tests/integration/_support/hook_count_probe.py @@ -75,6 +75,24 @@ class BlockingPostCallGuardrail(CustomGuardrail): raise HTTPException(status_code=400, detail="blocked by the hook-count guardrail") +class UsageRecordingPostCallGuardrail(CustomGuardrail): + """Records the token usage of the response it is handed in the request's guardrail information, so the spend row + shows which response the guardrail saw.""" + + async def async_post_call_success_hook( + self, data: dict[str, object], user_api_key_dict: object, response: object + ) -> None: + usage: Final = getattr(response, "usage", None) + self.add_standard_logging_guardrail_information_to_request_data( + guardrail_json_response={ + "prompt_tokens": getattr(usage, "prompt_tokens", None), + "completion_tokens": getattr(usage, "completion_tokens", None), + }, + request_data=data, + guardrail_status="success", + ) + + probe: Final = HookCountProbe() sync_only_probe: Final = SyncOnlyHookCountProbe() diff --git a/tests/integration/mcp/test_mcp_llm_endpoints.py b/tests/integration/mcp/test_mcp_llm_endpoints.py index 4b967802893..f6743a8751a 100644 --- a/tests/integration/mcp/test_mcp_llm_endpoints.py +++ b/tests/integration/mcp/test_mcp_llm_endpoints.py @@ -46,9 +46,10 @@ from pydantic import JsonValue, TypeAdapter pytestmark: Final = [pytest.mark.xdist_group("mcp_llm_private_gateway"), pytest.mark.shares_worker_state] -@pytest.fixture(scope="session") -def _llm_gateway(tmp_path_factory: pytest.TempPathFactory) -> Iterator[tuple[Gateway, Mapping[str, str]]]: - directory: Final = tmp_path_factory.mktemp("mcp-llm-gateway") +@contextmanager +def _private_gateway( + directory: Path, guardrails: Sequence[Mapping[str, JsonValue]] = () +) -> Iterator[tuple[Gateway, Mapping[str, str]]]: stock: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) config: Final = directory / "stored-responses.yaml" (directory / f"{STREAM_BLOCK_MODULE}.py").write_text(STREAM_BLOCK_SOURCE) @@ -68,7 +69,8 @@ def _llm_gateway(tmp_path_factory: pytest.TempPathFactory) -> Iterator[tuple[Gat { "guardrail_name": STREAM_BLOCK, "litellm_params": {"guardrail": f"{STREAM_BLOCK_MODULE}.BlockGuard", "mode": "post_call"}, - } + }, + *guardrails, ], } ) @@ -81,15 +83,61 @@ def _llm_gateway(tmp_path_factory: pytest.TempPathFactory) -> Iterator[tuple[Gat yield private, services -@pytest.fixture -def gateway(_llm_gateway: tuple[Gateway, Mapping[str, str]]) -> Iterator[Gateway]: - private, services = _llm_gateway +@contextmanager +def _serving(served: tuple[Gateway, Mapping[str, str]]) -> Iterator[Gateway]: + private, services = served with pytest.MonkeyPatch.context() as environment: for name, value in services.items(): environment.setenv(name, value) yield private +@pytest.fixture(scope="session") +def _llm_gateway(tmp_path_factory: pytest.TempPathFactory) -> Iterator[tuple[Gateway, Mapping[str, str]]]: + with _private_gateway(tmp_path_factory.mktemp("mcp-llm-gateway")) as served: + yield served + + +@pytest.fixture +def gateway(_llm_gateway: tuple[Gateway, Mapping[str, str]]) -> Iterator[Gateway]: + with _serving(_llm_gateway) as private: + yield private + + +ROUND_USAGE_GUARDRAIL: Final = "mcp-llm-round-usage" +CONTENT_FILTER: Final = f"mcp-llm-content-filter-{uuid.uuid4().hex[:8]}" + + +@pytest.fixture(scope="module") +def _guarded_llm_gateway(tmp_path_factory: pytest.TempPathFactory) -> Iterator[tuple[Gateway, Mapping[str, str]]]: + round_usage: Final[Mapping[str, JsonValue]] = { + "guardrail_name": ROUND_USAGE_GUARDRAIL, + "litellm_params": { + "guardrail": "integration._support.hook_count_probe.UsageRecordingPostCallGuardrail", + "mode": "post_call", + "default_on": True, + }, + } + content_filter: Final[Mapping[str, JsonValue]] = { + "guardrail_name": CONTENT_FILTER, + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "post_call", + "default_on": False, + "blocked_words": [{"keyword": "forbidden-keeper", "action": "BLOCK"}], + }, + } + guardrails: Final = (round_usage, content_filter) + with _private_gateway(tmp_path_factory.mktemp("mcp-llm-guarded-gateway"), guardrails) as served: + yield served + + +@pytest.fixture +def guarded_gateway(_guarded_llm_gateway: tuple[Gateway, Mapping[str, str]]) -> Iterator[Gateway]: + with _serving(_guarded_llm_gateway) as private: + yield private + + Surface = Literal["chat", "responses", "messages", "messages_bridge", "responses_bridge"] SURFACES: Final[tuple[Surface, ...]] = ("chat", "responses", "messages", "messages_bridge") ADD: Final = {"a": 2, "b": 3} @@ -1032,6 +1080,184 @@ def test_nonstream_failed_follow_up_bills_round_one_on_the_failure_row( ) +def _streamed_chunks(response: httpx.Response) -> tuple[dict[str, object], ...]: + return tuple( + json.loads(line.removeprefix("data: ")) + for line in response.text.splitlines() + if line.startswith("data: ") and line != "data: [DONE]" + ) + + +def test_streaming_chat_auto_execute_logs_the_answer_and_bills_both_rounds(gateway: Gateway) -> None: + billing: Final = Billing(FIRST_ROUND, FOLLOW_UP_ROUND) + with _rig(gateway, "chat", billing=billing, prices=PRICES) as rig: + response: Final = rig.send(_granted_key(rig), [AUTO], stream=True, stream_options={"include_usage": True}) + assert response.status_code == 200, response.text + chunks: Final = _streamed_chunks(response) + text: Final = "".join( + str(chunk["choices"][0]["delta"].get("content") or "") for chunk in chunks if chunk.get("choices") + ) + assert text == ANSWER, response.text + usages: Final = tuple(chunk["usage"] for chunk in chunks if chunk.get("usage")) + assert usages, response.text + rows: Final = _spend_rows(response.headers["x-litellm-call-id"], "acompletion", "success") + assert len(rows) == 1, rows + row: Final = rows[0] + assert row["request_id"] == chunks[0]["id"], (row, chunks[0]) + assert (row["prompt_tokens"], row["completion_tokens"]) == ( + FIRST_ROUND[0] + FOLLOW_UP_ROUND[0], + FIRST_ROUND[1] + FOLLOW_UP_ROUND[1], + ), row + assert float(str(row["spend"])) == pytest.approx(_price(FIRST_ROUND, FOLLOW_UP_ROUND)), row + final_usage: Final = usages[-1] + assert (final_usage["prompt_tokens"], final_usage["completion_tokens"], final_usage["total_tokens"]) == ( + row["prompt_tokens"], + row["completion_tokens"], + sum((*FIRST_ROUND, *FOLLOW_UP_ROUND)), + ), usages + + +def test_streaming_chat_auto_execute_whose_follow_up_fails_bills_round_one_once(gateway: Gateway) -> None: + billing: Final = Billing(FIRST_ROUND, FOLLOW_UP_ROUND, follow_up_fails=True) + with _rig(gateway, "chat", billing=billing, prices=PRICES) as rig: + response: Final = rig.send(_granted_key(rig), [AUTO], stream=True, stream_options={"include_usage": True}) + assert FOLLOW_UP_OUTAGE in response.text, response.text + call_id: Final = response.headers["x-litellm-call-id"] + succeeded: Final = _spend_rows(call_id, "acompletion", "success") + assert len(succeeded) == 1, succeeded + assert (succeeded[0]["prompt_tokens"], succeeded[0]["completion_tokens"]) == FIRST_ROUND, succeeded + assert float(str(succeeded[0]["spend"])) == pytest.approx(_price(FIRST_ROUND)), succeeded + failed: Final = _spend_rows(call_id, "acompletion", "failure") + assert [float(str(row["spend"])) for row in failed] == [0.0], failed + + +def _guardrail_information(metadata: JsonValue) -> tuple[JsonValue, ...]: + entries: Final = object_value(metadata).get("guardrail_information") + return tuple(entries) if isinstance(entries, list) else () + + +def _round_usage_seen_by_the_guardrail(call_id: str) -> list[JsonValue]: + rows: Final = read_rows( + 'SELECT metadata FROM "LiteLLM_SpendLogs" WHERE litellm_call_id=%s AND call_type=%s AND status=%s', + (call_id, "acompletion", "success"), + ) + entries: Final = tuple(chain.from_iterable(_guardrail_information(row["metadata"]) for row in rows)) + return [ + entry.get("guardrail_response") + for entry in entries + if isinstance(entry, dict) and entry.get("guardrail_name") == ROUND_USAGE_GUARDRAIL + ] + + +def test_streaming_chat_auto_execute_whose_follow_up_fails_runs_the_post_call_guardrail_on_round_one( + guarded_gateway: Gateway, +) -> None: + billing: Final = Billing(FIRST_ROUND, FOLLOW_UP_ROUND, follow_up_fails=True) + with _rig(guarded_gateway, "chat", billing=billing, prices=PRICES) as rig: + response: Final = rig.send(_granted_key(rig), [AUTO], stream=True, stream_options={"include_usage": True}) + assert FOLLOW_UP_OUTAGE in response.text, response.text + call_id: Final = response.headers["x-litellm-call-id"] + succeeded: Final = _spend_rows(call_id, "acompletion", "success") + assert len(succeeded) == 1, succeeded + assert (succeeded[0]["prompt_tokens"], succeeded[0]["completion_tokens"]) == FIRST_ROUND, succeeded + assert _round_usage_seen_by_the_guardrail(call_id) == [ + {"prompt_tokens": FIRST_ROUND[0], "completion_tokens": FIRST_ROUND[1]} + ] + + +def _chat_rows(call_id: str, statuses: Sequence[str]) -> list[dict[str, JsonValue]]: + return eventually( + lambda: read_rows( + 'SELECT status, metadata FROM "LiteLLM_SpendLogs" WHERE litellm_call_id=%s AND call_type=%s', + (call_id, "acompletion"), + ), + lambda found: _statuses(found) == sorted(statuses), + seconds=70, + ) + + +def _entries_of(guardrail_name: str, row: Mapping[str, JsonValue]) -> tuple[dict[str, JsonValue], ...]: + return tuple( + entry + for entry in _guardrail_information(row["metadata"]) + if isinstance(entry, dict) and entry.get("guardrail_name") == guardrail_name + ) + + +def test_streaming_chat_auto_execute_whose_follow_up_fails_counts_its_post_call_content_filter_once( + guarded_gateway: Gateway, +) -> None: + billing: Final = Billing(FIRST_ROUND, FOLLOW_UP_ROUND, follow_up_fails=True) + with _rig(guarded_gateway, "chat", billing=billing, prices=PRICES) as rig: + response: Final = rig.send( + _granted_key(rig), + [AUTO], + stream=True, + stream_options={"include_usage": True}, + guardrails=[CONTENT_FILTER], + ) + assert FOLLOW_UP_OUTAGE in response.text, response.text + rows: Final = _chat_rows(response.headers["x-litellm-call-id"], ("success", "failure")) + filtered: Final = tuple( + (row["status"], entry.get("guardrail_status")) for row in rows for entry in _entries_of(CONTENT_FILTER, row) + ) + assert filtered == (("failure", "guardrail_failed_to_respond"),), rows + (guardrail_id,) = frozenset( + str(entry.get("guardrail_id") or CONTENT_FILTER) + for row in rows + for entry in _entries_of(CONTENT_FILTER, row) + ) + counted: Final = eventually( + lambda: read_rows( + 'SELECT requests_evaluated, flagged_count FROM "LiteLLM_DailyGuardrailMetrics" WHERE guardrail_id=%s', + (guardrail_id,), + ), + lambda found: bool(found), + seconds=70, + ) + assert counted == [{"requests_evaluated": 1, "flagged_count": 1}], counted + + +@pytest.mark.parametrize("include_usage", (True, False), ids=("include-usage", "no-usage")) +def test_streaming_chat_auto_execute_answered_in_one_round_ends_with_the_usage_it_billed( + gateway: Gateway, include_usage: bool +) -> None: + billing: Final = Billing(FIRST_ROUND, FOLLOW_UP_ROUND) + with _rig(gateway, "chat", billing=billing, prices=PRICES) as rig: + response: Final = rig.send( + _granted_key(rig), + [AUTO], + prompt=f"{DIRECT} {rig.alias}", + stream=True, + stream_options={"include_usage": include_usage}, + ) + assert response.status_code == 200, response.text + assert response.text.rstrip().endswith("data: [DONE]"), response.text + chunks: Final = _streamed_chunks(response) + text: Final = "".join( + str(chunk["choices"][0]["delta"].get("content") or "") for chunk in chunks if chunk.get("choices") + ) + assert text == ANSWER, response.text + rows: Final = _spend_rows(response.headers["x-litellm-call-id"], "acompletion", "success") + assert len(rows) == 1 and (rows[0]["prompt_tokens"], rows[0]["completion_tokens"]) == FIRST_ROUND, rows + usage_at: Final = tuple(index for index, chunk in enumerate(chunks) if chunk.get("usage")) + if not include_usage: + assert usage_at == (), response.text + return + assert usage_at == (len(chunks) - 1,), response.text + finished_at: Final = tuple( + index + for index, chunk in enumerate(chunks) + if chunk.get("choices") and chunk["choices"][0].get("finish_reason") is not None + ) + assert finished_at and finished_at[-1] < usage_at[0], response.text + usage: Final = chunks[-1]["usage"] + assert (usage["prompt_tokens"], usage["completion_tokens"], usage["total_tokens"]) == ( + *FIRST_ROUND, + sum(FIRST_ROUND), + ), usage + + NO_CACHE: Final = {"no-cache": True} @@ -1269,6 +1495,27 @@ def test_streaming_responses_follow_up_broken_mid_stream_bills_round_one_once_an assert float(str(failure["spend"])) == pytest.approx(_price(failed_round)), rows +def test_streaming_chat_follow_up_broken_mid_answer_bills_round_one_and_its_partial_answer_with_one_fixed_margin( + gateway: Gateway, +) -> None: + billing: Final = Billing(FIRST_ROUND, FOLLOW_UP_ROUND, follow_up_breaks=True) + with _rig(gateway, "chat", upstream="messages_bridge", billing=billing, prices=PRICES) as rig: + response: Final = rig.send(_granted_key(rig), [AUTO], stream=True, stream_options={"include_usage": True}) + assert "the sum" in response.text, response.text + assert ANSWER not in response.text, response.text + assert len(rig.upstream_tools()) == 2 + round_one: Final = _price(FIRST_ROUND) + FIXED_MARGIN + rows: Final = _model_rows(response.headers["x-litellm-call-id"], _logs_a_failure_billing(round_one)) + assert _statuses(rows) == ["failure", "success"], rows + (success,) = (row for row in rows if row["status"] == "success") + (failure,) = (row for row in rows if row["status"] == "failure") + assert (success["prompt_tokens"], success["completion_tokens"]) == FIRST_ROUND, rows + assert float(str(success["spend"])) == pytest.approx(round_one), rows + partial_answer: Final = (int(str(failure["prompt_tokens"])), int(str(failure["completion_tokens"]))) + assert min(partial_answer) > 0, rows + assert float(str(failure["spend"])) == pytest.approx(_price(partial_answer)), rows + + @pytest.mark.parametrize("upstream", STREAMED_UPSTREAMS, ids=STREAMED_IDS) def test_streaming_responses_blocked_by_a_post_call_guardrail_bills_every_round_once( gateway: Gateway, upstream: StreamedUpstream diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 2d08609c93b..2db8941d304 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -83,6 +83,7 @@ from litellm.proxy.hooks.model_max_budget_limiter import ( team_member_budget_entity_id, ) from litellm.proxy.utils import ProxyLogging +from litellm.responses.mcp.parked_stream_dispatch import IN_FLIGHT_ROUND from litellm.router import Router from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.utils import ModelResponseListIterator @@ -9051,6 +9052,33 @@ class TestStreamingClientDisconnectBilling: proxy_logging_obj.arelease_max_parallel_requests_on_disconnect.assert_awaited_once() + @pytest.mark.asyncio + async def test_disconnect_whose_in_flight_round_billing_raises_still_closes_the_stream_and_frees_the_slot(self): + import types + + response = await self._start_partial_stream() + closed: Final = AsyncMock() + + async def cannot_price_the_in_flight_round() -> None: + raise RuntimeError("the in-flight round could not be priced") + + setattr(response.logging_obj, IN_FLIGHT_ROUND, cannot_price_the_in_flight_round) + stream: Final = types.SimpleNamespace(chunks=[], messages=None, aclose=closed) + proxy_logging_obj: Final = types.SimpleNamespace(arelease_max_parallel_requests_on_disconnect=AsyncMock()) + + await ProxyBaseLLMRequestProcessing._finalize_streaming_generator_cleanup( + request=None, + request_data={"litellm_logging_obj": response.logging_obj}, + response=stream, + stream_completed=False, + client_disconnected=True, + user_api_key_dict=MagicMock(), + proxy_logging_obj=proxy_logging_obj, + ) + + closed.assert_awaited_once() + proxy_logging_obj.arelease_max_parallel_requests_on_disconnect.assert_awaited_once() + async def _bill_and_collect_success_event(self, prepare=None, request_data=None): recorder = _RecordingSuccessLogger() original_callbacks = litellm.callbacks diff --git a/tests/unit/responses/mcp/test_chat_completions_handler.py b/tests/unit/responses/mcp/test_chat_completions_handler.py index ec5968c2a1a..c0d32742e8a 100644 --- a/tests/unit/responses/mcp/test_chat_completions_handler.py +++ b/tests/unit/responses/mcp/test_chat_completions_handler.py @@ -1,17 +1,25 @@ import asyncio +import gc import json +import queue import sys +import threading import types -from collections.abc import Mapping, Sequence +import warnings +from collections.abc import AsyncIterator, Callable, Iterator, Mapping, Sequence +from contextlib import contextmanager from datetime import datetime from types import MappingProxyType from typing import Final, cast -from unittest.mock import AsyncMock, patch +from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest import respx from httpx import Response from mcp.types import Tool as MCPTool +from starlette.responses import StreamingResponse +from starlette.types import Message import litellm from litellm.integrations.custom_logger import CustomLogger @@ -25,7 +33,8 @@ from litellm.responses.mcp.litellm_proxy_mcp_handler import ( LiteLLM_Proxy_MCP_Handler, ) from litellm.responses.utils import ResponsesAPIRequestUtils -from litellm.types.utils import ModelResponse, StandardLoggingPayload +from litellm.types.utils import ModelResponse, ModelResponseStream, StandardLoggingPayload, Usage +from litellm.utils import CustomStreamWrapper @pytest.mark.asyncio @@ -1556,6 +1565,7 @@ _CHAT_ROUND_TWO: Final = MappingProxyType( ), } ) +_NEVER_LOGGED_SECONDS: Final = 60 _CONTEXT_WINDOW_ERROR: Final = MappingProxyType( { "error": MappingProxyType( @@ -1575,16 +1585,31 @@ class _ChatRoundRecorder(CustomLogger): super().__init__() self.logged: tuple[StandardLoggingPayload, ...] = () self.failed: tuple[StandardLoggingPayload, ...] = () + self._recorded: Final = asyncio.Condition() async def async_log_success_event( self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime ) -> None: self.logged = (*self.logged, cast(StandardLoggingPayload, kwargs["standard_logging_object"])) + await self._announce() async def async_log_failure_event( self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime ) -> None: self.failed = (*self.failed, cast(StandardLoggingPayload, kwargs["standard_logging_object"])) + await self._announce() + + async def _announce(self) -> None: + async with self._recorded: + self._recorded.notify_all() + + async def settled(self, logged: int = 0, failed: int = 0) -> None: + async with self._recorded: + await asyncio.wait_for( + self._recorded.wait_for(lambda: len(self.logged) >= logged and len(self.failed) >= failed), + timeout=_NEVER_LOGGED_SECONDS, + ) + await _settled() def _chat_round_cost(body: Mapping[str, object]) -> float: @@ -1600,18 +1625,20 @@ async def _settled() -> None: await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10) -async def _auto_execute_chat_echo( - respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, rounds: Sequence[Response], call_id: str -) -> tuple[_ChatRoundRecorder, ModelResponse | Exception]: +def _serve_chat_rounds( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, rounds: Sequence[Response] +) -> None: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") litellm.in_memory_llm_clients_cache.flush_cache() respx_mock.post("https://mcp-chat-rounds.example.com/v1/chat/completions").mock(side_effect=list(rounds)) - recorder: Final = _ChatRoundRecorder() - logging_obj: Final = Logging( + + +def _chat_rounds_logging_obj(recorder: _ChatRoundRecorder, call_id: str, stream: bool) -> Logging: + return Logging( model="openai/gpt-5.6", messages=[{"role": "user", "content": "Call echo with PING"}], - stream=False, + stream=stream, call_type="acompletion", start_time=datetime(2026, 1, 1), litellm_call_id=call_id, @@ -1619,6 +1646,10 @@ async def _auto_execute_chat_echo( dynamic_async_success_callbacks=[recorder], dynamic_async_failure_callbacks=[recorder], ) + + +@contextmanager +def _echo_tool_served() -> Iterator[None]: echo_tool: Final = MCPTool.model_validate( {"name": "echo", "inputSchema": {"type": "object", "properties": {"value": {"type": "string"}}}}, by_name=False, @@ -1635,6 +1666,16 @@ async def _auto_execute_chat_echo( AsyncMock(return_value=[{"tool_call_id": "call_echo", "result": "PING", "name": "echo"}]), ), ): + yield + + +async def _auto_execute_chat_echo( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, rounds: Sequence[Response], call_id: str +) -> tuple[_ChatRoundRecorder, ModelResponse | Exception]: + _serve_chat_rounds(respx_mock, monkeypatch, rounds) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, call_id, stream=False) + with _echo_tool_served(): try: response: Final = await litellm.acompletion( model="openai/gpt-5.6", @@ -1712,3 +1753,994 @@ async def test_non_stream_auto_execute_bills_the_paid_first_round_when_the_follo assert (recorder.failed[0]["prompt_tokens"], recorder.failed[0]["completion_tokens"]) == (71, 19) assert recorder.failed[0]["response_cost"] == pytest.approx(_chat_round_cost(_CHAT_ROUND_ONE)) assert recorder.failed[0]["response_cost"] > 0 + + +_FIXED_MARGIN: Final = 0.001 + + +def _chat_stream_events(body: Mapping[str, object]) -> tuple[Mapping[str, object], ...]: + completed: Final = json.loads(json.dumps(body, default=dict)) + choice: Final = completed["choices"][0] + message: Final = choice["message"] + calls: Final = message.get("tool_calls") or [] + delta: Final = ( + {**message, "tool_calls": [{**call, "index": index} for index, call in enumerate(calls)]} if calls else message + ) + head: Final = {"id": completed["id"], "object": "chat.completion.chunk", "created": 1, "model": completed["model"]} + return ( + {**head, "choices": [{"index": 0, "delta": delta, "finish_reason": None}]}, + {**head, "choices": [{"index": 0, "delta": {}, "finish_reason": choice["finish_reason"]}]}, + {**head, "choices": [], "usage": completed["usage"]}, + ) + + +def _sse(events: Sequence[Mapping[str, object]]) -> bytes: + return b"".join(f"data: {json.dumps(event)}\n\n".encode() for event in events) + + +def _streamed(body: Mapping[str, object]) -> Response: + return Response( + 200, + content=_sse(_chat_stream_events(body)) + b"data: [DONE]\n\n", + headers={"content-type": "text/event-stream"}, + ) + + +class _CutOnceTheAnswerStarts(httpx.AsyncByteStream): + async def __aiter__(self) -> AsyncIterator[bytes]: + yield _sse(_chat_stream_events(_CHAT_ROUND_TWO)[:1]) + raise httpx.ReadError("connection reset mid-stream") + + +def _cut_once_the_answer_starts() -> Response: + return Response(200, stream=_CutOnceTheAnswerStarts(), headers={"content-type": "text/event-stream"}) + + +async def _open_chat_echo_stream( + logging_obj: Logging, metadata: dict[str, object] | None = None +) -> CustomStreamWrapper: + stream: Final = await litellm.acompletion( + model="openai/gpt-5.6", + messages=[{"role": "user", "content": "Call echo with PING"}], + tools=[{"type": "mcp", "server_url": "litellm_proxy/mcp/demo", "require_approval": "never"}], + api_key="sk-mcp-chat-rounds", + api_base="https://mcp-chat-rounds.example.com/v1", + stream=True, + stream_options={"include_usage": True}, + litellm_logging_obj=logging_obj, + metadata=metadata, + ) + assert isinstance(stream, CustomStreamWrapper) + return stream + + +async def _read_all(stream: CustomStreamWrapper) -> tuple[ModelResponseStream, ...]: + return tuple([chunk async for chunk in stream]) + + +async def _read_until(stream: CustomStreamWrapper, content: str) -> None: + async for chunk in stream: + if chunk.choices and chunk.choices[0].delta.content == content: + return + pytest.fail(f"the stream never sent {content!r}") + + +def _streamed_usage(chunks: Sequence[ModelResponseStream]) -> Usage: + usages: Final = tuple(usage for chunk in chunks if isinstance(usage := getattr(chunk, "usage", None), Usage)) + assert len(usages) == 1, usages + return usages[0] + + +def _assert_one_stream_row_bills_both_rounds_once( + logged: Sequence[StandardLoggingPayload], chunks: Sequence[ModelResponseStream] +) -> None: + charged_once: Final = _chat_round_cost(_CHAT_ROUND_ONE) + _chat_round_cost(_CHAT_ROUND_TWO) - _FIXED_MARGIN + assert len(logged) == 1 + row: Final = logged[0] + assert row["id"] == chunks[0].id + assert (row["prompt_tokens"], row["completion_tokens"]) == (71 + 149, 19 + 5) + assert row["metadata"]["usage_object"]["prompt_tokens_details"]["cached_tokens"] == 32 + 64 + assert row["response_cost"] == pytest.approx(charged_once) + assert row["cost_breakdown"]["total_cost"] == pytest.approx(charged_once) + assert row["metadata"]["usage_object"]["cost"] == pytest.approx(row["response_cost"]) + usage: Final = _streamed_usage(chunks) + assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (71 + 149, 19 + 5, 90 + 154) + assert usage.cost == pytest.approx(row["response_cost"]) + + +def _prompt_tokens_the_follow_up_sent(respx_mock: respx.MockRouter) -> int: + follow_up_request: Final = json.loads(respx_mock.calls[1].request.content) + return litellm.token_counter(model="gpt-5.6", messages=follow_up_request["messages"]) + + +def _assert_one_row_bills_round_one(logged: Sequence[StandardLoggingPayload]) -> None: + assert len(logged) == 1 + assert (logged[0]["prompt_tokens"], logged[0]["completion_tokens"]) == (71, 19) + assert logged[0]["response_cost"] == pytest.approx(_chat_round_cost(_CHAT_ROUND_ONE)) + + +@pytest.mark.asyncio +async def test_stream_auto_execute_read_to_the_end_logs_one_row_whose_usage_cost_counts_the_fixed_margin_once( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"fixed_amount": _FIXED_MARGIN}}) + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _streamed(_CHAT_ROUND_TWO))) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-read-to-the-end", stream=True) + + with _echo_tool_served(): + chunks: Final = await _read_all(await _open_chat_echo_stream(logging_obj)) + await recorder.settled(logged=1) + + _assert_one_stream_row_bills_both_rounds_once(recorder.logged, chunks) + + +@pytest.mark.asyncio +async def test_stream_auto_execute_deferred_for_a_post_call_guardrail_logs_both_rounds_once_when_the_proxy_releases( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.proxy.utils import ProxyLogging + + monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"fixed_amount": _FIXED_MARGIN}}) + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _streamed(_CHAT_ROUND_TWO))) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-rounds-deferred", stream=True) + + with _echo_tool_served(): + stream: Final = await _open_chat_echo_stream(logging_obj) + ProxyBaseLLMRequestProcessing(data={})._arm_deferred_stream_dispatch( + response=stream, route_type="acompletion", user_api_key_dict=MagicMock(), logging_obj=logging_obj + ) + chunks: Final = await _read_all(stream) + await _settled() + logged_before_release: Final = recorder.logged + ProxyLogging.fire_deferred_stream_logging({"litellm_logging_obj": logging_obj}) + await recorder.settled(logged=1) + + assert logged_before_release == () + _assert_one_stream_row_bills_both_rounds_once(recorder.logged, chunks) + + +@pytest.mark.asyncio +async def test_stream_auto_execute_blocked_by_a_post_call_guardrail_bills_both_rounds_on_the_failure_row( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + from litellm.exceptions import GuardrailRaisedException + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.proxy.utils import ProxyLogging + + monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"fixed_amount": _FIXED_MARGIN}}) + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _streamed(_CHAT_ROUND_TWO))) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-rounds-blocked", stream=True) + blocked: Final = GuardrailRaisedException(guardrail_name="output-policy", message="blocked") + + with _echo_tool_served(): + stream: Final = await _open_chat_echo_stream(logging_obj) + ProxyBaseLLMRequestProcessing(data={})._arm_deferred_stream_dispatch( + response=stream, route_type="acompletion", user_api_key_dict=MagicMock(), logging_obj=logging_obj + ) + await _read_all(stream) + assert ProxyLogging._discard_deferred_stream_logging_for_failure({"litellm_logging_obj": logging_obj}, blocked) + await logging_obj.async_failure_handler(blocked, "blocked by output-policy") + await recorder.settled(failed=1) + + charged_once: Final = _chat_round_cost(_CHAT_ROUND_ONE) + _chat_round_cost(_CHAT_ROUND_TWO) - _FIXED_MARGIN + assert recorder.logged == () + assert len(recorder.failed) == 1 + assert (recorder.failed[0]["prompt_tokens"], recorder.failed[0]["completion_tokens"]) == (71 + 149, 19 + 5) + assert recorder.failed[0]["response_cost"] == pytest.approx(charged_once) + + +@pytest.mark.asyncio +async def test_stream_auto_execute_rejected_by_the_proxy_during_the_follow_up_bills_round_one_with_its_own_breakdown( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + from fastapi import HTTPException + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.proxy.utils import ProxyLogging + + monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"fixed_amount": _FIXED_MARGIN}}) + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _streamed(_CHAT_ROUND_TWO))) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-rejected-mid-follow-up", stream=True) + rejected: Final = HTTPException(status_code=400, detail="rejected by a stream hook") + + with _echo_tool_served(): + stream: Final = await _open_chat_echo_stream(logging_obj) + ProxyBaseLLMRequestProcessing(data={})._arm_deferred_stream_dispatch( + response=stream, route_type="acompletion", user_api_key_dict=MagicMock(), logging_obj=logging_obj + ) + await _read_until(stream, "PING") + assert ProxyLogging._discard_deferred_stream_logging_for_failure({"litellm_logging_obj": logging_obj}, rejected) + await logging_obj.async_failure_handler(rejected, "rejected by a stream hook") + await recorder.settled(failed=1) + + assert recorder.logged == () + assert len(recorder.failed) == 1 + failed: Final = recorder.failed[0] + breakdown: Final = failed["cost_breakdown"] + assert (failed["prompt_tokens"], failed["completion_tokens"]) == (71, 19) + assert failed["response_cost"] == pytest.approx(_chat_round_cost(_CHAT_ROUND_ONE)) + assert breakdown is not None + assert breakdown["total_cost"] == pytest.approx(failed["response_cost"]) + + +@pytest.mark.asyncio +async def test_stream_auto_execute_answered_without_a_tool_call_logs_its_one_round_once_the_answer_finishes( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_TWO),)) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-one-round", stream=True) + + with _echo_tool_served(): + stream: Final = await _open_chat_echo_stream(logging_obj) + async for chunk in stream: + if chunk.choices and chunk.choices[0].finish_reason is not None: + break + await recorder.settled(logged=1) + logged_once_the_answer_finished: Final = recorder.logged + after_the_answer: Final = await _read_all(stream) + await _settled() + + assert len(logged_once_the_answer_finished) == 1 + assert recorder.logged == logged_once_the_answer_finished + assert recorder.logged[0]["id"] == _CHAT_ROUND_TWO["id"] + assert (recorder.logged[0]["prompt_tokens"], recorder.logged[0]["completion_tokens"]) == (149, 5) + assert after_the_answer[-1:] and isinstance(after_the_answer[-1].usage, Usage) + usage: Final = _streamed_usage(after_the_answer) + assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == ( + recorder.logged[0]["prompt_tokens"], + recorder.logged[0]["completion_tokens"], + recorder.logged[0]["total_tokens"], + ) + assert recorder.logged[0]["response_cost"] == pytest.approx(_chat_round_cost(_CHAT_ROUND_TWO)) + assert recorder.failed == () + + +@pytest.mark.asyncio +@pytest.mark.parametrize("closes", (True, False), ids=("aclose", "dropped")) +async def test_stream_auto_execute_bills_the_finished_first_round_when_the_caller_stops_reading_the_follow_up( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, closes: bool +): + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _streamed(_CHAT_ROUND_TWO))) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-caller-stops", stream=True) + + async def stop_reading_once_the_follow_up_answers() -> None: + stream: Final = await _open_chat_echo_stream(logging_obj) + await _read_until(stream, "PING") + if closes: + await stream.aclose() + + with _echo_tool_served(): + await stop_reading_once_the_follow_up_answers() + await recorder.settled(logged=1) + + _assert_one_row_bills_round_one(recorder.logged) + + +@pytest.mark.asyncio +async def test_stream_auto_execute_whose_first_round_breaks_mid_answer_bills_only_its_failure_row( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + _serve_chat_rounds(respx_mock, monkeypatch, (_cut_once_the_answer_starts(),)) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-first-round-breaks", stream=True) + + with _echo_tool_served(): + stream: Final = await _open_chat_echo_stream(logging_obj) + with pytest.raises(litellm.ServiceUnavailableError): + await _read_all(stream) + await recorder.settled(failed=1) + + assert recorder.logged == () + assert len(recorder.failed) == 1 + assert recorder.failed[0]["completion_tokens"] > 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("deferred_for_a_post_call_guardrail", [False, True]) +async def test_stream_auto_execute_bills_a_broken_follow_up_on_its_own_failure_row_with_no_second_fixed_margin( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, deferred_for_a_post_call_guardrail: bool +): + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.proxy.utils import ProxyLogging, _failure_fields_to_lift + + monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"fixed_amount": _FIXED_MARGIN}}) + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _cut_once_the_answer_starts())) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-follow-up-breaks", stream=True) + request_data: Final = MappingProxyType({"litellm_logging_obj": logging_obj}) + + with _echo_tool_served(): + stream: Final = await _open_chat_echo_stream(logging_obj) + if deferred_for_a_post_call_guardrail: + ProxyBaseLLMRequestProcessing(data={})._arm_deferred_stream_dispatch( + response=stream, route_type="acompletion", user_api_key_dict=MagicMock(), logging_obj=logging_obj + ) + with pytest.raises(litellm.ServiceUnavailableError) as broken: + await _read_all(stream) + if not ProxyLogging._discard_deferred_stream_logging_for_failure(dict(request_data), broken.value): + ProxyLogging.fire_deferred_stream_logging(dict(request_data)) + await recorder.settled(logged=1, failed=1) + + _assert_one_row_bills_round_one(recorder.logged) + assert len(recorder.failed) == 1 + failed: Final = recorder.failed[0] + unmargined: Final = ( + litellm.completion_cost( + completion_response=ModelResponse( + model="gpt-5.6", + usage=Usage( + prompt_tokens=failed["prompt_tokens"], + completion_tokens=failed["completion_tokens"], + total_tokens=failed["total_tokens"], + ), + ), + model="gpt-5.6", + custom_llm_provider="openai", + ) + - _FIXED_MARGIN + ) + assert failed["prompt_tokens"] == _prompt_tokens_the_follow_up_sent(respx_mock) + assert failed["completion_tokens"] > 0 + assert failed["response_cost"] == pytest.approx(unmargined) + request_failure: Final = _failure_fields_to_lift(request_data) + assert cast(StandardLoggingPayload, request_failure["standard_logging_object"])["id"] != recorder.logged[0]["id"] + assert request_failure["response_cost"] == pytest.approx(unmargined) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("deferred_for_a_post_call_guardrail", [False, True]) +async def test_stream_auto_execute_whose_follow_up_breaks_reports_the_request_guardrail_result_on_the_failure_row_only( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, deferred_for_a_post_call_guardrail: bool +): + from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.proxy.utils import ProxyLogging + + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _cut_once_the_answer_starts())) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-follow-up-breaks-guarded", stream=True) + metadata: Final[dict[str, object]] = {} + request_data: Final = {"metadata": metadata, "litellm_logging_obj": logging_obj} + content_filter: Final = CustomGuardrail(guardrail_name="request-content-filter", event_hook="post_call") + + with _echo_tool_served(): + stream: Final = await _open_chat_echo_stream(logging_obj, metadata) + if deferred_for_a_post_call_guardrail: + ProxyBaseLLMRequestProcessing(data={})._arm_deferred_stream_dispatch( + response=stream, route_type="acompletion", user_api_key_dict=MagicMock(), logging_obj=logging_obj + ) + with pytest.raises(litellm.ServiceUnavailableError) as broken: + await _read_all(stream) + content_filter.add_standard_logging_guardrail_information_to_request_data( + guardrail_json_response=str(broken.value), + request_data=request_data, + guardrail_status="guardrail_failed_to_respond", + ) + if not ProxyLogging._discard_deferred_stream_logging_for_failure(request_data, broken.value): + ProxyLogging.fire_deferred_stream_logging(request_data) + await recorder.settled(logged=1, failed=1) + + _assert_one_row_bills_round_one(recorder.logged) + assert recorder.logged[0]["guardrail_information"] is None + assert [entry["guardrail_name"] for entry in recorder.failed[0]["guardrail_information"] or ()] == [ + "request-content-filter" + ] + + +@pytest.mark.asyncio +async def test_stream_auto_execute_served_through_the_proxy_stream_hook_logs_both_rounds_once_when_it_ends( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + from litellm.caching.caching import DualCache + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.utils import ProxyLogging + + monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"fixed_amount": _FIXED_MARGIN}}) + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _streamed(_CHAT_ROUND_TWO))) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-proxy-stream-hook", stream=True) + + with _echo_tool_served(): + served: Final = ProxyLogging(user_api_key_cache=DualCache()).async_post_call_streaming_iterator_hook( + response=await _open_chat_echo_stream(logging_obj), + user_api_key_dict=UserAPIKeyAuth(), + request_data={"litellm_logging_obj": logging_obj}, + ) + chunks: Final = tuple([chunk async for chunk in served]) + await recorder.settled(logged=1) + + _assert_one_stream_row_bills_both_rounds_once(recorder.logged, chunks) + + +@pytest.mark.asyncio +async def test_stream_auto_execute_whose_follow_up_streams_nothing_still_bills_the_first_round_once( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + nothing_streamed: Final = Response(200, content=b"data: [DONE]\n\n", headers={"content-type": "text/event-stream"}) + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), nothing_streamed)) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-empty-follow-up", stream=True) + + with _echo_tool_served(): + await _read_all(await _open_chat_echo_stream(logging_obj)) + await recorder.settled(logged=1) + + _assert_one_row_bills_round_one(recorder.logged) + + +_ANSWER_WITH_UNREADABLE_AUDIO: Final = MappingProxyType( + { + **_CHAT_ROUND_TWO, + "choices": ( + MappingProxyType( + { + "index": 0, + "finish_reason": "stop", + "message": MappingProxyType( + { + "role": "assistant", + "content": "PING", + "audio": MappingProxyType({"id": "audio_echo", "data": "not base64", "transcript": "PING"}), + } + ), + } + ), + ), + } +) + + +@pytest.mark.asyncio +async def test_stream_auto_execute_whose_follow_up_cannot_be_assembled_ends_with_the_usage_of_the_round_it_billed( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _streamed(_ANSWER_WITH_UNREADABLE_AUDIO))) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-follow-up-unassembled", stream=True) + + with _echo_tool_served(): + chunks: Final = await _read_all(await _open_chat_echo_stream(logging_obj)) + await recorder.settled(logged=1) + + _assert_one_row_bills_round_one(recorder.logged) + assert any(chunk.choices and chunk.choices[0].delta.content == "PING" for chunk in chunks) + usage: Final = _streamed_usage(chunks) + assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (71, 19, 90) + assert usage.cost == pytest.approx(recorder.logged[0]["response_cost"]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("deferred_for_a_post_call_guardrail", [False, True]) +@pytest.mark.parametrize("answer", [_CHAT_ROUND_TWO, _ANSWER_WITH_UNREADABLE_AUDIO], ids=("assembled", "unassembled")) +async def test_stream_auto_execute_sends_its_usage_chunk_under_the_id_of_the_row_it_billed( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + deferred_for_a_post_call_guardrail: bool, + answer: Mapping[str, object], +): + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.proxy.utils import ProxyLogging + + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _streamed(answer))) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-usage-chunk-id", stream=True) + + with _echo_tool_served(): + stream: Final = await _open_chat_echo_stream(logging_obj) + if deferred_for_a_post_call_guardrail: + ProxyBaseLLMRequestProcessing(data={})._arm_deferred_stream_dispatch( + response=stream, route_type="acompletion", user_api_key_dict=MagicMock(), logging_obj=logging_obj + ) + chunks: Final = await _read_all(stream) + ProxyLogging.fire_deferred_stream_logging({"litellm_logging_obj": logging_obj}) + await recorder.settled(logged=1) + + usage_chunks: Final = tuple(chunk for chunk in chunks if isinstance(getattr(chunk, "usage", None), Usage)) + answer_chunks: Final = tuple(chunk for chunk in chunks if chunk.choices and chunk.choices[0].delta.content == "PING") + assert len(recorder.logged) == 1 + assert [chunk.id for chunk in answer_chunks] == [_CHAT_ROUND_TWO["id"]] + assert len(usage_chunks) == 1 + assert usage_chunks[0].id == chunks[0].id == recorder.logged[0]["id"] == _CHAT_ROUND_ONE["id"] + + +def _assert_one_row_bills_round_one_and_the_partial_answer( + recorder: _ChatRoundRecorder, respx_mock: respx.MockRouter +) -> None: + assert len(recorder.logged) == 1 + row: Final = recorder.logged[0] + assert row["id"] == _CHAT_ROUND_ONE["id"] + partial_prompt, partial_completion = (row["prompt_tokens"] - 71, row["completion_tokens"] - 19) + assert partial_prompt == _prompt_tokens_the_follow_up_sent(respx_mock) + assert partial_completion > 0 + partial_answer: Final = litellm.completion_cost( + completion_response=ModelResponse( + model="gpt-5.6", + usage=Usage( + prompt_tokens=partial_prompt, + completion_tokens=partial_completion, + total_tokens=partial_prompt + partial_completion, + ), + ), + model="gpt-5.6", + custom_llm_provider="openai", + ) + assert row["response_cost"] == pytest.approx(_chat_round_cost(_CHAT_ROUND_ONE) + partial_answer - _FIXED_MARGIN) + assert recorder.failed == () + + +@pytest.mark.asyncio +@pytest.mark.parametrize("deferred_for_a_post_call_guardrail", [False, True]) +async def test_stream_auto_execute_hung_up_during_the_follow_up_bills_round_one_and_the_partial_answer_on_one_row( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, deferred_for_a_post_call_guardrail: bool +): + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + + monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"fixed_amount": _FIXED_MARGIN}}) + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _streamed(_CHAT_ROUND_TWO))) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-hung-up-mid-answer", stream=True) + + with _echo_tool_served(): + stream: Final = await _open_chat_echo_stream(logging_obj) + if deferred_for_a_post_call_guardrail: + ProxyBaseLLMRequestProcessing(data={})._arm_deferred_stream_dispatch( + response=stream, route_type="acompletion", user_api_key_dict=MagicMock(), logging_obj=logging_obj + ) + await _read_until(stream, "PING") + await ProxyBaseLLMRequestProcessing.finalize_streaming_generator_cleanup( + request=None, request_data={"litellm_logging_obj": logging_obj}, response=stream, client_disconnected=True + ) + await recorder.settled(logged=1) + + _assert_one_row_bills_round_one_and_the_partial_answer(recorder, respx_mock) + + +def _chat_rounds_router() -> litellm.Router: + return litellm.Router( + model_list=[ + { + "model_name": "mcp-chat-rounds", + "litellm_params": { + "model": "openai/gpt-5.6", + "api_key": "sk-mcp-chat-rounds", + "api_base": "https://mcp-chat-rounds.example.com/v1", + }, + } + ] + ) + + +class _StallsOnceTheAnswerStarts(httpx.AsyncByteStream): + async def __aiter__(self) -> AsyncIterator[bytes]: + yield _sse(_chat_stream_events(_CHAT_ROUND_TWO)[:1]) + await asyncio.Event().wait() + + +def _streams_content(body: object, content: str) -> bool: + lines: Final = body.decode().splitlines() if isinstance(body, bytes) else () + events: Final = tuple(json.loads(line.removeprefix("data: ")) for line in lines if line.startswith("data: {")) + return any(event["choices"] and event["choices"][0]["delta"].get("content") == content for event in events) + + +async def _serve_until_the_client_hangs_up_after(response: StreamingResponse, content: str) -> None: + read: Final = asyncio.Event() + + async def receive() -> Message: + await read.wait() + return {"type": "http.disconnect"} + + async def send(message: Message) -> None: + if message["type"] == "http.response.body" and _streams_content(message.get("body"), content): + read.set() + + uvicorn_scope: Final = {"type": "http", "asgi": {"version": "3.0", "spec_version": "2.3"}, "headers": []} + await response(uvicorn_scope, receive, send) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("deferred_for_a_post_call_guardrail", [False, True]) +async def test_stream_auto_execute_routed_through_the_proxy_bills_the_partial_answer_when_the_client_hangs_up( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, deferred_for_a_post_call_guardrail: bool +): + from litellm.proxy import proxy_server + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing, create_response + + monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"fixed_amount": _FIXED_MARGIN}}) + _serve_chat_rounds( + respx_mock, + monkeypatch, + ( + _streamed(_CHAT_ROUND_ONE), + Response(200, stream=_StallsOnceTheAnswerStarts(), headers={"content-type": "text/event-stream"}), + ), + ) + router: Final = _chat_rounds_router() + monkeypatch.setattr(proxy_server, "llm_router", router) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-routed-hang-up", stream=True) + + with _echo_tool_served(): + stream: Final = await router.acompletion( + model="mcp-chat-rounds", + messages=[{"role": "user", "content": "Call echo with PING"}], + tools=[{"type": "mcp", "server_url": "litellm_proxy/mcp/demo", "require_approval": "never"}], + stream=True, + stream_options={"include_usage": True}, + litellm_logging_obj=logging_obj, + ) + if deferred_for_a_post_call_guardrail: + ProxyBaseLLMRequestProcessing(data={})._arm_deferred_stream_dispatch( + response=stream, route_type="acompletion", user_api_key_dict=MagicMock(), logging_obj=logging_obj + ) + served: Final = await create_response( + proxy_server.async_data_generator( + response=stream, + user_api_key_dict=UserAPIKeyAuth(), + request_data={"litellm_logging_obj": logging_obj, "model": "mcp-chat-rounds", "metadata": {}}, + ), + "text/event-stream", + {}, + ) + assert isinstance(served, StreamingResponse) + await _serve_until_the_client_hangs_up_after(served, "PING") + await recorder.settled(logged=1) + + _assert_one_row_bills_round_one_and_the_partial_answer(recorder, respx_mock) + + +async def _stop_reading_once_the_proxy_sent(served: AsyncIterator[object], content: str) -> None: + async for frame in served: + if _streams_content(frame, content): + return + pytest.fail(f"the proxy never sent {content!r}") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("deferred_for_a_post_call_guardrail", [False, True]) +async def test_stream_auto_execute_routed_through_the_proxy_bills_the_partial_answer_when_hung_up_between_reads( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, deferred_for_a_post_call_guardrail: bool +): + from litellm.proxy import proxy_server + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + + monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"fixed_amount": _FIXED_MARGIN}}) + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _streamed(_CHAT_ROUND_TWO))) + router: Final = _chat_rounds_router() + monkeypatch.setattr(proxy_server, "llm_router", router) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-routed-hang-up-between-reads", stream=True) + + with _echo_tool_served(): + stream: Final = await router.acompletion( + model="mcp-chat-rounds", + messages=[{"role": "user", "content": "Call echo with PING"}], + tools=[{"type": "mcp", "server_url": "litellm_proxy/mcp/demo", "require_approval": "never"}], + stream=True, + stream_options={"include_usage": True}, + litellm_logging_obj=logging_obj, + ) + if deferred_for_a_post_call_guardrail: + ProxyBaseLLMRequestProcessing(data={})._arm_deferred_stream_dispatch( + response=stream, route_type="acompletion", user_api_key_dict=MagicMock(), logging_obj=logging_obj + ) + served: Final = proxy_server.async_data_generator( + response=stream, + user_api_key_dict=UserAPIKeyAuth(), + request_data={"litellm_logging_obj": logging_obj, "model": "mcp-chat-rounds", "metadata": {}}, + ) + await _stop_reading_once_the_proxy_sent(served, "PING") + await served.aclose() + await recorder.settled(logged=1) + + _assert_one_row_bills_round_one_and_the_partial_answer(recorder, respx_mock) + + +_LONGEST_WAIT_FOR_THE_LOOP_TO_MOVE_SECONDS: Final = 10 + + +class _CountsTheToolResultWhileTheLoopRuns: + def __init__(self, loop: asyncio.AbstractEventLoop, count: Callable[..., int]) -> None: + self._loop: Final = loop + self._count: Final = count + self._counting: Final = asyncio.Event() + self._loop_moved: Final = threading.Event() + self.loop_moved_while_counting: tuple[bool, ...] = () + + def __call__(self, model: str | None = None, messages: list[object] | None = None, **kwargs: object) -> int: + if any(isinstance(message, Mapping) and message.get("role") == "tool" for message in messages or ()): + self._loop.call_soon_threadsafe(self._counting.set) + moved: Final = self._loop_moved.wait(timeout=_LONGEST_WAIT_FOR_THE_LOOP_TO_MOVE_SECONDS) + self.loop_moved_while_counting = (*self.loop_moved_while_counting, moved) + return self._count(model=model, messages=messages, **kwargs) + + async def keep_the_loop_moving(self) -> None: + await self._counting.wait() + self._loop_moved.set() + + +@contextmanager +def _partial_count_watched(monkeypatch: pytest.MonkeyPatch) -> Iterator[_CountsTheToolResultWhileTheLoopRuns]: + from litellm.litellm_core_utils import streaming_chunk_builder_utils + + counter: Final = _CountsTheToolResultWhileTheLoopRuns( + asyncio.get_running_loop(), streaming_chunk_builder_utils.token_counter + ) + monkeypatch.setattr(streaming_chunk_builder_utils, "token_counter", counter) + other_task: Final = asyncio.create_task(counter.keep_the_loop_moving()) + try: + yield counter + finally: + other_task.cancel() + + +class _StallsAfterTheAnswerStarts(httpx.AsyncByteStream): + def __init__(self) -> None: + self.stalled: Final = asyncio.Event() + + async def __aiter__(self) -> AsyncIterator[bytes]: + yield _sse(_chat_stream_events(_CHAT_ROUND_TWO)[:1]) + self.stalled.set() + await asyncio.Event().wait() + + +@pytest.mark.asyncio +async def test_stream_auto_execute_cancelled_mid_answer_counts_the_partial_off_the_loop_and_bills_it_once( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"fixed_amount": _FIXED_MARGIN}}) + answer: Final = _StallsAfterTheAnswerStarts() + _serve_chat_rounds( + respx_mock, + monkeypatch, + (_streamed(_CHAT_ROUND_ONE), Response(200, stream=answer, headers={"content-type": "text/event-stream"})), + ) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-cancelled-off-the-loop", stream=True) + + with _echo_tool_served(), _partial_count_watched(monkeypatch) as counter: + stream: Final = await _open_chat_echo_stream(logging_obj) + reading: Final = asyncio.create_task(_read_all(stream)) + await asyncio.wait_for(answer.stalled.wait(), timeout=10) + reading.cancel() + with pytest.raises(asyncio.CancelledError): + await reading + await stream.aclose() + await recorder.settled(logged=1) + + assert counter.loop_moved_while_counting == (True,) + _assert_one_row_bills_round_one_and_the_partial_answer(recorder, respx_mock) + + +@pytest.mark.asyncio +async def test_stream_auto_execute_routed_through_the_proxy_and_closed_between_reads_counts_the_partial_off_the_loop( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + from litellm.proxy import proxy_server + from litellm.proxy._types import UserAPIKeyAuth + + monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"fixed_amount": _FIXED_MARGIN}}) + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _streamed(_CHAT_ROUND_TWO))) + router: Final = _chat_rounds_router() + monkeypatch.setattr(proxy_server, "llm_router", router) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-routed-closed-off-the-loop", stream=True) + + with _echo_tool_served(), _partial_count_watched(monkeypatch) as counter: + stream: Final = await router.acompletion( + model="mcp-chat-rounds", + messages=[{"role": "user", "content": "Call echo with PING"}], + tools=[{"type": "mcp", "server_url": "litellm_proxy/mcp/demo", "require_approval": "never"}], + stream=True, + stream_options={"include_usage": True}, + litellm_logging_obj=logging_obj, + ) + served: Final = proxy_server.async_data_generator( + response=stream, + user_api_key_dict=UserAPIKeyAuth(), + request_data={"litellm_logging_obj": logging_obj, "model": "mcp-chat-rounds", "metadata": {}}, + ) + await _stop_reading_once_the_proxy_sent(served, "PING") + await served.aclose() + await recorder.settled(logged=1) + + assert counter.loop_moved_while_counting == (True,) + _assert_one_row_bills_round_one_and_the_partial_answer(recorder, respx_mock) + + +class _StallsBeforeTheAnswerStarts(httpx.AsyncByteStream): + def __init__(self) -> None: + self.opened: Final = asyncio.Event() + + async def __aiter__(self) -> AsyncIterator[bytes]: + yield b": the answer has not started\n\n" + self.opened.set() + await asyncio.Event().wait() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("deferred_for_a_post_call_guardrail", [False, True]) +async def test_stream_auto_execute_hung_up_before_the_answer_starts_bills_round_one_alone_on_one_row( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, deferred_for_a_post_call_guardrail: bool +): + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + + answer: Final = _StallsBeforeTheAnswerStarts() + _serve_chat_rounds( + respx_mock, + monkeypatch, + (_streamed(_CHAT_ROUND_ONE), Response(200, stream=answer, headers={"content-type": "text/event-stream"})), + ) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-hung-up-before-answer", stream=True) + + with _echo_tool_served(): + stream: Final = await _open_chat_echo_stream(logging_obj) + if deferred_for_a_post_call_guardrail: + ProxyBaseLLMRequestProcessing(data={})._arm_deferred_stream_dispatch( + response=stream, route_type="acompletion", user_api_key_dict=MagicMock(), logging_obj=logging_obj + ) + reading: Final = asyncio.create_task(_read_all(stream)) + await asyncio.wait_for(answer.opened.wait(), timeout=10) + reading.cancel() + with pytest.raises(asyncio.CancelledError): + await reading + await ProxyBaseLLMRequestProcessing.finalize_streaming_generator_cleanup( + request=None, request_data={"litellm_logging_obj": logging_obj}, response=stream, client_disconnected=True + ) + await recorder.settled(logged=1) + + _assert_one_row_bills_round_one(recorder.logged) + assert recorder.failed == () + + +async def _read_through_the_usage_chunk( + chunks: AsyncIterator[ModelResponseStream], read: tuple[ModelResponseStream, ...] = () +) -> tuple[ModelResponseStream, ...]: + chunk: Final = await anext(chunks) + if isinstance(getattr(chunk, "usage", None), Usage): + return (*read, chunk) + return await _read_through_the_usage_chunk(chunks, (*read, chunk)) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("deferred_for_a_post_call_guardrail", [False, True]) +async def test_stream_auto_execute_hung_up_after_its_whole_answer_bills_both_rounds_once( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, deferred_for_a_post_call_guardrail: bool +): + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + + monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"fixed_amount": _FIXED_MARGIN}}) + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _streamed(_CHAT_ROUND_TWO))) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-hung-up-after-answer", stream=True) + + with _echo_tool_served(): + stream: Final = await _open_chat_echo_stream(logging_obj) + if deferred_for_a_post_call_guardrail: + ProxyBaseLLMRequestProcessing(data={})._arm_deferred_stream_dispatch( + response=stream, route_type="acompletion", user_api_key_dict=MagicMock(), logging_obj=logging_obj + ) + chunks: Final = await _read_through_the_usage_chunk(aiter(stream)) + await ProxyBaseLLMRequestProcessing.finalize_streaming_generator_cleanup( + request=None, request_data={"litellm_logging_obj": logging_obj}, response=stream, client_disconnected=True + ) + await recorder.settled(logged=1) + + _assert_one_stream_row_bills_both_rounds_once(recorder.logged, chunks) + assert recorder.failed == () + + +@pytest.mark.asyncio +async def test_stream_auto_execute_bridged_to_messages_bills_round_one_when_its_follow_up_breaks( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _cut_once_the_answer_starts())) + recorder: Final = _ChatRoundRecorder() + logging_obj: Final = _chat_rounds_logging_obj(recorder, "mcp-chat-stream-bridged-to-messages", stream=True) + + async def bridged(stream: CustomStreamWrapper) -> AsyncIterator[ModelResponseStream]: + async for chunk in stream: + yield chunk + + with _echo_tool_served(): + relayed: Final = bridged(await _open_chat_echo_stream(logging_obj)) + ProxyBaseLLMRequestProcessing(data={})._arm_deferred_stream_dispatch( + response=relayed, route_type="anthropic_messages", user_api_key_dict=MagicMock(), logging_obj=logging_obj + ) + with pytest.raises(litellm.ServiceUnavailableError): + _ = [chunk async for chunk in relayed] + await recorder.settled(logged=1, failed=1) + + _assert_one_row_bills_round_one(recorder.logged) + + +def _read_synchronously_until_the_follow_up_answers( + logging_obj: Logging, loop: asyncio.AbstractEventLoop, before_dropping: Callable[[], None] = lambda: None +) -> None: + stream: Final = loop.run_until_complete(_open_chat_echo_stream(logging_obj)) + for chunk in stream: + if chunk.choices and chunk.choices[0].delta.content == "PING": + before_dropping() + return + pytest.fail("the stream never sent 'PING'") + + +_CHAT_STREAM_BILLING_NAMES: Final = ("MCPStreamingIterator", "ChatStreamRoundBilling", "_log_success") + + +def _names_chat_stream_billing(text: str) -> bool: + return any(name in text for name in _CHAT_STREAM_BILLING_NAMES) + + +class _UnraisableRecorder: + def __init__(self) -> None: + self.raised: tuple[str, ...] = () + + def __call__(self, unraisable: "sys.UnraisableHookArgs") -> None: + self.raised = (*self.raised, f"{unraisable.object!r}: {unraisable.exc_value!r}") + + +def test_stream_auto_execute_read_synchronously_and_dropped_mid_answer_bills_the_finished_first_round( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _streamed(_CHAT_ROUND_TWO))) + recorder: Final = _ChatRoundRecorder() + unraisable: Final = _UnraisableRecorder() + loop: Final = asyncio.new_event_loop() + gc.collect() + monkeypatch.setattr(sys, "unraisablehook", unraisable) + asyncio.set_event_loop(loop) + + try: + with _echo_tool_served(), warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + _read_synchronously_until_the_follow_up_answers( + _chat_rounds_logging_obj(recorder, "mcp-chat-stream-sync-dropped", stream=True), loop + ) + gc.collect() + loop.run_until_complete(recorder.settled(logged=1)) + finally: + asyncio.set_event_loop(None) + loop.close() + + _assert_one_row_bills_round_one(recorder.logged) + assert [raised for raised in unraisable.raised if _names_chat_stream_billing(raised)] == [] + assert [str(warning.message) for warning in caught if _names_chat_stream_billing(str(warning.message))] == [] + + +def test_stream_auto_execute_read_synchronously_and_dropped_after_its_loop_closed_drops_without_errors( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + _serve_chat_rounds(respx_mock, monkeypatch, (_streamed(_CHAT_ROUND_ONE), _streamed(_CHAT_ROUND_TWO))) + recorder: Final = _ChatRoundRecorder() + unraisable: Final = _UnraisableRecorder() + loop: Final = asyncio.new_event_loop() + gc.collect() + monkeypatch.setattr(sys, "unraisablehook", unraisable) + asyncio.set_event_loop(loop) + + try: + with _echo_tool_served(), warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + _read_synchronously_until_the_follow_up_answers( + _chat_rounds_logging_obj(recorder, "mcp-chat-stream-sync-dropped-after-close", stream=True), + loop, + before_dropping=loop.close, + ) + gc.collect() + finally: + asyncio.set_event_loop(None) + loop.close() + + assert [raised for raised in unraisable.raised if _names_chat_stream_billing(raised)] == [] + assert [str(warning.message) for warning in caught if _names_chat_stream_billing(str(warning.message))] == [] diff --git a/tests/unit/responses/mcp/test_parked_stream_dispatch.py b/tests/unit/responses/mcp/test_parked_stream_dispatch.py new file mode 100644 index 00000000000..dd380c582c8 --- /dev/null +++ b/tests/unit/responses/mcp/test_parked_stream_dispatch.py @@ -0,0 +1,17 @@ +from typing import Final + +import pytest + +from litellm.responses.mcp.parked_stream_dispatch import parked_success +from litellm.types.utils import ModelResponse + +_ASSEMBLED: Final = ModelResponse(id="chatcmpl-parked", model="gpt-5.6") + + +@pytest.mark.parametrize( + "parked", + [(), (_ASSEMBLED,), (_ASSEMBLED, False, "not a logger"), (_ASSEMBLED, False, None, None)], + ids=("empty", "assembled-alone", "third-part-is-not-a-logger", "four-parts"), +) +def test_a_parked_dispatch_of_an_unknown_shape_is_never_dispatched(parked: tuple[object, ...]): + assert parked_success(parked) is None From 4786dd861b1b8c001117e0403cc2c606c193896c Mon Sep 17 00:00:00 2001 From: zoveki <349028329+zoveki@users.noreply.github.com> Date: Fri, 30 Oct 2026 07:36:01 -0700 Subject: [PATCH 04/74] fix(proxy)!: scope /health by wildcard keys and team model allowlists the way auth does (#49155) * test(proxy): cover /health scoping for wildcard keys and team model allowlists * fix(proxy): scope /health with the model access check auth runs on a request * fix(proxy): scope /health to the model group an allowed name routes to A key alias, team alias, or router model_group_alias lets auth admit a requested name by checking its target, so /health also kept the deployment under the requested name, one the key's requests never reach. Keep only the served model group each admitted name dispatches to, through the same alias layers /v1/models mirrors, and let alias_map read its own read-only output. * test(proxy): cover /health skipping a deployment that has no model name for a restricted key * fix(health): show deployments reached through wildcard and alias allowlists, and by id when an alias shadows the name * chore(router): drop should_include_deployment from the outside-use allowlist now that /health no longer calls it * fix(health): never list a default-fallback deployment for an unserved grant, and keep access-group keys to their group's deployments * fix(health): resolve ?model= through the caller's aliases the way chat does * fix(proxy): apply user and agent model limits to /health and yield between checks /health now runs the same user and agent model checks chat runs, so a personal key whose owner may call only some models, or an agent key limited by its access group, no longer sees or probes other deployments. Each per-name check yields to the event loop, and ?model= reaches the wildcard deployment serving an allowed name * fix(proxy): check owner and agent grants for wildcard routes in /health and fail on non-access auth errors * fix(proxy): answer /health auth lookup failures the way chat does and record ?model= checks under the requested name * test(proxy): pass recorded_model_name in the direct health save tests * fix(proxy): count alias names as /health candidates so aliased names a key may call get probed * fix(health): narrow wildcard-served deployments to the key's access groups * fix(health): narrow every deployment serving a name to the key's access groups, team-owned ones included * fix(health): narrow a team deployment's internal name to the key's access groups like its public name * fix(health): keep a team deployment's internal name narrow when a limit holds only that name An agent or key limit that holds one team deployment's internal name no longer reaches every deployment behind the team's public name. The internal name maps to its public name only when the public name is also admitted. * test(health): cover all-proxy-models next to * for a key that also holds an access group * refactor(auth): keep the _as_proxy_exception name so main's tests still find it * fix(health): keep an internal team name a wildcard grant admits as its own target --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/proxy/auth/auth_checks.py | 47 +- .../proxy/common_utils/model_listing_utils.py | 11 +- .../health_endpoints/_health_endpoints.py | 427 ++++++++++- ...st_auth_checks_object_access_and_lookup.py | 10 + .../health_endpoints/test_health_endpoints.py | 720 +++++++++++++++++- .../unit/proxy/test_health_check_functions.py | 2 + 6 files changed, 1164 insertions(+), 53 deletions(-) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index b8d9530d2eb..f80296f4bd7 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -84,6 +84,7 @@ from litellm.proxy._types import ( UserAPIKeyAuth, ) from litellm.proxy.agent_endpoints.auth.agent_access_groups import ( + AgentAccessGroupCeiling, CeilingResolver, resolve_agent_access_group_ceiling, ) @@ -1684,6 +1685,7 @@ def get_actual_routes(allowed_routes: list) -> list: KEY_END_USER_BUDGET_ID_METADATA_FIELD: Final = "end_user_budget_id" _KEY_METADATA_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) +_OBJECTS_ADAPTER: Final[TypeAdapter[tuple[object, ...]]] = TypeAdapter(tuple[object, ...]) def get_key_end_user_budget_id(key_metadata: Mapping[str, object] | None) -> str | None: @@ -5085,6 +5087,36 @@ async def can_agent_call_model( ) +async def _agent_access_group_ceilings( + valid_token: UserAPIKeyAuth, resolve_ceiling: CeilingResolver +) -> tuple[AgentAccessGroupCeiling, ...]: + if not valid_token.agent_id: + return () + + from litellm.proxy.agent_endpoints.auth.agent_access_groups import resolve_managed_agent_ceilings + + managed: Final = managed_agent_policy(valid_token) + if managed is not None: + return await resolve_managed_agent_ceilings(managed) + unmanaged: Final = await resolve_ceiling(valid_token.agent_id) + return (unmanaged,) if unmanaged is not None else () + + +def _managed_agent_model_grants(valid_token: UserAPIKeyAuth) -> frozenset[str]: + managed: Final = managed_agent_policy(valid_token) + granted: Final = (managed.object_permission or {}).get("models") if managed is not None else None + if not isinstance(granted, (list, tuple)): + return frozenset() + return frozenset(name for name in _OBJECTS_ADAPTER.validate_python(granted) if isinstance(name, str)) + + +async def agent_model_grants( + valid_token: UserAPIKeyAuth, resolve_ceiling: CeilingResolver = resolve_agent_access_group_ceiling +) -> frozenset[str]: + ceilings: Final = await _agent_access_group_ceilings(valid_token, resolve_ceiling) + return _managed_agent_model_grants(valid_token).union(*(ceiling.models for ceiling in ceilings)) + + async def _check_agent_access_group_model_access( model: str | list[str] | None, # mutable-ok: _can_object_call_model and the client message helper take list[str] valid_token: UserAPIKeyAuth | None, @@ -5092,20 +5124,9 @@ async def _check_agent_access_group_model_access( resolve_ceiling: CeilingResolver = resolve_agent_access_group_ceiling, ) -> Literal[True]: """Attached groups naming no model deny every model; the empty allowlist in ``_can_object_call_model`` allows.""" - if not model or valid_token is None or not valid_token.agent_id: + if not model or valid_token is None: return True - - from litellm.proxy.agent_endpoints.auth.agent_access_groups import resolve_managed_agent_ceilings - - managed: Final = managed_agent_policy(valid_token) - unmanaged: Final = await resolve_ceiling(valid_token.agent_id) if managed is None else None - ceilings: Final = ( - await resolve_managed_agent_ceilings(managed) - if managed is not None - else (unmanaged,) - if unmanaged is not None - else () - ) + ceilings: Final = await _agent_access_group_ceilings(valid_token, resolve_ceiling) dispatched: Final = _resolve_team_alias( model, team_model_aliases_for_auth_check(valid_token), valid_token.team_id, llm_router ) diff --git a/litellm/proxy/common_utils/model_listing_utils.py b/litellm/proxy/common_utils/model_listing_utils.py index ad4c3bd8770..a9424018d9a 100644 --- a/litellm/proxy/common_utils/model_listing_utils.py +++ b/litellm/proxy/common_utils/model_listing_utils.py @@ -219,7 +219,7 @@ def caller_alias_maps( def alias_map(aliases: object) -> Mapping[str, str]: try: - entries: Final = _ALIAS_ENTRIES.validate_python(aliases, strict=True) + entries: Final = _ALIAS_ENTRIES.validate_python(aliases) except ValidationError: return _NO_ALIASES return MappingProxyType( @@ -287,6 +287,15 @@ def _dispatch(model_id: str, layers: Sequence[_AliasLayer]) -> tuple[str, bool]: return reduce(_through_layer, layers, (model_id, False)) +def dispatched_model(model_id: str, aliases: CallerAliases) -> str: + target, _ = _dispatch(model_id, _alias_layers(aliases)) + return target + + +def requestable_alias_names(aliases: CallerAliases) -> frozenset[str]: + return frozenset(chain.from_iterable(layer.aliases for layer in _alias_layers(aliases))) + + def _lookup_id(name: str, lookup_by_response: Mapping[str, str], lookup_ids: Container[str]) -> str | None: if name in lookup_by_response: return lookup_by_response[name] diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 59c63a18f0f..8545e0609db 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -7,13 +7,15 @@ import secrets import time import traceback from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass from datetime import datetime, timedelta, timezone +from itertools import chain, groupby from types import MappingProxyType from typing import Final, Literal, TypedDict import fastapi from fastapi import APIRouter, Depends, HTTPException, Request, Response, status -from pydantic import TypeAdapter +from pydantic import BaseModel, ConfigDict, TypeAdapter from typing_extensions import ReadOnly import litellm @@ -31,7 +33,9 @@ from litellm.proxy._types import ( CallInfo, EnterpriseLicenseData, Litellm_EntityType, + LiteLLM_UserTable, LitellmUserRoles, + ModelAccessDeniedProxyException, ProxyException, SpecialModelNames, UserAPIKeyAuth, @@ -39,13 +43,28 @@ from litellm.proxy._types import ( ) from litellm.proxy.auth.auth_checks import ( _resolve_key_models_for_auth_check, # pyright: ignore[reportPrivateUsage] # the auth layer's sentinel resolution, reused so /health scopes exactly like a request + agent_model_grants, + can_agent_call_model, + can_key_call_resolved_model, + can_user_call_model, + get_user_object, + key_model_aliases_for_auth_check, + model_matches_any_wildcard_pattern_in_list, + team_model_aliases_for_auth_check, +) +from litellm.proxy.auth.auth_exception_handler import ( + _as_proxy_exception, # pyright: ignore[reportPrivateUsage] # one auth-failure mapping, shared with user_api_key_auth ) from litellm.proxy.auth.auth_utils import ( _BANNED_REQUEST_BODY_PARAMS, # pyright: ignore[reportPrivateUsage] # one canonical list, shared with the request-body check ) -from litellm.proxy.auth.model_checks import get_key_models from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils import http_parsing_utils +from litellm.proxy.common_utils.model_listing_utils import ( + CallerAliases, + dispatched_model, + requestable_alias_names, +) from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for from litellm.proxy.db.db_lookup_gate import db_lookup_stall_tracker from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler @@ -79,6 +98,7 @@ from litellm.router_utils.clientside_credential_handler import ( _ADMIN_CONFIG_FIELDS_TO_CLEAR_ON_BASE_OVERRIDE, # pyright: ignore[reportPrivateUsage] # one canonical list, shared with the router path clientside_credential_keys, ) +from litellm.router_utils.common_utils import resolve_model_group_alias from litellm.secret_managers.main import get_secret_bool from litellm.types.proxy.litellm_pre_call_utils import RedactedDict, SecretFields from litellm.types.router import reject_server_owned_wif_fields @@ -253,6 +273,28 @@ def get_callback_identifier(callback): router: Final = APIRouter() _OBJECT_MAPPING: Final = TypeAdapter(Mapping[str, object]) +_OBJECTS: Final = TypeAdapter(tuple[object, ...]) +_DEPLOYMENTS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) + + +class _ModelGrants(BaseModel): + model_config = ConfigDict(frozen=True, from_attributes=True) + + models: tuple[object, ...] = () + + +class _ModelInfoAccessGroups(BaseModel): + model_config = ConfigDict(frozen=True) + + access_groups: tuple[object, ...] + + +class _DeploymentAccessGroups(BaseModel): + model_config = ConfigDict(frozen=True) + + model_info: _ModelInfoAccessGroups + + services = ( Literal[ "slack_budget_alerts", @@ -1045,42 +1087,339 @@ def _strip_admin_only_fields_from_health_result(result: dict) -> dict: return out -def _health_accessible_model_names( - user_api_key_dict: UserAPIKeyAuth, llm_router: Router | None -) -> frozenset[str] | None: - """Model names the caller may health-check, or None when the key is unrestricted.""" +def _key_models_unrestricted(user_api_key_dict: UserAPIKeyAuth) -> bool: granted_models: Final = _resolve_key_models_for_auth_check(user_api_key_dict) - if not granted_models or SpecialModelNames.all_proxy_models.value in granted_models: + return not granted_models or SpecialModelNames.all_proxy_models.value in granted_models + + +def _own_team_names(deployment: object, team_id: str | None) -> tuple[str, str] | None: + match deployment: + case { + "model_name": str(internal_name), + "model_info": {"team_id": str(owner_team_id), "team_public_model_name": str(public_name)}, + } if owner_team_id == team_id: + return (internal_name, public_name) + case _: + return None + + +def _names_caller_may_request(deployment: object, team_id: str | None) -> frozenset[str]: + team_names: Final = _own_team_names(deployment, team_id) + if team_names is not None: + return frozenset(team_names) + match deployment: + case {"model_name": str(name)}: + return frozenset((name,)) + case _: + return frozenset() + + +def _public_names_by_internal_name(llm_model_list: Sequence[object], team_id: str | None) -> Mapping[str, str]: + team_names: Final = (_own_team_names(deployment, team_id) for deployment in llm_model_list) + return MappingProxyType(dict(names for names in team_names if names is not None)) + + +async def _key_owner(user_api_key_dict: UserAPIKeyAuth) -> LiteLLM_UserTable | None: + if user_api_key_dict.team_id is not None or user_api_key_dict.user_id is None: return None + from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache + + try: + return await get_user_object( + user_id=user_api_key_dict.user_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + user_id_upsert=False, + parent_otel_span=user_api_key_dict.parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + ) + except Exception: # noqa: BLE001 # auth reads an unloadable user as no user limit, so /health does too + return None + + +async def _key_may_call_model( + model_name: str, + user_api_key_dict: UserAPIKeyAuth, + owner: LiteLLM_UserTable | None, + llm_router: Router | None, +) -> bool: + from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache + + await asyncio.sleep(0) + try: + await can_key_call_resolved_model( + model=model_name, llm_model_list=None, valid_token=user_api_key_dict, llm_router=llm_router + ) + await can_agent_call_model( + model=model_name, + valid_token=user_api_key_dict, + llm_router=llm_router, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + await can_user_call_model( + model=model_name, + llm_router=llm_router, + user_object=owner, + key_model_aliases=key_model_aliases_for_auth_check(user_api_key_dict), + ) + except ModelAccessDeniedProxyException: + return False + except ProxyException: + raise + except Exception as failure: + if isinstance(failure, HTTPException) and failure.status_code == status.HTTP_403_FORBIDDEN: + return False + raise _as_proxy_exception(failure) from failure + return True + + +async def _caller_aliases(user_api_key_dict: UserAPIKeyAuth) -> CallerAliases: + from litellm.proxy.proxy_server import prisma_client, proxy_config, proxy_logging_obj + + router_settings: Final = await proxy_config.get_hierarchical_router_settings( + user_api_key_dict, prisma_client, proxy_logging_obj + ) + return CallerAliases( + team_model_aliases_for_auth_check(user_api_key_dict), + key_model_aliases_for_auth_check(user_api_key_dict), + router_settings.get("model_group_alias") if router_settings is not None else None, + ) + + +def _model_auth_checks(model_name: str, aliases: CallerAliases) -> str: + return resolve_model_group_alias(aliases.router, model_name) or model_name + + +def _deployment_id(deployment: object) -> str | None: + match deployment: + case {"model_info": {"id": str(model_id)}}: + return model_id + case _: + return None + + +def _granted_names(user_api_key_dict: UserAPIKeyAuth) -> frozenset[str]: + grants: Final = ( + *_ModelGrants.model_validate(user_api_key_dict).models, + *_OBJECTS.validate_python(user_api_key_dict.team_models), + ) + return frozenset(name for name in grants if isinstance(name, str)) + + +async def _names_any_limit_grants(user_api_key_dict: UserAPIKeyAuth, owner: LiteLLM_UserTable | None) -> frozenset[str]: + owner_grants: Final = _ModelGrants.model_validate(owner).models if owner is not None else () + return ( + _granted_names(user_api_key_dict) + | frozenset(name for name in owner_grants if isinstance(name, str)) + | await agent_model_grants(user_api_key_dict) + ) + + +def _alias_names_caller_may_request(aliases: CallerAliases, llm_router: Router | None) -> frozenset[str]: + router_aliases: Final = frozenset(llm_router.registry().aliases()) if llm_router is not None else frozenset[str]() + return requestable_alias_names(aliases) | router_aliases + + +def _served_group(model_name: str, aliases: CallerAliases, llm_router: Router | None) -> str: + dispatched: Final = dispatched_model(model_name, aliases) + return llm_router.registry().resolve(dispatched) if llm_router is not None else dispatched + + +def _name_id_pairs(deployment: object, team_id: str | None) -> tuple[tuple[str, str], ...]: + model_id: Final = _deployment_id(deployment) + if model_id is None: + return () + return tuple((name, model_id) for name in _names_caller_may_request(deployment, team_id) | {model_id}) + + +def _ids_by_requested_name(llm_model_list: Sequence[object], team_id: str | None) -> Mapping[str, frozenset[str]]: + pairs: Final = sorted(chain.from_iterable(_name_id_pairs(deployment, team_id) for deployment in llm_model_list)) + return MappingProxyType( + {name: frozenset(model_id for _, model_id in group) for name, group in groupby(pairs, key=lambda pair: pair[0])} + ) + + +def _ids_of(deployments: Iterable[object]) -> frozenset[str]: + return frozenset(model_id for model_id in map(_deployment_id, deployments) if model_id is not None) + + +def _access_groups_of(deployment: object) -> frozenset[str]: + match deployment: + case {"model_info": {"access_groups": list()}}: + access_groups: Final = _DeploymentAccessGroups.model_validate(deployment).model_info.access_groups + return frozenset(access_group for access_group in access_groups if isinstance(access_group, str)) + case _: + return frozenset() + + +@dataclass(frozen=True, slots=True) +class _AccessGroupGrants: + granted_names: frozenset[str] + ids_in_granted_groups: frozenset[str] + + def requested_name(self, name: str, public_names: Mapping[str, str], admitted_names: frozenset[str]) -> str: + public_name: Final = public_names.get(name) + if public_name is None or public_name not in admitted_names or self._grants(name): + return name + return public_name + + def _grants(self, name: str) -> bool: + return name in self.granted_names or model_matches_any_wildcard_pattern_in_list(name, tuple(self.granted_names)) + + def narrow(self, serving_ids: frozenset[str], served_names: frozenset[str]) -> frozenset[str]: + in_granted_groups: Final = serving_ids & self.ids_in_granted_groups + granted_directly: Final = not self.granted_names.isdisjoint(served_names) + return serving_ids if granted_directly or not in_granted_groups else in_granted_groups + + +def _access_group_grants(llm_model_list: Sequence[object], user_api_key_dict: UserAPIKeyAuth) -> _AccessGroupGrants: + granted: Final = _granted_names(user_api_key_dict) + in_granted_groups: Final = ( + deployment for deployment in llm_model_list if not granted.isdisjoint(_access_groups_of(deployment)) + ) + return _AccessGroupGrants(granted, _ids_of(in_granted_groups)) + + +def _ids_serving_group( + group: str, + ids_by_name: Mapping[str, frozenset[str]], + user_api_key_dict: UserAPIKeyAuth, + llm_router: Router | None, + access_group_grants: _AccessGroupGrants, +) -> frozenset[str]: + named: Final = ids_by_name.get(group) + if named is not None: + return access_group_grants.narrow(named, frozenset((group,))) if llm_router is None: - return frozenset(granted_models) - registry: Final = llm_router.registry() + return frozenset() + team_id: Final = user_api_key_dict.team_id + wildcard_served: Final = llm_router.registry().deployments(group, source="wildcard", team_id=team_id) + served_names: Final = frozenset((group,)).union( + *(_names_caller_may_request(deployment, team_id) for deployment in wildcard_served) + ) + return access_group_grants.narrow(_ids_of(wildcard_served), served_names) + + +async def _names_key_may_request( + names: Iterable[str], + aliases: CallerAliases, + user_api_key_dict: UserAPIKeyAuth, + owner: LiteLLM_UserTable | None, + llm_router: Router | None, +) -> frozenset[str]: return frozenset( - get_key_models( - user_api_key_dict=user_api_key_dict, - proxy_model_list=registry.names(team_id=user_api_key_dict.team_id), - model_access_groups=registry.access_groups(), + [ + name + for name in names + if await _key_may_call_model(_model_auth_checks(name, aliases), user_api_key_dict, owner, llm_router) + ] + ) + + +def _ids_serving( + groups: Iterable[str], + ids_by_name: Mapping[str, frozenset[str]], + user_api_key_dict: UserAPIKeyAuth, + llm_router: Router | None, + access_group_grants: _AccessGroupGrants, +) -> frozenset[str]: + return frozenset[str]().union( + *( + _ids_serving_group(group, ids_by_name, user_api_key_dict, llm_router, access_group_grants) + for group in groups ) ) -def _caller_may_probe_deployment( - deployment: Mapping[str, object], - allowed_models: frozenset[str] | None, +def _shadowed_ids( + llm_model_list: Sequence[object], admitted_names: frozenset[str], reached_ids: frozenset[str], team_id: str | None +) -> frozenset[str]: + ids_of_admitted_names: Final = ( + _deployment_id(deployment) + for deployment in llm_model_list + if not admitted_names.isdisjoint(_names_caller_may_request(deployment, team_id)) + ) + return frozenset(model_id for model_id in ids_of_admitted_names if model_id is not None) - reached_ids + + +@dataclass(frozen=True, slots=True) +class _ReachableDeployments: + model_ids: frozenset[str] + groups: frozenset[str] + + def include(self, deployment: object, team_id: str | None) -> bool: + model_id: Final = _deployment_id(deployment) + if model_id is None: + return not self.groups.isdisjoint(_names_caller_may_request(deployment, team_id)) + return model_id in self.model_ids + + +_NOTHING_REACHABLE: Final = _ReachableDeployments(frozenset(), frozenset()) + + +async def _deployments_key_may_reach( + llm_model_list: Sequence[object], + aliases: CallerAliases, + user_api_key_dict: UserAPIKeyAuth, + owner: LiteLLM_UserTable | None, llm_router: Router | None, +) -> _ReachableDeployments: + candidate_names: Final = (await _names_any_limit_grants(user_api_key_dict, owner)).union( + _alias_names_caller_may_request(aliases, llm_router), + *(_names_caller_may_request(deployment, user_api_key_dict.team_id) for deployment in llm_model_list), + ) + admitted_names: Final = await _names_key_may_request(candidate_names, aliases, user_api_key_dict, owner, llm_router) + access_group_grants: Final = _access_group_grants(llm_model_list, user_api_key_dict) + public_names: Final = _public_names_by_internal_name(llm_model_list, user_api_key_dict.team_id) + groups: Final = frozenset( + _served_group(access_group_grants.requested_name(name, public_names, admitted_names), aliases, llm_router) + for name in admitted_names + ) + ids_by_name: Final = _ids_by_requested_name(llm_model_list, user_api_key_dict.team_id) + reached_ids: Final = _ids_serving(groups, ids_by_name, user_api_key_dict, llm_router, access_group_grants) + admitted_ids: Final = await _names_key_may_request( + _shadowed_ids(llm_model_list, admitted_names, reached_ids, user_api_key_dict.team_id), + aliases, + user_api_key_dict, + owner, + llm_router, + ) + id_groups: Final = frozenset(_served_group(model_id, aliases, llm_router) for model_id in admitted_ids) + return _ReachableDeployments( + reached_ids | _ids_serving(id_groups, ids_by_name, user_api_key_dict, llm_router, access_group_grants), groups + ) + + +def _model_param_target( + model: str, + aliases: CallerAliases, + llm_model_list: Sequence[Mapping[str, object]], team_id: str | None, - caller_is_admin: bool, + llm_router: Router | None, +) -> str: + group: Final = _served_group(model, aliases, llm_router) + if llm_router is None or deployments_targeted_by_name(llm_model_list, group, team_id): + return group + patterns: Final = frozenset( + deployment.get("model_name") + for deployment in llm_router.registry().deployments(group, source="wildcard", team_id=team_id) + ) + match tuple(patterns): + case (str(pattern),): + return pattern + case _: + return group + + +def _caller_may_probe_deployment( + deployment: Mapping[str, object], reachable: _ReachableDeployments, team_id: str | None, caller_is_admin: bool ) -> bool: """Same deployment visibility rule as routing: another team's deployment is never in scope, team-less callers included.""" if not caller_is_admin and not deployment_usable_by_team(deployment, team_id): return False - if allowed_models is None: - return True - if llm_router is None: - return deployment.get("model_name") in allowed_models - model: Final = dict(deployment) - registry: Final = llm_router.registry() - return any(registry.includes(model, model_name=name, team_id=team_id) for name in allowed_models) + return reachable.include(deployment, team_id) def _resolve_targeted_model_ids( @@ -1137,6 +1476,7 @@ def _filter_health_check_results_by_model_ids(results: dict, allowed_model_ids: async def _perform_health_check_and_save( model_list, target_model, + recorded_model_name: str | None, cli_model, details, prisma_client, @@ -1159,8 +1499,7 @@ async def _perform_health_check_and_save( # Optionally save health check result to database (non-blocking) if prisma_client is not None: - # For CLI model, use cli_model name; for router models, use target_model - model_name_for_db: Final = cli_model if cli_model is not None else target_model + model_name_for_db: Final = cli_model if cli_model is not None else recorded_model_name if model_name_for_db is not None: asyncio.create_task( _save_health_check_to_db( @@ -1290,6 +1629,7 @@ async def health_endpoint( cli_result: Final = await _perform_health_check_and_save( model_list=[], target_model=None, + recorded_model_name=None, cli_model=user_model, details=health_check_details, prisma_client=prisma_client, @@ -1304,15 +1644,39 @@ async def health_endpoint( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail={"error": "Model list not initialized"}, ) - allowed_models: Final = _health_accessible_model_names(user_api_key_dict, llm_router) - restrict_to_allowed_models: Final = not is_admin or allowed_models is not None + restrict_to_allowed_models: Final = not is_admin or not _key_models_unrestricted(user_api_key_dict) + aliases: Final = await _caller_aliases(user_api_key_dict) if restrict_to_allowed_models else None + reachable: Final = ( + await _deployments_key_may_reach( + _OBJECTS.validate_python(llm_model_list), + aliases, + user_api_key_dict, + await _key_owner(user_api_key_dict), + llm_router, + ) + if aliases is not None + else _NOTHING_REACHABLE + ) + requested_model: Final = ( + _model_param_target( + model, + aliases, + _DEPLOYMENTS.validate_python(llm_model_list), + user_api_key_dict.team_id, + llm_router, + ) + if model is not None and aliases is not None + else model + ) _llm_model_list: Final = [ m for m in copy.deepcopy(llm_model_list) if not restrict_to_allowed_models - or _caller_may_probe_deployment(m, allowed_models, llm_router, user_api_key_dict.team_id, is_admin) + or _caller_may_probe_deployment(m, reachable, user_api_key_dict.team_id, is_admin) ] - targeted_ids: Final = _resolve_targeted_model_ids(_llm_model_list, model, model_id, user_api_key_dict.team_id) + targeted_ids: Final = _resolve_targeted_model_ids( + _llm_model_list, requested_model, model_id, user_api_key_dict.team_id + ) if restrict_to_allowed_models and targeted_ids is not None and not targeted_ids: raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, @@ -1370,7 +1734,8 @@ async def health_endpoint( else: router_result: Final = await _perform_health_check_and_save( model_list=_llm_model_list, - target_model=target_model, + target_model=target_model if model_id else requested_model, + recorded_model_name=target_model, cli_model=None, details=health_check_details, prisma_client=prisma_client, diff --git a/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py b/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py index 0bdc5f1910a..6b59f6d40dc 100644 --- a/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py +++ b/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py @@ -47,6 +47,7 @@ from litellm.proxy.auth.auth_checks import ( can_object_call_model, _can_object_call_vector_stores, _check_agent_access_group_model_access, + agent_model_grants, check_end_user_budget, _check_team_member_budget, personal_key_owner_teams, @@ -11294,6 +11295,15 @@ async def test_agent_access_groups_naming_no_model_deny_every_model(): assert exc_info.value.type == ProxyErrorTypes.agent_model_access_denied +@pytest.mark.asyncio +async def test_agent_model_grants_names_every_model_the_agents_access_groups_allow(): + agent_key: Final = UserAPIKeyAuth(token="agent-token", agent_id="agent-1", models=[]) + resolve, asked = _agent_model_ceiling_resolver(frozenset({"gpt-5", "openai/gpt-5.6"})) + + assert await agent_model_grants(agent_key, resolve) == frozenset({"gpt-5", "openai/gpt-5.6"}) + assert asked == ["agent-1"] + + @pytest.mark.asyncio async def test_agent_without_access_groups_adds_no_model_ceiling(): agent_key: Final = UserAPIKeyAuth(token="agent-token", agent_id="agent-1", models=["gpt-5", "claude-sonnet"]) diff --git a/tests/unit/proxy/health_endpoints/test_health_endpoints.py b/tests/unit/proxy/health_endpoints/test_health_endpoints.py index 9847787efce..2258eb95fb8 100644 --- a/tests/unit/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/unit/proxy/health_endpoints/test_health_endpoints.py @@ -1,8 +1,9 @@ import asyncio import copy +import itertools import json import time -from collections.abc import Awaitable, Iterator, Mapping, Sequence +from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence from contextlib import contextmanager from datetime import datetime, timedelta from importlib import import_module @@ -22,7 +23,7 @@ import litellm import litellm.proxy.health_endpoints._health_endpoints as _health_endpoints_module from litellm.litellm_core_utils.health_check_helpers import TEST_IMAGE_BASE64 from litellm.models.credentials import CredentialItem -from litellm.proxy._types import LitellmUserRoles, ProxyException, UserAPIKeyAuth +from litellm.proxy._types import LiteLLM_UserTable, LitellmUserRoles, ProxyErrorTypes, ProxyException, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.health_endpoints._health_endpoints import ( _db_health_readiness_check, @@ -35,6 +36,7 @@ from litellm.proxy.health_endpoints._health_endpoints import ( test_model_connection as health_test_model_connection, ) from litellm.router import Router +from litellm.types.router import RouterModelGroupAliasItem from litellm.types.utils import oauth_token_exchange_litellm_params from litellm.types.workload_identity import ( ANTHROPIC_WIF_KWARGS_KEYS, @@ -925,7 +927,11 @@ async def test_test_model_connection_stored_audio_base_model_follows_the_stored_ deployment: Final = MappingProxyType( { "model_name": "custom-name-audio", - "litellm_params": {**deployment_params, "api_key": "fake-azure-key", "api_base": "https://x.openai.azure.com"}, + "litellm_params": { + **deployment_params, + "api_key": "fake-azure-key", + "api_base": "https://x.openai.azure.com", + }, "model_info": deployment_model_info, } ) @@ -1948,7 +1954,6 @@ def test_health_liveness_endpoint(proxy_client): assert duration_ms < 100, f"Health check took {duration_ms:.2f}ms, expected < 100ms for a simple endpoint" - def test_proxy_client_serves_full_proxy_routes_after_gateway_import(monkeypatch: pytest.MonkeyPatch) -> None: from litellm.proxy.proxy_server import app @@ -2472,6 +2477,7 @@ def _proxy_health_globals( llm_router: object, use_background_health_checks: bool = False, health_check_results: Mapping[str, object] | None = None, + prisma_client: object = None, ) -> Iterator[None]: with ( patch( # test-quality-ok: proxy module global, no injection seam @@ -2481,7 +2487,7 @@ def _proxy_health_globals( "litellm.proxy.proxy_server.llm_router", llm_router ), patch( # test-quality-ok: proxy module global, no injection seam - "litellm.proxy.proxy_server.prisma_client", None + "litellm.proxy.proxy_server.prisma_client", prisma_client ), patch( # test-quality-ok: proxy module global, no injection seam "litellm.proxy.proxy_server.use_background_health_checks", use_background_health_checks @@ -3812,7 +3818,11 @@ def test_clean_endpoint_data_never_displays_credential_fields(credential_field, async def _live_probed_model_ids( - model_list: Sequence[Mapping[str, object]], user_api_key_dict: UserAPIKeyAuth, model: str | None = None + model_list: Sequence[Mapping[str, object]], + user_api_key_dict: UserAPIKeyAuth, + model: str | None = None, + llm_router: Router | None = None, + prisma_client: object = None, ) -> set[str]: from fastapi import Response @@ -3825,7 +3835,7 @@ async def _live_probed_model_ids( return {"healthy_endpoints": [], "unhealthy_endpoints": [], "healthy_count": 0, "unhealthy_count": 0} with ( - _proxy_health_globals(model_list, _router_for(model_list)), + _proxy_health_globals(model_list, llm_router or _router_for(model_list), prisma_client=prisma_client), patch( # test-quality-ok: the model list handed to the probe is the assertion; no injection seam "litellm.proxy.health_endpoints._health_endpoints._perform_health_check_and_save", side_effect=fake_perform, @@ -4120,6 +4130,7 @@ async def _live_narrowed_model_ids( user_api_key_dict: UserAPIKeyAuth, model: str | None = None, model_id: str | None = None, + llm_router: Router | None = None, ) -> set[str]: from fastapi import Response @@ -4130,7 +4141,7 @@ async def _live_narrowed_model_ids( return probed, [], {} with ( - _proxy_health_globals(model_list, _router_for(model_list)), + _proxy_health_globals(model_list, llm_router or _router_for(model_list)), patch( # test-quality-ok: the probe is the provider edge; which deployments reach it is the assertion "litellm.proxy.health_check._perform_health_check", side_effect=fake_probe ), @@ -4289,6 +4300,699 @@ async def test_health_endpoint_keeps_a_team_only_public_name_off_a_team_less_key probe.assert_not_awaited() +_PROVIDER_PREFIXED_MODEL_LIST: Final = [ + { + "model_name": "openai/gpt-5.4-nano", + "litellm_params": {"model": "openai/gpt-5.4-nano"}, + "model_info": {"id": "id-prefixed-nano"}, + }, + { + "model_name": "gpt-5.4-mini", + "litellm_params": {"model": "openai/gpt-5.4-mini"}, + "model_info": {"id": "id-mini"}, + }, + { + "model_name": "claude-haiku-5-5", + "litellm_params": {"model": "anthropic/claude-haiku-5-5"}, + "model_info": {"id": "id-haiku"}, + }, +] +_PROVIDER_PREFIXED_CACHED_RESULTS: Final = { + "healthy_endpoints": [ + {"model": "openai/gpt-5.4-nano", "model_id": "id-prefixed-nano"}, + {"model": "openai/gpt-5.4-mini", "model_id": "id-mini"}, + {"model": "anthropic/claude-haiku-5-5", "model_id": "id-haiku"}, + ], + "unhealthy_endpoints": [], + "healthy_count": 3, + "unhealthy_count": 0, +} +_PROVIDER_WILDCARD_KEY: Final = UserAPIKeyAuth(api_key="hashed-test-key", models=["openai/*"]) +_RESTRICTED_TEAM_KEY: Final = UserAPIKeyAuth( + api_key="hashed-test-key", models=[], team_id="team-a", team_models=["gpt-5.4-mini"] +) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("caller", "expected_ids"), + [(_PROVIDER_WILDCARD_KEY, {"id-prefixed-nano", "id-mini"}), (_RESTRICTED_TEAM_KEY, {"id-mini"})], +) +async def test_health_endpoint_probes_the_deployments_auth_lets_the_key_request( + caller: UserAPIKeyAuth, expected_ids: set[str] +): + probed: Final = await _live_probed_model_ids(_PROVIDER_PREFIXED_MODEL_LIST, caller) + + assert probed == expected_ids + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("caller", "expected_ids"), + [(_PROVIDER_WILDCARD_KEY, ["id-prefixed-nano", "id-mini"]), (_RESTRICTED_TEAM_KEY, ["id-mini"])], +) +async def test_health_endpoint_returns_the_deployments_auth_lets_the_key_request_on_background_cache_path( + caller: UserAPIKeyAuth, expected_ids: list[str] +): + from fastapi import Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + with _proxy_health_globals( + _PROVIDER_PREFIXED_MODEL_LIST, + _router_for(_PROVIDER_PREFIXED_MODEL_LIST), + use_background_health_checks=True, + health_check_results=_PROVIDER_PREFIXED_CACHED_RESULTS, + ): + result: Final = await health_endpoint(response=Response(), user_api_key_dict=caller, model=None, model_id=None) + + assert [ep["model_id"] for ep in result["healthy_endpoints"]] == expected_ids + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_background_health_checks", [False, True]) +async def test_health_endpoint_refuses_a_deployment_the_team_allowlist_forbids(use_background_health_checks: bool): + from fastapi import HTTPException, Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + with ( + _proxy_health_globals( + _PROVIDER_PREFIXED_MODEL_LIST, + _router_for(_PROVIDER_PREFIXED_MODEL_LIST), + use_background_health_checks=use_background_health_checks, + health_check_results=_PROVIDER_PREFIXED_CACHED_RESULTS, + ), + patch( # test-quality-ok: the probe must never run; the endpoint has no injection seam for it + "litellm.proxy.health_endpoints._health_endpoints._perform_health_check_and_save", new_callable=AsyncMock + ) as probe, + pytest.raises(HTTPException) as refused, + ): + await health_endpoint( + response=Response(), user_api_key_dict=_RESTRICTED_TEAM_KEY, model=None, model_id="id-prefixed-nano" + ) + + assert refused.value.status_code == 403 + probe.assert_not_awaited() + + +_KEY_ALIASED_AWAY_FROM_HAIKU: Final = UserAPIKeyAuth( + api_key="hashed-test-key", models=["gpt-5.4-mini"], aliases={"claude-haiku-5-5": "gpt-5.4-mini"} +) +_TEAM_ALIASED_AWAY_FROM_HAIKU: Final = UserAPIKeyAuth( + api_key="hashed-test-key", + models=["claude-haiku-5-5"], + team_id="team-a", + team_model_aliases={"claude-haiku-5-5": "gpt-5.4-mini"}, +) +_MINI_ONLY_KEY: Final = UserAPIKeyAuth(api_key="hashed-test-key", models=["gpt-5.4-mini"]) +_ALIAS_SHADOW_CASES: Final = [ + pytest.param(_KEY_ALIASED_AWAY_FROM_HAIKU, {}, id="key-alias"), + pytest.param(_TEAM_ALIASED_AWAY_FROM_HAIKU, {}, id="team-alias"), + pytest.param(_MINI_ONLY_KEY, {"claude-haiku-5-5": "gpt-5.4-mini"}, id="router-alias"), +] + + +def _router_with_aliases(model_group_alias: Mapping[str, str]) -> Router: + return Router(model_list=copy.deepcopy(_PROVIDER_PREFIXED_MODEL_LIST), model_group_alias=dict(model_group_alias)) + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("caller", "model_group_alias"), _ALIAS_SHADOW_CASES) +async def test_health_endpoint_probes_the_deployment_an_alias_sends_the_key_to_not_the_one_it_shadows( + caller: UserAPIKeyAuth, model_group_alias: Mapping[str, str] +): + probed: Final = await _live_probed_model_ids( + _PROVIDER_PREFIXED_MODEL_LIST, caller, llm_router=_router_with_aliases(model_group_alias) + ) + + assert probed == {"id-mini"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("caller", "model_group_alias"), _ALIAS_SHADOW_CASES) +async def test_health_endpoint_returns_the_deployment_an_alias_sends_the_key_to_on_background_cache_path( + caller: UserAPIKeyAuth, model_group_alias: Mapping[str, str] +): + from fastapi import Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + with _proxy_health_globals( + _PROVIDER_PREFIXED_MODEL_LIST, + _router_with_aliases(model_group_alias), + use_background_health_checks=True, + health_check_results=_PROVIDER_PREFIXED_CACHED_RESULTS, + ): + result: Final = await health_endpoint(response=Response(), user_api_key_dict=caller, model=None, model_id=None) + + assert [ep["model_id"] for ep in result["healthy_endpoints"]] == ["id-mini"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_background_health_checks", [False, True]) +async def test_health_endpoint_refuses_the_deployment_a_key_alias_shadows(use_background_health_checks: bool): + from fastapi import HTTPException, Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + with ( + _proxy_health_globals( + _PROVIDER_PREFIXED_MODEL_LIST, + _router_for(_PROVIDER_PREFIXED_MODEL_LIST), + use_background_health_checks=use_background_health_checks, + health_check_results=_PROVIDER_PREFIXED_CACHED_RESULTS, + ), + patch( # test-quality-ok: the probe must never run; the endpoint has no injection seam for it + "litellm.proxy.health_endpoints._health_endpoints._perform_health_check_and_save", new_callable=AsyncMock + ) as probe, + pytest.raises(HTTPException) as refused, + ): + await health_endpoint( + response=Response(), user_api_key_dict=_KEY_ALIASED_AWAY_FROM_HAIKU, model=None, model_id="id-haiku" + ) + + assert refused.value.status_code == 403 + probe.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("caller", "model_group_alias"), _ALIAS_SHADOW_CASES) +async def test_health_endpoint_targets_the_deployment_an_alias_name_sends_the_key_to( + caller: UserAPIKeyAuth, model_group_alias: Mapping[str, str] +): + probed: Final = await _live_narrowed_model_ids( + _PROVIDER_PREFIXED_MODEL_LIST, + caller, + model="claude-haiku-5-5", + llm_router=_router_with_aliases(model_group_alias), + ) + + assert probed == {"id-mini"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("caller", "model_group_alias"), _ALIAS_SHADOW_CASES) +async def test_health_endpoint_records_an_alias_check_under_the_name_the_caller_asked_for( + caller: UserAPIKeyAuth, model_group_alias: Mapping[str, str] +): + from fastapi import Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + async def fake_probe(model_list, details=True, max_concurrency=None, instrumentation_context=None): + return [{"model": m["litellm_params"]["model"], "model_id": m["model_info"]["id"]} for m in model_list], [], {} + + with ( + _proxy_health_globals( + _PROVIDER_PREFIXED_MODEL_LIST, _router_with_aliases(model_group_alias), prisma_client=MagicMock() + ), + patch( # test-quality-ok: the probe is the provider edge; the endpoint has no injection seam for it + "litellm.proxy.health_check._perform_health_check", side_effect=fake_probe + ), + patch( # test-quality-ok: the history row is the assertion; the endpoint has no injection seam for its writer + "litellm.proxy.health_endpoints._health_endpoints._save_health_check_to_db", new_callable=AsyncMock + ) as save, + ): + result: Final = await health_endpoint( + response=Response(), user_api_key_dict=caller, model="claude-haiku-5-5", model_id=None + ) + + assert [ep["model_id"] for ep in result["healthy_endpoints"]] == ["id-mini"] + assert save.call_args.args[1] == "claude-haiku-5-5" + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("caller", "model_group_alias"), _ALIAS_SHADOW_CASES) +async def test_health_endpoint_targets_the_deployment_an_alias_name_sends_the_key_to_on_background_cache_path( + caller: UserAPIKeyAuth, model_group_alias: Mapping[str, str] +): + from fastapi import Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + with _proxy_health_globals( + _PROVIDER_PREFIXED_MODEL_LIST, + _router_with_aliases(model_group_alias), + use_background_health_checks=True, + health_check_results=_PROVIDER_PREFIXED_CACHED_RESULTS, + ): + result: Final = await health_endpoint( + response=Response(), user_api_key_dict=caller, model="claude-haiku-5-5", model_id=None + ) + + assert [ep["model_id"] for ep in result["healthy_endpoints"]] == ["id-mini"] + + +@pytest.mark.asyncio +async def test_health_endpoint_never_probes_a_deployment_without_a_model_name_for_a_restricted_key(): + nameless_deployment: Final = { + "litellm_params": {"model": "openai/gpt-5.4-mini"}, + "model_info": {"id": "id-nameless"}, + } + + probed: Final = await _live_probed_model_ids( + [*_PROVIDER_PREFIXED_MODEL_LIST, nameless_deployment], + _MINI_ONLY_KEY, + llm_router=_router_for(_PROVIDER_PREFIXED_MODEL_LIST), + ) + + assert probed == {"id-mini"} + + +@pytest.mark.asyncio +async def test_health_endpoint_never_probes_the_deployment_a_keys_router_alias_sends_it_to_but_forbids(): + caller: Final = UserAPIKeyAuth( + api_key="hashed-test-key", + models=["gpt-5.4-mini", "openai/gpt-5.4-nano"], + router_settings={"model_group_alias": {"gpt-5.4-mini": "claude-haiku-5-5"}}, + ) + + probed: Final = await _live_probed_model_ids(_PROVIDER_PREFIXED_MODEL_LIST, caller) + + assert probed == {"id-prefixed-nano"} + + +_WILDCARD_DEPLOYMENT: Final = { + "model_name": "openai/*", + "litellm_params": {"model": "openai/*"}, + "model_info": {"id": "id-wild"}, +} +_CATCH_ALL_DEPLOYMENT: Final = {"model_name": "*", "litellm_params": {"model": "*"}, "model_info": {"id": "id-star"}} +_TEAM_WILDCARD_DEPLOYMENT: Final = { + "model_name": "openai-wild_team-a_7c1e", + "litellm_params": {"model": "openai/*"}, + "model_info": {"id": "id-team-wild", "team_id": "team-a", "team_public_model_name": "openai/*"}, +} + + +def _team_key(team_models: list[str], team_model_aliases: Mapping[str, str] | None = None) -> UserAPIKeyAuth: + return UserAPIKeyAuth( + api_key="hashed-test-key", + models=[], + team_id="team-a", + team_models=team_models, + team_model_aliases=dict(team_model_aliases) if team_model_aliases is not None else None, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model_list", "model_group_alias", "caller", "expected_ids"), + [ + pytest.param( + [_WILDCARD_DEPLOYMENT, _PROVIDER_PREFIXED_MODEL_LIST[2]], + {}, + _team_key(["openai/gpt-5.4-nano"]), + {"id-wild"}, + id="team-name-served-by-provider-wildcard", + ), + pytest.param( + [_TEAM_WILDCARD_DEPLOYMENT, _PROVIDER_PREFIXED_MODEL_LIST[2]], + {}, + _team_key(["openai/gpt-5.4-nano"]), + {"id-team-wild"}, + id="team-name-served-by-teams-own-wildcard", + ), + pytest.param( + [_CATCH_ALL_DEPLOYMENT], + {}, + _team_key(["gpt-5.4-mini"]), + {"id-star"}, + id="team-name-served-by-catch-all", + ), + pytest.param( + _PROVIDER_PREFIXED_MODEL_LIST, + {"gpt-4": "gpt-5.4-mini"}, + _team_key(["gpt-4"]), + {"id-mini"}, + id="team-allows-router-alias", + ), + pytest.param( + _PROVIDER_PREFIXED_MODEL_LIST, + {}, + _team_key(["my-alias"], {"my-alias": "gpt-5.4-mini"}), + {"id-mini"}, + id="team-allows-team-alias", + ), + pytest.param( + _PROVIDER_PREFIXED_MODEL_LIST, + {"friendly-mini": RouterModelGroupAliasItem(model="gpt-5.4-mini", hidden=True)}, + UserAPIKeyAuth(api_key="hashed-test-key", models=["friendly-*"]), + {"id-mini"}, + id="wildcard-key-allows-hidden-router-alias", + ), + pytest.param( + _PROVIDER_PREFIXED_MODEL_LIST, + {}, + _team_key(["friendly-*"], {"friendly-mini": "gpt-5.4-mini"}), + {"id-mini"}, + id="wildcard-team-allows-team-alias", + ), + pytest.param( + _PROVIDER_PREFIXED_MODEL_LIST, + {"claude-haiku-5-5": "gpt-5.4-mini"}, + UserAPIKeyAuth(api_key="hashed-test-key", models=[]), + {"id-prefixed-nano", "id-mini", "id-haiku"}, + id="unrestricted-key-calls-shadowed-deployment-by-id", + ), + ], +) +async def test_health_endpoint_probes_every_deployment_the_callers_allowed_names_route_to( + model_list: Sequence[Mapping[str, object]], + model_group_alias: Mapping[str, str | RouterModelGroupAliasItem], + caller: UserAPIKeyAuth, + expected_ids: set[str], +): + llm_router: Final = Router( + model_list=copy.deepcopy([dict(deployment) for deployment in model_list]), + model_group_alias=dict(model_group_alias), + ) + + probed: Final = await _live_probed_model_ids(llm_router.get_model_list() or [], caller, llm_router=llm_router) + + assert probed == expected_ids + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "caller", + [ + pytest.param( + UserAPIKeyAuth(api_key="hashed-test-key", models=["gpt-5.4-mini", "gpt-4-retired"]), id="key-grant" + ), + pytest.param(_team_key(["gpt-5.4-mini", "gpt-4-retired"]), id="team-grant"), + ], +) +async def test_health_endpoint_never_probes_the_default_fallback_for_a_granted_name_no_deployment_serves( + caller: UserAPIKeyAuth, +) -> None: + llm_router: Final = Router( + model_list=copy.deepcopy(_PROVIDER_PREFIXED_MODEL_LIST), default_fallbacks=["claude-haiku-5-5"] + ) + + probed: Final = await _live_probed_model_ids(_PROVIDER_PREFIXED_MODEL_LIST, caller, llm_router=llm_router) + + assert probed == {"id-mini"} + + +def _personal_key_whose_owner_may_call(models: list[str]) -> UserAPIKeyAuth: + from litellm.proxy.proxy_server import user_api_key_cache + + owner_id: Final = "health-scope-owner-" + "-".join(models) + user_api_key_cache.set_cache(key=owner_id, value=LiteLLM_UserTable(user_id=owner_id, models=models)) + return UserAPIKeyAuth(api_key="hashed-test-key", models=[], user_id=owner_id) + + +def _managed_agent_key_granted(models: list[str]) -> UserAPIKeyAuth: + from litellm.types.agents import AgentResponse + + policy: Final = AgentResponse( + agent_id="managed", agent_name="Managed", agent_card_params={}, object_permission={"models": models} + ) + return UserAPIKeyAuth(api_key="hashed-test-key", models=[]).model_copy(update={"managed_agent_policy": policy}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("make_caller", "models", "expected_ids"), + [ + pytest.param(_personal_key_whose_owner_may_call, ["gpt-5.4-mini"], {"id-mini"}, id="key-owners-models"), + pytest.param(_managed_agent_key_granted, ["gpt-5.4-mini"], {"id-mini"}, id="managed-agents-models"), + pytest.param(_managed_agent_key_granted, [], set(), id="managed-agent-without-model-grants"), + ], +) +async def test_health_endpoint_probes_only_the_models_the_keys_user_or_agent_may_call( + make_caller: Callable[[list[str]], UserAPIKeyAuth], models: list[str], expected_ids: set[str] +): + probed: Final = await _live_probed_model_ids(_PROVIDER_PREFIXED_MODEL_LIST, make_caller(models)) + + assert probed == expected_ids + + +@pytest.mark.asyncio +async def test_health_endpoint_refuses_a_model_id_the_keys_owner_may_not_call(): + from fastapi import Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + with ( + _proxy_health_globals(_PROVIDER_PREFIXED_MODEL_LIST, _router_for(_PROVIDER_PREFIXED_MODEL_LIST)), + patch( # test-quality-ok: the probe must never run; the endpoint has no injection seam for it + "litellm.proxy.health_endpoints._health_endpoints._perform_health_check_and_save", new_callable=AsyncMock + ) as probe, + pytest.raises(HTTPException) as refused, + ): + await health_endpoint( + response=Response(), + user_api_key_dict=_personal_key_whose_owner_may_call(["gpt-5.4-mini"]), + model=None, + model_id="id-prefixed-nano", + ) + + assert refused.value.status_code == 403 + probe.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "make_caller", + [ + pytest.param(_personal_key_whose_owner_may_call, id="key-owner"), + pytest.param(_managed_agent_key_granted, id="managed-agent"), + ], +) +async def test_health_endpoint_keeps_the_wildcard_serving_a_name_the_keys_user_or_agent_may_call( + make_caller: Callable[[list[str]], UserAPIKeyAuth], +): + model_list: Final = [_WILDCARD_DEPLOYMENT, *_PROVIDER_PREFIXED_MODEL_LIST] + + probed: Final = await _live_probed_model_ids(model_list, make_caller(["openai/gpt-5.6"])) + + assert probed == {"id-wild"} + + +@pytest.mark.asyncio +async def test_health_endpoint_answers_an_agent_key_acting_for_an_unknown_user_the_way_chat_does(): + from litellm.types.agents import AgentCaller + + users_table: Final = MagicMock() + users_table.db.litellm_usertable.find_unique = AsyncMock(return_value=None) + agent_key: Final = UserAPIKeyAuth(api_key="hashed-test-key", models=["gpt-5.4-mini"]).model_copy( + update={"agent_caller": AgentCaller(user_id="user-deleted-before-this-request")} + ) + + with pytest.raises(ProxyException) as failed: + await _live_probed_model_ids(_PROVIDER_PREFIXED_MODEL_LIST, agent_key, prisma_client=users_table) + + assert (failed.value.code, failed.value.type) == ("401", ProxyErrorTypes.auth_error) + assert "user-deleted-before-this-request" in failed.value.message + + +@pytest.mark.asyncio +async def test_health_endpoint_lets_other_requests_run_while_it_scopes_a_large_model_list(): + model_list: Final = [ + {"model_name": f"model-{n}", "litellm_params": {"model": f"openai/model-{n}"}, "model_info": {"id": f"id-{n}"}} + for n in range(50) + ] + other_request_turns: Final = itertools.count() + scoping_done: Final = asyncio.Event() + + async def scope() -> set[str]: + try: + return await _live_probed_model_ids( + model_list, UserAPIKeyAuth(api_key="hashed-test-key", models=["model-0"]) + ) + finally: + scoping_done.set() + + async def other_request() -> None: + while not scoping_done.is_set(): + next(other_request_turns) + await asyncio.sleep(0) + + probed, _ = await asyncio.gather(scope(), other_request()) + + assert probed == {"id-0"} + assert next(other_request_turns) >= len(model_list) + + +_WILDCARD_CACHED_RESULTS: Final = { + "healthy_endpoints": [ + {"model": "openai/gpt-5.6", "model_id": "id-wild"}, + {"model": "openai/gpt-5.6", "model_id": "id-team-wild"}, + {"model": "anthropic/claude-haiku-5-5", "model_id": "id-haiku"}, + ], + "unhealthy_endpoints": [], + "healthy_count": 3, + "unhealthy_count": 0, +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_background_health_checks", [False, True]) +@pytest.mark.parametrize( + ("wildcard_deployment", "expected_id"), + [ + pytest.param(_WILDCARD_DEPLOYMENT, "id-wild", id="proxy-wildcard"), + pytest.param(_TEAM_WILDCARD_DEPLOYMENT, "id-team-wild", id="teams-own-wildcard"), + ], +) +async def test_health_endpoint_model_param_reaches_the_wildcard_serving_a_team_allowed_name( + wildcard_deployment: Mapping[str, object], expected_id: str, use_background_health_checks: bool +): + from fastapi import Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + model_list: Final = [wildcard_deployment, _PROVIDER_PREFIXED_MODEL_LIST[2]] + + async def fake_probe(model_list, details=True, max_concurrency=None, instrumentation_context=None): + return [{"model": m["litellm_params"]["model"], "model_id": m["model_info"]["id"]} for m in model_list], [], {} + + with ( + _proxy_health_globals( + model_list, + _router_for(model_list), + use_background_health_checks=use_background_health_checks, + health_check_results=_WILDCARD_CACHED_RESULTS, + ), + patch( # test-quality-ok: the probe is the provider edge; which deployments reach it is the assertion + "litellm.proxy.health_check._perform_health_check", side_effect=fake_probe + ), + ): + result: Final = await health_endpoint( + response=Response(), user_api_key_dict=_team_key(["openai/gpt-5.6"]), model="openai/gpt-5.6", model_id=None + ) + + assert [ep["model_id"] for ep in result["healthy_endpoints"]] == [expected_id] + + +_ACCESS_GROUP_SPLIT_MODEL_LIST: Final = [ + { + "model_name": "gpt-5.4-mini", + "litellm_params": {"model": "openai/gpt-5.4-mini"}, + "model_info": {"id": "id-mini-premium", "access_groups": ["premium"]}, + }, + { + "model_name": "gpt-5.4-mini", + "litellm_params": {"model": "openai/gpt-5.4-mini"}, + "model_info": {"id": "id-mini-standard"}, + }, +] + + +@pytest.mark.asyncio +async def test_health_endpoint_probes_only_the_access_groups_deployments_of_a_shared_name() -> None: + caller: Final = UserAPIKeyAuth(api_key="hashed-test-key", models=["premium"]) + + probed: Final = await _live_probed_model_ids(_ACCESS_GROUP_SPLIT_MODEL_LIST, caller) + + assert probed == {"id-mini-premium"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("direct_grant", ["gpt-5.4-mini", "*", "all-proxy-models"]) +async def test_health_endpoint_probes_every_deployment_of_a_name_the_key_also_holds_directly(direct_grant: str) -> None: + caller: Final = UserAPIKeyAuth(api_key="hashed-test-key", models=["premium", direct_grant]) + + probed: Final = await _live_probed_model_ids(_ACCESS_GROUP_SPLIT_MODEL_LIST, caller) + + assert probed == {"id-mini-premium", "id-mini-standard"} + + +_ACCESS_GROUP_SPLIT_WILDCARD_MODEL_LIST: Final = [ + { + "model_name": "openai/*", + "litellm_params": {"model": "openai/*"}, + "model_info": {"id": "id-wild-premium", "access_groups": ["premium"]}, + }, + { + "model_name": "openai/*", + "litellm_params": {"model": "openai/*"}, + "model_info": {"id": "id-wild-standard", "access_groups": ["standard"]}, + }, +] + + +@pytest.mark.asyncio +async def test_health_endpoint_probes_only_the_access_groups_wildcard_deployments_serving_a_name() -> None: + caller: Final = _personal_key_whose_owner_may_call(["openai/gpt-5.4-nano"]).model_copy( + update={"models": ["premium"]} + ) + + probed: Final = await _live_probed_model_ids(_ACCESS_GROUP_SPLIT_WILDCARD_MODEL_LIST, caller) + + assert probed == {"id-wild-premium"} + + +def _team_deployment( + internal_name: str, public_name: str, model: str, model_id: str, access_groups: list[str] +) -> Mapping[str, object]: + return { + "model_name": internal_name, + "litellm_params": {"model": model}, + "model_info": { + "id": model_id, + "team_id": "team-a", + "team_public_model_name": public_name, + "access_groups": access_groups, + }, + } + + +def _team_split_model_list(public_name: str, model: str) -> list[Mapping[str, object]]: + return [ + _team_deployment("model_name_team-a_7c1e4b2a", public_name, model, "id-team-premium", ["premium"]), + _team_deployment("model_name_team-a_9d2f6e01", public_name, model, "id-team-standard", ["standard"]), + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("public_name", "model"), + [ + pytest.param("openai/*", "openai/*", id="team-wildcard"), + pytest.param("gpt-5.4-mini", "openai/gpt-5.4-mini", id="team-model"), + ], +) +async def test_health_endpoint_probes_only_the_access_groups_team_deployments_serving_a_name( + public_name: str, model: str +) -> None: + probed: Final = await _live_probed_model_ids(_team_split_model_list(public_name, model), _team_key(["premium"])) + + assert probed == {"id-team-premium"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("internal_name_grant", ["model_name_team-a_9d2f6e01", "model_name_team-a_*"]) +async def test_health_endpoint_probes_a_team_deployment_whose_internal_name_the_key_holds( + internal_name_grant: str, +) -> None: + caller: Final = _team_key(["premium", internal_name_grant]) + + probed: Final = await _live_probed_model_ids(_team_split_model_list("gpt-5.4-mini", "openai/gpt-5.4-mini"), caller) + + assert probed == {"id-team-premium", "id-team-standard"} + + +@pytest.mark.asyncio +async def test_health_endpoint_probes_only_the_team_deployment_whose_internal_name_an_agent_holds() -> None: + from litellm.types.agents import AgentResponse + + policy: Final = AgentResponse( + agent_id="managed", + agent_name="Managed", + agent_card_params={}, + object_permission={"models": ["model_name_team-a_9d2f6e01"]}, + ) + caller: Final = _team_key(["gpt-5.4-mini"]).model_copy(update={"managed_agent_policy": policy}) + + probed: Final = await _live_probed_model_ids(_team_split_model_list("gpt-5.4-mini", "openai/gpt-5.4-mini"), caller) + + assert probed == {"id-team-standard"} + + def test_resolve_targeted_model_ids_lets_model_id_win_over_model(): resolve = _health_endpoints_module._resolve_targeted_model_ids diff --git a/tests/unit/proxy/test_health_check_functions.py b/tests/unit/proxy/test_health_check_functions.py index 2c0198cd538..79eec44958f 100644 --- a/tests/unit/proxy/test_health_check_functions.py +++ b/tests/unit/proxy/test_health_check_functions.py @@ -741,6 +741,7 @@ async def test_perform_health_check_and_save_passes_model_id_to_perform_health_c result = await _perform_health_check_and_save( model_list=model_list, target_model=None, + recorded_model_name=None, cli_model=None, details=True, prisma_client=None, @@ -777,6 +778,7 @@ async def test_perform_health_check_and_save_forwards_skip_disabled_background_f await _perform_health_check_and_save( model_list=model_list, target_model=None, + recorded_model_name=None, cli_model=None, details=True, prisma_client=None, From 43c07a0f1246c95da710408df4d7846bc5f848e2 Mon Sep 17 00:00:00 2001 From: tavren27 <348969968+tavren27@users.noreply.github.com> Date: Fri, 30 Oct 2026 07:48:32 -0700 Subject: [PATCH 05/74] fix(router)!: check and count deployment itpm and otpm on every Responses WebSocket turn (#49069) * fix(router): check and count deployment itpm and otpm on every Responses WebSocket turn The success hook only reconciled an ITPM/OTPM reservation found in top-level metadata or the standard logging object, but on the proxy the reservation lives in litellm_params.metadata, so no reservation was ever settled there. The hook now always calls the reconcile, which already returns early without a reservation and clears what it settled, so a second success event is a no-op. A Responses WebSocket turn after the first now has its deployment's current-minute ITPM and OTPM counters checked and is refused with a 429 frame once a counter is at its limit. An admitted turn carries a zero reservation on the current minute's keys, so its actual tokens land on those counters on both the native and the managed relay. * fix(responses): settle the socket open's itpm/otpm reservation on the first turn only After the first turn takes its logging copy, both websocket relays drop the open's ITPM/OTPM reservation keys from the connection metadata, the same way the budget reservation is detached. Later turns, unsolicited backend events, turns after limits are removed mid-socket, and sockets with no turn admission no longer settle or refund the open's reservation a second time. * refactor(responses): read the itpm/otpm reservation keys from core metadata helpers instead of router_utils * fix(responses): keep a socket open's hold for its first real turn and check tokens before request limits A turn that will settle the socket open's itpm/otpm reservation no longer counts that reservation against itself or swaps it for a zero one, so Codex's warmup frame on a managed provider, or a refused first frame on the native socket, no longer blocks the first real prompt or strands the hold for the minute. The token check now runs before the rpm/rpd checks, so a turn refused for tokens never uses up a request slot. Token-check tests pin the clock, and the managed relay settle test drives a real streamed call through a custom provider. * test(router): pin the router clock in the socket turn admission tests Both tests seeded this minute and the next to dodge a minute boundary. They now use the file's pinned-minute fixture and seed only that minute, so neither depends on when the test runs. * fix(router): check a responses socket's open without reserving and read shared token counters first A Responses WebSocket open now runs the same itpm/otpm check a turn runs and reserves nothing. It used to reserve an estimate that falls back to the model's max output, so a socket could not open under any otpm below that. The open still records a zero reservation for its own minute, so the first frame, which the relay forwards without a turn check, counts its actual tokens like every later turn. This drops the plumbing that only carried the open's hold into the first turn. The turn admission callable takes no arguments again, and the native relay detaches only the budget reservation, as before this change. The itpm/otpm turn check reads Redis first and falls back to this instance's in-memory count only without Redis or when the circuit breaker is open. These counters go down when a call settles, so an in-memory copy at the limit could refuse turns after the shared count dropped. The tpm check keeps its in-memory shortcut, since its counter only grows within a minute. Responses usage that reports input_tokens_details.cached_tokens now leaves cached input out of itpm, as chat usage already did. * fix(router): leave a socket turn's cached input out of itpm The native Responses WebSocket relay logs a turn with its list of events, so the deployment rate limit hook finds no usage on the response and falls back to the standard logging payload. That fallback read cached input from metadata.cache_read_input_tokens, a key the payload's cleaned metadata never carries, so every cached token of a socket turn was counted against itpm. The fallback now reads the cached count from the payload's usage_object, the same usage the logging path already built for cost, so a native socket turn counts only its uncached input. The managed relay and HTTP /v1/responses already counted correctly through the response's own usage * fix(router): mark only responses socket opens as check-only and count a turn on the minute it finishes The check-only branch for a socket open keyed on a websocket value in the request kwargs, which any HTTP body can carry and which the realtime route also sets. The router now passes an opens_responses_socket flag into set_io_token_rate_limit_request_kwargs only for its _aresponses_websocket call, and the flag lives in the same context value as the request kwargs, so no request body can set it and realtime opens reserve and get refused as before A zero reservation holds nothing to refund, so a socket turn now counts its real tokens on the minute the call finishes, read from the clock at settle time. A non-zero reservation still settles on the keys it reserved on so its refund hits the same counter When the shared Redis read returns nothing, the check now falls back to this instance's in-memory count instead of admitting the turn with no count at all * fix(router): keep io token reservation keys out of the metadata sent to providers The itpm/otpm pre-call check wrote its reservation keys into both metadata and litellm_metadata. On the routes the router serves through its generic helper (Responses, Anthropic Messages, and the rest), metadata is the client's own field and goes to the provider as is. So _litellm_itpm_cache_key and _litellm_otpm_cache_key, which carry the deployment id and the model name, reached the provider, came back in the echoed response metadata, and counted toward the provider's 16-key cap. The managed Responses socket relay merged each turn's reservation into the frame's metadata the same way, and a native socket opened with metadata sent the keys on every turn through its request defaults The router now passes the limiter the metadata field it uses for its own fields on that call (metadata for chat, litellm_metadata for the generic routes), and the reservation goes only there. metadata_channel is a required keyword, so a new caller has to name the field it writes. The managed relay puts each turn's reservation only on litellm_metadata With the reservation only on litellm_metadata, a retried call on the generic routes refunded its first attempt twice and pushed the deployment's counters below zero. The failure hook read the reservation from the logging copy in litellm_params.metadata and cleared only that copy, so the router's litellm_metadata still held it and the retry refunded it again. The hooks now also read and clear litellm_params.litellm_metadata, which is the router's own dict The proxy does not strip client keys that start with _litellm_ on the way in, so dropping that prefix at the provider boundary would also drop keys clients send today The io token tests now run Redis in process with fakeredis instead of a TCP fake server * test(responses): count socket turns through real logging callbacks instead of a Logging subclass The native relay test now hands the rate limit check to a real Logging object as its success and failure callback, so the turn's tokens go through the same dispatch path production uses. --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../internal_call_metadata.py | 6 + litellm/llms/custom_httpx/llm_http_handler.py | 4 +- litellm/responses/main.py | 6 +- litellm/responses/streaming_iterator.py | 103 ++-- litellm/router.py | 26 +- .../io_token_rate_limit_check.py | 209 ++++++--- ...test_responses_websocket_turn_admission.py | 113 ++++- .../test_responses_websocket_all_providers.py | 441 +++++++++++++++++- .../test_router/test_io_token_rate_limits.py | 356 ++++++++++++-- tests/unit/test_router/test_router.py | 6 +- .../test_router/test_router_helper_utils.py | 374 ++++++++++++++- 11 files changed, 1492 insertions(+), 152 deletions(-) diff --git a/litellm/litellm_core_utils/internal_call_metadata.py b/litellm/litellm_core_utils/internal_call_metadata.py index f3dbd612bee..453d4358b35 100644 --- a/litellm/litellm_core_utils/internal_call_metadata.py +++ b/litellm/litellm_core_utils/internal_call_metadata.py @@ -35,6 +35,12 @@ from litellm.types.utils import ( BUDGET_RESERVATION_METADATA_KEYS: Final = frozenset({"user_api_key_budget_reservation"}) +ITPM_RESERVED_KEY: Final = "_litellm_itpm_reserved" +OTPM_RESERVED_KEY: Final = "_litellm_otpm_reserved" +ITPM_CACHE_KEY: Final = "_litellm_itpm_cache_key" +OTPM_CACHE_KEY: Final = "_litellm_otpm_cache_key" +IO_TOKEN_RESERVATION_KEYS: Final = frozenset({ITPM_RESERVED_KEY, OTPM_RESERVED_KEY, ITPM_CACHE_KEY, OTPM_CACHE_KEY}) + MODEL_ACCESS_GROUP_METADATA_KEY: Final = "user_api_key_matched_model_access_groups" """Where auth records the model access groups that authorized the request, for the spend writer. diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 61fc23589ac..9a2b352dd86 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -249,7 +249,7 @@ if TYPE_CHECKING: FakeAnthropicMessagesStreamIterator, ) from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig - from litellm.responses.streaming_iterator import ResponsesWsFrameGuard + from litellm.responses.streaming_iterator import ResponsesWsFrameGuard, ResponsesWsTurnAdmission from litellm.types.google_genai.main import GenerateContentResponse from litellm.types.llms.openai_evals import ( CancelEvalResponse, @@ -7269,7 +7269,7 @@ class BaseLLMHTTPHandler: custom_llm_provider: str | None = None, first_message: str | None = None, request_defaults: ResponsesWebSocketRequestDefaults | None = None, - turn_admission: Callable[[], Awaitable[None]] | None = None, + turn_admission: "ResponsesWsTurnAdmission | None" = None, frame_guard: "ResponsesWsFrameGuard | None" = None, **kwargs: object, ) -> Exception | None: diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 5c8350589b8..9259103cb5c 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -1,7 +1,7 @@ import asyncio import contextvars import json -from collections.abc import Awaitable, Callable, Coroutine, Generator, Iterable, Mapping, Sequence +from collections.abc import Coroutine, Generator, Iterable, Mapping, Sequence from contextlib import contextmanager from dataclasses import dataclass from functools import partial @@ -81,7 +81,7 @@ from litellm.types.llms.openai import ( # Handle ResponseText import with fallback if TYPE_CHECKING: - from litellm.responses.streaming_iterator import ResponsesWsFrameGuard + from litellm.responses.streaming_iterator import ResponsesWsFrameGuard, ResponsesWsTurnAdmission from litellm.types.llms.openai import ResponseText else: ResponseText = str # Fallback for ResponseText import @@ -2867,7 +2867,7 @@ async def _aresponses_websocket( api_base: str | None = None, api_key: str | None = None, timeout: float | None = None, - turn_admission: Callable[[], Awaitable[None]] | None = None, + turn_admission: "ResponsesWsTurnAdmission | None" = None, responses_ws_frame_guard: "ResponsesWsFrameGuard | None" = None, **kwargs, ) -> Exception | None: diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index f0bb2201162..fb91ba6a618 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -2145,18 +2145,32 @@ async def _enforce_frame_project_quota( ) -async def _admit_frame_turn(turn_admission: Callable[[], Awaitable[None]] | None, raw_message: str) -> Exception | None: +@dataclass(frozen=True, slots=True) +class _AdmittedTurn: + reservation: Mapping[str, object] = field(default_factory=lambda: MappingProxyType({})) + + +@dataclass(frozen=True, slots=True) +class _RefusedTurn: + rejection: Exception + + +ResponsesWsTurnAdmission: TypeAlias = Callable[[], Awaitable[Mapping[str, object]]] + + +async def _admit_frame_turn( + turn_admission: ResponsesWsTurnAdmission | None, raw_message: str +) -> _AdmittedTurn | _RefusedTurn: frame: Final = _json_object_or_none(raw_message) if turn_admission is None or frame is None or frame.get("type") != "response.create": - return None + return _AdmittedTurn() try: - await turn_admission() + return _AdmittedTurn(await turn_admission()) except RateLimitError as refusal: - return refusal + return _RefusedTurn(refusal) except Exception as e: verbose_logger.exception("Responses WS turn admission failed: %s", e) - return e - return None + return _RefusedTurn(e) def _turn_refusal_frame(refusal: RateLimitError) -> str: @@ -2191,6 +2205,7 @@ RESPONSES_WS_MASKABLE_TEXT_BLOCK_TYPES: Final = frozenset({"input_text", "output _RESPONSES_WS_FAILURE_EVENT_TYPES: Final = frozenset({"error", "response.failed"}) _PER_TURN_METADATA_KEYS: Final = frozenset({"metadata", "litellm_metadata"}) +_NO_IO_TOKEN_RESERVATION: Final[Mapping[str, object]] = MappingProxyType({}) _GUARDRAIL_INFORMATION_KEY: Final = "standard_logging_guardrail_information" @@ -2318,6 +2333,17 @@ def _set_turn_guardrail_information( _replace_guardrail_information(litellm_params.get(key), guardrail_information) +def _set_turn_io_token_reservation(turn_logging_obj: LiteLLMLoggingObj, reservation: Mapping[str, object]) -> None: + litellm_params: Final = _json_object_or_none( + turn_logging_obj.litellm_params # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # Logging.litellm_params is untyped + ) + if litellm_params is None or not reservation: + return + for bucket in (litellm_params.get(key) for key in _PER_TURN_METADATA_KEYS): + if _is_json_object(bucket): + bucket.update(reservation) + + def _replace_guardrail_information(bucket: object, guardrail_information: tuple[Mapping[str, object], ...]) -> None: if not _is_json_object(bucket): return @@ -2458,7 +2484,7 @@ class ResponsesWebSocketStreaming: authorized_model: str | None = None, custom_llm_provider: str | None = None, request_defaults: ResponsesWebSocketRequestDefaults | None = None, - turn_admission: Callable[[], Awaitable[None]] | None = None, + turn_admission: ResponsesWsTurnAdmission | None = None, frame_guard: ResponsesWsFrameGuard | None = None, ): self.websocket = websocket @@ -2480,10 +2506,13 @@ class ResponsesWebSocketStreaming: # response.create frame to prevent deployment-substitution attacks. self.authorized_model: str | None = authorized_model self.request_defaults: ResponsesWebSocketRequestDefaults | None = request_defaults - self.turn_admission: Callable[[], Awaitable[None]] | None = turn_admission + self.turn_admission: ResponsesWsTurnAdmission | None = turn_admission def _start_turn( - self, client_frame: str | None, guardrail_information: tuple[Mapping[str, object], ...] = () + self, + client_frame: str | None, + guardrail_information: tuple[Mapping[str, object], ...] = (), + io_token_reservation: Mapping[str, object] = _NO_IO_TOKEN_RESERVATION, ) -> None: start_time: Final = datetime.now() # noqa: DTZ005 # naive to match the logging pipeline start_time turn_logging_obj: Final = self.logging_obj.copy_for_new_call( @@ -2491,6 +2520,7 @@ class ResponsesWebSocketStreaming: ) self._detach_budget_reservation() _set_turn_guardrail_information(turn_logging_obj, guardrail_information) + _set_turn_io_token_reservation(turn_logging_obj, io_token_reservation) turn_logging_obj.model_call_details["messages"] = list(_logged_user_messages(client_frame)) turn_logging_obj.pre_call(input=client_frame, api_key="") self._open_turns = (*self._open_turns, _ResponsesWsTurn(logging_obj=turn_logging_obj)) @@ -2510,11 +2540,14 @@ class ResponsesWebSocketStreaming: } def _record_client_frame( - self, client_frame: str, guardrail_information: tuple[Mapping[str, object], ...] = () + self, + client_frame: str, + guardrail_information: tuple[Mapping[str, object], ...] = (), + io_token_reservation: Mapping[str, object] = _NO_IO_TOKEN_RESERVATION, ) -> None: frame: Final = _json_object_or_none(client_frame) if frame is not None and frame.get("type") == "response.create": - self._start_turn(client_frame, guardrail_information) + self._start_turn(client_frame, guardrail_information, io_token_reservation) async def _guard_frame(self, message: str) -> GuardedResponsesWsFrame | None: """Run the proxy's pre_call guardrails on one ``response.create`` frame. @@ -2533,15 +2566,18 @@ class ResponsesWebSocketStreaming: return None async def _forward_guarded_frame( - self, message: str, turn_admission: Callable[[], Awaitable[None]] | None = None + self, message: str, turn_admission: ResponsesWsTurnAdmission | None = None ) -> None: guarded: Final = await self._guard_frame(message) - if guarded is None or not await self._admit_or_reject_turn(guarded, message, turn_admission): + if guarded is None: + return + admitted: Final = await self._admit_or_reject_turn(guarded, message, turn_admission) + if admitted is None: return if guarded.pii_tokens: self._remember_pii_tokens(guarded.pii_tokens) masked: Final = await self._mask_response_create(guarded.frame) - self._record_client_frame(masked, guarded.guardrail_information) + self._record_client_frame(masked, guarded.guardrail_information, admitted.reservation) await self.backend_ws.send(masked) def _remember_pii_tokens(self, pii_tokens: Mapping[str, str]) -> None: @@ -2941,15 +2977,17 @@ class ResponsesWebSocketStreaming: return True async def _admit_or_reject_turn( - self, guarded: GuardedResponsesWsFrame, message: str, turn_admission: Callable[[], Awaitable[None]] | None - ) -> bool: - rejection: Final = await _admit_frame_turn(turn_admission, message) - if rejection is None: - return True - if self._failure is None: - self._failure = rejection - await _reject_guarded_turn(self.websocket, guarded, rejection) - return False + self, guarded: GuardedResponsesWsFrame, message: str, turn_admission: ResponsesWsTurnAdmission | None + ) -> _AdmittedTurn | None: + verdict: Final = await _admit_frame_turn(turn_admission, message) + match verdict: + case _AdmittedTurn(): + return verdict + case _RefusedTurn(rejection=rejection): + if self._failure is None: + self._failure = rejection + await _reject_guarded_turn(self.websocket, guarded, rejection) + return None async def _send_turn_refusal(self, refusal: RateLimitError) -> None: try: @@ -3052,7 +3090,7 @@ class ManagedResponsesWebSocketHandler: custom_llm_provider: str | None = None, first_message: str | None = None, quota_callbacks: Sequence[ProjectQuotaCallback] | None = None, - turn_admission: Callable[[], Awaitable[None]] | None = None, + turn_admission: ResponsesWsTurnAdmission | None = None, frame_guard: ResponsesWsFrameGuard | None = None, guardrail_callbacks: Sequence[PresidioGuardrailCallback] | None = None, **kwargs: object, @@ -3076,7 +3114,7 @@ class ManagedResponsesWebSocketHandler: self._connection_provider = self._resolve_provider(model) or custom_llm_provider self.first_message = first_message self.quota_callbacks: tuple[ProjectQuotaCallback, ...] = tuple(quota_callbacks) if quota_callbacks else () - self.turn_admission: Callable[[], Awaitable[None]] | None = turn_admission + self.turn_admission: ResponsesWsTurnAdmission | None = turn_admission self._failure: Exception | None = None # Carry through safe pass-through kwargs (e.g. extra_headers) self.extra_kwargs: dict[str, object] = {k: v for k, v in kwargs.items() if k not in _MANAGED_WS_SKIP_KWARGS} @@ -3473,7 +3511,7 @@ class ManagedResponsesWebSocketHandler: # ------------------------------------------------------------------ async def _process_response_create( - self, raw_message: str, turn_admission: Callable[[], Awaitable[None]] | None = None + self, raw_message: str, turn_admission: ResponsesWsTurnAdmission | None = None ) -> None: """ Parse one ``response.create`` event, call ``litellm.aresponses(stream=True)``, @@ -3520,11 +3558,11 @@ class ManagedResponsesWebSocketHandler: guarded: Final = await self._guard_frame(raw_message) if guarded is None: return - rejection: Final = await _admit_frame_turn(turn_admission, raw_message) - if rejection is not None: + verdict: Final = await _admit_frame_turn(turn_admission, raw_message) + if isinstance(verdict, _RefusedTurn): if self._failure is None: - self._failure = rejection - await _reject_guarded_turn(self.websocket, guarded, rejection) + self._failure = verdict.rejection + await _reject_guarded_turn(self.websocket, guarded, verdict.rejection) return if guarded.pii_tokens: self._pii_tokens = {**self._pii_tokens, **guarded.pii_tokens} @@ -3555,6 +3593,11 @@ class ManagedResponsesWebSocketHandler: ) self._update_proxy_request(call_kwargs, requested_model or self.model_group or model) call_kwargs.update(self.extra_kwargs) + turn_litellm_metadata: Final = call_kwargs.get("litellm_metadata") + call_kwargs["litellm_metadata"] = { + **(turn_litellm_metadata if _is_json_object(turn_litellm_metadata) else {}), + **verdict.reservation, + } try: terminal_event: Final = await self._stream_and_forward(model, call_kwargs) diff --git a/litellm/router.py b/litellm/router.py index af2181cedb5..233e7581a40 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -378,6 +378,7 @@ from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check impor EncryptedContentAffinityCheck, ) from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import ( + async_admit_io_token_turn, async_io_token_refund_failure, build_io_token_rate_limit_headers, deployment_has_io_token_limits, @@ -5113,6 +5114,7 @@ class Router(DeprecatedRouterMembers): kwargs: dict, function_name: str | None = None, splits_stream_timeout: bool = False, + opens_responses_socket: bool = False, ) -> None: """ 5 jobs: @@ -5202,7 +5204,12 @@ class Router(DeprecatedRouterMembers): # here before it's wiped below, instead of relying on that attempt's # (possibly still-pending) failure event to do it. refund_stale_reservation_before_retry(self._core_cache, kwargs) - set_io_token_rate_limit_request_kwargs(kwargs, store_in_context=deployment_has_io_token_limits(deployment)) + set_io_token_rate_limit_request_kwargs( + kwargs, + store_in_context=deployment_has_io_token_limits(deployment), + opens_responses_socket=opens_responses_socket, + metadata_channel=metadata_variable_name, + ) kwargs[metadata_variable_name].setdefault( ROUTING_REQUEST_TAGS_METADATA_KEY, @@ -6615,6 +6622,7 @@ class Router(DeprecatedRouterMembers): kwargs=kwargs, function_name=function_name, splits_stream_timeout=handler_name in _STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS, + opens_responses_socket=handler_name == "_aresponses_websocket", ) data: Final = deployment["litellm_params"].copy() @@ -10465,9 +10473,20 @@ class Router(DeprecatedRouterMembers): return deployment if current is None else current.model_dump(exclude_none=True) async def async_admit_deployment_turn( - self, model_group: str, opened_on: Mapping[str, object], request_kwargs: Mapping[str, object] - ) -> None: + self, + model_group: str, + opened_on: Mapping[str, object], + request_kwargs: Mapping[str, object], + ) -> Mapping[str, object]: deployment: Final = self._deployment_as_configured_now(opened_on) + rate_limit_check: Final = next( + (callback for callback in self._routing_callbacks() if isinstance(callback, ModelRateLimitingCheck)), None + ) + io_token_admission: Final[Mapping[str, object]] = ( + MappingProxyType({}) + if rate_limit_check is None + else await async_admit_io_token_turn(rate_limit_check.dual_cache, deployment, get_utc_datetime()) + ) budget_limiter: Final = self._core_get_router_deployment_budget_limiter() if budget_limiter is not None: try: @@ -10488,6 +10507,7 @@ class Router(DeprecatedRouterMembers): await self.async_routing_strategy_pre_call_checks( # pyright: ignore[reportUnknownMemberType] # legacy signature takes an untyped deployment dict deployment=dict(without_io_token_limits(deployment)), parent_otel_span=None ) + return io_token_admission async def _admit_turn_through_routing_strategy( self, model_group: str, deployment: Mapping[str, object], request_kwargs: Mapping[str, object] diff --git a/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py b/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py index 7580c7dadd3..80e1f3fe29c 100644 --- a/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py +++ b/litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py @@ -14,6 +14,9 @@ from __future__ import annotations import contextlib import contextvars from collections.abc import Mapping, MutableMapping +from dataclasses import dataclass +from datetime import datetime +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final import httpx @@ -23,7 +26,15 @@ from litellm import token_counter from litellm._internal_context import with_service_target from litellm._logging import verbose_router_logger from litellm.caching.dual_cache import DualCache +from litellm.caching.redis_cache import RedisCache, RedisCircuitBreakerOpenError from litellm.constants import NO_MAX_TOKENS_TPM_RESERVATION_FRACTION +from litellm.litellm_core_utils.internal_call_metadata import ( + IO_TOKEN_RESERVATION_KEYS, + ITPM_CACHE_KEY, + ITPM_RESERVED_KEY, + OTPM_CACHE_KEY, + OTPM_RESERVED_KEY, +) from litellm.litellm_core_utils.token_counter import deployment_openai_model, offload_token_count from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping from litellm.router_utils.routing_read_batch import ROUTER_USAGE_TARGET @@ -39,18 +50,27 @@ else: RoutingArgsTTL: Final = 60 -_io_token_rate_limit_request_kwargs: Final[contextvars.ContextVar[dict[str, object] | None]] = contextvars.ContextVar( - "io_token_rate_limit_request_kwargs", + +@dataclass(frozen=True, slots=True) +class _IoTokenRateLimitRequest: + kwargs: dict[str, object] | None + opens_responses_socket: bool + metadata_channel: str + + +_io_token_rate_limit_request: Final[contextvars.ContextVar[_IoTokenRateLimitRequest | None]] = contextvars.ContextVar( + "io_token_rate_limit_request", default=None, ) -ITPM_RESERVED_KEY: Final = "_litellm_itpm_reserved" -OTPM_RESERVED_KEY: Final = "_litellm_otpm_reserved" -ITPM_CACHE_KEY: Final = "_litellm_itpm_cache_key" -OTPM_CACHE_KEY: Final = "_litellm_otpm_cache_key" - -def set_io_token_rate_limit_request_kwargs(kwargs: dict[str, object] | None, store_in_context: bool = True) -> None: +def set_io_token_rate_limit_request_kwargs( + kwargs: dict[str, object] | None, + store_in_context: bool = True, + *, + opens_responses_socket: bool = False, + metadata_channel: str, +) -> None: # The reservation sentinels are server-only, but `metadata` is caller # controlled on proxy requests. Strip any client-supplied copies here (this # runs before the router stashes its own reservation) so a forged @@ -64,11 +84,14 @@ def set_io_token_rate_limit_request_kwargs(kwargs: dict[str, object] | None, sto # every other deployment overwrite the slot with None instead of the # kwargs; overwriting (rather than skipping) also releases a previous # request's kwargs when a context is reused. - _io_token_rate_limit_request_kwargs.set(kwargs if store_in_context else None) + _io_token_rate_limit_request.set( + _IoTokenRateLimitRequest(kwargs, opens_responses_socket, metadata_channel) if store_in_context else None + ) def get_io_token_rate_limit_request_kwargs() -> dict[str, object] | None: - return _io_token_rate_limit_request_kwargs.get() + request: Final = _io_token_rate_limit_request.get() + return None if request is None else request.kwargs def seconds_until_minute_reset() -> int: @@ -224,27 +247,32 @@ def _otpm_reservation( return default if from_deployment is None else min(from_deployment, default) +def _usage_field(container: object, name: str) -> object: + if is_str_keyed_mapping(container): + return container.get(name) + return getattr(container, name, None) + + +def _cached_prompt_tokens(usage: object) -> int: + reported: Final = ( + _usage_field(_usage_field(usage, "prompt_tokens_details"), "cached_tokens"), + _usage_field(_usage_field(usage, "input_tokens_details"), "cached_tokens"), + _usage_field(usage, "cache_read_input_tokens"), + ) + return next((int(tokens) for tokens in reported if isinstance(tokens, int | float) and tokens), 0) + + def _get_usage_tokens(usage: object) -> tuple[int, int, int]: if usage is None: return 0, 0, 0 if hasattr(usage, "prompt_tokens") or hasattr(usage, "input_tokens"): prompt = int(getattr(usage, "prompt_tokens", None) or getattr(usage, "input_tokens", 0) or 0) completion = int(getattr(usage, "completion_tokens", None) or getattr(usage, "output_tokens", 0) or 0) - cached = 0 - details: object = getattr(usage, "prompt_tokens_details", None) - if details is not None: - cached = int(getattr(details, "cached_tokens", 0) or 0) - if not cached: - cached = int(getattr(usage, "cache_read_input_tokens", 0) or 0) - return prompt, completion, cached + return prompt, completion, _cached_prompt_tokens(usage) if isinstance(usage, dict): prompt = int(usage.get("prompt_tokens") or usage.get("input_tokens") or 0) completion = int(usage.get("completion_tokens") or usage.get("output_tokens") or 0) - details = usage.get("prompt_tokens_details") or {} - cached = int(details.get("cached_tokens", 0) or 0) if isinstance(details, dict) else 0 - if not cached: - cached = int(usage.get("cache_read_input_tokens") or 0) - return prompt, completion, cached + return prompt, completion, _cached_prompt_tokens(usage) return 0, 0, 0 @@ -298,19 +326,36 @@ def _resolve_reconcile_usage_tokens( if isinstance(standard_logging_object, dict): prompt_tokens = int(standard_logging_object.get("prompt_tokens") or 0) completion_tokens = int(standard_logging_object.get("completion_tokens") or 0) - cached_tokens = 0 - metadata: Final = standard_logging_object.get("metadata") - if isinstance(metadata, dict): - cached_tokens = int(metadata.get("cache_read_input_tokens") or 0) + logged_usage: Final = _usage_field(standard_logging_object.get("metadata"), "usage_object") + logged_cached_tokens: Final = _cached_prompt_tokens(logged_usage) # Same rationale as _usage_is_present: a bare total_tokens with no # prompt/completion breakdown can't be split, so it isn't treated # as resolved usage - the reservation is kept instead of refunded. if prompt_tokens or completion_tokens: - return max(0, prompt_tokens - cached_tokens), completion_tokens, True + return max(0, prompt_tokens - logged_cached_tokens), completion_tokens, True return 0, 0, False +def _reservation( + itpm_reserved: int, otpm_reserved: int, itpm_cache_key: str | None, otpm_cache_key: str | None +) -> Mapping[str, object]: + return MappingProxyType( + { + ITPM_RESERVED_KEY: itpm_reserved, + OTPM_RESERVED_KEY: otpm_reserved, + ITPM_CACHE_KEY: itpm_cache_key, + OTPM_CACHE_KEY: otpm_cache_key, + } + ) + + +def _stash_reservation_in_metadata(request: _IoTokenRateLimitRequest | None, reservation: Mapping[str, object]) -> None: + metadata: Final = None if request is None or not request.kwargs else request.kwargs.get(request.metadata_channel) + if isinstance(metadata, dict): + metadata.update(reservation) + + def _key_for_this_minute(reserved_key: str) -> str: counter, _, _ = reserved_key.rpartition(":") return f"{counter}:{get_utc_datetime().strftime('%H-%M')}" @@ -326,30 +371,6 @@ def _reconcile_increments( ) -def _stash_reservation_in_metadata( - request_kwargs: dict[str, object] | None, - *, - itpm_reserved: int, - otpm_reserved: int, - itpm_cache_key: str | None, - otpm_cache_key: str | None, -) -> None: - if not request_kwargs: - return - reservation: Final = { - ITPM_RESERVED_KEY: itpm_reserved, - OTPM_RESERVED_KEY: otpm_reserved, - ITPM_CACHE_KEY: itpm_cache_key, - OTPM_CACHE_KEY: otpm_cache_key, - } - for channel in ("metadata", "litellm_metadata"): - existing = request_kwargs.get(channel) - if isinstance(existing, dict): - existing.update(reservation) - elif channel == "metadata": - request_kwargs[channel] = dict(reservation) - - def _extract_reservation(reservation: Mapping[str, int | str | None]) -> tuple[int, int, str | None, str | None]: itpm_cache_key: Final = reservation.get(ITPM_CACHE_KEY) otpm_cache_key: Final = reservation.get(OTPM_CACHE_KEY) @@ -399,7 +420,7 @@ def _clear_reservation_from_kwargs(kwargs: Mapping[str, object] | None) -> None: for channel in _reservation_channels(kwargs): channel_dict = _as_mutable_mapping(channel) if channel_dict is not None: - for key in (ITPM_RESERVED_KEY, OTPM_RESERVED_KEY, ITPM_CACHE_KEY, OTPM_CACHE_KEY): + for key in IO_TOKEN_RESERVATION_KEYS: channel_dict.pop(key, None) @@ -499,7 +520,8 @@ def io_token_pre_call_check( if itpm_limit is None and otpm_limit is None: return deployment - request_kwargs: Final = get_io_token_rate_limit_request_kwargs() + request: Final = _io_token_rate_limit_request.get() + request_kwargs: Final = None if request is None else request.kwargs _model, served_model = _tokenizer_and_served_models(deployment) estimated_input: Final = _estimate_input_tokens(request_kwargs, model=_model) otpm_to_reserve: Final = ( @@ -546,11 +568,13 @@ def io_token_pre_call_check( raise _stash_reservation_in_metadata( - request_kwargs, - itpm_reserved=itpm_reserved, - otpm_reserved=otpm_reserved, - itpm_cache_key=itpm_key if itpm_limit is not None else None, - otpm_cache_key=otpm_key if otpm_limit is not None else None, + request, + _reservation( + itpm_reserved, + otpm_reserved, + itpm_key if itpm_limit is not None else None, + otpm_key if otpm_limit is not None else None, + ), ) return deployment @@ -565,7 +589,14 @@ async def async_io_token_pre_call_check( if itpm_limit is None and otpm_limit is None: return deployment - request_kwargs: Final = get_io_token_rate_limit_request_kwargs() + request: Final = _io_token_rate_limit_request.get() + request_kwargs: Final = None if request is None else request.kwargs + if request is not None and request.opens_responses_socket: + _stash_reservation_in_metadata( + request, + await async_admit_io_token_turn(dual_cache, deployment, get_utc_datetime(), parent_otel_span), + ) + return deployment _model, served_model = _tokenizer_and_served_models(deployment) estimated_input: Final = await offload_token_count(_estimate_input_tokens)(request_kwargs, model=_model) otpm_to_reserve: Final = ( @@ -618,15 +649,65 @@ async def async_io_token_pre_call_check( raise _stash_reservation_in_metadata( - request_kwargs, - itpm_reserved=itpm_reserved, - otpm_reserved=otpm_reserved, - itpm_cache_key=itpm_key if itpm_limit is not None else None, - otpm_cache_key=otpm_key if otpm_limit is not None else None, + request, + _reservation( + itpm_reserved, + otpm_reserved, + itpm_key if itpm_limit is not None else None, + otpm_key if otpm_limit is not None else None, + ), ) return deployment +async def _async_redis_usage(redis_cache: RedisCache | None, key: str, parent_otel_span: Span | None) -> int | None: + if redis_cache is None: + return None + with contextlib.suppress(RedisCircuitBreakerOpenError): + return await redis_cache.async_get_cache(key=key, parent_otel_span=parent_otel_span) + return None + + +@with_service_target(ROUTER_USAGE_TARGET) +async def _async_shared_usage(dual_cache: DualCache, key: str, parent_otel_span: Span | None) -> int | None: + shared: Final = await _async_redis_usage(dual_cache.redis_cache, key, parent_otel_span) + if shared is not None: + return shared + return await dual_cache.async_get_cache(key=key, local_only=True) + + +async def _raise_if_at_limit( + dual_cache: DualCache, key: str, limit: int | None, limit_label: str, parent_otel_span: Span | None +) -> None: + if limit is None: + return + usage: Final = await _async_shared_usage(dual_cache, key, parent_otel_span) + if usage is not None and usage >= limit: + raise _rate_limit_error(limit_label, limit, usage) + + +@with_service_target(ROUTER_USAGE_TARGET) +async def async_admit_io_token_turn( + dual_cache: DualCache, + deployment: Mapping[str, object], + now: datetime, + parent_otel_span: Span | None = None, +) -> Mapping[str, object]: + configured: Final = dict(deployment) + itpm_limit, otpm_limit = get_deployment_io_token_limits(configured) + if itpm_limit is None and otpm_limit is None: + return MappingProxyType({}) + cache_keys: Final = _get_cache_keys(configured, now.strftime("%H-%M")) + if cache_keys is None: + return MappingProxyType({}) + itpm_key, otpm_key = cache_keys + await _raise_if_at_limit(dual_cache, itpm_key, itpm_limit, "ITPM", parent_otel_span) + await _raise_if_at_limit(dual_cache, otpm_key, otpm_limit, "OTPM", parent_otel_span) + return _reservation( + 0, 0, itpm_key if itpm_limit is not None else None, otpm_key if otpm_limit is not None else None + ) + + @with_service_target(ROUTER_USAGE_TARGET) def io_token_reconcile_success( dual_cache: DualCache, diff --git a/tests/integration/routing/test_responses_websocket_turn_admission.py b/tests/integration/routing/test_responses_websocket_turn_admission.py index 87b56a92979..7b0e14d044f 100644 --- a/tests/integration/routing/test_responses_websocket_turn_admission.py +++ b/tests/integration/routing/test_responses_websocket_turn_admission.py @@ -518,9 +518,7 @@ def test_a_tpm_limit_lowered_while_the_socket_is_open_applies_to_its_next_turn( assert deployment.forwarded_turns() == 1 -def test_a_socket_on_an_otpm_limited_deployment_reserves_once_and_serves_every_turn( - limited: Gateway, cert: tuple[Path, Path] -) -> None: +def test_a_socket_on_an_otpm_limited_deployment_serves_every_turn(limited: Gateway, cert: tuple[Path, Path]) -> None: with responses_peer(cert) as peer, limited.scenario() as scenario: deployment: Final = _native(scenario, peer, otpm=10_000) key: Final = scenario.key(models=[deployment.name]) @@ -536,6 +534,115 @@ def test_a_socket_on_an_otpm_limited_deployment_reserves_once_and_serves_every_t assert deployment.forwarded_turns() == 5 +IO_TOKEN_LIMIT: Final = 10_000 +TOKENS_PER_TURN: Final = MappingProxyType({"itpm": USAGE["input_tokens"], "otpm": USAGE["output_tokens"]}) + + +def _io_tokens_counted(identity: str, kind: str, minute: str) -> int: + with _redis() as redis: + return sum(int(redis.get(key) or 0) for key in redis.scan_iter(f"global_router:{identity}:*:{kind}:{minute}")) + + +def _add_io_tokens(deployment: Deployment, kind: str, minute: str, tokens: int) -> None: + with _redis() as redis: + redis.incrby(f"global_router:{deployment.identity}:openai/{PROVIDER_MODEL}:{kind}:{minute}", tokens) + + +async def _say_until_counted( + connection: websockets.ClientConnection, deployment: Deployment, counted: Callable[[], float], expected: int +) -> tuple[dict[str, JsonValue], ...]: + received: Final = await _say(connection, deployment.name) + await _until(counted, lambda tokens: tokens == expected) + return received + + +@pytest.mark.parametrize("kind", ("otpm", "itpm")) +def test_a_turn_is_refused_once_earlier_turns_fill_the_deployment_io_token_limit( + limited: Gateway, cert: tuple[Path, Path], kind: str +) -> None: + per_turn: Final = TOKENS_PER_TURN[kind] + with responses_peer(cert) as peer, limited.scenario() as scenario: + deployment: Final = _native(scenario, peer, **{kind: IO_TOKEN_LIMIT}) + key: Final = scenario.key(models=[deployment.name]) + minute: Final = _fresh_minute() + counted: Final = partial(_io_tokens_counted, deployment.identity, kind, minute) + + async def session() -> tuple[tuple[dict[str, JsonValue], ...], ...]: + async with _socket(limited, key, deployment.name) as connection: + first: Final = await _say_until_counted(connection, deployment, counted, per_turn) + await asyncio.to_thread(_add_io_tokens, deployment, kind, minute, IO_TOKEN_LIMIT - 5 * per_turn) + answered: Final = tuple( + [ + await _say_until_counted( + connection, deployment, counted, IO_TOKEN_LIMIT - (5 - number) * per_turn + ) + for number in range(2, 6) + ] + ) + return (first, *answered, await _say(connection, deployment.name)) + + turns: Final = asyncio.run(session()) + + assert [completed(received)["status"] for received in turns[:5]] == ["completed"] * 5 + _assert_refused(turns[5]) + assert deployment.forwarded_turns() == 5 + assert held(counted, lambda tokens: tokens == IO_TOKEN_LIMIT, holding=3, seconds=20) == IO_TOKEN_LIMIT + + +def test_a_socket_at_half_its_deployment_otpm_serves_ten_turns(limited: Gateway, cert: tuple[Path, Path]) -> None: + with responses_peer(cert) as peer, limited.scenario() as scenario: + deployment: Final = _native(scenario, peer, otpm=IO_TOKEN_LIMIT) + key: Final = scenario.key(models=[deployment.name]) + minute: Final = _fresh_minute() + _add_io_tokens(deployment, "otpm", minute, IO_TOKEN_LIMIT // 2) + + async def session() -> tuple[tuple[dict[str, JsonValue], ...], ...]: + async with _socket(limited, key, deployment.name) as connection: + return tuple([await _say(connection, deployment.name) for _ in range(10)]) + + turns: Final = asyncio.run(session()) + + assert [completed(received)["status"] for received in turns] == ["completed"] * 10 + assert deployment.forwarded_turns() == 10 + expected: Final = IO_TOKEN_LIMIT // 2 + 10 * USAGE["output_tokens"] + counted: Final = partial(_io_tokens_counted, deployment.identity, "otpm", minute) + assert held(counted, lambda tokens: tokens == expected, holding=3, seconds=20) == expected + + +@pytest.mark.parametrize("relay", ("native", "managed")) +def test_every_turn_counts_its_actual_tokens_toward_the_deployment_itpm_and_otpm( + limited: Gateway, cert: tuple[Path, Path], relay: str +) -> None: + with responses_peer(cert) as peer, limited.scenario() as scenario: + limits: Final = {"itpm": 1_000_000, "otpm": 1_000_000} + deployment: Final = _native(scenario, peer, **limits) if relay == "native" else _managed(scenario, **limits) + key: Final = scenario.key(models=[deployment.name]) + minute: Final = _fresh_minute() + + async def session() -> tuple[tuple[dict[str, JsonValue], ...], ...]: + async with _socket(limited, key, deployment.name) as connection: + return tuple([await _say(connection, deployment.name) for _ in range(3)]) + + turns: Final = asyncio.run(session()) + + usages: Final = tuple(object_value(completed(received)["usage"]) for received in turns) + expected: Final = { + "itpm": sum(int(str(usage["input_tokens"])) for usage in usages), + "otpm": sum(int(str(usage["output_tokens"])) for usage in usages), + } + assert any(tokens > 0 for tokens in expected.values()), usages + counted: Final = { + kind: held( + partial(_io_tokens_counted, deployment.identity, kind, minute), + lambda tokens, kind=kind: tokens == expected[kind], + holding=3, + seconds=20, + ) + for kind in expected + } + assert counted == expected + + def test_usage_based_routing_refuses_a_turn_only_once_earlier_turns_used_up_the_deployment_tpm( usage_routed: Gateway, cert: tuple[Path, Path] ) -> None: diff --git a/tests/unit/responses/test_responses_websocket_all_providers.py b/tests/unit/responses/test_responses_websocket_all_providers.py index ab52e6d29c3..d0e26798090 100644 --- a/tests/unit/responses/test_responses_websocket_all_providers.py +++ b/tests/unit/responses/test_responses_websocket_all_providers.py @@ -8,15 +8,21 @@ Tests that: """ import asyncio +import itertools import json -from collections.abc import Mapping -from typing import Final -from unittest.mock import MagicMock +from collections.abc import AsyncIterator, Mapping +from datetime import datetime, timezone +from functools import partial +from typing import TYPE_CHECKING, Final +from unittest.mock import AsyncMock, MagicMock +import httpx import pytest +import respx from litellm.llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig from litellm.llms.chatgpt.responses.transformation import ChatGPTResponsesAPIConfig +from litellm.llms.custom_llm import CustomLLM from litellm.llms.databricks.responses.transformation import ( DatabricksResponsesAPIConfig, ) @@ -42,6 +48,13 @@ from litellm.llms.volcengine.responses.transformation import ( VolcEngineResponsesAPIConfig, ) from litellm.llms.xai.responses.transformation import XAIResponsesAPIConfig +from litellm.types.utils import GenericStreamingChunk + +if TYPE_CHECKING: + from litellm.caching.dual_cache import DualCache + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.responses.streaming_iterator import ResponsesWsTurnAdmission + from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck class TestResponsesAPIWebSocketSupport: @@ -1591,7 +1604,7 @@ def _admission_refusing_the_next_turn(): from litellm.exceptions import RateLimitError - return AsyncMock(side_effect=[RateLimitError(message="crossed budget", llm_provider="", model=""), None]) + return AsyncMock(side_effect=[RateLimitError(message="crossed budget", llm_provider="", model=""), {}]) class _ClientSocket: @@ -1630,7 +1643,7 @@ class _StalledClientSocket(_ClientSocket): def _admission_crashing_the_next_turn(): from unittest.mock import AsyncMock - return AsyncMock(side_effect=[ValueError("custom check failed"), None]) + return AsyncMock(side_effect=[ValueError("custom check failed"), {}]) def _guard_blocking(word: str): @@ -1655,6 +1668,229 @@ def _guard_recording(bill_refusal: MagicMock): return guard +_OPENED_ON: Final = "global_router:ws-dep:openai/gpt-5-mini:otpm:12-34" +_TURNED_ON: Final = "global_router:ws-dep:openai/gpt-5-mini:otpm:12-35" +_OPENED_AT: Final = datetime(2026, 1, 1, 12, 34, 56, tzinfo=timezone.utc) +_TURNED_AT: Final = datetime(2026, 1, 1, 12, 35, 5, tzinfo=timezone.utc) +_WS_DEPLOYMENT: Final = { + "litellm_params": {"model": "openai/gpt-5-mini", "otpm": 4096}, + "model_info": {"id": "ws-dep"}, +} +_OTHER_USAGE: Final = 10 +_OUTPUT_TOKENS_PER_TURN: Final = 2 +_UPSTREAM_ERROR: Final = json.dumps({"type": "error", "error": {"type": "server_error", "message": "upstream failed"}}) + + +def _turn_completed() -> str: + return json.dumps( + { + "type": "response.completed", + "response": { + "id": "resp_turn", + "status": "completed", + "output": [], + "usage": {"input_tokens": 5, "output_tokens": _OUTPUT_TOKENS_PER_TURN, "total_tokens": 7}, + }, + } + ) + + +async def _socket_opened_on(dual_cache: "DualCache") -> Mapping[str, object]: + from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import async_admit_io_token_turn + + await dual_cache.async_increment_cache(key=_OPENED_ON, value=_OTHER_USAGE, ttl=60) + await dual_cache.async_increment_cache(key=_TURNED_ON, value=_OTHER_USAGE, ttl=60) + return await async_admit_io_token_turn(dual_cache, _WS_DEPLOYMENT, _OPENED_AT) + + +def _turns_finish_at(monkeypatch: pytest.MonkeyPatch, finished_at: datetime) -> None: + monkeypatch.setattr( + "litellm.router_utils.pre_call_checks.io_token_rate_limit_check.get_utc_datetime", lambda: finished_at + ) + + +def _later_turns_admitted_through(dual_cache: "DualCache") -> "ResponsesWsTurnAdmission": + from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import async_admit_io_token_turn + + return partial(async_admit_io_token_turn, dual_cache, _WS_DEPLOYMENT, _TURNED_AT) + + +_ITPM_OPENED_ON: Final = "global_router:ws-dep:openai/gpt-5-mini:itpm:12-34" +_ITPM_DEPLOYMENT: Final = { + "litellm_params": {"model": "openai/gpt-5-mini", "itpm": 40_000}, + "model_info": {"id": "ws-dep"}, +} +_TURN_INPUT_TOKENS: Final = 1_000 + + +def _turn_completed_reading_cache(cached_tokens: int) -> str: + return json.dumps( + { + "type": "response.completed", + "response": { + "id": "resp_cached_turn", + "status": "completed", + "output": [], + "usage": { + "input_tokens": _TURN_INPUT_TOKENS, + "input_tokens_details": {"cached_tokens": cached_tokens}, + "output_tokens": _OUTPUT_TOKENS_PER_TURN, + "total_tokens": _TURN_INPUT_TOKENS + _OUTPUT_TOKENS_PER_TURN, + }, + }, + } + ) + + +def _connection_logged_through(rate_limits: "ModelRateLimitingCheck", opened_with: Mapping[str, object]) -> "Logging": + import time + + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.types.utils import CallTypes + + connection: Final = Logging( + model="gpt-5-mini", + messages=[], + stream=False, + call_type=CallTypes.aresponses_websocket.value, + start_time=time.time(), + litellm_call_id="connection-call", + function_id="connection-call", + dynamic_async_success_callbacks=[rate_limits], + dynamic_async_failure_callbacks=[rate_limits], + ) + connection.update_environment_variables( + litellm_params={"metadata": dict(opened_with), CallTypes.aresponses_websocket.value: True}, + optional_params={}, + ) + return connection + + +def _provider_turn_stream(_request: httpx.Request) -> httpx.Response: + completed: Final = { + "id": "resp_turn", + "object": "response", + "created_at": 1767271505, + "status": "completed", + "model": "gpt-5-mini", + "output": [], + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + "usage": {"input_tokens": 5, "output_tokens": _OUTPUT_TOKENS_PER_TURN, "total_tokens": 7}, + } + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + text="".join( + f"event: {kind}\ndata: {json.dumps({'type': kind, 'response': completed})}\n\n" + for kind in ("response.created", "response.completed") + ), + ) + + +class _ProviderResponsesSocket: + """A provider's native Responses socket that answers each ``response.create`` with a completed turn.""" + + def __init__(self) -> None: + self.sent: Final[list[dict[str, object]]] = [] + self._replies: Final[asyncio.Queue[str]] = asyncio.Queue() + + def connect(self, _url: str, **_kwargs: object) -> "_ProviderResponsesSocket": + return self + + async def __aenter__(self) -> "_ProviderResponsesSocket": + return self + + async def __aexit__(self, *_exc_info: object) -> None: + return None + + async def send(self, message: str) -> None: + self.sent.append(json.loads(message)) + await self._replies.put(_turn_completed()) + + async def recv(self, decode: bool = False) -> str: + return await self._replies.get() + + async def close(self) -> None: + return None + + +class _TwoOutputTokenTurns(CustomLLM): + def __init__(self, failing_turns: frozenset[int] = frozenset()) -> None: + super().__init__() + self._failing_turns: Final = failing_turns + self._turns: Final = itertools.count(1) + + async def astreaming(self, *args: object, **kwargs: object) -> AsyncIterator[GenericStreamingChunk]: + if next(self._turns) in self._failing_turns: + raise RuntimeError("upstream failed") + yield GenericStreamingChunk( + text="hi", + is_finished=True, + finish_reason="stop", + index=0, + tool_use=None, + usage={"prompt_tokens": 5, "completion_tokens": _OUTPUT_TOKENS_PER_TURN, "total_tokens": 7}, + ) + + +async def _run_managed_relay( + monkeypatch: pytest.MonkeyPatch, + provider: CustomLLM, + rate_limits: "ModelRateLimitingCheck", + frames: tuple[str, ...], + turn_admission: "ResponsesWsTurnAdmission | None", + opened_with: Mapping[str, object], +) -> _ClientSocket: + import litellm + + monkeypatch.setattr(litellm, "custom_provider_map", [{"provider": "ws-turn-llm", "custom_handler": provider}]) + return await _relay_turns_to(monkeypatch, "ws-turn-llm/turns", rate_limits, frames, turn_admission, opened_with) + + +async def _relay_turns_to( + monkeypatch: pytest.MonkeyPatch, + model: str, + rate_limits: "ModelRateLimitingCheck", + frames: tuple[str, ...], + turn_admission: "ResponsesWsTurnAdmission | None", + opened_with: Mapping[str, object], + **connection: object, +) -> _ClientSocket: + import litellm + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER + from litellm.responses.streaming_iterator import ManagedResponsesWebSocketHandler, drain_logging_tasks + + monkeypatch.setattr(litellm, "callbacks", [rate_limits]) + first, *later = frames + websocket: Final = _ClientSocket(*later) + handler: Final = ManagedResponsesWebSocketHandler( + websocket=websocket, + model=model, + logging_obj=Logging( + model=model, + messages=[], + stream=True, + call_type="aresponses", + start_time=0, + litellm_call_id="test-id", + function_id="test-func", + ), + litellm_metadata={"user_api_key_hash": "k1", **opened_with}, + first_message=first, + turn_admission=turn_admission, + **connection, + ) + + await handler.run() + await drain_logging_tasks() + await asyncio.sleep(0) + await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10.0) + return websocket + + def _errors(websocket: _ClientSocket) -> list[tuple[object, object]]: return [(frame.get("status"), frame["error"]) for frame in websocket.sent if frame.get("type") == "error"] @@ -1722,6 +1958,195 @@ class TestWebSocketTurnAdmission: assert isinstance(failure, RateLimitError) assert "crossed budget" in str(failure) + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("first_turn_end", "first_turn_tokens"), + [ + pytest.param(_turn_completed(), _OUTPUT_TOKENS_PER_TURN, id="first_turn_completed"), + pytest.param(_UPSTREAM_ERROR, 0, id="first_turn_failed"), + ], + ) + async def test_native_relay_counts_every_turn_on_the_minute_it_finishes( + self, first_turn_end: str, first_turn_tokens: int, monkeypatch: pytest.MonkeyPatch + ) -> None: + import websockets.exceptions # noqa: F401 (lazy submodule must be importable) + + from litellm.caching.dual_cache import DualCache + from litellm.responses.streaming_iterator import drain_logging_tasks + from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck + + _turns_finish_at(monkeypatch, _TURNED_AT) + dual_cache: Final = DualCache() + opened_with: Final = await _socket_opened_on(dual_cache) + first, second, third = _turn_frames() + backend_ws: Final = MagicMock() + backend_ws.send = AsyncMock() + backend_ws.recv = AsyncMock( + side_effect=[first_turn_end, _turn_completed(), _turn_completed(), Exception("stop")] + ) + handler: Final = _make_streaming( + websocket=_ClientSocket(second, third), + backend_ws=backend_ws, + request_data={}, + logging_obj=_connection_logged_through(ModelRateLimitingCheck(dual_cache), opened_with), + first_message=first, + turn_admission=_later_turns_admitted_through(dual_cache), + ) + + await handler.client_to_backend() + await handler.backend_to_client() + await drain_logging_tasks() + + assert await dual_cache.async_get_cache(key=_OPENED_ON) == _OTHER_USAGE + assert ( + await dual_cache.async_get_cache(key=_TURNED_ON) + == _OTHER_USAGE + first_turn_tokens + 2 * _OUTPUT_TOKENS_PER_TURN + ) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("cached_tokens", "counted_input_tokens"), + [ + pytest.param(800, _TURN_INPUT_TOKENS - 800, id="cached_input"), + pytest.param(0, _TURN_INPUT_TOKENS, id="uncached_input"), + ], + ) + async def test_native_relay_leaves_a_turn_s_cached_input_out_of_the_deployment_s_itpm( + self, cached_tokens: int, counted_input_tokens: int, monkeypatch: pytest.MonkeyPatch + ) -> None: + import websockets.exceptions # noqa: F401 (lazy submodule must be importable) + + from litellm.caching.dual_cache import DualCache + from litellm.responses.streaming_iterator import drain_logging_tasks + from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import async_admit_io_token_turn + from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck + + _turns_finish_at(monkeypatch, _OPENED_AT) + dual_cache: Final = DualCache() + opened_with: Final = await async_admit_io_token_turn(dual_cache, _ITPM_DEPLOYMENT, _OPENED_AT) + first, _, _ = _turn_frames() + backend_ws: Final = MagicMock() + backend_ws.send = AsyncMock() + backend_ws.recv = AsyncMock(side_effect=[_turn_completed_reading_cache(cached_tokens), Exception("stop")]) + handler: Final = _make_streaming( + websocket=_ClientSocket(), + backend_ws=backend_ws, + request_data={}, + logging_obj=_connection_logged_through(ModelRateLimitingCheck(dual_cache), opened_with), + first_message=first, + ) + + await handler.client_to_backend() + await handler.backend_to_client() + await drain_logging_tasks() + + assert await dual_cache.async_get_cache(key=_ITPM_OPENED_ON) == counted_input_tokens + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("failing_turns", "first_turn_tokens"), + [ + pytest.param(frozenset(), _OUTPUT_TOKENS_PER_TURN, id="first_turn_completed"), + pytest.param(frozenset({1}), 0, id="first_turn_failed"), + ], + ) + async def test_managed_relay_counts_every_turn_on_the_minute_it_finishes( + self, monkeypatch: pytest.MonkeyPatch, failing_turns: frozenset[int], first_turn_tokens: int + ) -> None: + from litellm.caching.dual_cache import DualCache + from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck + + _turns_finish_at(monkeypatch, _TURNED_AT) + dual_cache: Final = DualCache() + opened_with: Final = await _socket_opened_on(dual_cache) + + await _run_managed_relay( + monkeypatch, + _TwoOutputTokenTurns(failing_turns), + ModelRateLimitingCheck(dual_cache), + _turn_frames(), + _later_turns_admitted_through(dual_cache), + opened_with, + ) + + assert await dual_cache.async_get_cache(key=_OPENED_ON) == _OTHER_USAGE + assert ( + await dual_cache.async_get_cache(key=_TURNED_ON) + == _OTHER_USAGE + first_turn_tokens + 2 * _OUTPUT_TOKENS_PER_TURN + ) + + @pytest.mark.asyncio + async def test_managed_relay_sends_the_provider_only_the_metadata_each_turn_carries( + self, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter + ) -> None: + import litellm + from litellm.caching.dual_cache import DualCache + from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + _turns_finish_at(monkeypatch, _TURNED_AT) + provider: Final = respx_mock.post("https://api.openai.com/v1/responses").mock(side_effect=_provider_turn_stream) + client_metadata: Final = {"session": "abc", "team": "search"} + dual_cache: Final = DualCache() + + await _relay_turns_to( + monkeypatch, + "openai/gpt-5-mini", + ModelRateLimitingCheck(dual_cache), + tuple( + json.dumps({"type": "response.create", "input": word, "metadata": client_metadata}) + for word in ("one", "two") + ), + _later_turns_admitted_through(dual_cache), + await _socket_opened_on(dual_cache), + api_key="sk-fake", + ) + + assert [json.loads(call.request.read()).get("metadata") for call in provider.calls] == [ + client_metadata, + client_metadata, + ] + assert await dual_cache.async_get_cache(key=_TURNED_ON) == _OTHER_USAGE + 2 * _OUTPUT_TOKENS_PER_TURN + + @pytest.mark.asyncio + async def test_native_relay_sends_the_provider_only_the_metadata_the_socket_opened_with( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + from unittest.mock import patch + + from litellm import Router + + monkeypatch.setenv("LITELLM_RUST", "false") + provider: Final = _ProviderResponsesSocket() + client_metadata: Final = {"session": "abc", "team": "search"} + first, second, _ = _turn_frames() + router: Final = Router( + model_list=[ + { + "model_name": "ws-io", + "litellm_params": { + "model": "openai/gpt-5-mini", + "api_key": "sk-fake", + "itpm": 40_000, + "otpm": 4096, + }, + "model_info": {"id": "ws-dep"}, + } + ], + optional_pre_call_checks=["enforce_model_rate_limits"], + ) + + with patch("websockets.connect", provider.connect): + await router._aresponses_websocket( + model="ws-io", + websocket=_ClientSocket(second), + first_message=first, + metadata=dict(client_metadata), + ) + + assert [frame.get("metadata") for frame in provider.sent] == [client_metadata, client_metadata] + @pytest.mark.asyncio async def test_managed_relay_skips_a_later_turn_the_deployment_refuses(self, monkeypatch: pytest.MonkeyPatch): from unittest.mock import AsyncMock @@ -1875,7 +2300,7 @@ class TestWebSocketTurnAdmission: request_data={}, first_message=first, frame_guard=_guard_recording(bill_refusal), - turn_admission=AsyncMock(side_effect=[refusal, None]), + turn_admission=AsyncMock(side_effect=[refusal, {}]), ) close_time_failure: Final = await handler.bidirectional_forward() @@ -1904,7 +2329,7 @@ class TestWebSocketTurnAdmission: request_data={}, first_message=first, frame_guard=_guard_recording(bill_refusal), - turn_admission=AsyncMock(side_effect=[refusal, None]), + turn_admission=AsyncMock(side_effect=[refusal, {}]), ) relay: Final = asyncio.ensure_future(handler.bidirectional_forward()) await asyncio.wait_for(client.send_started.wait(), timeout=1) @@ -1955,7 +2380,7 @@ class TestWebSocketTurnAdmission: ), first_message=first, frame_guard=_guard_recording(bill_refusal), - turn_admission=AsyncMock(side_effect=[refusal, None]), + turn_admission=AsyncMock(side_effect=[refusal, {}]), ) close_time_failure: Final = await handler.run() diff --git a/tests/unit/test_router/test_io_token_rate_limits.py b/tests/unit/test_router/test_io_token_rate_limits.py index e5ff4531f84..819c6b9d4f5 100644 --- a/tests/unit/test_router/test_io_token_rate_limits.py +++ b/tests/unit/test_router/test_io_token_rate_limits.py @@ -4,14 +4,19 @@ Tests for separate ITPM/OTPM deployment rate limits (enforce_model_rate_limits). import asyncio from collections.abc import Awaitable, Callable, Mapping -from datetime import datetime +from datetime import datetime, timezone from typing import Final +from unittest.mock import AsyncMock, MagicMock import pytest +from fakeredis import FakeRedis, FakeServer +from fakeredis.aioredis import FakeRedis as FakeAsyncRedis import litellm from litellm import Router from litellm.caching.dual_cache import DualCache +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.caching.redis_cache import RedisCache, RedisCircuitBreakerOpenError from litellm.constants import NO_MAX_TOKENS_TPM_RESERVATION_FRACTION from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER @@ -22,6 +27,7 @@ from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import ( OTPM_RESERVED_KEY, _explicit_max_tokens, _reservation_value, + async_admit_io_token_turn, async_io_token_pre_call_check, async_io_token_reconcile_success, build_io_token_rate_limit_headers, @@ -35,6 +41,7 @@ from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import ( from litellm.router_utils.pre_call_checks.model_rate_limit_check import ( ModelRateLimitingCheck, ) +from litellm.types.llms.openai import InputTokensDetails, ResponseAPIUsage, ResponsesAPIResponse from litellm.types.router import RouterCacheEnum from litellm.types.utils import ModelResponse, Usage from litellm.utils import get_utc_datetime @@ -115,7 +122,7 @@ class TestModelRateLimitingCheckIOTokens: "max_tokens": 10, "metadata": {}, } - set_io_token_rate_limit_request_kwargs(request_kwargs) + set_io_token_rate_limit_request_kwargs(request_kwargs, metadata_channel="metadata") await check.async_pre_call_check(deployment) minute = get_utc_datetime().strftime("%H-%M") @@ -173,7 +180,8 @@ class TestModelRateLimitingCheckIOTokens: ], "max_tokens": 10, "metadata": {}, - } + }, + metadata_channel="metadata", ) with pytest.raises(litellm.RateLimitError) as exc_info: @@ -203,7 +211,8 @@ class TestModelRateLimitingCheckIOTokens: "messages": [{"role": "user", "content": "hi"}], "max_tokens": max_tokens, "metadata": {}, - } + }, + metadata_channel="metadata", ) async def _attempt(): @@ -255,7 +264,8 @@ class TestModelRateLimitingCheckIOTokens: { "max_tokens": 5, "metadata": {}, - } + }, + metadata_channel="metadata", ) async def _attempt(): @@ -320,7 +330,7 @@ class TestModelRateLimitingCheckIOTokens: } request_kwargs = {"max_tokens": 5, "metadata": {}} - set_io_token_rate_limit_request_kwargs(request_kwargs) + set_io_token_rate_limit_request_kwargs(request_kwargs, metadata_channel="metadata") await check.async_pre_call_check(deployment) minute = get_utc_datetime().strftime("%H-%M") @@ -363,15 +373,15 @@ class TestModelRateLimitingCheckIOTokens: "model_name": "opus", } request_kwargs = {"max_tokens": 5, "metadata": {}} - set_io_token_rate_limit_request_kwargs(request_kwargs) + set_io_token_rate_limit_request_kwargs(request_kwargs, metadata_channel="metadata") await check.async_pre_call_check(deployment) # Second zero-estimate call still fits within the itpm=2 limit. - set_io_token_rate_limit_request_kwargs({"max_tokens": 5, "metadata": {}}) + set_io_token_rate_limit_request_kwargs({"max_tokens": 5, "metadata": {}}, metadata_channel="metadata") await check.async_pre_call_check(deployment) # A third exceeds the limit and is rejected. - set_io_token_rate_limit_request_kwargs({"max_tokens": 5, "metadata": {}}) + set_io_token_rate_limit_request_kwargs({"max_tokens": 5, "metadata": {}}, metadata_channel="metadata") with pytest.raises(litellm.RateLimitError): await check.async_pre_call_check(deployment) @@ -392,7 +402,7 @@ class TestModelRateLimitingCheckIOTokens: "max_tokens": 0, "metadata": {}, } - set_io_token_rate_limit_request_kwargs(zero_output_kwargs) + set_io_token_rate_limit_request_kwargs(zero_output_kwargs, metadata_channel="metadata") await check.async_pre_call_check(deployment) zero_output_otpm_key = zero_output_kwargs["metadata"][OTPM_CACHE_KEY] @@ -404,7 +414,7 @@ class TestModelRateLimitingCheckIOTokens: "max_tokens": 5, "metadata": {}, } - set_io_token_rate_limit_request_kwargs(normal_output_kwargs) + set_io_token_rate_limit_request_kwargs(normal_output_kwargs, metadata_channel="metadata") await check.async_pre_call_check(deployment) normal_output_otpm_key = normal_output_kwargs["metadata"][OTPM_CACHE_KEY] @@ -432,7 +442,7 @@ class TestModelRateLimitingCheckIOTokens: "model_name": "no-max-tokens", } request_kwargs: Final = {"messages": [{"role": "user", "content": "hello"}], "metadata": {}} - set_io_token_rate_limit_request_kwargs(request_kwargs) + set_io_token_rate_limit_request_kwargs(request_kwargs, metadata_channel="metadata") if use_sync_check: check.pre_call_check(deployment) @@ -462,7 +472,7 @@ class TestModelRateLimitingCheckIOTokens: async def _attempt() -> dict[str, object] | None: request_kwargs: Final = {"messages": [{"role": "user", "content": "hi"}], "metadata": {}} - set_io_token_rate_limit_request_kwargs(request_kwargs) + set_io_token_rate_limit_request_kwargs(request_kwargs, metadata_channel="metadata") try: await check.async_pre_call_check(deployment) except litellm.RateLimitError: @@ -493,7 +503,7 @@ class TestModelRateLimitingCheckIOTokens: "model_name": "roomy", } request_kwargs: Final = {"messages": [{"role": "user", "content": "hello"}], "metadata": {}} - set_io_token_rate_limit_request_kwargs(request_kwargs) + set_io_token_rate_limit_request_kwargs(request_kwargs, metadata_channel="metadata") await check.async_pre_call_check(deployment) @@ -507,7 +517,7 @@ class TestModelRateLimitingCheckIOTokens: async def _reserved(model_info: dict[str, str]) -> int: request_kwargs: Final = {"messages": [{"role": "user", "content": "hello"}], "metadata": {}} - set_io_token_rate_limit_request_kwargs(request_kwargs) + set_io_token_rate_limit_request_kwargs(request_kwargs, metadata_channel="metadata") deployment: Final = { "litellm_params": {"model": "azure/litellm-io-custom-deployment", "otpm": otpm_limit}, "model_info": model_info, @@ -536,7 +546,7 @@ class TestModelRateLimitingCheckIOTokens: } async def _pre_call(request_kwargs: dict[str, object]) -> None: - set_io_token_rate_limit_request_kwargs(request_kwargs) + set_io_token_rate_limit_request_kwargs(request_kwargs, metadata_channel="metadata") if use_sync_check: check.pre_call_check(deployment) else: @@ -563,12 +573,14 @@ class TestModelRateLimitingCheckIOTokens: "model_name": "opus", } request_kwargs: Final = {"messages": [{"role": "user", "content": "hello"}], "max_tokens": 100, "metadata": {}} - set_io_token_rate_limit_request_kwargs(request_kwargs) + set_io_token_rate_limit_request_kwargs(request_kwargs, metadata_channel="metadata") await check.async_pre_call_check(deployment) assert request_kwargs["metadata"][OTPM_RESERVED_KEY] == 100 - set_io_token_rate_limit_request_kwargs({"messages": [{"role": "user", "content": "hi"}], "metadata": {}}) + set_io_token_rate_limit_request_kwargs( + {"messages": [{"role": "user", "content": "hi"}], "metadata": {}}, metadata_channel="metadata" + ) with pytest.raises(litellm.RateLimitError): await check.async_pre_call_check(deployment) @@ -585,7 +597,7 @@ class TestModelRateLimitingCheckIOTokens: with_cap: Final = {"messages": [{"role": "user", "content": "hello"}], "max_tokens": 300, "metadata": {}} for request_kwargs in (without_cap, with_cap): - set_io_token_rate_limit_request_kwargs(request_kwargs) + set_io_token_rate_limit_request_kwargs(request_kwargs, metadata_channel="metadata") if use_sync_check: check.pre_call_check(deployment) else: @@ -602,7 +614,7 @@ class TestModelRateLimitingCheckIOTokens: async def reserved(deployment_cap: Mapping[str, int], request_shape: Mapping[str, object]) -> int: request_kwargs: Final = {**request_shape, "metadata": {}} - set_io_token_rate_limit_request_kwargs(request_kwargs) + set_io_token_rate_limit_request_kwargs(request_kwargs, metadata_channel="metadata") await check.async_pre_call_check( { "litellm_params": {"model": MANTLE_OPUS, "otpm": 4000, **deployment_cap}, @@ -631,7 +643,7 @@ class TestModelRateLimitingCheckIOTokens: async def reserved(deployment_cap: Mapping[str, int]) -> int: request_kwargs: Final = {"messages": [{"role": "user", "content": "hello"}], "metadata": {}} - set_io_token_rate_limit_request_kwargs(request_kwargs) + set_io_token_rate_limit_request_kwargs(request_kwargs, metadata_channel="metadata") deployment: Final = { "litellm_params": {"model": UNMAPPED_MODEL, "otpm": otpm_limit, **deployment_cap}, "model_info": {"id": f"io-deployment-cap-above-default-{deployment_cap}"}, @@ -667,7 +679,7 @@ class TestModelRateLimitingCheckIOTokens: "max_tokens": 10, "metadata": {}, } - set_io_token_rate_limit_request_kwargs(request_kwargs) + set_io_token_rate_limit_request_kwargs(request_kwargs, metadata_channel="metadata") check.pre_call_check(deployment) minute = get_utc_datetime().strftime("%H-%M") @@ -803,7 +815,7 @@ class TestModelRateLimitingCheckIOTokens: # The retry proceeds to stash deployment B's own reservation on the # same dict; it starts clean, unaffected by A's cleared sentinels. - set_io_token_rate_limit_request_kwargs(shared_kwargs) + set_io_token_rate_limit_request_kwargs(shared_kwargs, metadata_channel="metadata") itpm_key_b = f"global_router:io-retry-b:bedrock_mantle/test-b:itpm:{minute}" shared_kwargs["metadata"][ITPM_RESERVED_KEY] = 4 shared_kwargs["metadata"][ITPM_CACHE_KEY] = itpm_key_b @@ -819,7 +831,7 @@ class TestModelRateLimitingCheckIOTokens: "litellm_metadata": {OTPM_RESERVED_KEY: 7}, "litellm_params": {"metadata": {OTPM_CACHE_KEY: "attacker:key:otpm:00-00"}}, } - set_io_token_rate_limit_request_kwargs(forged) + set_io_token_rate_limit_request_kwargs(forged, metadata_channel="metadata") stored = get_io_token_rate_limit_request_kwargs() assert ITPM_RESERVED_KEY not in stored["metadata"] @@ -845,7 +857,7 @@ class TestModelRateLimitingCheckIOTokens: }, } # The router sanitizes the request kwargs before the call runs. - set_io_token_rate_limit_request_kwargs(kwargs) + set_io_token_rate_limit_request_kwargs(kwargs, metadata_channel="metadata") response = ModelResponse( choices=[{"message": {"role": "assistant", "content": "ok"}, "index": 0, "finish_reason": "stop"}], usage=Usage(prompt_tokens=1, completion_tokens=1, total_tokens=2), @@ -880,7 +892,8 @@ class TestModelRateLimitingCheckIOTokens: "messages": [{"role": "user", "content": "hello world"}], "max_tokens": 5, "metadata": {}, - } + }, + metadata_channel="metadata", ) with pytest.raises(RuntimeError): @@ -945,7 +958,7 @@ class TestModelRateLimitingCheckIOTokens: "max_tokens": 5, "metadata": {}, } - set_io_token_rate_limit_request_kwargs(request_kwargs) + set_io_token_rate_limit_request_kwargs(request_kwargs, metadata_channel="metadata") result = await async_io_token_pre_call_check(dual_cache, deployment) @@ -1147,7 +1160,7 @@ class TestModelRateLimitingCheckIOTokens: "max_tokens": 5, "metadata": {}, } - set_io_token_rate_limit_request_kwargs(request_kwargs) + set_io_token_rate_limit_request_kwargs(request_kwargs, metadata_channel="metadata") with caplog.at_level(logging.WARNING, logger="LiteLLM Router"): with pytest.raises(litellm.RateLimitError): @@ -1495,15 +1508,15 @@ class TestContextSlotRetention: "messages": [{"role": "user", "content": "x" * 1000}], "metadata": {ITPM_RESERVED_KEY: 999, ITPM_CACHE_KEY: "forged"}, } - set_io_token_rate_limit_request_kwargs(kwargs, store_in_context=False) + set_io_token_rate_limit_request_kwargs(kwargs, store_in_context=False, metadata_channel="metadata") assert get_io_token_rate_limit_request_kwargs() is None assert ITPM_RESERVED_KEY not in kwargs["metadata"] assert ITPM_CACHE_KEY not in kwargs["metadata"] - set_io_token_rate_limit_request_kwargs(kwargs, store_in_context=True) + set_io_token_rate_limit_request_kwargs(kwargs, store_in_context=True, metadata_channel="metadata") assert get_io_token_rate_limit_request_kwargs() is kwargs - set_io_token_rate_limit_request_kwargs(kwargs, store_in_context=False) + set_io_token_rate_limit_request_kwargs(kwargs, store_in_context=False, metadata_channel="metadata") assert get_io_token_rate_limit_request_kwargs() is None @pytest.mark.asyncio @@ -1516,7 +1529,7 @@ class TestContextSlotRetention: } ] ) - set_io_token_rate_limit_request_kwargs(None) + set_io_token_rate_limit_request_kwargs(None, metadata_channel="metadata") kwargs = {"messages": [{"role": "user", "content": "hello"}], "metadata": {}} deployment = router.get_deployment_by_model_group_name("plain") assert deployment is not None @@ -1534,7 +1547,7 @@ class TestContextSlotRetention: ], optional_pre_call_checks=["enforce_model_rate_limits"], ) - set_io_token_rate_limit_request_kwargs(None) + set_io_token_rate_limit_request_kwargs(None, metadata_channel="metadata") kwargs = {"messages": [{"role": "user", "content": "hello"}], "metadata": {}} deployment = router.get_deployment_by_model_group_name("limited") assert deployment is not None @@ -1551,7 +1564,7 @@ async def test_custom_azure_deployment_name_reserves_itpm_on_its_base_models_tok "max_tokens": 5, "metadata": metadata, } - set_io_token_rate_limit_request_kwargs(request_kwargs) + set_io_token_rate_limit_request_kwargs(request_kwargs, metadata_channel="metadata") await async_io_token_pre_call_check(DualCache(), deployment) return metadata[ITPM_RESERVED_KEY] @@ -1571,3 +1584,278 @@ async def test_custom_azure_deployment_name_reserves_itpm_on_its_base_models_tok ) assert named == served + + +def _reply_using(prompt_tokens: int, completion_tokens: int) -> ModelResponse: + return ModelResponse( + choices=[{"message": {"role": "assistant", "content": "hi"}, "index": 0, "finish_reason": "stop"}], + usage=Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + ), + ) + + +def _proxy_success_kwargs(reservation: Mapping[str, object]) -> dict[str, object]: + return { + "litellm_params": {"metadata": {**reservation, "user_api_key_hash": "hashed-key"}}, + "standard_logging_object": { + "hidden_params": {"litellm_model_name": "openai/gpt-5-mini"}, + "metadata": {"user_api_key_hash": "hashed-key"}, + }, + } + + +async def _deliver_success( + check: ModelRateLimitingCheck, delivery: str, kwargs: dict[str, object], reply: object +) -> None: + if delivery == "async": + await check.async_log_success_event(kwargs, reply, None, None) + return + check.log_success_event(kwargs, reply, None, None) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "deliveries", + [("async",), ("sync",), ("async", "async"), ("async", "sync"), ("sync", "async"), ("sync", "sync")], +) +async def test_success_settles_a_reservation_held_only_in_litellm_params_metadata_exactly_once( + deliveries: tuple[str, ...], +) -> None: + dual_cache: Final = DualCache() + check: Final = ModelRateLimitingCheck(dual_cache=dual_cache) + itpm_key: Final = "global_router:proxy-shape:openai/gpt-5-mini:itpm:12-34" + otpm_key: Final = "global_router:proxy-shape:openai/gpt-5-mini:otpm:12-34" + await dual_cache.async_increment_cache(key=itpm_key, value=40, ttl=60) + await dual_cache.async_increment_cache(key=otpm_key, value=4096, ttl=60) + kwargs: Final = _proxy_success_kwargs( + {ITPM_RESERVED_KEY: 40, OTPM_RESERVED_KEY: 4096, ITPM_CACHE_KEY: itpm_key, OTPM_CACHE_KEY: otpm_key} + ) + + for delivery in deliveries: + await _deliver_success(check, delivery, kwargs, _reply_using(prompt_tokens=7, completion_tokens=3)) + + assert await dual_cache.async_get_cache(key=itpm_key) == 7 + assert await dual_cache.async_get_cache(key=otpm_key) == 3 + + +_TURN_AT: Final = datetime(2026, 1, 1, 12, 34, 56, tzinfo=timezone.utc) +_ACTUAL_TOKENS: Final = {"itpm": 5, "otpm": 2} + + +def _turn_deployment(kind: str) -> dict[str, object]: + return {"litellm_params": {"model": "openai/gpt-5-mini", kind: 100}, "model_info": {"id": f"ws-turn-{kind}"}} + + +def _turn_key(kind: str, minute: str = "12-34") -> str: + return f"global_router:ws-turn-{kind}:openai/gpt-5-mini:{kind}:{minute}" + + +@pytest.fixture +def settled_on_the_turn_s_minute(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + "litellm.router_utils.pre_call_checks.io_token_rate_limit_check.get_utc_datetime", lambda: _TURN_AT + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["itpm", "otpm"]) +@pytest.mark.parametrize("used", [100, 101, 250]) +async def test_turn_admission_refuses_once_the_minutes_tokens_reach_the_limit(kind: str, used: int) -> None: + dual_cache: Final = DualCache() + await dual_cache.async_increment_cache(key=_turn_key(kind), value=used, ttl=60) + + with pytest.raises(litellm.RateLimitError, match=f"{kind.upper()} limit=100, current usage={used}"): + await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), _TURN_AT) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("settled_on_the_turn_s_minute") +@pytest.mark.parametrize("kind", ["itpm", "otpm"]) +@pytest.mark.parametrize("used", [0, 1, 99]) +async def test_an_admitted_turn_reserves_nothing_and_counts_its_actual_tokens(kind: str, used: int) -> None: + dual_cache: Final = DualCache() + await dual_cache.async_increment_cache(key=_turn_key(kind), value=used, ttl=60) + actual: Final = _ACTUAL_TOKENS + + reservation: Final = await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), _TURN_AT) + counter_key: Final = reservation[ITPM_CACHE_KEY if kind == "itpm" else OTPM_CACHE_KEY] + assert isinstance(counter_key, str) + assert await dual_cache.async_get_cache(key=counter_key) == used + await async_io_token_reconcile_success( + dual_cache, _proxy_success_kwargs(reservation), _reply_using(actual["itpm"], actual["otpm"]) + ) + + assert await dual_cache.async_get_cache(key=counter_key) == used + actual[kind] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "deployment", + [ + {"litellm_params": {"model": "openai/gpt-5-mini", "tpm": 1}, "model_info": {"id": "ws-turn-unlimited"}}, + {"litellm_params": {"model": "openai/gpt-5-mini", "otpm": 1}}, + ], +) +async def test_a_turn_without_io_token_counters_is_admitted_with_nothing_to_settle( + deployment: dict[str, object], +) -> None: + assert await async_admit_io_token_turn(DualCache(), deployment, _TURN_AT) == {} + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("settled_on_the_turn_s_minute") +@pytest.mark.parametrize("kind", ["itpm", "otpm"]) +@pytest.mark.parametrize("used_by_others", [0, 1, 99]) +async def test_a_socket_open_reserves_nothing_and_its_first_turn_counts_its_actual_tokens_on_its_own_minute( + kind: str, used_by_others: int +) -> None: + dual_cache: Final = DualCache() + await dual_cache.async_increment_cache(key=_turn_key(kind), value=used_by_others, ttl=60) + socket_open: Final[dict[str, object]] = {"metadata": {}} + set_io_token_rate_limit_request_kwargs(socket_open, opens_responses_socket=True, metadata_channel="metadata") + + assert await async_io_token_pre_call_check(dual_cache, _turn_deployment(kind)) == _turn_deployment(kind) + assert await dual_cache.async_get_cache(key=_turn_key(kind)) == used_by_others + + opened_with: Final = socket_open["metadata"] + assert isinstance(opened_with, dict) + await async_io_token_reconcile_success( + dual_cache, _proxy_success_kwargs(opened_with), _reply_using(_ACTUAL_TOKENS["itpm"], _ACTUAL_TOKENS["otpm"]) + ) + + assert await dual_cache.async_get_cache(key=_turn_key(kind)) == used_by_others + _ACTUAL_TOKENS[kind] + + +class _RedisCacheOnFakeServer(RedisCache): + def __init__(self, server: FakeServer) -> None: + super().__init__(host="io-token-turn-redis", connection_pool=FakeRedis(server=server).connection_pool) + self._fake_async_client: Final = FakeAsyncRedis(server=server) + + def init_async_client(self) -> FakeAsyncRedis: + return self._fake_async_client + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["itpm", "otpm"]) +async def test_a_turn_is_refused_on_this_instance_s_count_when_reading_the_shared_counter_fails( + kind: str, +) -> None: + server: Final = FakeServer() + dual_cache: Final = DualCache(in_memory_cache=InMemoryCache(), redis_cache=_RedisCacheOnFakeServer(server)) + server.connected = False + await dual_cache.in_memory_cache.async_increment(key=_turn_key(kind), value=100) + + with pytest.raises(litellm.RateLimitError, match=f"{kind.upper()} limit=100, current usage=100"): + await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), _TURN_AT) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["itpm", "otpm"]) +@pytest.mark.parametrize("delivery", ["async", "sync"]) +async def test_a_turn_finishing_in_the_next_minute_counts_its_actual_tokens_on_that_minute( + kind: str, delivery: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr( + "litellm.router_utils.pre_call_checks.io_token_rate_limit_check.get_utc_datetime", + lambda: datetime(2026, 1, 1, 12, 35, 10, tzinfo=timezone.utc), + ) + dual_cache: Final = DualCache() + reservation: Final = await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), _TURN_AT) + + await _deliver_success( + ModelRateLimitingCheck(dual_cache=dual_cache), + delivery, + _proxy_success_kwargs(reservation), + _reply_using(_ACTUAL_TOKENS["itpm"], _ACTUAL_TOKENS["otpm"]), + ) + + assert [await dual_cache.async_get_cache(key=_turn_key(kind, minute)) for minute in ("12-34", "12-35")] == [ + None, + _ACTUAL_TOKENS[kind], + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["itpm", "otpm"]) +async def test_a_turn_is_admitted_once_the_shared_counter_drops_below_the_limit_this_instance_last_read( + kind: str, +) -> None: + redis_cache: Final = _RedisCacheOnFakeServer(FakeServer()) + dual_cache: Final = DualCache(in_memory_cache=InMemoryCache(), redis_cache=redis_cache) + await dual_cache.in_memory_cache.async_increment(key=_turn_key(kind), value=100) + await redis_cache.async_increment(key=_turn_key(kind), value=99) + + admitted: Final = await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), _TURN_AT) + + assert admitted[ITPM_CACHE_KEY if kind == "itpm" else OTPM_CACHE_KEY] == _turn_key(kind) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "responses_reply", + [ + pytest.param( + ResponsesAPIResponse( + id="resp_cached", + created_at=0, + output=[], + usage=ResponseAPIUsage( + input_tokens=50, + output_tokens=5, + total_tokens=55, + input_tokens_details=InputTokensDetails(cached_tokens=30), + ), + ), + id="usage_object", + ), + pytest.param( + { + "usage": { + "input_tokens": 50, + "output_tokens": 5, + "total_tokens": 55, + "input_tokens_details": {"cached_tokens": 30}, + } + }, + id="usage_dict", + ), + ], +) +@pytest.mark.usefixtures("settled_on_the_turn_s_minute") +async def test_a_turn_counts_only_its_uncached_responses_input_toward_itpm(responses_reply: object) -> None: + dual_cache: Final = DualCache() + await dual_cache.async_increment_cache(key=_turn_key("itpm"), value=10, ttl=60) + + reservation: Final = await async_admit_io_token_turn(dual_cache, _turn_deployment("itpm"), _TURN_AT) + await async_io_token_reconcile_success(dual_cache, _proxy_success_kwargs(reservation), responses_reply) + + assert await dual_cache.async_get_cache(key=_turn_key("itpm")) == 10 + 50 - 30 + + +def _shared_counter_answering(answer: int | Exception) -> DualCache: + redis_cache: Final = MagicMock(spec=RedisCache) + redis_cache.async_get_cache = AsyncMock(side_effect=[answer]) + return DualCache(in_memory_cache=InMemoryCache(), redis_cache=redis_cache) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["itpm", "otpm"]) +async def test_a_later_turn_is_refused_when_another_instance_filled_the_shared_minute(kind: str) -> None: + with pytest.raises(litellm.RateLimitError, match=f"{kind.upper()} limit=100, current usage=100"): + await async_admit_io_token_turn(_shared_counter_answering(100), _turn_deployment(kind), _TURN_AT) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["itpm", "otpm"]) +async def test_a_later_turn_is_admitted_on_this_instance_s_count_when_the_shared_counter_is_unreachable( + kind: str, +) -> None: + dual_cache: Final = _shared_counter_answering(RedisCircuitBreakerOpenError("circuit open")) + await dual_cache.in_memory_cache.async_increment(key=_turn_key(kind), value=99) + + admitted: Final = await async_admit_io_token_turn(dual_cache, _turn_deployment(kind), _TURN_AT) + + assert admitted[ITPM_CACHE_KEY if kind == "itpm" else OTPM_CACHE_KEY] == _turn_key(kind) diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 1ac44b5976d..a6436e47507 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -2200,7 +2200,11 @@ async def test_ageneric_api_call_deployment_model_overrides_alias(): ) def inject_alias_into_kwargs( - deployment: dict, kwargs: dict, function_name: str | None = None, splits_stream_timeout: bool = False + deployment: dict, + kwargs: dict, + function_name: str | None = None, + splits_stream_timeout: bool = False, + opens_responses_socket: bool = False, ) -> None: # Simulate the alias leaking into kwargs (as happens when # _ageneric_api_call_with_fallbacks sets kwargs["model"] = alias before diff --git a/tests/unit/test_router/test_router_helper_utils.py b/tests/unit/test_router/test_router_helper_utils.py index 5f1b6032ca8..5f31ded626d 100644 --- a/tests/unit/test_router/test_router_helper_utils.py +++ b/tests/unit/test_router/test_router_helper_utils.py @@ -4,13 +4,15 @@ import gc import json import os import weakref -from collections.abc import Iterator, Mapping +from collections.abc import Awaitable, Callable, Iterator, Mapping from datetime import datetime, timezone from types import MethodType from typing import Final from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest +import respx import litellm from litellm import Router @@ -538,14 +540,14 @@ async def test_deployment_callback_on_success_counts_a_responses_websocket_turn( assert await router.cache.async_get_cache(key=tpm_key) == 17 -async def _kwargs_routed_to(router: Router, model: str, function_name: str) -> dict[str, object]: +async def _kwargs_routed_to(router: Router, model: str, function_name: str, **request: object) -> dict[str, object]: routed: Final[dict[str, object]] = {} async def call(**kwargs: object) -> None: routed.update(kwargs) call.__name__ = function_name - await router._ageneric_api_call_with_fallbacks_helper(model=model, original_generic_function=call) + await router._ageneric_api_call_with_fallbacks_helper(model=model, original_generic_function=call, **request) return routed @@ -715,6 +717,363 @@ async def test_responses_websocket_turns_do_not_reserve_deployment_otpm(): await admit_turn() +def _io_limited_router(kind: str, **litellm_params: object) -> Router: + return Router( + model_list=[ + { + "model_name": "ws-io", + "litellm_params": { + "model": "openai/ws-io-unpriced", + "api_key": "sk-fake", + kind: 10_000, + **litellm_params, + }, + "model_info": {"id": "ws-io-dep"}, + } + ], + optional_pre_call_checks=["enforce_model_rate_limits"], + ) + + +def _minutes_counter_key(kind: str, minute: datetime, model: str = "openai/ws-io-unpriced") -> str: + return f"global_router:ws-io-dep:{model}:{kind}:{minute.strftime('%H-%M')}" + + +async def _count_minutes_tokens( + router: Router, kind: str, tokens: int, minute: datetime, model: str = "openai/ws-io-unpriced" +) -> None: + await router.cache.async_increment_cache(key=_minutes_counter_key(kind, minute, model), value=tokens, ttl=120) + + +async def _settle_turn( + router: Router, reservation: Mapping[str, object], input_tokens: int, output_tokens: int +) -> None: + from litellm.cost_calculator import ResponsesWebSocketTokenUsageProcessor + from litellm.litellm_core_utils.get_litellm_params import get_litellm_params + from litellm.router_utils.pre_call_checks.model_rate_limit_check import ModelRateLimitingCheck + + rate_limits: Final = next( + check for check in router.optional_callbacks or () if isinstance(check, ModelRateLimitingCheck) + ) + await rate_limits.async_log_success_event( + {"litellm_params": get_litellm_params(litellm_metadata=dict(reservation))}, + ResponsesWebSocketTokenUsageProcessor.create_logging_object( + [ + { + "type": "response.completed", + "response": {"usage": {"input_tokens": input_tokens, "output_tokens": output_tokens}}, + } + ] + ), + None, + None, + ) + + +_LONG_OUTPUT_MODEL: Final = "openai/gpt-5.4-mini" + + +def _long_output_router() -> tuple[Router, Mapping[str, int]]: + max_output: Final = litellm.get_model_info(_LONG_OUTPUT_MODEL)["max_output_tokens"] + assert isinstance(max_output, int) + limits: Final = {"itpm": 10_000, "otpm": max_output // 2} + return _io_limited_router("itpm", model=_LONG_OUTPUT_MODEL, otpm=limits["otpm"]), limits + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["itpm", "otpm"]) +async def test_responses_websocket_opens_without_counting_tokens_until_the_minute_reaches_deployment_io_limit( + kind: str, router_minute_pinned: datetime +): + router, limits = _long_output_router() + + await _kwargs_routed_to(router, "ws-io", "_aresponses_websocket", websocket=object()) + + assert [ + await router.cache.async_get_cache(key=_minutes_counter_key(counted, router_minute_pinned, _LONG_OUTPUT_MODEL)) + for counted in ("itpm", "otpm") + ] == [None, None] + await _count_minutes_tokens(router, kind, limits[kind], router_minute_pinned, _LONG_OUTPUT_MODEL) + with pytest.raises(litellm.RateLimitError, match=f"{kind.upper()} limit={limits[kind]}, current usage="): + await _kwargs_routed_to(router, "ws-io", "_aresponses_websocket", websocket=object()) + + +@pytest.mark.asyncio +async def test_responses_websocket_first_turn_counts_its_actual_tokens_toward_deployment_itpm_and_otpm( + router_minute_pinned: datetime, +): + router, _ = _long_output_router() + socket_metadata: Final = (await _kwargs_routed_to(router, "ws-io", "_aresponses_websocket", websocket=object()))[ + "litellm_metadata" + ] + assert isinstance(socket_metadata, dict) + + await _settle_turn(router, socket_metadata, input_tokens=20, output_tokens=15) + + assert [ + await router.cache.async_get_cache(key=_minutes_counter_key(counted, router_minute_pinned, _LONG_OUTPUT_MODEL)) + for counted in ("itpm", "otpm") + ] == [20, 15] + + +_ADMITTED_LATE_IN_THE_MINUTE: Final = datetime(2026, 1, 1, 12, 0, 50, tzinfo=timezone.utc) +_FINISHED_IN_THE_NEXT_MINUTE: Final = datetime(2026, 1, 1, 12, 1, 15, tzinfo=timezone.utc) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["itpm", "otpm"]) +@pytest.mark.parametrize("turn", ["first", "later"]) +async def test_responses_websocket_turn_finishing_in_the_next_minute_counts_its_tokens_toward_that_minute( + kind: str, turn: str, monkeypatch: pytest.MonkeyPatch +): + _pin_router_clock(monkeypatch, _ADMITTED_LATE_IN_THE_MINUTE) + router: Final = _io_limited_router(kind) + socket: Final = await _kwargs_routed_to(router, "ws-io", "_aresponses_websocket", websocket=object()) + admit_turn: Final = socket["turn_admission"] + assert callable(admit_turn) + reservation: Final = socket["litellm_metadata"] if turn == "first" else await admit_turn() + assert isinstance(reservation, Mapping) + _pin_router_clock(monkeypatch, _FINISHED_IN_THE_NEXT_MINUTE) + + await _settle_turn(router, reservation, input_tokens=10_000, output_tokens=10_000) + + with pytest.raises(litellm.RateLimitError, match=f"{kind.upper()} limit=10000, current usage=10000"): + await admit_turn() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("body", [{}, {"websocket": True}], ids=["plain", "websocket_key"]) +async def test_messages_call_reserves_max_tokens_against_deployment_otpm_whatever_websocket_key_its_body_carries( + body: Mapping[str, object], router_minute_pinned: datetime +): + router: Final = _io_limited_router("otpm") + + for _ in range(3): + await _kwargs_routed_to(router, "ws-io", "anthropic_messages", max_tokens=3_000, **body) + + assert await router.cache.async_get_cache(key=_minutes_counter_key("otpm", router_minute_pinned)) == 9_000 + with pytest.raises(litellm.RateLimitError, match="OTPM limit=10000, current usage=12000"): + await _kwargs_routed_to(router, "ws-io", "anthropic_messages", max_tokens=3_000, **body) + + +async def _realtime_opens_until_refused(monkeypatch: pytest.MonkeyPatch, **socket: object) -> tuple[int, str]: + monkeypatch.setattr(litellm, "callbacks", []) + router: Final = _io_limited_router("otpm", otpm=50) + for opened in range(51): + try: + await _kwargs_routed_to(router, "ws-io", "_arealtime", **socket) + except litellm.RateLimitError as refused: + return opened, str(refused) + pytest.fail("realtime opens never reached the deployment's otpm") + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("router_minute_pinned") +async def test_realtime_open_reserves_and_is_refused_at_deployment_otpm_like_an_open_without_a_socket( + monkeypatch: pytest.MonkeyPatch, +): + with_socket: Final = await _realtime_opens_until_refused(monkeypatch, websocket=object()) + + assert "OTPM limit=50, current usage=" in with_socket[1] + assert with_socket == await _realtime_opens_until_refused(monkeypatch) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["itpm", "otpm"]) +async def test_responses_websocket_turn_is_refused_once_the_minutes_tokens_reach_deployment_io_limit( + kind: str, router_minute_pinned: datetime +): + router: Final = _io_limited_router(kind) + admit_turn: Final = (await _kwargs_routed_to(router, "ws-io", "_aresponses_websocket"))["turn_admission"] + assert callable(admit_turn) + await _count_minutes_tokens(router, kind, 10_000, router_minute_pinned) + + with pytest.raises(litellm.RateLimitError, match=f"{kind.upper()} limit=10000, current usage="): + await admit_turn() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ["itpm", "otpm"]) +async def test_responses_websocket_turn_refused_for_tokens_leaves_the_daily_request_limit_unused( + kind: str, router_minute_pinned: datetime +): + router: Final = _io_limited_router(kind, rpd=2) + admit_turn: Final = (await _kwargs_routed_to(router, "ws-io", "_aresponses_websocket"))["turn_admission"] + assert callable(admit_turn) + await _count_minutes_tokens(router, kind, 10_000, router_minute_pinned) + with pytest.raises(litellm.RateLimitError, match=f"{kind.upper()} limit=10000"): + await admit_turn() + await _count_minutes_tokens(router, kind, -10_000, router_minute_pinned) + + await admit_turn() + + with pytest.raises(litellm.RateLimitError, match="RPD"): + await admit_turn() + + +_PROVIDER_RESPONSE: Final = { + "id": "resp_io", + "object": "response", + "created_at": 1767268830, + "status": "completed", + "model": "ws-io-unpriced", + "output": [ + { + "type": "message", + "id": "msg_io", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "Done.", "annotations": []}], + } + ], + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + "usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15}, +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "client_metadata", [None, {"session": "abc", "team": "search"}], ids=["without_metadata", "with_metadata"] +) +async def test_responses_call_sends_the_provider_only_the_client_metadata_and_settles_deployment_io_tokens( + client_metadata: Mapping[str, str] | None, + router_minute_pinned: datetime, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +): + from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER + from litellm.responses.streaming_iterator import drain_logging_tasks + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + provider: Final = respx_mock.post("https://api.openai.com/v1/responses").mock( + return_value=httpx.Response(200, json=_PROVIDER_RESPONSE) + ) + router: Final = _io_limited_router("itpm", otpm=10_000) + + await router.aresponses( + model="ws-io", + input="hi", + max_output_tokens=50, + **({} if client_metadata is None else {"metadata": dict(client_metadata)}), + ) + await drain_logging_tasks() + await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10.0) + + assert [json.loads(call.request.read()).get("metadata") for call in provider.calls] == [client_metadata] + assert [ + await router.cache.async_get_cache(key=_minutes_counter_key(kind, router_minute_pinned)) + for kind in ("itpm", "otpm") + ] == [10, 5] + + +@pytest.mark.parametrize( + "client_metadata", [None, {"session": "abc", "team": "search"}], ids=["without_metadata", "with_metadata"] +) +def test_sync_responses_call_sends_the_provider_only_the_client_metadata( + client_metadata: Mapping[str, str] | None, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +): + litellm.in_memory_llm_clients_cache.flush_cache() + provider: Final = respx_mock.post("https://api.openai.com/v1/responses").mock( + return_value=httpx.Response(200, json=_PROVIDER_RESPONSE) + ) + + _io_limited_router("itpm", otpm=10_000).responses( + model="ws-io", + input="hi", + max_output_tokens=50, + **({} if client_metadata is None else {"metadata": dict(client_metadata)}), + ) + + assert [json.loads(call.request.read()).get("metadata") for call in provider.calls] == [client_metadata] + + +_ANTHROPIC_PROVIDER_RESPONSE: Final = { + "id": "msg_io", + "type": "message", + "role": "assistant", + "model": "ws-io-unpriced", + "content": [{"type": "text", "text": "Done."}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 10, "output_tokens": 5}, +} + + +async def _responses_call(router: Router) -> object: + return await router.aresponses(model="ws-io", input="hi", max_output_tokens=50) + + +async def _messages_call(router: Router) -> object: + return await router.aanthropic_messages(model="ws-io", messages=[{"role": "user", "content": "hi"}], max_tokens=50) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model", "provider_url", "provider_response", "call"), + [ + pytest.param( + "openai/ws-io-unpriced", + "https://api.openai.com/v1/responses", + _PROVIDER_RESPONSE, + _responses_call, + id="responses", + ), + pytest.param( + "anthropic/ws-io-unpriced", + "https://api.anthropic.com/v1/messages", + _ANTHROPIC_PROVIDER_RESPONSE, + _messages_call, + id="messages", + ), + ], +) +async def test_retried_call_settles_deployment_io_tokens_to_the_successful_attempt_s_tokens( + model: str, + provider_url: str, + provider_response: Mapping[str, object], + call: Callable[[Router], Awaitable[object]], + router_minute_pinned: datetime, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +): + from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER + from litellm.responses.streaming_iterator import drain_logging_tasks + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + provider: Final = respx_mock.post(provider_url).mock( + side_effect=[ + httpx.Response(500, json={"error": {"message": "upstream failed", "type": "server_error"}}), + httpx.Response(200, json=provider_response), + ] + ) + router: Final = Router( + model_list=[ + { + "model_name": "ws-io", + "litellm_params": {"model": model, "api_key": "sk-fake", "itpm": 10_000, "otpm": 10_000}, + "model_info": {"id": "ws-io-dep"}, + } + ], + optional_pre_call_checks=["enforce_model_rate_limits"], + num_retries=1, + retry_after=0, + ) + + await call(router) + await drain_logging_tasks() + await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10.0) + + assert provider.call_count == 2 + assert [ + await router.cache.async_get_cache(key=_minutes_counter_key(kind, router_minute_pinned, model)) + for kind in ("itpm", "otpm") + ] == [10, 5] + + @pytest.mark.asyncio async def test_deployment_callback_on_success_tracks_tpm_for_io_deployment(): """ @@ -1194,10 +1553,17 @@ def _rpm_tpm_router(model_id: str) -> Router: ) +def _pin_router_clock(monkeypatch: pytest.MonkeyPatch, pinned: datetime) -> None: + monkeypatch.setattr("litellm.router.get_utc_datetime", lambda: pinned) + monkeypatch.setattr( + "litellm.router_utils.pre_call_checks.io_token_rate_limit_check.get_utc_datetime", lambda: pinned + ) + + @pytest.fixture def router_minute_pinned(monkeypatch: pytest.MonkeyPatch) -> datetime: pinned: Final = datetime(2026, 1, 1, 12, 0, 30, tzinfo=timezone.utc) - monkeypatch.setattr("litellm.router.get_utc_datetime", lambda: pinned) + _pin_router_clock(monkeypatch, pinned) return pinned From df47ec01717cef4e5614d34081a568be4e5de4c6 Mon Sep 17 00:00:00 2001 From: quietjules <349048734+quietjules@users.noreply.github.com> Date: Fri, 30 Oct 2026 08:27:35 -0700 Subject: [PATCH 06/74] fix(spend): keep a NaN or infinite cost from wedging daily spend writes (#48829) * fix(spend): keep a NaN or infinite cost from wedging daily spend writes A non-finite provider-reported cost now counts as not reported, so the token price bills the request. Any other non-finite cost is logged and recorded as 0, and spend log JSON writes NaN and Infinity as null so Postgres accepts the row. * fix(spend): price non-finite provider costs on every route and drain parked NaN batches Perplexity, OpenRouter pass-through, and Responses streaming now treat a NaN or infinite reported cost as unreported, so the token price bills the request. A daily spend batch parked in the Redis buffer by an older version with a NaN spend now drains as 0 instead of failing every flush. The cost header never carries NaN or Infinity, and a set holding a non-finite float no longer breaks spend log JSON * fix(spend): record non-finite savings as 0 and warn when a parked NaN batch drains An overflowing price on a compressed request made compression savings infinite, which still wedged the daily spend batch. Every savings column now records a non-finite value as 0 with a warning. The Redis drain warns when it reads a NaN an older version parked, since the spend summed into it is lost from daily spend. The writer test now runs the real payload builder and spend log insert * fix(spend): keep non-finite costs out of headers, responses, and admin JSON Writing NaN as null is now opt-in for spend log columns, the Redis buffer, and streamed frames, so admin writes and logging integrations keep their output. The cost calculator records an overflowing total as 0 with a warning, cost headers leave out non-finite values, a non-streaming NaN usage.cost is dropped instead of failing the response, and a parked NaN spend log row reads back as 0. * fix(spend): keep caller-set call ids from splitting the non-finite cost warning The overflow warning strips newlines from litellm_call_id, and neither non-finite cost warning prints the raw value, which said nothing beyond "non-finite" * fix(spend): drop non-finite Responses and OpenRouter costs and the breakdown of an unpriceable request * fix(types): drop a usage cost breakdown that holds a non-finite number * fix(streaming): keep a dropped non-finite final usage cost from falling back to an earlier chunk's cost * fix(spend): keep the dropped-cost flag out of reach of provider usage keys A provider usage key named dropped_non_finite_cost shadowed the Usage method on the instance, so the stream builder raised and dropped the request's spend. The builder now reads the flag through the class, and Usage sets the flag after copying provider keys, so a _dropped_non_finite_cost key cannot set it either. --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/litellm_core_utils/core_helpers.py | 21 ++- litellm/litellm_core_utils/litellm_logging.py | 25 ++- litellm/litellm_core_utils/safe_json_dumps.py | 23 ++- .../streaming_chunk_builder_utils.py | 9 +- .../llms/openrouter/chat/transformation.py | 8 +- .../pass_through/typesafe_logging_handler.py | 6 +- litellm/llms/perplexity/cost_calculator.py | 8 +- litellm/proxy/common_request_processing.py | 15 +- litellm/proxy/db/db_spend_update_writer.py | 12 +- .../redis_update_buffer.py | 24 ++- .../proxy/hooks/proxy_track_cost_callback.py | 3 +- litellm/proxy/spend_tracking/savings.py | 6 +- .../spend_tracking/spend_tracking_utils.py | 11 +- litellm/types/llms/openai.py | 10 +- litellm/types/utils.py | 30 +++- .../test_non_finite_cost_spend_writes.py | 166 ++++++++++++++++++ .../test_convert_dict_to_response.py | 44 +++++ .../litellm_core_utils/test_core_helpers.py | 28 ++- .../test_litellm_logging.py | 54 ++++++ .../test_safe_json_dumps.py | 25 +++ .../test_streaming_chunk_builder_utils.py | 41 +++++ .../test_openrouter_chat_transformation.py | 29 +++ ...est_openrouter_responses_transformation.py | 25 +++ .../test_perplexity_cost_calculator.py | 22 +++ .../test_redis_update_buffer.py | 130 ++++++++++++++ .../proxy/db/test_db_spend_update_writer.py | 54 ++++++ ...st_typesafe_passthrough_logging_handler.py | 29 +++ .../unit/proxy/spend_tracking/test_savings.py | 28 +++ .../proxy/test_common_request_processing.py | 49 ++++++ tests/unit/test_main.py | 22 +++ 30 files changed, 906 insertions(+), 51 deletions(-) create mode 100644 tests/integration/spend/test_non_finite_cost_spend_writes.py diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 28f45d2bb2c..2f81154ab83 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -982,25 +982,36 @@ def _usage_field(usage: object, name: str) -> object: return getattr(usage, name, None) -def _cost_number(value: object) -> float | None: - if isinstance(value, bool) or not isinstance(value, int | float): +def finite_cost(value: object) -> float | None: + if isinstance(value, bool) or not isinstance(value, int | float) or not math.isfinite(value): return None return float(value) +def single_log_line(value: object) -> str: + return str(value).replace("\r", "").replace("\n", "") + + +def finite_reported_cost(value: object) -> float | None: + cost: Final = finite_cost(value) + if cost is None and isinstance(value, float): + verbose_logger.warning("Ignoring a non-finite provider-reported cost %s", value) + return cost + + def _reported_cost_value(value: object) -> float | None: """A reported cost is a number, or a breakdown whose total lives under ``total_cost``.""" raw: Final[object] = value if isinstance(value, Mapping): - return _cost_number(_usage_field(raw, "total_cost")) - return _cost_number(value) + return finite_reported_cost(_usage_field(raw, "total_cost")) + return finite_reported_cost(value) def response_cost_in_hidden_params(response: _CarriesHiddenParams) -> float | None: additional_headers: Final[object] = response.hidden_params.get("additional_headers") if not isinstance(additional_headers, Mapping): return None - return _cost_number(_USAGE_BLOCK_ADAPTER.validate_python(additional_headers).get(RESPONSE_COST_HEADER)) + return finite_reported_cost(_USAGE_BLOCK_ADAPTER.validate_python(additional_headers).get(RESPONSE_COST_HEADER)) def override_stamped_response_cost(response: _CarriesHiddenParams, cost: float) -> None: diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 8bfb0b60693..cfe8c1739c6 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -76,6 +76,7 @@ from litellm.litellm_core_utils.classifier_logging import ( is_classifier_call, ) from litellm.litellm_core_utils.core_helpers import ( + finite_cost, get_provider_response_headers_from_hidden_params, is_expected_client_error, native_messages_reported_cost, @@ -84,6 +85,7 @@ from litellm.litellm_core_utils.core_helpers import ( proxy_stamped_used_client_oauth_token, reconstruct_model_name, set_response_cost_in_hidden_params, + single_log_line, ) from litellm.litellm_core_utils.error_normalization import normalize_error from litellm.litellm_core_utils.get_litellm_params import get_litellm_params @@ -2093,7 +2095,7 @@ class Logging(LiteLLMLoggingBaseClass): litellm_model_name=litellm_model_name, router_model_id=router_model_id or hidden_params.get("model_id"), ) - return hidden_params["response_cost"] + return self._recorded_response_cost(hidden_params["response_cost"]) elif router_model_id is None and "model_id" in hidden_params: # use model_id if not already set router_model_id = hidden_params["model_id"] @@ -2193,7 +2195,7 @@ class Logging(LiteLLMLoggingBaseClass): litellm_model_name=litellm_model_name, router_model_id=router_model_id, ) - return total_response_cost + return self._recorded_response_cost(total_response_cost) except Exception as e: # error calculating cost debug_info = StandardLoggingModelCostFailureDebugInformation( error_str=str(e), @@ -2219,6 +2221,15 @@ class Logging(LiteLLMLoggingBaseClass): _response_cost_calculator = response_cost_calculator + def _recorded_response_cost(self, cost: float | None) -> float | None: + if cost is None or finite_cost(cost) is not None: + return cost + verbose_logger.warning( + "Recording a non-finite response_cost as 0 for litellm_call_id=%s", single_log_line(self.litellm_call_id) + ) + self.cost_breakdown = None + return 0.0 + def uncached_price(self, result: object) -> UncachedPrice: previous_cost_breakdown: Final = self.cost_breakdown previous_billed_token_rates: Final = self.billed_token_rates @@ -7582,7 +7593,15 @@ def get_standard_logging_object_payload( ) custom_pricing: Final = False if spilled_over else logging_obj.custom_pricing_for(init_response_obj) raw_response_cost: Final = kwargs.get("response_cost") - llm_response_cost: Final[float] = raw_response_cost or 0.0 + numeric_response_cost: Final = raw_response_cost if isinstance(raw_response_cost, int | float) else None + priced_response_cost: Final = finite_cost(numeric_response_cost) + if numeric_response_cost is not None and priced_response_cost is None: + verbose_logger.warning( + "Logging a non-finite response_cost=%s as 0 for litellm_call_id=%s", + numeric_response_cost, + single_log_line(logging_obj.litellm_call_id), + ) + llm_response_cost: Final[float] = priced_response_cost or 0.0 guardrail_cost: Final = guardrail_information_cost(metadata.get("standard_logging_guardrail_information")) finalized_cost: Final = _finalized_cost_of(logging_obj) response_cost: Final[float] = ( diff --git a/litellm/litellm_core_utils/safe_json_dumps.py b/litellm/litellm_core_utils/safe_json_dumps.py index 24c401281ef..909e1222ae6 100644 --- a/litellm/litellm_core_utils/safe_json_dumps.py +++ b/litellm/litellm_core_utils/safe_json_dumps.py @@ -1,4 +1,5 @@ import json +import math from collections.abc import Callable from typing import Any, Final @@ -14,16 +15,23 @@ def strip_null_bytes(value: str) -> str: return value.replace("\x00", "") +def _finite_or_none(value: float | None) -> float | None: + return None if isinstance(value, float) and not math.isfinite(value) else value + + def safe_json_structure( data: object, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH, value_transform: Callable[[str | None, str], str] | None = None, key: str | None = None, + *, + non_finite_as_null: bool = False, ) -> object: """ Rebuild data out of JSON-native pieces while detecting circular references. If a circular reference is detected then a marker string is returned. - NUL bytes are stripped from strings to prevent PostgreSQL 22P05 errors. + NUL bytes are stripped from strings to prevent PostgreSQL 22P05 errors. With + non_finite_as_null, NaN and infinite floats become null, since jsonb rejects them. value_transform, when given, is applied to every string leaf (and to the str() fallback for non-serializable objects) with the mapping key the leaf @@ -43,7 +51,7 @@ def safe_json_structure( cleaned = obj.replace("\x00", "") if "\x00" in obj else obj return _transform(key, cleaned) if isinstance(obj, (int, float, bool, type(None))): - return obj + return _finite_or_none(obj) if non_finite_as_null else obj # Check for circular reference. if id(obj) in seen: return "CircularReference Detected" @@ -65,8 +73,9 @@ def safe_json_structure( seen.remove(id(obj)) return tuple_result elif isinstance(obj, set): - sortable_values: Final[list[Any]] = [_serialize(item, seen, depth + 1, key) for item in obj] - set_result: Final = sorted(sortable_values) + members: Final[list[Any]] = [_serialize(item, seen, depth + 1, key) for item in obj] + present: Final = sorted(member for member in members if member is not None) + set_result: Final = present + [None] * members.count(None) seen.remove(id(obj)) return set_result elif isinstance(obj, BaseModel): @@ -88,6 +97,10 @@ def safe_dumps( data: object, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH, value_transform: Callable[[str | None, str], str] | None = None, + *, + non_finite_as_null: bool = False, ) -> str: """Serialize data to JSON text through safe_json_structure.""" - return json.dumps(safe_json_structure(data, max_depth, value_transform), default=str) + return json.dumps( + safe_json_structure(data, max_depth, value_transform, non_finite_as_null=non_finite_as_null), default=str + ) diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 94351e72ab1..574f1103c1a 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -15,6 +15,7 @@ from litellm._logging import verbose_logger from litellm.litellm_core_utils.core_helpers import ( PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS, chunk_usage, + finite_cost, provider_reported_usage_cost, stamp_litellm_bill_on_usage, ) @@ -1014,7 +1015,9 @@ class ChunkProcessor: prompt_tokens_details, cache_creation_token_details ) - if usage_chunk_dict["cost"] is not None: + if Usage.dropped_non_finite_cost(usage_chunk): + cost = None + elif usage_chunk_dict["cost"] is not None: cost = usage_chunk_dict["cost"] prompt_tokens_details = attach_cache_creation_token_details(prompt_tokens_details, cache_creation_token_details) @@ -1444,8 +1447,8 @@ def _set_stream_builder_response_cost( def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None: if logging_obj is None: return - if _logged_provider(logging_obj) in PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS and isinstance( - getattr(usage, "cost", None), (int, float) + if _logged_provider(logging_obj) in PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS and ( + finite_cost(getattr(usage, "cost", None)) is not None ): return computed_cost: Final = logging_obj.response_cost_calculator(result=response) diff --git a/litellm/llms/openrouter/chat/transformation.py b/litellm/llms/openrouter/chat/transformation.py index 9aaf552e79a..cca6815399f 100644 --- a/litellm/llms/openrouter/chat/transformation.py +++ b/litellm/llms/openrouter/chat/transformation.py @@ -13,6 +13,7 @@ from typing import TYPE_CHECKING, Final, cast import httpx from pydantic import ConfigDict, TypeAdapter +from litellm.litellm_core_utils.core_helpers import finite_cost from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.chat.transformation import BaseLLMException @@ -225,8 +226,11 @@ class OpenrouterConfig(OpenAIGPTConfig): additional_headers: Final = cast( # cast-ok: preserve mapping operations on response metadata dict[str, object], hidden_params["additional_headers"] ) - if isinstance(response_cost, str | int | float): - additional_headers["llm_provider-x-litellm-response-cost"] = float(response_cost) + reported_cost: Final = ( + finite_cost(float(response_cost)) if isinstance(response_cost, str | int | float) else None + ) + if reported_cost is not None: + additional_headers["llm_provider-x-litellm-response-cost"] = reported_cost except Exception: # If we can't extract cost, continue without it - don't fail the response pass diff --git a/litellm/llms/pass_through/typesafe_logging_handler.py b/litellm/llms/pass_through/typesafe_logging_handler.py index 9eae57774c2..7a41d2a5d14 100644 --- a/litellm/llms/pass_through/typesafe_logging_handler.py +++ b/litellm/llms/pass_through/typesafe_logging_handler.py @@ -6,6 +6,7 @@ import httpx from pydantic import TypeAdapter, ValidationError import litellm +from litellm.litellm_core_utils.core_helpers import finite_reported_cost from litellm.litellm_core_utils.decisions_pricing import apply_decisions_pricing from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.litellm_logging import ( @@ -100,9 +101,10 @@ class TypeSafePassthroughLoggingHandler: f"{custom_llm_provider}/{model}" for model in candidate_models if model is not None ) pricing: Final = _pricing_for(candidate_model_keys) + reported_cost: Final = finite_reported_cost(usage.cost) if custom_llm_provider == "openrouter" else None response_cost: Final = ( - usage.cost - if custom_llm_provider == "openrouter" and usage.cost is not None + reported_cost + if reported_cost is not None else input_tokens * pricing.input_cost_per_token + output_tokens * pricing.output_cost_per_token ) usage_object: Final = Usage( diff --git a/litellm/llms/perplexity/cost_calculator.py b/litellm/llms/perplexity/cost_calculator.py index 2d762adf958..c3d178771a5 100644 --- a/litellm/llms/perplexity/cost_calculator.py +++ b/litellm/llms/perplexity/cost_calculator.py @@ -6,6 +6,7 @@ Helper util for handling perplexity-specific cost calculation from datetime import datetime from typing import Final +from litellm.litellm_core_utils.core_helpers import finite_reported_cost from litellm.litellm_core_utils.llm_cost_calc.utils import TokenRates, apply_off_peak_pricing from litellm.types.utils import Usage from litellm.utils import get_model_info @@ -34,10 +35,9 @@ def cost_per_token(model: str, usage: Usage, current_time: datetime | None = Non total_cost = cost_info.get("total_cost") elif isinstance(cost_info, (int, float)) and not isinstance(cost_info, bool): total_cost = float(cost_info) - if total_cost is not None: - # Return total cost as completion_cost (prompt_cost=0) since Perplexity - # doesn't break down by input/output in their cost object - return (0.0, float(total_cost)) + reported_cost: Final = None if total_cost is None else finite_reported_cost(float(total_cost)) + if reported_cost is not None: + return (0.0, reported_cost) ## FALLBACK: Calculate cost manually if Perplexity doesn't provide it ## GET MODEL INFO diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 48eb58adaf2..762f552dadc 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -60,6 +60,7 @@ from litellm.litellm_core_utils.bug_report import ( strip_bug_report_notice, ) from litellm.litellm_core_utils.core_helpers import ( + finite_cost, get_or_create_metadata_bucket, independent_snapshot, is_expected_client_error, @@ -1812,14 +1813,16 @@ def _get_cost_breakdown_from_logging_obj( def _header_value(value: float | None) -> str | None: - return str(value) if value is not None else None + return str(value) if finite_cost(value) is not None else None def _response_cost_header_value(hidden_params: Mapping[str, object]) -> float | str: if hidden_params.get("cache_hit") is True: return 0.0 response_cost: Final = hidden_params.get("response_cost") - return response_cost if isinstance(response_cost, (int, float, str)) and response_cost else "" + if isinstance(response_cost, str): + return response_cost + return response_cost if isinstance(response_cost, int | float) and finite_cost(response_cost) else "" def cost_headers( @@ -4509,7 +4512,7 @@ class ProxyBaseLLMRequestProcessing: """ if isinstance(chunk, dict): # Use safe_dumps for proper JSON serialization with circular reference detection - chunk_str: Final = safe_dumps(chunk) + chunk_str: Final = safe_dumps(chunk, non_finite_as_null=True) return f"{STREAM_SSE_DATA_PREFIX}{chunk_str}\n\n" else: return chunk @@ -4885,7 +4888,11 @@ class ProxyBaseLLMRequestProcessing: obj, model_name, litellm_logging_obj ) if maybe_modified is not None: - lines[idx] = "data: " + safe_dumps(maybe_modified) + ("\r" if ln.endswith("\r") else "") + lines[idx] = ( + "data: " + + safe_dumps(maybe_modified, non_finite_as_null=True) + + ("\r" if ln.endswith("\r") else "") + ) return "\n".join(lines) return None except Exception: diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index fe804a82e96..d9fb236632b 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -34,6 +34,7 @@ from litellm.constants import ( DB_SPEND_UPDATE_JOB_NAME, INTERNAL_CALL_ORIGIN_METADATA_KEY, ) +from litellm.litellm_core_utils.core_helpers import finite_cost, single_log_line from litellm.litellm_core_utils.litellm_logging import coerce_model_access_groups from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.proxy._types import ( @@ -652,7 +653,14 @@ class DBSpendUpdateWriter: end_time=end_time, llm_router=llm_router, ) - payload["spend"] = response_cost or 0.0 + billable_cost: Final = finite_cost(response_cost) + if response_cost is not None and billable_cost is None: + verbose_proxy_logger.warning( + "Spend tracking - recording a non-finite response_cost=%s as 0 for request_id=%s", + response_cost, + single_log_line(payload["request_id"]), + ) + payload["spend"] = billable_cost or 0.0 if isinstance(payload["startTime"], datetime): payload["startTime"] = payload["startTime"].isoformat() if isinstance(payload["endTime"], datetime): @@ -702,7 +710,7 @@ class DBSpendUpdateWriter: spawn_logging_task( asyncio.get_running_loop(), self._batch_database_updates( - response_cost=response_cost, + response_cost=billable_cost, user_id=user_id, hashed_token=hashed_token, team_id=team_id, diff --git a/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py b/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py index 3a4375ac54a..459f95972ce 100644 --- a/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py +++ b/litellm/proxy/db/db_transaction_queue/redis_update_buffer.py @@ -8,7 +8,7 @@ import asyncio import json from collections.abc import Mapping, Sequence from datetime import datetime -from functools import reduce +from functools import partial, reduce from itertools import chain from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast @@ -109,6 +109,16 @@ _ValueT = TypeVar("_ValueT") _DECODED_JSON: Final = TypeAdapter(object) +def _zero_for_non_finite_constant(constant: str) -> float: + verbose_proxy_logger.warning( + "Redis spend buffer - reading a parked %s spend as 0, the spend it held is lost", constant + ) + return 0.0 + + +_load_buffered_transaction: Final = partial(json.loads, parse_constant=_zero_for_non_finite_constant) + + def _spend_log_json_default(value: object) -> str: return value.isoformat() if isinstance(value, datetime) else str(value) @@ -118,7 +128,7 @@ def _encode_spend_log_row(row: SpendLogRow) -> str: def _decode_spend_log_row(encoded: str) -> dict[str, object] | None: - decoded: Final = _DECODED_JSON.validate_python(json.loads(encoded)) + decoded: Final = _DECODED_JSON.validate_python(_load_buffered_transaction(encoded)) return decoded if isinstance(decoded, dict) else None @@ -825,13 +835,13 @@ class RedisUpdateBuffer: if slot_result is None: daily_results.append(None) else: - list_of_daily = [json.loads(t) for t in slot_result] + list_of_daily = [_load_buffered_transaction(t) for t in slot_result] aggregated = DailySpendUpdateQueue.get_aggregated_daily_spend_update_transactions(list_of_daily) daily_results.append(aggregated) window_spend: Final = ( WindowSpendUpdateQueue.get_aggregated_window_spend_transactions( - tuple(json.loads(transaction) for transaction in raw_results[6]) + tuple(_load_buffered_transaction(transaction) for transaction in raw_results[6]) ) if raw_results[6] is not None else None @@ -880,7 +890,7 @@ class RedisUpdateBuffer: ) if list_of_transactions is None: return None - return [json.loads(transaction) for transaction in list_of_transactions] + return [_load_buffered_transaction(transaction) for transaction in list_of_transactions] async def get_all_daily_spend_update_transactions_from_redis_buffer( self, @@ -1010,9 +1020,9 @@ class RedisUpdateBuffer: Parses the list of transactions from Redis """ if isinstance(list_of_transactions, list): - return [json.loads(transaction) for transaction in list_of_transactions] + return [_load_buffered_transaction(transaction) for transaction in list_of_transactions] else: - return [json.loads(list_of_transactions)] + return [_load_buffered_transaction(list_of_transactions)] @staticmethod def _combine_list_of_transactions( diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 9c4a4bc775b..0d333d4fd08 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -16,6 +16,7 @@ from litellm.constants import BACKGROUND_INTERACTION_COST_POLLING_ENABLED, BILLE from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import ( budget_reservation_from_metadata, + finite_cost, get_litellm_metadata_from_kwargs, get_metadata_variable_name_from_kwargs, get_parent_otel_span_from_kwargs, @@ -703,7 +704,7 @@ class _ProxyDBLogger(CustomLogger): raw_cost: Final = _str_keyed_fields( sl_object if sl_object is not None else kwargs # pyright: ignore[reportUnknownArgumentType] # untyped success-callback payload ).get("response_cost") - priced_cost: Final = raw_cost if isinstance(raw_cost, (int, float)) else None + priced_cost: Final = finite_cost(raw_cost) response_cost: Final = ( await apply_search_free_tier( kwargs=kwargs, # pyright: ignore[reportUnknownArgumentType] # untyped success-callback payload diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index 28602fbbefc..8e79ea5255f 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -689,9 +689,13 @@ def compute_savings_spend( cost_breakdown=cost_breakdown, ) ) - return SavingsSpend( + savings: Final = SavingsSpend( compression=compression, prompt_caching=prompt_caching, autorouter=0.0 if autorouter is None else autorouter, gateway_injected_caching=gateway_injected_caching, ) + if all(isfinite(value) for value in savings): + return savings + verbose_proxy_logger.warning("Spend tracking - recording non-finite savings %s as 0 for model=%s", savings, model) + return SavingsSpend(*(value if isfinite(value) else 0.0 for value in savings)) diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 09b7ce7ee9a..f8d5cba4229 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -912,7 +912,7 @@ def get_logging_payload( user=metadata.get("user_api_key_user_id", "") or "", team_id=metadata.get("user_api_key_team_id", "") or "", organization_id=metadata.get("user_api_key_org_id") or "", - metadata=safe_dumps(clean_metadata), + metadata=safe_dumps(clean_metadata, non_finite_as_null=True), cache_key=cache_key, spend=kwargs.get("response_cost", 0), total_tokens=usage.get("total_tokens", standard_logging_total_tokens), @@ -1224,7 +1224,7 @@ def _get_messages_for_spend_logs_payload( messages: Final = standard_logging_payload.get("messages") if messages is not None: try: - return safe_dumps(messages) + return safe_dumps(messages, non_finite_as_null=True) except Exception: return "{}" return "{}" @@ -1919,7 +1919,8 @@ def _get_proxy_server_request_for_spend_logs_payload( _request_body_json_str: Final = safe_dumps( _stored_request_without_requester_ip_address(stored_request_body) if drop_requester_ip_address - else stored_request_body + else stored_request_body, + non_finite_as_null=True, ) if LITELLM_TRUNCATED_PAYLOAD_FIELD in _request_body_json_str: verbose_proxy_logger.info( @@ -2000,7 +2001,7 @@ def _get_response_for_spend_logs_payload( if isinstance(sanitized_response, str): result_str = strip_null_bytes(sanitized_response) else: - result_str = safe_dumps(sanitized_response) + result_str = safe_dumps(sanitized_response, non_finite_as_null=True) if LITELLM_TRUNCATED_PAYLOAD_FIELD in result_str: verbose_proxy_logger.info( "Spend Log: response was truncated before storing in DB. %s", @@ -2071,7 +2072,7 @@ def spend_log_row_with_retained_metadata( return row metadata: Final = _SPEND_LOGS_METADATA_ADAPTER.validate_json(metadata_json) retained: Final = {name: value for name, value in metadata.items() if fields.keeps(name)} - return {**row, "metadata": safe_dumps(retained)} + return {**row, "metadata": safe_dumps(retained, non_finite_as_null=True)} def _get_status_for_spend_log( diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index d9d86cf54b3..a54d66c4beb 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1,4 +1,5 @@ import builtins +import math from collections.abc import Iterable, Mapping, Sequence from enum import Enum from os import PathLike @@ -83,6 +84,7 @@ from typing_extensions import ( override, ) +from litellm._logging import verbose_logger from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject, LiteLLMBaseModel from litellm.types.responses.main import ( CustomToolCallOutputItem, @@ -1446,9 +1448,11 @@ class ResponseAPIUsage(BaseLiteLLMOpenAIResponseObject): @classmethod def parse_cost(cls, v: object) -> object: """Normalise cost: accept either a float or a dict with a ``total_cost`` key.""" - if isinstance(v, dict): - return v.get("total_cost") - return v + cost: Final = v.get("total_cost") if isinstance(v, dict) else v + if isinstance(cost, float) and not math.isfinite(cost): + verbose_logger.warning("Dropping a non-finite provider usage cost") + return None + return cost model_config = ConfigDict(extra="allow") diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 780c9ef6824..55aaf606175 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -1,4 +1,5 @@ import json +import math import re import time from collections.abc import Collection, Mapping, Sequence @@ -1971,6 +1972,19 @@ class ServerToolUse(LiteLLMBaseModel): return getattr(self, key) +_COST_BREAKDOWN_ADAPTER: Final = TypeAdapter(Mapping[object, object]) + + +def _is_non_finite_float(value: object) -> bool: + return isinstance(value, float) and not math.isfinite(value) + + +def _carries_non_finite_number(cost: object) -> bool: + if not isinstance(cost, Mapping): + return _is_non_finite_float(cost) + return any(_is_non_finite_float(member) for member in _COST_BREAKDOWN_ADAPTER.validate_python(cost).values()) + + class Usage(SafeAttributeModel, CompletionUsage): _cache_creation_input_tokens: int = PrivateAttr( 0 @@ -1978,6 +1992,7 @@ class Usage(SafeAttributeModel, CompletionUsage): _cache_read_input_tokens: int = PrivateAttr( 0 ) # hidden param for prompt caching. Might change, once openai introduces their equivalent. + _dropped_non_finite_cost: bool = PrivateAttr(False) server_tool_use: ServerToolUse | None = None cost: float | None = None @@ -2084,10 +2099,14 @@ class Usage(SafeAttributeModel, CompletionUsage): else: # maintain openai compatibility in usage object if possible del self.server_tool_use - if cost is not None: - self.cost = cost - else: + dropped_non_finite_cost: Final = cost is not None and _carries_non_finite_number(cost) + if cost is None: del self.cost + elif dropped_non_finite_cost: + verbose_logger.warning("Dropping a non-finite provider usage cost") + del self.cost + else: + self.cost = cost ## ANTHROPIC MAPPING ## if "cache_creation_input_tokens" in params and isinstance(params["cache_creation_input_tokens"], int): @@ -2102,6 +2121,11 @@ class Usage(SafeAttributeModel, CompletionUsage): for k, v in params.items(): setattr(self, k, v) + self._dropped_non_finite_cost = dropped_non_finite_cost + + @staticmethod + def dropped_non_finite_cost(usage: "Usage") -> bool: + return usage._dropped_non_finite_cost def __contains__(self, key) -> bool: # Define custom behavior for the 'in' operator diff --git a/tests/integration/spend/test_non_finite_cost_spend_writes.py b/tests/integration/spend/test_non_finite_cost_spend_writes.py new file mode 100644 index 00000000000..9a4d2dd45a4 --- /dev/null +++ b/tests/integration/spend/test_non_finite_cost_spend_writes.py @@ -0,0 +1,166 @@ +import json +from hashlib import sha256 +from pathlib import Path +from typing import Final, Literal +from uuid import uuid4 + +import pytest +from integration._support.client import Gateway, eventually +from integration._support.database import read_rows +from integration._support.openai_wire import answering_model_discovery +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue + +pytestmark: Final = pytest.mark.shares_worker_state + +PROMPT_TOKENS: Final = 5 +COMPLETION_TOKENS: Final = 3 +INPUT_RATE: Final = 0.001 +OUTPUT_RATE: Final = 0.002 +TOKEN_PRICE: Final = PROMPT_TOKENS * INPUT_RATE + COMPLETION_TOKENS * OUTPUT_RATE +OVERFLOWING_INPUT_RATE: Final = 1e308 +POISONED_UPSTREAM: Final = "poisoned-upstream" +CONTROL_UPSTREAM: Final = "control-upstream" +DAILY_TOTALS: Final = "SELECT model_group, SUM(spend)::float8 AS spend, SUM(api_requests)::int8 AS api_requests" +DAILY_USER_QUERY: Final = f'{DAILY_TOTALS} FROM "LiteLLM_DailyUserSpend" WHERE api_key=%s GROUP BY model_group' +DAILY_TEAM_QUERY: Final = f'{DAILY_TOTALS} FROM "LiteLLM_DailyTeamSpend" WHERE api_key=%s GROUP BY model_group' +DAILY_TAG_QUERY: Final = f'{DAILY_TOTALS} FROM "LiteLLM_DailyTagSpend" WHERE api_key=%s AND tag=%s GROUP BY model_group' + +Poison = Literal["reported_nan_cost", "overflowing_price"] + + +def _frame(payload: dict[str, JsonValue]) -> bytes: + return b"data: " + json.dumps(payload).encode() + b"\n\n" + + +def _stream(upstream_model: str, reported_cost: float | None) -> tuple[bytes, ...]: + chunk: Final = {"id": f"chatcmpl-{uuid4().hex}", "object": "chat.completion.chunk", "created": 1} + usage: Final[dict[str, JsonValue]] = { + "prompt_tokens": PROMPT_TOKENS, + "completion_tokens": COMPLETION_TOKENS, + "total_tokens": PROMPT_TOKENS + COMPLETION_TOKENS, + } + reported_usage: Final = usage if reported_cost is None else {**usage, "cost": reported_cost} + return ( + _frame({**chunk, "model": upstream_model, "choices": [{"index": 0, "delta": {"content": "ok"}}]}), + _frame( + { + **chunk, + "model": upstream_model, + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + "usage": reported_usage, + } + ), + b"data: [DONE]\n\n", + ) + + +def _deployment(alias: str, upstream_model: str, wire_url: str, input_rate: float) -> dict[str, JsonValue]: + return { + "model_name": alias, + "litellm_params": { + "model": f"openai/{upstream_model}", + "api_base": f"{wire_url}/v1", + "api_key": "integration-provider-key", + "input_cost_per_token": input_rate, + "output_cost_per_token": OUTPUT_RATE, + }, + } + + +def _proxy_config(directory: Path, deployments: tuple[dict[str, JsonValue], ...]) -> Path: + config: Final = directory / "non_finite_cost_config.yaml" + config.write_text( + json.dumps( + { + "model_list": list(deployments), + "general_settings": { + "master_key": "os.environ/LITELLM_MASTER_KEY", + "database_url": "os.environ/DATABASE_URL", + "disable_spend_logs": False, + "proxy_batch_write_at": 1, + "proxy_batch_polling_interval": 1, + }, + "router_settings": {"disable_cooldowns": True}, + } + ) + ) + return config + + +def _by_model_group(rows: list[dict[str, JsonValue]]) -> dict[str, float]: + return {str(row["model_group"]): float(str(row["spend"])) for row in rows} + + +def _both_requests_rows(query: str, parameters: tuple[str, ...]) -> list[dict[str, JsonValue]]: + return eventually(lambda: read_rows(query, parameters), lambda rows: len(rows) == 2, seconds=70) + + +@pytest.mark.parametrize("poison", ["reported_nan_cost", "overflowing_price"]) +@pytest.mark.timeout(240) +def test_a_non_finite_cost_never_blocks_the_next_requests_spend_rows( + gateway: Gateway, tmp_path: Path, poison: Poison +) -> None: + poisoned_alias: Final = f"non-finite-cost-{uuid4().hex}" + control_alias: Final = f"finite-cost-{uuid4().hex}" + tag: Final = f"non-finite-cost-{uuid4().hex}" + poisoned_spend: Final = TOKEN_PRICE if poison == "reported_nan_cost" else 0.0 + poisoned_input_rate: Final = INPUT_RATE if poison == "reported_nan_cost" else OVERFLOWING_INPUT_RATE + + def respond(request: Request) -> Reply: + assert request.target.endswith("/chat/completions"), request.target + upstream_model: Final = json.loads(request.body)["model"] + reported_cost: Final = ( + float("nan") if upstream_model == POISONED_UPSTREAM and poison == "reported_nan_cost" else None + ) + return Reply(content_type="text/event-stream", chunks=_stream(upstream_model, reported_cost)) + + with wire_server(answering_model_discovery(respond)) as wire: + config: Final = _proxy_config( + tmp_path, + ( + _deployment(poisoned_alias, POISONED_UPSTREAM, wire.url, poisoned_input_rate), + _deployment(control_alias, CONTROL_UPSTREAM, wire.url, INPUT_RATE), + ), + ) + with owned_proxy(gateway, tmp_path, {}, config=config) as candidate, candidate.scenario() as scenario: + team: Final = scenario.team() + key: Final = scenario.key(user_id=f"non-finite-cost-{uuid4().hex}", team_id=team) + digest: Final = sha256(key.encode()).hexdigest() + responses: Final = tuple( + candidate.request( + "POST", + "/v1/chat/completions", + { + "model": alias, + "messages": [{"role": "user", "content": f"non-finite cost {uuid4().hex}"}], + "stream": True, + "metadata": {"tags": [tag]}, + }, + key=key, + ) + for alias in (poisoned_alias, control_alias) + ) + assert all(response.status_code == 200 for response in responses), [response.text for response in responses] + + expected: Final = {poisoned_alias: pytest.approx(poisoned_spend), control_alias: pytest.approx(TOKEN_PRICE)} + spend_logs: Final = _both_requests_rows( + 'SELECT model_group, spend FROM "LiteLLM_SpendLogs" WHERE api_key=%s', (digest,) + ) + assert _by_model_group(spend_logs) == expected, spend_logs + daily_rows: Final = ( + _both_requests_rows(DAILY_USER_QUERY, (digest,)), + _both_requests_rows(DAILY_TEAM_QUERY, (digest,)), + _both_requests_rows(DAILY_TAG_QUERY, (digest, tag)), + ) + assert [_by_model_group(rows) for rows in daily_rows] == [expected] * 3, daily_rows + assert [{row["api_requests"] for row in rows} for rows in daily_rows] == [{1}] * 3, daily_rows + key_total: Final = poisoned_spend + TOKEN_PRICE + key_spend: Final = eventually( + lambda: read_rows('SELECT spend FROM "LiteLLM_VerificationToken" WHERE token=%s', (digest,)), + lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) == pytest.approx(key_total), + seconds=70, + return_last_on_timeout=True, + ) + assert float(str(key_spend[0]["spend"])) == pytest.approx(key_total), key_spend diff --git a/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py b/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py index de8d6a7a8c5..d0f5bfc6b9f 100644 --- a/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py +++ b/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py @@ -3089,6 +3089,50 @@ def test_convert_to_model_response_object_function_output(): assert result._hidden_params is not None +@pytest.mark.parametrize( + "reported_cost", + [ + float("nan"), + float("inf"), + {"input_tokens_cost": 0.0001, "request_cost": 0.005, "total_cost": float("nan")}, + {"input_tokens_cost": float("inf"), "request_cost": 0.005, "total_cost": 0.0051}, + ], + ids=["nan", "inf", "breakdown-with-nan-total", "breakdown-with-inf-part"], +) +def test_a_non_finite_provider_usage_cost_is_dropped_so_the_response_is_strict_json(reported_cost: object): + response: Final = convert_to_model_response_object( + response_object={ + "id": "chatcmpl-non-finite-cost", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4, "cost": reported_cost}, + }, + model_response_object=ModelResponse(), + ) + + usage: Final = json.loads(json.dumps(response.model_dump(), allow_nan=False))["usage"] + assert "cost" not in usage + assert usage["total_tokens"] == 4 + + +def test_a_finite_provider_cost_breakdown_with_non_string_keys_is_kept(): + response: Final = convert_to_model_response_object( + response_object={ + "id": "chatcmpl-int-key-cost", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4, "cost": {1: 0.5}}, + }, + model_response_object=ModelResponse(), + ) + + assert response.usage.cost == {1: 0.5} + + _URL_CITATION: Final = { "type": "url_citation", "url_citation": { diff --git a/tests/unit/litellm_core_utils/test_core_helpers.py b/tests/unit/litellm_core_utils/test_core_helpers.py index 1c86c4e30d8..93defc0f04f 100644 --- a/tests/unit/litellm_core_utils/test_core_helpers.py +++ b/tests/unit/litellm_core_utils/test_core_helpers.py @@ -26,15 +26,24 @@ from litellm.litellm_core_utils.core_helpers import ( normalize_drop_params, preserve_upstream_non_openai_attributes, process_response_headers, + provider_reported_usage_cost, qualify_agentic_followup_model, qualify_provider_stripped_model, reconstruct_model_name, redact_nested_match_and_regex_keys, + response_cost_in_hidden_params, set_provider_response_headers_in_hidden_params, unbind_budget_reservation_from_callbacks, ) from litellm.proxy._types import UserAPIKeyAuth -from litellm.types.utils import Delta, ImageResponse, ModelResponseStream, StreamingChoices, TranscriptionResponse +from litellm.types.utils import ( + Delta, + ImageResponse, + ModelResponse, + ModelResponseStream, + StreamingChoices, + TranscriptionResponse, +) def test_parent_otel_span_reads_dict_like_metadata_by_index(): @@ -790,6 +799,23 @@ class TestReceiveStampSurvivesASecondMetadataBucket: assert get_litellm_metadata_from_kwargs(kwargs)["litellm_received_at"] is own +@pytest.mark.parametrize("reported", [float("nan"), float("inf"), {"total_cost": float("nan")}]) +def test_a_non_finite_provider_reported_cost_counts_as_unreported(reported: object) -> None: + assert provider_reported_usage_cost({"cost": reported}, "openrouter") is None + assert provider_reported_usage_cost({"cost": {"total_cost": 0.25}}, "openrouter") == 0.25 + + +@pytest.mark.parametrize("header_cost", [float("nan"), float("inf")]) +def test_a_non_finite_response_cost_header_counts_as_unreported(header_cost: float) -> None: + response: Final = ModelResponse() + response._hidden_params = {"additional_headers": {RESPONSE_COST_HEADER: header_cost}} + finite: Final = ModelResponse() + finite._hidden_params = {"additional_headers": {RESPONSE_COST_HEADER: 0.25}} + + assert response_cost_in_hidden_params(response) is None + assert response_cost_in_hidden_params(finite) == 0.25 + + def test_billed_usage_cost_bills_any_chat_upstream_without_its_own_reader() -> None: from types import SimpleNamespace diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index b3ae3bf6240..336e9f7d24a 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -6308,6 +6308,60 @@ def test_standard_logging_payload_resolves_kubernetes_pod_routing_from_metadata_ assert payload["metadata"][KUBERNETES_POD_ROUTING_KEY] == expected +@pytest.mark.parametrize("response_cost", [float("nan"), float("inf"), float("-inf")]) +def test_get_standard_logging_object_payload_logs_a_non_finite_response_cost_as_zero( + logging_obj: LitellmLogging, response_cost: float +) -> None: + from datetime import datetime + + from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload + + now: Final = datetime(2026, 1, 1) + payload: Final = get_standard_logging_object_payload( + kwargs={"model": "gpt-4o", "messages": [], "litellm_params": {}, "response_cost": response_cost}, + init_response_obj={}, + start_time=now, + end_time=now, + logging_obj=logging_obj, + status="success", + ) + + assert payload is not None + assert payload["response_cost"] == 0 + + +def test_an_overflowing_custom_price_records_a_zero_response_cost(caplog: pytest.LogCaptureFixture) -> None: + messages: Final = [{"role": "user", "content": "hello world"}] + start_time: Final = datetime.datetime.now() + logging_obj: Final = Logging( + model="gpt-4o", + messages=messages, + stream=False, + call_type="completion", + start_time=start_time, + litellm_call_id="overflow\nforged log line", + function_id="overflow", + ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + response: Final = litellm.completion( + model="gpt-4o", + mock_response="hi there", + messages=messages, + input_cost_per_token=1e308, + output_cost_per_token=1e308, + litellm_logging_obj=logging_obj, + ) + logging_obj._process_hidden_params_and_response_cost( + response, start_time=start_time, end_time=datetime.datetime.now() + ) + + assert response._hidden_params["response_cost"] == 0.0 + payload: Final = logging_obj.model_call_details["standard_logging_object"] + assert json.loads(json.dumps(payload, allow_nan=False, default=str))["response_cost"] == 0.0 + warnings: Final = [r.getMessage() for r in caplog.records if "non-finite response_cost" in r.getMessage()] + assert warnings == ["Recording a non-finite response_cost as 0 for litellm_call_id=overflowforged log line"] + + def test_get_standard_logging_object_payload_carries_matched_access_groups(logging_obj): """Access groups stamped at auth time reach the logging payload, so integrations see what a request billed.""" from datetime import datetime diff --git a/tests/unit/litellm_core_utils/test_safe_json_dumps.py b/tests/unit/litellm_core_utils/test_safe_json_dumps.py index 1f2664f33cb..1b8bdd21326 100644 --- a/tests/unit/litellm_core_utils/test_safe_json_dumps.py +++ b/tests/unit/litellm_core_utils/test_safe_json_dumps.py @@ -169,6 +169,31 @@ def test_null_byte_stripped_in_fallback_str(): assert json.loads(out)["obj"] == "objrepr" +@pytest.mark.parametrize("value", [float("nan"), float("inf"), float("-inf")]) +def test_non_finite_floats_are_written_as_json_null(value: float) -> None: + out = safe_dumps( + {"cost_breakdown": {"total_cost": value, "output_cost": 0.006}, "costs": [value]}, non_finite_as_null=True + ) + + assert json.loads(out, parse_constant=lambda token: pytest.fail(f"non-JSON token {token}")) == { + "cost_breakdown": {"total_cost": None, "output_cost": 0.006}, + "costs": [None], + } + + +def test_a_set_holding_a_non_finite_float_sorts_its_finite_members_and_puts_null_last() -> None: + assert json.loads(safe_dumps({"costs": {2.0, float("inf"), 1.0}}, non_finite_as_null=True)) == { + "costs": [1.0, 2.0, None] + } + + +def test_non_finite_floats_keep_their_value_unless_the_caller_asks_for_null() -> None: + assert json.loads(safe_dumps({"timeout": float("inf"), "costs": [float("-inf")]})) == { + "timeout": float("inf"), + "costs": [float("-inf")], + } + + def test_clean_strings_are_not_run_through_replace(): """Regression for LIT-3910. diff --git a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py index 28587bf4749..c8fa9c94c17 100644 --- a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -1077,6 +1077,47 @@ def test_cost_field_in_usage_chunks(): assert usage.completion_tokens == 5 + +def _perplexity_usage_chunk(cost: Mapping[str, float], finish_reason: str | None) -> ModelResponseStream: + return ModelResponseStream( + id="chatcmpl-pplx", + created=1745513206, + model="sonar", + choices=[StreamingChoices(finish_reason=finish_reason, index=0, delta=Delta(content="ok"))], + usage={"prompt_tokens": 26, "completion_tokens": 1, "total_tokens": 27, "cost": dict(cost)}, + ) + + +@pytest.mark.parametrize("poisoned_field", ["request_cost", "total_cost"]) +def test_a_non_finite_final_stream_cost_does_not_fall_back_to_an_earlier_chunk_cost(poisoned_field: str): + final_cost: Final = {"input_tokens_cost": 0.0012, "output_tokens_cost": 1e-05, "request_cost": 0.0025} + chunks: Final = [ + _perplexity_usage_chunk({"input_tokens_cost": 0, "output_tokens_cost": 0, "total_cost": 0}, None), + _perplexity_usage_chunk({**final_cost, "total_cost": 0.00371, poisoned_field: float("nan")}, "stop"), + ] + + response: Final = stream_chunk_builder(chunks=chunks) + + assert response is not None + assert getattr(response.usage, "cost", None) is None + assert response.usage.total_tokens == 27 + + +@pytest.mark.parametrize("provider_key", ["dropped_non_finite_cost", "_dropped_non_finite_cost"]) +def test_a_provider_usage_key_named_like_the_dropped_cost_flag_keeps_the_stream_cost(provider_key: str): + chunk: Final = ModelResponseStream( + id="chatcmpl-flag-key", + created=1745513206, + model="sonar", + choices=[StreamingChoices(finish_reason="stop", index=0, delta=Delta(content="ok"))], + usage={"prompt_tokens": 26, "completion_tokens": 1, "total_tokens": 27, "cost": 0.00371, provider_key: True}, + ) + + response: Final = stream_chunk_builder(chunks=[chunk]) + + assert response is not None + assert getattr(response.usage, "cost", None) == 0.00371 + def test_stream_chunk_builder_tolerates_trailing_chunk_without_choices(): """Regression for https://github.com/BerriAI/litellm/issues/32051 diff --git a/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py b/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py index 59355dfabcb..888c47505a4 100644 --- a/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py +++ b/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py @@ -1,6 +1,7 @@ import json from typing import Final +from unittest.mock import MagicMock import httpx import pytest @@ -464,6 +465,34 @@ def test_openrouter_cost_tracking_non_streaming(): ) + +@pytest.mark.parametrize("cost", [float("nan"), float("inf")]) +def test_openrouter_chat_leaves_a_non_finite_provider_cost_out_of_the_response_headers(cost: float) -> None: + body: Final = { + "id": "gen-123", + "object": "chat.completion", + "created": 1234567890, + "model": "openai/gpt-5.6-luna", + "choices": [{"message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop", "index": 0}], + "usage": {"prompt_tokens": 10, "completion_tokens": 1, "total_tokens": 11, "cost": cost}, + } + + result: Final = OpenrouterConfig().transform_response( + model="openrouter/openai/gpt-5.6-luna", + raw_response=httpx.Response( + 200, content=json.dumps(body).encode(), headers={"content-type": "application/json"} + ), + model_response=litellm.ModelResponse(), + logging_obj=MagicMock(), + request_data={}, + messages=[{"role": "user", "content": "Reply with ok"}], + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert "llm_provider-x-litellm-response-cost" not in result._hidden_params.get("additional_headers", {}) + def test_openrouter_cost_tracking_streaming(): """ Test OpenRouter cost tracking for streaming completions. diff --git a/tests/unit/llms/openrouter/responses/test_openrouter_responses_transformation.py b/tests/unit/llms/openrouter/responses/test_openrouter_responses_transformation.py index 19517e58b31..78d7161c388 100644 --- a/tests/unit/llms/openrouter/responses/test_openrouter_responses_transformation.py +++ b/tests/unit/llms/openrouter/responses/test_openrouter_responses_transformation.py @@ -9,6 +9,8 @@ reasoning.encrypted_content for multi-turn stateless workflows. Related issue: https://github.com/BerriAI/litellm/issues/22189 """ +import json +from typing import Final from unittest.mock import MagicMock import httpx @@ -157,6 +159,29 @@ def test_openrouter_responses_returns_provider_reported_cost(cost): assert get_response_cost_from_hidden_params(response.hidden_params) == cost +@pytest.mark.parametrize("cost", [float("nan"), float("inf")]) +def test_openrouter_responses_drops_a_non_finite_provider_cost(cost: float) -> None: + body: Final = { + "id": "resp_test", + "created_at": 1, + "model": "unmapped-model", + "object": "response", + "output": [], + "status": "completed", + "usage": {"input_tokens": 2, "output_tokens": 3, "total_tokens": 5, "cost": cost}, + } + response: Final = OpenRouterResponsesAPIConfig().transform_response_api_response( + model="openrouter/unmapped-model", + raw_response=httpx.Response( + 200, content=json.dumps(body).encode(), headers={"content-type": "application/json"} + ), + logging_obj=MagicMock(), + ) + + assert get_response_cost_from_hidden_params(response.hidden_params) is None + assert json.loads(json.dumps(response.model_dump(), allow_nan=False, default=str))["usage"]["cost"] is None + + @pytest.mark.parametrize( "event_type", ["response.completed", "response.incomplete", "response.failed"], diff --git a/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py b/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py index 83c71479311..380f85445e5 100644 --- a/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py +++ b/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py @@ -6,9 +6,11 @@ search queries, and reasoning tokens. """ import json +import logging import math import os from datetime import datetime, timezone +from typing import Final from unittest.mock import patch import pytest @@ -140,6 +142,26 @@ class TestPerplexityCostCalculator: assert prompt_cost == 0.0 assert completion_cost == 0.008 + @pytest.mark.parametrize( + "reported_cost", + [float("nan"), float("inf"), {"total_cost": float("nan")}], + ids=["nan", "inf", "nan-total-cost-dict"], + ) + def test_a_non_finite_perplexity_cost_is_priced_like_no_reported_cost_with_a_warning( + self, reported_cost: float | dict, caplog: pytest.LogCaptureFixture + ): + unreported: Final = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) + reported: Final = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) + reported.cost = reported_cost + + token_priced: Final = perplexity_cost_per_token(model="sonar-pro", usage=unreported) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + reported_priced: Final = perplexity_cost_per_token(model="sonar-pro", usage=reported) + + assert sum(token_priced) > 0 + assert reported_priced == token_priced + assert [record for record in caplog.records if "non-finite provider-reported cost" in record.getMessage()] + OFF_PEAK_MODEL = "sonar-off-peak-test" OFF_PEAK_WINDOW = "14:00-00:00" INSIDE_WINDOW = datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc) diff --git a/tests/unit/proxy/db/db_transaction_queue/test_redis_update_buffer.py b/tests/unit/proxy/db/db_transaction_queue/test_redis_update_buffer.py index 24c12e06c9c..65da5630f33 100644 --- a/tests/unit/proxy/db/db_transaction_queue/test_redis_update_buffer.py +++ b/tests/unit/proxy/db/db_transaction_queue/test_redis_update_buffer.py @@ -1,10 +1,12 @@ import json +import logging from datetime import datetime, timezone from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest +from litellm._logging import verbose_proxy_logger from litellm.constants import REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.proxy._types import DailyTagSpendTransaction @@ -379,6 +381,104 @@ async def test_org_member_spend_is_summed_across_pods_and_restored_on_rpush_fail assert restored_spend["org_member_list_transactions"] == {member_key: 1.5} +@pytest.mark.asyncio +async def test_a_single_parked_batch_with_a_nan_spend_drains_as_zero( + redis_update_buffer: RedisUpdateBuffer, mock_redis_cache: AsyncMock +): + mock_redis_cache.async_lpop = AsyncMock( + return_value=json.dumps({"key_list_transactions": {"nan-key": float("nan"), "ok-key": 0.5}}, allow_nan=True) + ) + + drained: Final = await redis_update_buffer.get_all_update_transactions_from_redis_buffer() + + assert drained is not None + assert drained["key_list_transactions"] == {"nan-key": 0.0, "ok-key": 0.5} + + +PARKED_DAILY_KEY: Final = "user-1_2025-01-15_api-key-123_gpt-4o_openai_/chat/completions" + + +def _parked_daily_batch(spend: float) -> str: + return json.dumps( + { + PARKED_DAILY_KEY: { + "spend": spend, + "prompt_tokens": 12, + "completion_tokens": 7, + "api_requests": 1, + "successful_requests": 1, + "failed_requests": 0, + } + }, + allow_nan=True, + ) + + +@pytest.mark.asyncio +async def test_a_nan_spend_parked_by_an_older_version_drains_from_the_pipeline_as_zero( + redis_update_buffer: RedisUpdateBuffer, mock_redis_cache: AsyncMock +): + entity_batch: Final = json.dumps({"key_list_transactions": {"hashed-key": float("nan")}}, allow_nan=True) + window_batches: Final = tuple( + json.dumps( + [ + { + "entity_type": "key", + "entity_id": "hashed-key", + "window_duration": "30d", + "window_start": "2026-08-01T00:00:00.000000", + "spend": spend, + "started_at": None, + } + ], + allow_nan=True, + ) + for spend in (float("nan"), 3.0) + ) + mock_redis_cache.async_lpop_pipeline = AsyncMock( + return_value=[ + [entity_batch], + [_parked_daily_batch(float("nan")), _parked_daily_batch(0.0042)], + None, + None, + None, + None, + list(window_batches), + ] + ) + + ( + db_spend, + daily_user, + *_rest, + window_spend, + ) = await redis_update_buffer.get_all_transactions_from_redis_buffer_pipeline() + + assert db_spend is not None + assert db_spend["key_list_transactions"] == {"hashed-key": 0} + assert daily_user is not None + assert daily_user[PARKED_DAILY_KEY]["spend"] == 0.0042 + assert daily_user[PARKED_DAILY_KEY]["api_requests"] == 2 + assert window_spend is not None + assert [transaction["spend"] for transaction in window_spend] == [3.0] + + +@pytest.mark.asyncio +async def test_a_nan_spend_parked_by_an_older_version_drains_from_a_daily_key_as_zero( + redis_update_buffer: RedisUpdateBuffer, mock_redis_cache: AsyncMock, caplog: pytest.LogCaptureFixture +): + mock_redis_cache.async_lpop = AsyncMock( + return_value=[_parked_daily_batch(float("inf")), _parked_daily_batch(0.0042)] + ) + + with caplog.at_level(logging.WARNING, logger=verbose_proxy_logger.name): + daily_tag: Final = await redis_update_buffer.get_all_daily_tag_spend_update_transactions_from_redis_buffer() + + assert daily_tag is not None + assert daily_tag[PARKED_DAILY_KEY]["spend"] == 0.0042 + assert [record.getMessage() for record in caplog.records if "Infinity" in record.getMessage()] != [] + + @pytest.mark.asyncio async def test_get_all_transactions_from_redis_buffer_pipeline_no_redis(): """When redis_cache is None, should return all Nones""" @@ -724,6 +824,10 @@ class _ListRedis: def __init__(self) -> None: self.rows: list[str] = [] + async def async_rpush(self, key: str, values: list[str]) -> int: + self.rows.extend(values) + return len(self.rows) + async def async_rpush_and_trim(self, key: str, values: list[str], max_len: int) -> int: self.rows.extend(values) pushed_len = len(self.rows) @@ -752,6 +856,32 @@ async def test_store_spend_logs_in_redis_drops_oldest_rows_past_the_cap(): assert await buffer.get_spend_logs_from_redis_buffer(limit=10) == () +@pytest.mark.asyncio +@pytest.mark.parametrize("spend", [float("nan"), float("inf")]) +async def test_a_non_finite_daily_spend_buffered_in_redis_drains_as_zero(spend: float): + buffer: Final = RedisUpdateBuffer(redis_cache=_ListRedis()) + queue: Final = DailySpendUpdateQueue() + await queue.add_update(json.loads(_parked_daily_batch(spend))) + await queue.add_update(json.loads(_parked_daily_batch(0.0042))) + + await buffer.store_in_memory_daily_tag_spend_updates_in_redis(queue) + drained: Final = await buffer.get_all_daily_tag_spend_update_transactions_from_redis_buffer() + + assert drained is not None + assert drained[PARKED_DAILY_KEY]["spend"] == 0.0 + assert drained[PARKED_DAILY_KEY]["api_requests"] == 2 + + +@pytest.mark.asyncio +async def test_a_spend_log_row_parked_with_a_nan_spend_is_read_back_with_zero_spend(): + buffer: Final = RedisUpdateBuffer(redis_cache=_ListRedis()) + buffer.should_commit_spend_updates_to_redis = MagicMock(return_value=True) + + assert await buffer.store_spend_logs_in_redis([{"request_id": "nan-row", "spend": float("nan")}]) is True + + assert await buffer.get_spend_logs_from_redis_buffer(limit=10) == ({"request_id": "nan-row", "spend": 0.0},) + + @pytest.mark.asyncio async def test_store_spend_logs_in_redis_reports_failure_without_redis(): buffer = RedisUpdateBuffer(redis_cache=None) diff --git a/tests/unit/proxy/db/test_db_spend_update_writer.py b/tests/unit/proxy/db/test_db_spend_update_writer.py index c3a93fd89be..298b153f621 100644 --- a/tests/unit/proxy/db/test_db_spend_update_writer.py +++ b/tests/unit/proxy/db/test_db_spend_update_writer.py @@ -2226,6 +2226,60 @@ async def test_shutdown_drain_finishes_the_rollups_update_database_started(): assert (team_row["api_requests"], team_row["spend"]) == (1, 0.1) +@pytest.mark.asyncio +@pytest.mark.parametrize("response_cost", [float("nan"), float("inf"), float("-inf")]) +async def test_update_database_records_a_non_finite_response_cost_as_zero(response_cost: float) -> None: + db_writer: Final = DBSpendUpdateWriter() + prisma: Final = _tool_usage_prisma() + prisma.get_request_status = MagicMock(return_value="success") + response: Final = litellm.ModelResponse( + model="gpt-5.6-luna", + usage=litellm.Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15), + ) + with ( + patch("litellm.proxy.proxy_server.disable_spend_logs", False), + patch("litellm.proxy.proxy_server.prisma_client", prisma), + patch("litellm.proxy.proxy_server.user_api_key_cache", MagicMock()), + patch("litellm.proxy.proxy_server.litellm_proxy_budget_name", "test-budget"), + ): + await db_writer.update_database( + token="test-token", + user_id="test-user", + end_user_id=None, + team_id="test-team", + org_id=None, + kwargs={ + "model": "gpt-5.6-luna", + "custom_llm_provider": "openai", + "call_type": "acompletion", + "litellm_call_id": "req-non-finite-cost", + "response_cost": response_cost, + "litellm_params": {"metadata": {"user_api_key": "hashed-key", "user_api_key_team_id": "test-team"}}, + }, + completion_response=response, + start_time=datetime.now(timezone.utc), + end_time=datetime.now(timezone.utc), + response_cost=response_cost, + ) + await drain_logging_tasks() + + assert [row["spend"] for row in prisma.spend_log_transactions] == [0] + spend: Final = await db_writer.spend_update_queue.flush_and_get_aggregated_db_spend_update_transactions() + assert spend["key_list_transactions"] == {"test-token": 0} + assert spend["team_list_transactions"] == {"test-team": 0} + daily_user: Final = ( + await db_writer.daily_spend_update_queue.flush_and_get_aggregated_daily_spend_update_transactions() + ) + daily_team: Final = ( + await db_writer.daily_team_spend_update_queue.flush_and_get_aggregated_daily_spend_update_transactions() + ) + assert daily_user is not None and daily_team is not None + (user_row,) = daily_user.values() + (team_row,) = daily_team.values() + assert (user_row["api_requests"], user_row["spend"]) == (1, 0) + assert (team_row["api_requests"], team_row["spend"]) == (1, 0) + + @pytest.mark.asyncio async def test_shutdown_drain_waits_for_an_in_flight_rollup_enqueue(): """ diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py index 4011bc0294a..7f8aac983f5 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py @@ -323,6 +323,35 @@ def test_openrouter_decisions_response_uses_provider_cost_without_cost_map_row( assert response["kwargs"]["combined_usage_object"].total_tokens == 302 +@pytest.mark.parametrize("reported_cost", [float("nan"), float("inf")]) +def test_openrouter_non_finite_cost_logs_the_token_price(monkeypatch: pytest.MonkeyPatch, reported_cost: float) -> None: + monkeypatch.setitem( + litellm.model_cost, + "openrouter/typesafe/jev-1.13", + {"input_cost_per_token": 1.0, "output_cost_per_token": 3.0, "litellm_provider": "openrouter", "mode": "chat"}, + ) + logging_obj: Final = _logging_obj() + + result: Final = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler( + httpx_response=_response(), + response_body={ + "model": "typesafe/jev-1.13-20260917", + "usage": {"input_tokens": 2, "output_tokens": 1, "cost": reported_cost}, + }, + logging_obj=logging_obj, + url_route="https://openrouter.ai/api/alpha/decisions", + result='{"answers": {}}', + start_time=datetime(2000, 1, 1), + end_time=datetime(2000, 1, 1), + cache_hit=False, + request_body={"model": "typesafe/jev-1.13"}, + custom_llm_provider="openrouter", + ) + + assert result["kwargs"]["response_cost"] == pytest.approx(5.0) + assert logging_obj.model_call_details["response_cost"] == pytest.approx(5.0) + + def test_success_handler_dispatches_openrouter_to_the_shared_handler(): logging_obj = _logging_obj() normalized = PassThroughEndpointLogging().normalize_llm_passthrough_logging_payload( diff --git a/tests/unit/proxy/spend_tracking/test_savings.py b/tests/unit/proxy/spend_tracking/test_savings.py index f9566fd1ba0..8503a124b01 100644 --- a/tests/unit/proxy/spend_tracking/test_savings.py +++ b/tests/unit/proxy/spend_tracking/test_savings.py @@ -137,6 +137,34 @@ def test_compression_savings_priced_at_input_rate(): assert result.prompt_caching == 0.0 +def test_an_overflowing_deployment_price_records_finite_savings() -> None: + router: Final = Router( + model_list=[ + { + "model_name": "overflowing-sonnet", + "litellm_params": { + "model": "anthropic/claude-sonnet-5-5", + "input_cost_per_token": 1e308, + "cache_read_input_token_cost": 1e-07, + }, + }, + ] + ) + deployment_id: Final = router.get_model_list(model_name="overflowing-sonnet")[0]["model_info"]["id"] + + result: Final = compute_savings_spend( + model="claude-sonnet-5-5", + custom_llm_provider="anthropic", + compression_saved_tokens=4389, + gateway_injected_cache=True, + usage_object=_caching_usage(read=1000, written=0), + model_id=deployment_id, + llm_router=lambda: router, + ) + + assert result == (0.0, 0.0, 0.0, 0.0) + + def test_prompt_caching_savings_priced_at_input_minus_cache_read(): input_cost, cache_read_cost = _anthropic_costs("claude-sonnet-5") # A model that supports prompt caching must charge less to read from cache; diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 2db8941d304..a7596b51e3e 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -12496,6 +12496,55 @@ class TestResponseCostFromLoggingObjForCacheHits: ) assert {name: headers.get(name) for name in cost_headers} == dict.fromkeys(cost_headers, expected_header) + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("response_cost", "expected_header"), + ((float("nan"), None), (float("inf"), None), (0.0042, "0.0042"), (2, "2")), + ids=("nan", "inf", "finite", "int"), + ) + async def test_a_non_finite_response_cost_sends_no_cost_header( + self, response_cost: float, expected_header: str | None + ): + response: Final = litellm.ModelResponse() + response._hidden_params = {"response_cost": response_cost} + + headers: Final = await ProxyBaseLLMRequestProcessing.build_litellm_proxy_success_headers_from_llm_response( + response=response, + request_data={"model": "gpt-5.4-mini"}, + request=Request({"type": "http", "method": "POST", "headers": []}), + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"), + logging_obj=self._logging_obj(), + version=None, + proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()), + ) + + assert headers.get("x-litellm-response-cost") == expected_header + + @pytest.mark.asyncio + async def test_an_overflowing_cost_breakdown_sends_no_infinite_cost_header(self): + response: Final = litellm.ModelResponse() + response._hidden_params = {"response_cost": 0.0} + logging_obj: Final = self._logging_obj() + logging_obj.cost_breakdown = { + "original_cost": float("inf"), + "input_cost": float("inf"), + "output_cost": 0.0012, + "total_cost": float("inf"), + } + + headers: Final = await ProxyBaseLLMRequestProcessing.build_litellm_proxy_success_headers_from_llm_response( + response=response, + request_data={"model": "gpt-5.4-mini"}, + request=Request({"type": "http", "method": "POST", "headers": []}), + user_api_key_dict=ProxyUserAPIKeyAuth(api_key="sk-test"), + logging_obj=logging_obj, + version=None, + proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()), + ) + + cost_headers: Final = {name: value for name, value in headers.items() if name.startswith("x-litellm-response-cost")} + assert cost_headers == {"x-litellm-response-cost-output": "0.0012"} + def test_a_fresh_zero_cost_v1_messages_call_still_sends_no_cost_header(self): logging_obj: Final = self._logging_obj() logging_obj.model_call_details["response_cost"] = 0.0 diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index b0bf1cb879e..aaec486b60f 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -5900,6 +5900,28 @@ def test_stream_chunk_builder_keeps_provider_reported_usage_cost(): assert response._hidden_params["response_cost"] == pytest.approx(0.5) +@pytest.mark.parametrize("reported_cost", [float("nan"), float("inf")]) +def test_stream_chunk_builder_bills_the_token_price_when_the_provider_cost_is_not_finite(reported_cost: float): + usage_chunk: Final = _stream_builder_text_chunk("gpt-4o", "") + usage_chunk.usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15, cost=reported_cost) + chunks: Final = [ + _stream_builder_text_chunk("gpt-4o", "Hello "), + _stream_builder_text_chunk("gpt-4o", "world.", finish_reason="stop"), + usage_chunk, + ] + + response: Final = litellm.stream_chunk_builder( + chunks=chunks, messages=[{"role": "user", "content": "hi"}], logging_obj=_stream_builder_logging_obj() + ) + + assert response is not None + prompt_cost, completion_cost = litellm.cost_per_token(model="gpt-4o", usage_object=response.usage) + token_price: Final = prompt_cost + completion_cost + assert token_price > 0 + assert getattr(response.usage, "cost", None) == pytest.approx(token_price) + assert response._hidden_params["response_cost"] == pytest.approx(token_price) + + def test_completion_cost_on_a_stamped_stream_still_bills_the_reported_charge(): usage_chunk: Final = _stream_builder_text_chunk("gpt-4o", "") usage_chunk.usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15, cost=0.5) From 5d5e3685c1d52841082c199a88f8190f35e7be9b Mon Sep 17 00:00:00 2001 From: tessellark <349031000+tessellark@users.noreply.github.com> Date: Fri, 30 Oct 2026 08:48:43 -0700 Subject: [PATCH 07/74] fix(logging): warn when drop_params drops a requested reasoning level on the remaining paths (#48933) * fix(logging): warn when drop_params drops a requested reasoning level on the remaining paths * fix(logging): keep guardrail and token-count reads of a request from logging drop warnings Only the real Anthropic-messages adapter request reports a dropped thinking budget. Guardrail, shadow-eval and token-count translations of the same request now drop it silently, so one request logs one warning. The 422 retry also no longer hashes each loc entry, so a list inside loc cannot raise TypeError. * fix(bedrock): keep the DeepSeek R1 Converse reasoning_effort drop at debug level Only allowed_openai_params reaches that branch, and it puts reasoning_effort back after the provider mapping, so a warning there claimed a drop that never happened * fix(logging): warn on Gemini non-integer budget drops and keep the auto-router baseline silent Gemini 2.x now logs the drop of a non-integer thinking budget. The auto-router savings baseline builds a body it never sends, so it no longer warns. Logged values are capped with reprlib, Azure AI marks only its drop_params retry as gated, and Bedrock Chat Completions names the model without its route prefix. * fix(logging): warn on the Gemini 3, Magistral, Ollama, Fireworks and DeepSeek ARN reasoning drops Gemini 3 and later now log a dropped thinking budget, Magistral logs the level it swaps for a reasoning prompt, Ollama logs a level it maps to think off, Fireworks logs chat_template_kwargs effort keys dropped for a model the map does not list as reasoning, and DeepSeek on Bedrock logs a drop when the model is addressed by an ARN that advertises the param. * fix(logging): warn on the remaining Ollama, DeepSeek and Tencent reasoning drops The ollama/ completion route now shares the chat route's think mapping, so a level it turns off logs the same warning. DeepSeek warns when it drops thinking.budget_tokens, and Tencent warns when an adaptive-only model gets a non-integer budget. Magistral no longer warns on low, medium, high or enabled thinking, the values its reasoning prompt stands in for. The Gemini 3 budget warning no longer claims the model uses its default level. * fix(reasoning): warn when Claude drops a non-string reasoning_effort Claude chat and native /v1/messages dropped a reasoning_effort like 5 or {"effort": 5} with no log line. Both now log one warn-only WARNING. A Magistral thinking.type sent as a list no longer raises TypeError. --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../reasoning_effort_utils.py | 77 ++++++++++- litellm/llms/anthropic/chat/transformation.py | 2 + litellm/llms/anthropic/common_utils.py | 33 +++-- .../messages/adapters/transformation.py | 18 ++- .../llms/anthropic/messages/native/handler.py | 8 +- .../messages/native/transformation.py | 11 +- .../responses_adapters/transformation.py | 4 +- .../llms/anthropic/prompt_cache_prediction.py | 8 +- litellm/llms/azure_ai/chat/transformation.py | 25 +++- .../chat/chat_completions/transformation.py | 17 +++ .../bedrock/chat/converse_transformation.py | 17 ++- litellm/llms/deepseek/chat/transformation.py | 13 ++ litellm/llms/fal_ai/chat/transformation.py | 13 +- .../llms/fireworks_ai/chat/transformation.py | 14 +- .../fireworks_ai/completion/transformation.py | 7 +- .../llms/hosted_vllm/chat/transformation.py | 2 +- litellm/llms/mistral/chat/transformation.py | 25 +++- litellm/llms/oci/chat/transformation.py | 16 +++ litellm/llms/ollama/chat/transformation.py | 8 +- litellm/llms/ollama/common_utils.py | 12 ++ .../llms/ollama/completion/transformation.py | 8 +- litellm/llms/openai/common_utils.py | 22 +-- litellm/llms/openai/openai.py | 8 +- litellm/llms/tencent/chat/transformation.py | 7 + .../vertex_and_google_ai_studio_gemini.py | 26 +++- .../reasoning_effort_capability.py | 8 +- .../test_anthropic_guardrail_handler.py | 16 +++ .../test_anthropic_chat_transformation.py | 42 ++++++ ...hropic_messages_adapters_transformation.py | 16 ++- ..._handler_reasoning_effort_normalization.py | 26 ++-- .../native/test_anthropic_messages_handler.py | 10 +- .../native/test_anthropic_messages_router.py | 17 ++- .../test_reasoning_effort_translation.py | 46 +++++++ .../test_responses_adapters_transformation.py | 33 +++-- .../messages/test_reasoning_effort_fields.py | 5 +- .../anthropic/test_anthropic_common_utils.py | 18 ++- .../test_anthropic_prompt_cache_prediction.py | 40 ++++++ .../chat/test_azure_ai_transformation.py | 61 +++++++++ ...bedrock_chat_completions_transformation.py | 52 ++++--- .../chat/test_converse_transformation.py | 47 +++++++ .../chat/test_deepseek_chat_transformation.py | 41 ++++++ .../chat/test_fal_ai_chat_transformation.py | 33 ++++- .../test_fireworks_ai_chat_transformation.py | 35 +++++ ...works_ai_text_completion_transformation.py | 37 +++++ .../test_mistral_chat_transformation.py | 105 +++++++++++++++ .../oci/chat/test_oci_chat_transformation.py | 41 +++--- .../ollama/test_ollama_chat_transformation.py | 35 +++++ .../test_ollama_completion_transformation.py | 39 ++++++ .../llms/openai/test_openai_common_utils.py | 127 +++++++++++++++++- .../chat/test_tencent_chat_transformation.py | 26 ++++ ...test_vertex_and_google_ai_studio_gemini.py | 72 ++++++++++ tests/unit/test_main.py | 36 ++++- 52 files changed, 1318 insertions(+), 147 deletions(-) diff --git a/litellm/litellm_core_utils/reasoning_effort_utils.py b/litellm/litellm_core_utils/reasoning_effort_utils.py index acd20aeb37d..e9c94e32f8f 100644 --- a/litellm/litellm_core_utils/reasoning_effort_utils.py +++ b/litellm/litellm_core_utils/reasoning_effort_utils.py @@ -1,6 +1,12 @@ +import reprlib +from collections.abc import Collection, Mapping +from dataclasses import dataclass from typing import Final, Literal +from pydantic import TypeAdapter + import litellm +from litellm._logging import verbose_logger from litellm.constants import ( DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET, @@ -11,6 +17,54 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping OpenAIStyleReasoningEffort = Literal["minimal", "low", "medium", "high"] +_DROP_WARNING: Final = "Dropping %s=%s for model=%r because %s" +_DROP_PARAMS_DROP_WARNING: Final = "Dropping %s=%s for model=%r (drop_params=True) because %s" +_NON_NUMERIC_BUDGET_REASON: Final = "it is not a number. Send budget_tokens as an integer to set a reasoning level." +_NON_STRING_EFFORT_REASON: Final = ( + "it is not a string. Send reasoning_effort as a level name such as 'high' to set a reasoning level." +) +_REASONING_PARAMS: Final = ("reasoning", "reasoning_effort", "thinking") +OPAQUE_VALUE: Final = TypeAdapter(object) + + +def warn_dropped_reasoning( + param: str, value: object, model: str | None, reason: str, *, drop_params_gated: bool +) -> None: + verbose_logger.warning( + _DROP_PARAMS_DROP_WARNING if drop_params_gated else _DROP_WARNING, param, reprlib.repr(value), model, reason + ) + + +def warn_non_string_reasoning_effort(value: object, model: str | None) -> None: + effort: Final = as_str_mapping(value) + requested_level: Final = effort.get("effort") if effort is not None else value + if requested_level is None or isinstance(value, str): + return + warn_dropped_reasoning("reasoning_effort", value, model, _NON_STRING_EFFORT_REASON, drop_params_gated=False) + + +def warn_dropped_reasoning_params( + dropped: Collection[object], + request: Mapping[str, object], + model: str | None, + reason: str, + *, + drop_params_gated: bool, +) -> None: + for param in _REASONING_PARAMS: + if param in dropped and param in request: + warn_dropped_reasoning(param, request[param], model, reason, drop_params_gated=drop_params_gated) + + +@dataclass(frozen=True, slots=True) +class DroppedThinkingBudget: + budget_tokens: object + + def warn(self, model: str) -> None: + warn_dropped_reasoning( + "thinking.budget_tokens", self.budget_tokens, model, _NON_NUMERIC_BUDGET_REASON, drop_params_gated=True + ) + def reasoning_effort_from_thinking_budget( budget_tokens: float, @@ -29,7 +83,9 @@ def reasoning_effort_from_thinking_budget( return "minimal" -def reasoning_effort_from_thinking(thinking: object, *, drop_params: bool) -> OpenAIStyleReasoningEffort | None: +def _effort_or_dropped_budget( + thinking: object, *, drop_params: bool +) -> OpenAIStyleReasoningEffort | DroppedThinkingBudget | None: fields: Final = as_str_mapping(thinking) if fields is None or fields.get("type") != "enabled": return None @@ -37,10 +93,27 @@ def reasoning_effort_from_thinking(thinking: object, *, drop_params: bool) -> Op if isinstance(budget, (int, float)) and not isinstance(budget, bool): return reasoning_effort_from_thinking_budget(budget) if drop_params or litellm.drop_params is True: - return None + return DroppedThinkingBudget(budget) raise non_numeric_thinking_budget_error(budget) +def reasoning_effort_from_thinking( + thinking: object, *, model: str, drop_params: bool +) -> OpenAIStyleReasoningEffort | None: + effort: Final = _effort_or_dropped_budget(thinking, drop_params=drop_params) + if not isinstance(effort, DroppedThinkingBudget): + return effort + effort.warn(model) + return None + + +def reasoning_effort_from_thinking_silently( + thinking: object, *, drop_params: bool +) -> OpenAIStyleReasoningEffort | None: + effort: Final = _effort_or_dropped_budget(thinking, drop_params=drop_params) + return None if isinstance(effort, DroppedThinkingBudget) else effort + + def non_numeric_thinking_budget_error(budget: object) -> UnsupportedParamsError: return UnsupportedParamsError( message=( diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 865247afc63..d773e31c733 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -48,6 +48,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import place_mid_conversation_system, split_leading_system_run, ) +from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_non_string_reasoning_effort from litellm.llms.base_llm.base_utils import ( ResponseFormatSchema, as_json_object, @@ -1951,6 +1952,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if isinstance(effort_value, dict): effort_value = effort_value.get("effort") if not isinstance(effort_value, str): + warn_non_string_reasoning_effort(OPAQUE_VALUE.validate_python(value), model) continue mapped_thinking = AnthropicConfig._map_reasoning_effort( reasoning_effort=effort_value, diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 82977f97854..6d1ac24af55 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -37,7 +37,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of, ) -from litellm.litellm_core_utils.reasoning_effort_utils import non_numeric_thinking_budget_error +from litellm.litellm_core_utils.reasoning_effort_utils import DroppedThinkingBudget, non_numeric_thinking_budget_error from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.anthropic.wif import ( aget_anthropic_wif_token, @@ -610,11 +610,11 @@ def _numeric_budget_tokens(budget_tokens: object) -> int | None: return None -def _legacy_budget_tokens(budget_tokens: object, *, drop_params: bool) -> int | None: +def _legacy_budget_tokens(budget_tokens: object, *, drop_params: bool) -> int | DroppedThinkingBudget: if (numeric := _numeric_budget_tokens(budget_tokens)) is not None: return numeric if drop_params or litellm.drop_params is True: - return None + return DroppedThinkingBudget(budget_tokens) raise non_numeric_thinking_budget_error(budget_tokens) @@ -1097,6 +1097,22 @@ class AnthropicModelInfo(BaseLLMModelInfo): custom_llm_provider: str, drop_params: bool, ) -> None: + dropped: Final = AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model_silently( + model=model, + optional_params=optional_params, + custom_llm_provider=custom_llm_provider, + drop_params=drop_params, + ) + if dropped is not None: + dropped.warn(model) + + @staticmethod + def translate_legacy_thinking_for_adaptive_model_silently( + model: str, + optional_params: MutableMapping[str, object], # mutable-ok: in-place out-param like the sibling helpers + custom_llm_provider: str, + drop_params: bool, + ) -> DroppedThinkingBudget | None: """Translate legacy ``thinking.type=enabled`` to adaptive for the adaptive-thinking models that reject it (4.7+ and the 5 families). Models flagged ``supports_legacy_thinking`` (the 4.6 family) accept the @@ -1105,12 +1121,12 @@ class AnthropicModelInfo(BaseLLMModelInfo): ``output_config.effort`` is never overridden. """ if not AnthropicModelInfo._is_adaptive_thinking_model(model, custom_llm_provider): - return + return None if AnthropicModelInfo._supports_legacy_thinking(model, custom_llm_provider): - return + return None thinking: Final = validated_object_mapping(optional_params.get("thinking")) if thinking is None or thinking.get("type") != "enabled": - return + return None budget_tokens: Final = _legacy_budget_tokens(thinking.get("budget_tokens", 0), drop_params=drop_params) display: Final = thinking.get("display") @@ -1119,8 +1135,8 @@ class AnthropicModelInfo(BaseLLMModelInfo): thinking, rebuilt_keys=frozenset(("type", "budget_tokens", "display")), drop_params=drop_params ) optional_params["thinking"] = {**carried_keys, "type": "adaptive", **supported_display} - if budget_tokens is None: - return + if isinstance(budget_tokens, DroppedThinkingBudget): + return budget_tokens effort: Final = AnthropicModelInfo._legacy_budget_to_effort( model=model, budget_tokens=budget_tokens, custom_llm_provider=custom_llm_provider ) @@ -1129,6 +1145,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): "effort": effort, **(existing_output_config if isinstance(existing_output_config, dict) else MappingProxyType({})), } + return None @staticmethod def _legacy_budget_to_effort(model: str, budget_tokens: int, custom_llm_provider: str) -> str: diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index 8ea6293bbe6..bc9649ed867 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -134,6 +134,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( ) from litellm.litellm_core_utils.reasoning_effort_utils import ( reasoning_effort_from_thinking, + reasoning_effort_from_thinking_silently, ) from litellm.llms.anthropic.common_utils import ( eager_input_streaming_flag, @@ -297,6 +298,7 @@ class AnthropicAdapter: custom_llm_provider=custom_llm_provider, api_base=api_base, drop_params=drop_params, + report_reasoning_drops=True, ) return translated_body, tool_name_mapping @@ -666,7 +668,9 @@ class LiteLLMAnthropicMessagesAdapter: def translate_anthropic_thinking_to_reasoning_effort( thinking: AnthropicThinkingParam, *, + model: str, drop_params: bool = True, + report_reasoning_drops: bool = False, ) -> str | None: """ Translate Anthropic's thinking parameter to OpenAI's reasoning_effort. @@ -685,7 +689,9 @@ class LiteLLMAnthropicMessagesAdapter: if thinking_type == "disabled": return "none" elif thinking_type == "enabled": - return reasoning_effort_from_thinking(thinking, drop_params=drop_params) + if report_reasoning_drops: + return reasoning_effort_from_thinking(thinking, model=model, drop_params=drop_params) + return reasoning_effort_from_thinking_silently(thinking, drop_params=drop_params) elif thinking_type == "adaptive": # Adaptive thinking: effort is controlled by output_config.effort, # not budget_tokens. Return a default; caller should override with @@ -752,7 +758,7 @@ class LiteLLMAnthropicMessagesAdapter: return {"thinking": thinking} else: reasoning_effort: Final = LiteLLMAnthropicMessagesAdapter.translate_anthropic_thinking_to_reasoning_effort( - thinking + thinking, model=model ) if reasoning_effort: return { @@ -1147,6 +1153,7 @@ class LiteLLMAnthropicMessagesAdapter: *, custom_llm_provider: str | None = None, drop_params: bool = True, + report_reasoning_drops: bool = False, ) -> None: """Translate Anthropic thinking to either thinking or reasoning_effort. @@ -1207,7 +1214,10 @@ class LiteLLMAnthropicMessagesAdapter: return reasoning_effort: Final = declared_effort or self.translate_anthropic_thinking_to_reasoning_effort( - cast(AnthropicThinkingParam, thinking), drop_params=drop_params + cast(AnthropicThinkingParam, thinking), + model=model, + drop_params=drop_params, + report_reasoning_drops=report_reasoning_drops, ) if not reasoning_effort: return @@ -1267,6 +1277,7 @@ class LiteLLMAnthropicMessagesAdapter: api_base: str | None = None, preserve_midturn_system: bool = False, drop_params: bool = True, + report_reasoning_drops: bool = False, ) -> tuple[ChatCompletionRequest, dict[str, str]]: """ This is used by the beta Anthropic Adapter, for translating anthropic `/v1/messages` requests to the openai format. @@ -1321,6 +1332,7 @@ class LiteLLMAnthropicMessagesAdapter: new_kwargs=new_kwargs, custom_llm_provider=custom_llm_provider, drop_params=drop_params, + report_reasoning_drops=report_reasoning_drops, ) ## CONVERT STOP_SEQUENCES self._translate_stop_sequences_to_openai( diff --git a/litellm/llms/anthropic/messages/native/handler.py b/litellm/llms/anthropic/messages/native/handler.py index 6f97a019a96..1da7c2efdda 100644 --- a/litellm/llms/anthropic/messages/native/handler.py +++ b/litellm/llms/anthropic/messages/native/handler.py @@ -20,6 +20,7 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.exception_mapping_utils import exception_type from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.anthropic.common_utils import ( + AnthropicModelInfo, strip_provider_specific_fields_from_anthropic_messages, tools_marked_for_a_tool_config_point, ) @@ -36,7 +37,6 @@ from litellm.router_utils.reasoning_effort_capability import ( keeps_effort, model_group_from_metadata, resolve_deployment_reasoning_effort, - with_adaptive_thinking_as_effort, ) from litellm.types.llms.anthropic import AllAnthropicToolsValues from litellm.types.llms.anthropic_messages.anthropic_request import AnthropicMetadata @@ -185,9 +185,11 @@ def _with_deployment_output_effort( ).value def translate(source: Mapping[str, object]) -> Mapping[str, object]: - return with_adaptive_thinking_as_effort( - source, model=model, custom_llm_provider=custom_llm_provider, drop_params=drop_params + translated: Final = dict(source) + AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model( + model=model, optional_params=translated, custom_llm_provider=custom_llm_provider, drop_params=drop_params ) + return translated requested_output: Final = as_str_mapping(params.get("output_config")) caller_output: Final = None if requested_output is None else resolve(requested_output) diff --git a/litellm/llms/anthropic/messages/native/transformation.py b/litellm/llms/anthropic/messages/native/transformation.py index ef855eff180..c148a0defe8 100644 --- a/litellm/llms/anthropic/messages/native/transformation.py +++ b/litellm/llms/anthropic/messages/native/transformation.py @@ -21,6 +21,7 @@ from litellm.constants import ANTHROPIC_BILLING_METADATA_PREFIX from litellm.exceptions import AuthenticationError from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_non_string_reasoning_effort from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.anthropic.messages.convertible_blocks import drop_non_string_tool_result_texts from litellm.llms.anthropic.messages.safeguards import proxy_serves_safeguards @@ -103,6 +104,7 @@ _MESSAGES_ADAPTER: Final = TypeAdapter(list[dict[str, object]]) class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): _workload_identity_eligible: ClassVar[bool] = True + _report_reasoning_drops: ClassVar[bool] = True @property def custom_llm_provider(self) -> str | None: @@ -695,12 +697,17 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): ): anthropic_messages_optional_request_params.pop("output_format") # pyright: ignore[reportUnknownMemberType] # the request params arrive as a bare dict + requested_reasoning_effort: Final = OPAQUE_VALUE.validate_python( + anthropic_messages_optional_request_params.get("reasoning_effort") # pyright: ignore[reportUnknownMemberType] # the request params arrive as a bare dict + ) self._translate_reasoning_effort_to_anthropic( model=model, optional_params=anthropic_messages_optional_request_params, max_tokens=max_tokens, custom_llm_provider=self._resolved_provider, ) + if self._report_reasoning_drops: + warn_non_string_reasoning_effort(requested_reasoning_effort, model) AnthropicModelInfo.maybe_drop_disabled_thinking( model=model, @@ -708,12 +715,14 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): custom_llm_provider=self._resolved_provider, ) - AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model( + dropped_budget: Final = AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model_silently( model=model, optional_params=anthropic_messages_optional_request_params, custom_llm_provider=self._resolved_provider, drop_params=getattr(litellm_params, "drop_params", None) is True, ) + if dropped_budget is not None and self._report_reasoning_drops: + dropped_budget.warn(model) self._translate_adaptive_effort_for_non_adaptive_model( model=model, diff --git a/litellm/llms/anthropic/messages/responses_adapters/transformation.py b/litellm/llms/anthropic/messages/responses_adapters/transformation.py index 94c608f90e6..0630bef0311 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/messages/responses_adapters/transformation.py @@ -534,6 +534,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: output_config: dict[str, object] | None = None, auto_summary_level: str = "detailed", *, + model: str, drop_params: bool = True, ) -> dict[str, object] | None: """ @@ -555,7 +556,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: if isinstance(output_config, dict) and output_config.get("effort"): effort = output_config["effort"] elif thinking_type == "enabled": - budget_effort: Final = reasoning_effort_from_thinking(thinking, drop_params=drop_params) + budget_effort: Final = reasoning_effort_from_thinking(thinking, model=model, drop_params=drop_params) if budget_effort is None: return None effort = budget_effort @@ -667,6 +668,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: thinking, output_config=cast(dict[str, object] | None, output_config), auto_summary_level=auto_summary_level, + model=model, drop_params=drop_params, ) if reasoning: diff --git a/litellm/llms/anthropic/prompt_cache_prediction.py b/litellm/llms/anthropic/prompt_cache_prediction.py index 27e003ce8b6..2da071ebdd2 100644 --- a/litellm/llms/anthropic/prompt_cache_prediction.py +++ b/litellm/llms/anthropic/prompt_cache_prediction.py @@ -9,7 +9,7 @@ from dataclasses import dataclass, field from functools import reduce from itertools import accumulate, groupby from types import MappingProxyType -from typing import Annotated, Final, Literal, Protocol, TypeAlias +from typing import Annotated, ClassVar, Final, Literal, Protocol, TypeAlias import httpx from pydantic import ConfigDict, Field, JsonValue, StrictInt, TypeAdapter, ValidationError @@ -81,6 +81,10 @@ _DEPLOYMENT_OPTIONS: Final = frozenset( ) +class _BaselineAnthropicMessagesConfig(AnthropicMessagesConfig): + _report_reasoning_drops: ClassVar[bool] = False + + class _StrictModel(LiteLLMBaseModel): model_config = ConfigDict(extra="forbid", frozen=True, strict=True) @@ -661,7 +665,7 @@ def prepare_native_baseline_body(request: Mapping[str, object], model: str) -> M TypeAdapter(tuple[str, ...]).validate_python(owned.get("additional_drop_params") or ()), dict(options), ) - config: Final = AnthropicMessagesConfig() + config: Final = _BaselineAnthropicMessagesConfig() body: Final = config.transform_anthropic_messages_request( model=resolved_model, messages=strip_provider_specific_fields_from_anthropic_messages(messages), diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index 5381447f28f..5214d31fb2e 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -20,6 +20,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( prompt_cache_breakpoint_for_wire, with_prompt_cache_breakpoint, ) +from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning_params from litellm.llms.azure.common_utils import BaseAzureLLM from litellm.llms.azure_ai.common_utils import ( api_key_header_for_base, @@ -40,6 +41,14 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer +_REQUEST_BODY: Final = TypeAdapter(dict[str, object]) + + +def _request_model(request: Mapping[str, object]) -> str | None: + model: Final = request.get("model") + return model if isinstance(model, str) else None + + class AzureFoundryErrorStrings(str, enum.Enum): SET_EXTRA_PARAMETERS_TO_PASS_THROUGH = "Set extra-parameters to 'pass-through'" @@ -510,7 +519,13 @@ class AzureAIStudioConfig(OpenAIConfig): request_data = self._drop_extra_params_from_request_data(request_data, error_text) if "Extra inputs are not permitted" in error_text and self._error_has_tool_level_extra_fields(error_text): request_data = self._drop_tool_level_extra_fields(request_data, error_text) - data: Final = drop_params_from_unprocessable_entity_error(e=e, data=request_data) + data: Final = drop_params_from_unprocessable_entity_error( + e=e, + data=request_data, + model=_request_model(_REQUEST_BODY.validate_python(request_data)), + drop_params_gated="Extra inputs are not permitted" in error_text + and not self._error_has_tool_level_extra_fields(error_text), + ) return data def _drop_tool_level_extra_fields(self, request_data: dict, error_text: str) -> dict: @@ -526,6 +541,14 @@ class AzureAIStudioConfig(OpenAIConfig): def _drop_extra_params_from_request_data(self, request_data: dict, error_text: str) -> dict: params_to_drop: Final = self._extract_params_to_drop_from_error_text(error_text) if params_to_drop: + request: Final = _REQUEST_BODY.validate_python(request_data) + warn_dropped_reasoning_params( + params_to_drop, + request, + _request_model(request), + "Azure AI rejected it as an extra parameter its deployment does not pass through.", + drop_params_gated=False, + ) for param in params_to_drop: if param in request_data: request_data.pop(param, None) diff --git a/litellm/llms/bedrock/chat/chat_completions/transformation.py b/litellm/llms/bedrock/chat/chat_completions/transformation.py index 123e4a9e6b4..dd2277a6c35 100644 --- a/litellm/llms/bedrock/chat/chat_completions/transformation.py +++ b/litellm/llms/bedrock/chat/chat_completions/transformation.py @@ -33,6 +33,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import ( inline_remote_image_urls, inline_remote_media, ) +from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning from litellm.litellm_core_utils.url_utils import ensure_api_route, join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, bedrock_bearer_token @@ -42,6 +43,7 @@ from litellm.llms.bedrock.common_utils import ( bedrock_rejects_stop_sequences, bedrock_runtime_chat_completions_serves_reasoning_inline, split_bedrock_region_path, + strip_bedrock_routing_prefix, ) from litellm.llms.openai.chat.gpt_transformation import OpenAIChatCompletionStreamingHandler from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig @@ -121,6 +123,13 @@ def without_refused_reasoning_effort(model: str, params: Mapping[str, object]) - effort: Final = params.get("reasoning_effort") if not isinstance(effort, str) or effort not in chat_completions_reasoning_efforts_refused_for(model): return params + warn_dropped_reasoning( + "reasoning_effort", + effort, + strip_bedrock_routing_prefix(model), + "AWS's Chat Completions endpoint rejects that level for this model family, so AWS applies its default effort.", + drop_params_gated=False, + ) return _without_params(params, frozenset(("reasoning_effort",))) @@ -445,6 +454,14 @@ class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig): ), status_code=400, ) + if malformed_effort: + warn_dropped_reasoning( + "reasoning_effort", + raw_params["reasoning_effort"], + strip_bedrock_routing_prefix(model), + "Bedrock's Chat Completions endpoint takes it only as a string, so AWS applies its default effort.", + drop_params_gated=True, + ) rejected_stop: Final = frozenset(("stop",)) if bedrock_rejects_stop_sequences(model) else frozenset[str]() return dict( without_refused_reasoning_effort( diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 5e621a5f322..3800489ec05 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -50,6 +50,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of, ) +from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning from litellm.llms.anthropic.chat.transformation import ( DROP_UNSUPPORTED_ADAPTIVE_THINKING_WARNING, DROP_UNSUPPORTED_OUTPUT_CONFIG_WARNING, @@ -187,6 +188,8 @@ def _tools_the_model_accepts( _OUTPUT_CONFIG_BLOCK: Final = TypeAdapter(OutputConfigBlock) +_DEEPSEEK_THINKING_REASON: Final = "DeepSeek on Bedrock reasons natively and takes no thinking field." +_DEEPSEEK_R1_EFFORT_REASON: Final = "DeepSeek R1 on Bedrock always reasons and rejects reasoning_effort." def _has_native_text_format(output_config: object) -> bool: @@ -640,6 +643,10 @@ class AmazonConverseConfig(BaseConfig): or supports_reasoning(model=base_model, custom_llm_provider=self.custom_llm_provider) ) + def _warn_native_reasoning_drop(self, param: str, value: object, model: str, reason: str) -> None: + if param in self.get_supported_openai_params(model): + warn_dropped_reasoning(param, value, model, reason, drop_params_gated=False) + def _model_rejects_reasoning_effort_param(self, model: str, base_model: str) -> bool: """Whether the model returns a 400 for every ``reasoning_effort`` shape on Converse. @@ -1143,9 +1150,8 @@ class AmazonConverseConfig(BaseConfig): "tool_choice": {"type": "auto", "disable_parallel_tool_use": not value} } if param == "thinking" and drop_thinking_param: - verbose_logger.debug( - "Dropping unsupported `thinking` param for Bedrock model=%s; it reasons natively.", - model, + self._warn_native_reasoning_drop( + param, OPAQUE_VALUE.validate_python(value), model, _DEEPSEEK_THINKING_REASON ) elif param == "thinking" and not self._is_openai_gpt_reasoning_model(model): if ( @@ -1179,10 +1185,7 @@ class AmazonConverseConfig(BaseConfig): drop_params=drop_params, ) elif param == "reasoning_effort" and isinstance(value, str) and drop_reasoning_effort_param: - verbose_logger.debug( - "Dropping unsupported `reasoning_effort` param for Bedrock model=%s; it always reasons and rejects it.", - model, - ) + self._warn_native_reasoning_drop(param, value, model, _DEEPSEEK_R1_EFFORT_REASON) elif ( param == "reasoning_effort" and isinstance(value, str) diff --git a/litellm/llms/deepseek/chat/transformation.py b/litellm/llms/deepseek/chat/transformation.py index ecaa0a2ef16..5b7c2a2c410 100644 --- a/litellm/llms/deepseek/chat/transformation.py +++ b/litellm/llms/deepseek/chat/transformation.py @@ -6,10 +6,12 @@ from collections.abc import Coroutine, Mapping, Sequence from typing import Final, Literal, cast, overload import litellm +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.prompt_templates.common_utils import ( convert_content_list_to_str, extract_search_results_text, ) +from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues @@ -18,6 +20,16 @@ from litellm.utils import supports_reasoning, supports_vision from ...openai.chat.gpt_transformation import OpenAIGPTConfig SUPPORTED_REASONING_EFFORTS: Final = ("minimal", "low", "medium", "high", "xhigh", "max", "ultra") +_NO_BUDGET_REASON: Final = "DeepSeek takes thinking only as enabled or disabled, with no budget." + + +def _warn_on_dropped_budget(thinking: object, model: str) -> None: + fields: Final = as_str_mapping(thinking) + if fields is None or fields.get("type") != "enabled" or "budget_tokens" not in fields: + return + warn_dropped_reasoning( + "thinking.budget_tokens", fields["budget_tokens"], model, _NO_BUDGET_REASON, drop_params_gated=False + ) class DeepSeekChatConfig(OpenAIGPTConfig): @@ -53,6 +65,7 @@ class DeepSeekChatConfig(OpenAIGPTConfig): # Then re-add only if valid for DeepSeek thinking_value: Final = optional_params.pop("thinking", None) reasoning_effort: Final = optional_params.pop("reasoning_effort", None) + _warn_on_dropped_budget(OPAQUE_VALUE.validate_python(thinking_value), model) # Handle thinking parameter - accept both enabled and disabled, ignore budget_tokens if isinstance(thinking_value, dict) and thinking_value.get("type") in ("enabled", "disabled"): diff --git a/litellm/llms/fal_ai/chat/transformation.py b/litellm/llms/fal_ai/chat/transformation.py index 115b1f840de..848ae498cdd 100644 --- a/litellm/llms/fal_ai/chat/transformation.py +++ b/litellm/llms/fal_ai/chat/transformation.py @@ -12,7 +12,9 @@ from typing import TYPE_CHECKING, Final import httpx from pydantic import ConfigDict, TypeAdapter +import litellm from litellm.litellm_core_utils.core_helpers import map_finish_reason +from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.secret_managers.main import get_secret_str from litellm.types.llms.base import LiteLLMBaseModel @@ -29,6 +31,10 @@ PROVIDER_PREFIX: Final[str] = "fal_ai/" PASSTHROUGH_PARAMS: Final[frozenset[str]] = frozenset(("reasoning", "temperature", "top_p")) REASONING_DISABLED_EFFORTS: Final[frozenset[str]] = frozenset(("none", "minimal")) REASONING_ENABLED_EFFORTS: Final[frozenset[str]] = frozenset(("low", "medium", "high")) +_UNSUPPORTED_EFFORT_REASON: Final = ( + f"fal only turns reasoning off for {sorted(REASONING_DISABLED_EFFORTS)} " + f"and on for {sorted(REASONING_ENABLED_EFFORTS)}." +) class _FalUsage(LiteLLMBaseModel): @@ -120,9 +126,10 @@ class FalAIChatConfig(BaseConfig): return False if isinstance(value, str) and value in REASONING_ENABLED_EFFORTS: return True - if drop_params: - return None - raise FalAIError(status_code=400, message=f"Unsupported reasoning_effort {value!r} for {model}") + if not (drop_params or litellm.drop_params is True): + raise FalAIError(status_code=400, message=f"Unsupported reasoning_effort {value!r} for {model}") + warn_dropped_reasoning("reasoning_effort", value, model, _UNSUPPORTED_EFFORT_REASON, drop_params_gated=True) + return None def _translate_param(self, param: str, value: object, model: str, drop_params: bool) -> tuple[str, object] | None: if param in ("temperature", "top_p"): diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index fa5794d71b3..92c6c8439b3 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -17,6 +17,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( strip_litellm_response_only_fields, unpack_legacy_defs, ) +from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning from litellm.litellm_core_utils.url_utils import join_api_base from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( @@ -97,6 +98,7 @@ def _json_schema_response_format(schema: object, name: str) -> Mapping[str, obje EFFORT_KWARG_KEYS: Final = frozenset({"enable_thinking", "thinking", "reasoning_budget", "low_effort"}) +_NOT_A_REASONING_MODEL_REASON: Final = "litellm's model map does not list it as a reasoning model." def _bool_from_kwargs(kwargs: Mapping[str, object], keys: tuple[str, ...]) -> bool | None: @@ -107,6 +109,13 @@ def _bool_from_kwargs(kwargs: Mapping[str, object], keys: tuple[str, ...]) -> bo return None +def warn_dropped_effort_kwargs(chat_template_kwargs: object, effort: object, model: str) -> None: + if effort != "none": + warn_dropped_reasoning( + "chat_template_kwargs", chat_template_kwargs, model, _NOT_A_REASONING_MODEL_REASON, drop_params_gated=False + ) + + def effort_from_chat_template_kwargs(kwargs: Mapping[str, object]) -> object: enable_thinking: Final = _bool_from_kwargs(kwargs, ("enable_thinking", "thinking")) if enable_thinking is False: @@ -438,10 +447,7 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): if effort is None: return () if not supports_reasoning(model=model, custom_llm_provider="fireworks_ai"): - verbose_logger.debug( - "fireworks_ai model %r does not support reasoning; dropping chat_template_kwargs effort keys.", - model, - ) + warn_dropped_effort_kwargs(OPAQUE_VALUE.validate_python(chat_template_kwargs), effort, model) return () return (("reasoning_effort", effort),) diff --git a/litellm/llms/fireworks_ai/completion/transformation.py b/litellm/llms/fireworks_ai/completion/transformation.py index 1f5d4f306a9..611e60cedd4 100644 --- a/litellm/llms/fireworks_ai/completion/transformation.py +++ b/litellm/llms/fireworks_ai/completion/transformation.py @@ -2,6 +2,7 @@ from collections.abc import Mapping from typing import Final from litellm._logging import verbose_logger +from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE from litellm.types.llms.openai import AllMessageValues, OpenAITextCompletionUserMessage from litellm.utils import supports_reasoning @@ -12,6 +13,7 @@ from ..chat.transformation import ( NIM_VLLM_STRIP_PARAMS, FireworksAIConfig, effort_from_chat_template_kwargs, + warn_dropped_effort_kwargs, ) from ..common_utils import FireworksAIMixin, resolve_fireworks_resource_name @@ -129,10 +131,7 @@ class FireworksAITextCompletionConfig(FireworksAIMixin, BaseTextCompletionConfig ) return result if not supports_reasoning(model=model, custom_llm_provider="fireworks_ai"): - verbose_logger.debug( - "fireworks_ai model %r does not support reasoning; dropping chat_template_kwargs effort keys.", - model, - ) + warn_dropped_effort_kwargs(OPAQUE_VALUE.validate_python(chat_template_kwargs), effort, model) return result return {**result, "reasoning_effort": effort} diff --git a/litellm/llms/hosted_vllm/chat/transformation.py b/litellm/llms/hosted_vllm/chat/transformation.py index 63e6cb2b5da..9f9503f93b6 100644 --- a/litellm/llms/hosted_vllm/chat/transformation.py +++ b/litellm/llms/hosted_vllm/chat/transformation.py @@ -99,7 +99,7 @@ class HostedVLLMChatConfig(OpenAIGPTConfig): thinking_effort: Final = ( None if "reasoning_effort" in non_default_params - else reasoning_effort_from_thinking(thinking, drop_params=drop_params) + else reasoning_effort_from_thinking(thinking, model=model, drop_params=drop_params) ) if thinking_effort is not None: non_default_params["reasoning_effort"] = thinking_effort diff --git a/litellm/llms/mistral/chat/transformation.py b/litellm/llms/mistral/chat/transformation.py index 15e348453e0..a402e2c4048 100644 --- a/litellm/llms/mistral/chat/transformation.py +++ b/litellm/llms/mistral/chat/transformation.py @@ -25,6 +25,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( strip_litellm_internal_message_fields, strip_none_values_from_message, ) +from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.openai.chat.gpt_transformation import ( OpenAIChatCompletionStreamingHandler, @@ -74,6 +75,12 @@ _MISTRAL_REQUEST_ROUTING: Final = TypeAdapter(MistralRequestRouting) _MISTRAL_CACHE_KEY_PARAMS: Final = TypeAdapter(MistralCacheKeyParams) _MISTRAL_REASONING_EFFORT_PARAMS: Final = TypeAdapter(MistralReasoningEffortParams) _MISTRAL_FILE_PART: Final = TypeAdapter(MistralFilePart) +_REASONING_PROMPT_REASON: Final = "litellm sends Magistral a reasoning system prompt instead of the requested level." +_NON_LEVEL_EFFORT_REASON: Final = ( + "the Mistral API takes it only as a level such as 'high', so Mistral applies its default effort." +) +_DOCUMENTED_MAGISTRAL_EFFORTS: Final = frozenset({"low", "medium", "high"}) +_REASONING_ON_THINKING_TYPES: Final = frozenset({"enabled", "adaptive"}) def _reasoning_level_and_summary(reasoning_effort: object) -> tuple[object, object]: @@ -83,6 +90,16 @@ def _reasoning_level_and_summary(reasoning_effort: object) -> tuple[object, obje return requested.get("effort"), requested.get("summary") +def _warn_unless_reasoning_prompt_honors(param: str, value: object, model: str) -> None: + if param == "reasoning_effort" and isinstance(value, str) and value in _DOCUMENTED_MAGISTRAL_EFFORTS: + return + thinking: Final = as_str_mapping(value) + thinking_type: Final = thinking.get("type") if thinking is not None else None + if param == "thinking" and isinstance(thinking_type, str) and thinking_type in _REASONING_ON_THINKING_TYPES: + return + warn_dropped_reasoning(param, value, model, _REASONING_PROMPT_REASON, drop_params_gated=False) + + def _is_json_array(value: object) -> TypeIs[Sequence[object]]: # guard-ok: a decoded JSON array is a list return isinstance(value, list) @@ -253,6 +270,7 @@ class MistralConfig(OpenAIGPTConfig): if param in ("reasoning_effort", "thinking") and "magistral" in model.lower(): # Flag that we need to add reasoning system prompt optional_params["_add_reasoning_prompt"] = True + _warn_unless_reasoning_prompt_honors(param, OPAQUE_VALUE.validate_python(value), model) if param in ("parallel_tool_calls", "prompt_cache_key"): optional_params[param] = value return optional_params @@ -260,13 +278,18 @@ class MistralConfig(OpenAIGPTConfig): def _reasoning_level_to_send( self, model: str, requested: MistralReasoningEffortParams, drop_params: bool ) -> str | None: - level, summary = _reasoning_level_and_summary(requested.get("reasoning_effort")) + requested_effort: Final = requested.get("reasoning_effort") + level, summary = _reasoning_level_and_summary(requested_effort) sendable: Final = ( _accepted_reasoning_effort(model, level, self.custom_llm_provider) if isinstance(level, str) else None ) if sendable is not None and summary is None: return sendable if drop_params or litellm.drop_params is True: + if sendable is None and level is not None: + warn_dropped_reasoning( + "reasoning_effort", requested_effort, model, _NON_LEVEL_EFFORT_REASON, drop_params_gated=True + ) return sendable raise UnsupportedParamsError( message=( diff --git a/litellm/llms/oci/chat/transformation.py b/litellm/llms/oci/chat/transformation.py index 8257a75b4a5..9c827822dc1 100644 --- a/litellm/llms/oci/chat/transformation.py +++ b/litellm/llms/oci/chat/transformation.py @@ -21,6 +21,7 @@ from litellm._logging import verbose_logger from litellm.constants import DEFAULT_OCI_CHAT_MAX_TOKENS from litellm.litellm_core_utils.logging_utils import track_llm_api_timing from litellm.litellm_core_utils.prompt_templates.common_utils import messages_with_file_parts_as_image_urls +from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException @@ -428,6 +429,13 @@ class OCIChatConfig(BaseConfig): alias = param_map.get(key) if key == "reasoning_effort" and alias and not _model_supports_reasoning_effort(model): if drop_params or litellm.drop_params: + warn_dropped_reasoning( + key, + value, + model, + "the model map lists no reasoning_effort levels for it, so OCI would reject reasoningEffort.", + drop_params_gated=True, + ) continue raise OCIError( status_code=400, @@ -451,6 +459,14 @@ class OCIChatConfig(BaseConfig): if key == "n" and (value is None or value == 1): continue if drop_params or litellm.drop_params: + if key == "reasoning_effort": + warn_dropped_reasoning( + key, + value, + model, + "OCI's Cohere chat API has no reasoning effort field.", + drop_params_gated=True, + ) continue raise OCIError( status_code=400, diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index f9e7071db41..755d647da1d 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -14,6 +14,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( extract_images_from_message, extract_reasoning_content, ) +from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException @@ -29,7 +30,7 @@ from litellm.types.llms.openai import ( ) from litellm.types.utils import ModelResponse, ModelResponseStream -from ..common_utils import OllamaError +from ..common_utils import OllamaError, ollama_think if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -172,10 +173,7 @@ class OllamaChatConfig(BaseConfig): if value.get("json_schema") and value["json_schema"].get("schema"): optional_params["format"] = value["json_schema"]["schema"] if param == "reasoning_effort" and value is not None: - if model.startswith("gpt-oss"): - optional_params["think"] = value - else: - optional_params["think"] = value in {"low", "medium", "high"} + optional_params["think"] = ollama_think(model, OPAQUE_VALUE.validate_python(value)) ### FUNCTION CALLING LOGIC ### # Ollama 0.4+ supports native tool calling - pass tools directly # and let Ollama handle model capability detection diff --git a/litellm/llms/ollama/common_utils.py b/litellm/llms/ollama/common_utils.py index f313c62f026..fcf6a396a13 100644 --- a/litellm/llms/ollama/common_utils.py +++ b/litellm/llms/ollama/common_utils.py @@ -8,11 +8,14 @@ from pydantic import BaseModel, TypeAdapter from litellm import verbose_logger from litellm._uuid import uuid +from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.utils import ModelResponse, ModelResponseStream _PROVIDER_FIELDS: Final = TypeAdapter(dict[str, object]) +_THINK_ON_EFFORTS: Final = frozenset({"low", "medium", "high"}) +_THINK_OFF_REASON: Final = "Ollama takes think only as on or off, and litellm maps this level to off." _DECODED_JSON: Final = TypeAdapter(object) @@ -296,3 +299,12 @@ class OllamaModelInfo(BaseLLMModelInfo): Return the base model name for Ollama (no-op). """ return model + + +def ollama_think(model: str, reasoning_effort: object) -> object: + if model.startswith("gpt-oss"): + return reasoning_effort + think: Final = reasoning_effort in _THINK_ON_EFFORTS + if isinstance(reasoning_effort, str) and not think and reasoning_effort != "none": + warn_dropped_reasoning("reasoning_effort", reasoning_effort, model, _THINK_OFF_REASON, drop_params_gated=False) + return think diff --git a/litellm/llms/ollama/completion/transformation.py b/litellm/llms/ollama/completion/transformation.py index 239b4e32645..6dbcb050e11 100644 --- a/litellm/llms/ollama/completion/transformation.py +++ b/litellm/llms/ollama/completion/transformation.py @@ -23,6 +23,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import ( async_inline_remote_media, inline_remote_image_urls, ) +from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException @@ -43,7 +44,7 @@ from litellm.types.utils import ( generate_id, ) -from ..common_utils import OllamaError, OllamaModelInfo, convert_image +from ..common_utils import OllamaError, OllamaModelInfo, convert_image, ollama_think if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -249,10 +250,7 @@ class OllamaConfig(BaseConfig): elif param == "stop": optional_params["stop"] = value elif param == "reasoning_effort" and value is not None: - if model.startswith("gpt-oss"): - optional_params["think"] = value - else: - optional_params["think"] = value in {"low", "medium", "high"} + optional_params["think"] = ollama_think(model, OPAQUE_VALUE.validate_python(value)) elif param == "response_format" and isinstance(value, dict): if value["type"] == "json_object": optional_params["format"] = "json" diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index aa14a985afe..c113cb351d8 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -32,6 +32,7 @@ import litellm from litellm.exceptions import error_response_without_credentials from litellm.litellm_core_utils.completion_timeout import CompletionTimeout from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object +from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning_params from litellm.litellm_core_utils.token_counter import token_counter, usage_tokenizer_model from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.custom_httpx.http_handler import ( @@ -165,17 +166,11 @@ _DECODED_JSON: Final = TypeAdapter(object) def drop_params_from_unprocessable_entity_error( e: openai.UnprocessableEntityError | httpx.HTTPStatusError, data: Mapping[str, object], + *, + model: str | None, + drop_params_gated: bool, ) -> dict[str, object]: - """ - Helper function to read OpenAI UnprocessableEntityError and drop the params that raised an error from the error message. - - Args: - e (UnprocessableEntityError): The UnprocessableEntityError exception - data (Dict[str, Any]): The original data dictionary containing all parameters - - Returns: - Dict[str, Any]: A new dictionary with invalid parameters removed - """ + """Return a copy of ``data`` without the params a 422 response names as invalid.""" invalid_params: Final[list[str]] = [] if isinstance(e, httpx.HTTPStatusError): error_json: Final = decoded_json_object(_DECODED_JSON.validate_python(e.response.json())) @@ -203,6 +198,13 @@ def drop_params_from_unprocessable_entity_error( invalid_params.append(error_dict["loc"][1]) new_data: Final = {k: v for k, v in data.items() if k not in invalid_params} + warn_dropped_reasoning_params( + invalid_params, + data, + model, + "the provider answered 422 naming it an invalid param.", + drop_params_gated=drop_params_gated, + ) return new_data diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index bef99fd0719..bdd9f285823 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -940,7 +940,9 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): except openai.UnprocessableEntityError as e: ## check if body contains unprocessable params - related issue https://github.com/BerriAI/litellm/issues/4800 if litellm.drop_params is True or drop_params is True: - inference_params = drop_params_from_unprocessable_entity_error(e, inference_params) + inference_params = drop_params_from_unprocessable_entity_error( + e, inference_params, model=model, drop_params_gated=True + ) else: raise e # e.message @@ -1097,7 +1099,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): except openai.UnprocessableEntityError as e: ## check if body contains unprocessable params - related issue https://github.com/BerriAI/litellm/issues/4800 if litellm.drop_params is True or drop_params is True: - data = drop_params_from_unprocessable_entity_error(e, data) + data = drop_params_from_unprocessable_entity_error(e, data, model=model, drop_params_gated=True) else: raise e # e.message @@ -1252,7 +1254,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): except openai.UnprocessableEntityError as e: ## check if body contains unprocessable params - related issue https://github.com/BerriAI/litellm/issues/4800 if litellm.drop_params is True or drop_params is True: - data = drop_params_from_unprocessable_entity_error(e, data) + data = drop_params_from_unprocessable_entity_error(e, data, model=model, drop_params_gated=True) else: raise e except ( diff --git a/litellm/llms/tencent/chat/transformation.py b/litellm/llms/tencent/chat/transformation.py index 1f0bcd218d6..23a6ccd52c6 100644 --- a/litellm/llms/tencent/chat/transformation.py +++ b/litellm/llms/tencent/chat/transformation.py @@ -9,12 +9,15 @@ from typing import Final, TypedDict from typing_extensions import ReadOnly import litellm +from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning from litellm.litellm_core_utils.url_utils import ensure_api_route, join_api_base from litellm.secret_managers.main import get_secret_str from litellm.utils import supports_reasoning from ...openai.chat.gpt_transformation import OpenAIGPTConfig +_ADAPTIVE_BUDGET_REASON: Final = "it is not an integer, so litellm sends adaptive thinking with no budget." + class ThinkingPayload(TypedDict, total=False): """Tencent TokenHub `thinking` object. @@ -102,6 +105,10 @@ class TencentChatConfig(OpenAIGPTConfig): if isinstance(budget, int): coerced_with_budget: Final[ThinkingPayload] = {"type": "adaptive", "budget_tokens": budget} return coerced_with_budget + if budget is not None: + warn_dropped_reasoning( + "thinking.budget_tokens", budget, model, _ADAPTIVE_BUDGET_REASON, drop_params_gated=False + ) coerced: Final[ThinkingPayload] = {"type": "adaptive"} return coerced diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index c506192f253..1ef5267df14 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -34,6 +34,7 @@ from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAcc from litellm.litellm_core_utils.prompt_templates.factory import ( encode_tool_call_id_with_signature, ) +from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, @@ -201,6 +202,13 @@ def _unsupported_reasoning_effort(reasoning_effort: str) -> UnsupportedParamsErr _SAMPLING_DEPRECATION_UNMAPPED_MODELS: Final = "models outside litellm's model map" +_NON_INTEGER_BUDGET_REASON: Final = ( + "Gemini takes budget_tokens only as an integer, so it uses its default thinking budget." +) +_LEVEL_ONLY_BUDGET_REASON: Final = ( + "Gemini 3 and later set reasoning by thinking level, not budget, so litellm does not send the budget. " + "Send reasoning_effort to pick a level." +) _sampling_deprecation_lock: Final = threading.Lock() @@ -1466,6 +1474,14 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): params["includeThoughts"] = False else: params["includeThoughts"] = sends_thoughts + if thinking_budget is not None: + warn_dropped_reasoning( + "thinking.budget_tokens", + thinking_budget, + model, + _LEVEL_ONLY_BUDGET_REASON, + drop_params_gated=False, + ) # Follow provider defaults unless explicitly opted into legacy behavior. if litellm.enable_gemini_default_thinking_level_low is True: params["thinkingLevel"] = ( @@ -1478,8 +1494,16 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): # For older Gemini models, use thinkingBudget if thinking_enabled and not VertexGeminiConfig._is_thinking_budget_zero(thinking_budget): params["includeThoughts"] = sends_thoughts - if thinking_budget is not None and isinstance(thinking_budget, int): + if isinstance(thinking_budget, int): params["thinkingBudget"] = thinking_budget + elif thinking_budget is not None: + warn_dropped_reasoning( + "thinking.budget_tokens", + thinking_budget, + model, + _NON_INTEGER_BUDGET_REASON, + drop_params_gated=False, + ) return params diff --git a/litellm/router_utils/reasoning_effort_capability.py b/litellm/router_utils/reasoning_effort_capability.py index 3563fcd96ae..c315a31b65d 100644 --- a/litellm/router_utils/reasoning_effort_capability.py +++ b/litellm/router_utils/reasoning_effort_capability.py @@ -38,7 +38,7 @@ from litellm.litellm_core_utils.reasoning_effort_strength import ( ReasoningEffortRounding, nearest_declared_reasoning_effort, ) -from litellm.litellm_core_utils.reasoning_effort_utils import reasoning_effort_from_thinking +from litellm.litellm_core_utils.reasoning_effort_utils import reasoning_effort_from_thinking_silently from litellm.llms.anthropic.common_utils import AnthropicModelInfo, validated_object_mapping from litellm.types.llms.anthropic import AnthropicThinkingParam from litellm.types.llms.openai import REASONING_EFFORT @@ -143,7 +143,7 @@ def with_adaptive_thinking_as_effort( params: Mapping[str, object], *, model: str, custom_llm_provider: str, drop_params: bool ) -> Mapping[str, object]: translated: Final = dict(params) - AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model( + AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model_silently( model=model, optional_params=translated, custom_llm_provider=custom_llm_provider, drop_params=drop_params ) return translated @@ -151,7 +151,7 @@ def with_adaptive_thinking_as_effort( def thinking_budget_effort(thinking: object, *, model: str, custom_llm_provider: str, drop_params: bool) -> str | None: if custom_llm_provider in _PROVIDERS_SENDING_THINKING_AS_EFFORT: - return reasoning_effort_from_thinking(thinking, drop_params=drop_params) + return reasoning_effort_from_thinking_silently(thinking, drop_params=drop_params) adaptive_config: Final = as_str_mapping( with_adaptive_thinking_as_effort( {"thinking": thinking}, model=model, custom_llm_provider=custom_llm_provider, drop_params=drop_params @@ -162,7 +162,7 @@ def thinking_budget_effort(thinking: object, *, model: str, custom_llm_provider: return adaptive_effort if isinstance(adaptive_effort, str) else None if "thinking" in (get_supported_openai_params(model=model, custom_llm_provider=custom_llm_provider) or ()): return None - return reasoning_effort_from_thinking(thinking, drop_params=drop_params) + return reasoning_effort_from_thinking_silently(thinking, drop_params=drop_params) def provider_reads_thinking(thinking: object, *, model: str, custom_llm_provider: str, drop_params: bool) -> bool: diff --git a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index d815e1eedd3..c2ee58b67ed 100644 --- a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -6,6 +6,7 @@ with guardrail transformations, specifically testing edge cases with empty choic """ import json +import logging import re from typing import Any, Literal, Optional, Final from unittest.mock import MagicMock, patch @@ -3391,3 +3392,18 @@ class TestAnthropicMessagesHandlerPostCallHookResponse: native = {"type": "message", "role": "assistant", "content": [{"type": "text", "text": "hi"}]} assert AnthropicMessagesHandler().post_call_hook_response(native) is native + + +def test_reading_a_dropped_budget_for_a_guardrail_logs_no_drop_warning(caplog: pytest.LogCaptureFixture) -> None: + data: Final = { + "model": "gemini-2.5-flash", + "max_tokens": 64, + "thinking": {"type": "enabled", "budget_tokens": "high"}, + "messages": [{"role": "user", "content": "hi"}], + } + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + messages: Final = AnthropicMessagesHandler().get_structured_messages(data) + + assert messages == [{"role": "user", "content": "hi"}] + assert not [record for record in caplog.records if "Dropping" in record.getMessage()] diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index 4d5d6cc17a9..cbb60caa57b 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -1,5 +1,6 @@ import copy import json +import logging from itertools import chain from typing import Final from unittest.mock import MagicMock, patch @@ -3189,6 +3190,47 @@ def test_reasoning_effort_accepts_dict_shape_for_non_adaptive_model( ) +def _reasoning_effort_drop_warnings(caplog: pytest.LogCaptureFixture) -> tuple[str, ...]: + return tuple( + record.getMessage() for record in caplog.records if "Dropping reasoning_effort=" in record.getMessage() + ) + + +@pytest.mark.parametrize("reasoning_effort_value", [5, {"effort": 5, "summary": "concise"}]) +def test_reasoning_effort_that_is_not_a_level_name_is_dropped_with_one_warning( + reasoning_effort_value: object, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + result: Final = AnthropicConfig().map_openai_params( + non_default_params={"reasoning_effort": reasoning_effort_value}, + optional_params={}, + model="claude-sonnet-4-6-20260219", + drop_params=False, + ) + + warnings: Final = _reasoning_effort_drop_warnings(caplog) + assert "thinking" not in result + assert "output_config" not in result + assert len(warnings) == 1 + assert f"reasoning_effort={reasoning_effort_value!r}" in warnings[0] + assert "claude-sonnet-4-6-20260219" in warnings[0] + + +@pytest.mark.parametrize("reasoning_effort_value", [{"summary": "concise"}, {"effort": "low", "summary": "concise"}]) +def test_reasoning_effort_with_a_level_name_or_no_level_logs_no_drop( + reasoning_effort_value: object, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + AnthropicConfig().map_openai_params( + non_default_params={"reasoning_effort": reasoning_effort_value}, + optional_params={}, + model="claude-sonnet-4-6-20260219", + drop_params=False, + ) + + assert _reasoning_effort_drop_warnings(caplog) == () + + @pytest.mark.parametrize( "model,budget_tokens,expected", [ diff --git a/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py index c30644e7fdc..e0882653324 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py @@ -1,6 +1,7 @@ import base64 import functools import json +import logging from collections.abc import Mapping from types import MappingProxyType from typing import Any, Final, cast @@ -2871,14 +2872,21 @@ def test_translate_completion_input_params_rejects_a_non_numeric_budget_without_ @pytest.mark.parametrize("budget", ["2048", True]) def test_translate_completion_input_params_drops_a_non_numeric_budget_under_drop_params( - budget: object, monkeypatch: pytest.MonkeyPatch + budget: object, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture ): monkeypatch.setattr(litellm, "drop_params", False) - translated: Final = AnthropicAdapter().translate_completion_input_params( - _gemini_request_with_budget(budget, drop_params=True) - ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + translated: Final = AnthropicAdapter().translate_completion_input_params( + _gemini_request_with_budget(budget, drop_params=True) + ) + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] assert translated is not None assert not {"reasoning_effort", "thinking"} & translated.keys() + assert len(drop_warnings) == 1 + assert all( + fragment in drop_warnings[0] + for fragment in (f"thinking.budget_tokens={budget!r}", "gemini-2.5-flash", "(drop_params=True)") + ) def test_translate_openai_content_to_anthropic_reasoning_content_without_thinking_blocks(): diff --git a/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py b/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py index e504513dd36..27ed1f5ebd6 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py @@ -8,6 +8,7 @@ leaves the adapter, in the shape the target expects. import functools import json +import logging from collections.abc import Mapping from typing import Final @@ -510,26 +511,33 @@ class TestADeclaredDeploymentRoundsTheMessagesTierOntoItsLevels: @pytest.mark.asyncio @pytest.mark.parametrize("budget", ["2048", True]) @pytest.mark.parametrize("litellm_params, model_info, upstream_url, upstream_body", _MALFORMED_BUDGET_TARGETS) - async def test_drop_params_drops_a_non_numeric_budget( + async def test_drop_params_drops_a_non_numeric_budget_with_one_warning( self, local_model_cost_map: None, respx_mock: respx.MockRouter, + caplog: pytest.LogCaptureFixture, litellm_params: Mapping[str, object], model_info: Mapping[str, object], upstream_url: str, upstream_body: Mapping[str, object], budget: object, ) -> None: - sent: Final = await _body_sent_upstream( - respx_mock, - {**litellm_params, "drop_params": True}, - model_info, - upstream_url, - upstream_body, - thinking={"type": "enabled", "budget_tokens": budget}, - ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + sent: Final = await _body_sent_upstream( + respx_mock, + {**litellm_params, "drop_params": True}, + model_info, + upstream_url, + upstream_body, + thinking={"type": "enabled", "budget_tokens": budget}, + ) assert not {"reasoning_effort", "reasoning", "thinking"} & sent.keys() + dropped: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] + assert len(dropped) == 1 + assert f"thinking.budget_tokens={budget!r}" in dropped[0] + assert "(drop_params=True)" in dropped[0] + assert str(litellm_params["model"]).split("/", 1)[1] in dropped[0] @pytest.mark.asyncio @pytest.mark.parametrize( diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py index 5ce6d03f477..af3108e5072 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py @@ -159,7 +159,7 @@ async def test_openai_model_does_not_forward_stream_options_to_responses_api(): await litellm.anthropic.messages.acreate( max_tokens=100, messages=[{"role": "user", "content": "Hello, how are you?"}], - model="openai/gpt-5.5", + model="openai/gpt-6.1-sol", api_key="test-api-key", stream_options={"include_usage": True}, ) @@ -669,7 +669,9 @@ class TestThinkingSummaryPreservation: ) thinking = {"type": "enabled", "budget_tokens": 5000, "summary": "concise"} - result = LiteLLMAnthropicToResponsesAPIAdapter.translate_thinking_to_reasoning(thinking) + result = LiteLLMAnthropicToResponsesAPIAdapter.translate_thinking_to_reasoning( + thinking, model="openai/gpt-6.1-sol" + ) assert result == {"effort": "high", "summary": "concise"} def test_responses_adapter_no_summary_by_default(self): @@ -683,7 +685,9 @@ class TestThinkingSummaryPreservation: try: litellm.reasoning_auto_summary = False thinking = {"type": "enabled", "budget_tokens": 5000} - result = LiteLLMAnthropicToResponsesAPIAdapter.translate_thinking_to_reasoning(thinking) + result = LiteLLMAnthropicToResponsesAPIAdapter.translate_thinking_to_reasoning( + thinking, model="openai/gpt-6.1-sol" + ) assert result == {"effort": "high"} assert result is not None and "summary" not in result finally: diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_router.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_router.py index 596cb968a0e..1d0d70d8e60 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_router.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_router.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio import base64 import json +import logging import struct import time import uuid @@ -891,24 +892,32 @@ async def test_native_messages_send_no_effort_for_a_default_effort_beside_a_thin @pytest.mark.asyncio @pytest.mark.parametrize("model_info", [None, _DECLARING_LOW_AND_HIGH]) @pytest.mark.parametrize("drop_globally", [False, True]) -async def test_native_messages_drop_params_drops_a_non_numeric_thinking_budget_on_adaptive_claude( +async def test_native_messages_drop_params_drops_a_non_numeric_thinking_budget_on_adaptive_claude_with_one_warning( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, model_info: Mapping[str, object] | None, drop_globally: bool, ) -> None: monkeypatch.setattr(litellm, "drop_params", drop_globally) request_drop: Final = {} if drop_globally else {"drop_params": True} - dropped: Final = await _native_opus_body_sent( - respx_mock, model_info, thinking={"type": "enabled", "budget_tokens": "lots"}, **request_drop - ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + dropped: Final = await _native_opus_body_sent( + respx_mock, model_info, thinking={"type": "enabled", "budget_tokens": "lots"}, **request_drop + ) + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] adaptive_alone: Final = await _native_opus_body_sent( respx_mock, model_info, thinking={"type": "adaptive"}, **request_drop ) assert "output_config" not in adaptive_alone assert dropped == adaptive_alone + assert len(drop_warnings) == 1 + assert all( + fragment in drop_warnings[0] + for fragment in ("thinking.budget_tokens='lots'", "claude-opus-5-5", "(drop_params=True)") + ) _MIXED_GROUP: Final = "gpt-5.6-mixed" diff --git a/tests/unit/llms/anthropic/messages/native/test_reasoning_effort_translation.py b/tests/unit/llms/anthropic/messages/native/test_reasoning_effort_translation.py index 50dc0deabfd..28abb348371 100644 --- a/tests/unit/llms/anthropic/messages/native/test_reasoning_effort_translation.py +++ b/tests/unit/llms/anthropic/messages/native/test_reasoning_effort_translation.py @@ -1,5 +1,8 @@ """Tests for ``reasoning_effort`` translation on the Anthropic /v1/messages route.""" +import logging +from typing import Final + import pytest from litellm.constants import ( @@ -46,6 +49,49 @@ def test_reasoning_effort_maps_to_output_config_for_adaptive_model( assert result.get("output_config") == {"effort": expected_effort} +def _reasoning_effort_drop_warnings(caplog: pytest.LogCaptureFixture) -> tuple[str, ...]: + return tuple( + record.getMessage() for record in caplog.records if "Dropping reasoning_effort=" in record.getMessage() + ) + + +@pytest.mark.parametrize("reasoning_effort", [5, {"effort": "high"}]) +def test_reasoning_effort_that_is_not_a_string_is_dropped_with_one_warning( + reasoning_effort: object, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + result: Final = AnthropicMessagesConfig().transform_anthropic_messages_request( + model="claude-opus-4-7", + messages=[{"role": "user", "content": "Hello"}], + anthropic_messages_optional_request_params={"max_tokens": 1024, "reasoning_effort": reasoning_effort}, + litellm_params={}, + headers={}, + ) + + warnings: Final = _reasoning_effort_drop_warnings(caplog) + assert "reasoning_effort" not in result + assert "output_config" not in result + assert len(warnings) == 1 + assert f"reasoning_effort={reasoning_effort!r}" in warnings[0] + assert "claude-opus-4-7" in warnings[0] + + +@pytest.mark.parametrize("reasoning_effort", ["high", {"summary": "concise"}]) +def test_reasoning_effort_that_is_a_string_or_names_no_level_logs_no_drop( + reasoning_effort: object, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + AnthropicMessagesConfig().transform_anthropic_messages_request( + model="claude-opus-4-7", + messages=[{"role": "user", "content": "Hello"}], + anthropic_messages_optional_request_params={"max_tokens": 1024, "reasoning_effort": reasoning_effort}, + litellm_params={}, + headers={}, + ) + + assert _reasoning_effort_drop_warnings(caplog) == () + + def test_reasoning_effort_none_clears_thinking_and_output_config(): config = AnthropicMessagesConfig() optional_params = { diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py index f6fcb9e6f1e..ece0cf5d423 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py @@ -44,6 +44,7 @@ def _make_request(**overrides) -> AnthropicMessagesRequest: _ADAPTER = LiteLLMAnthropicToResponsesAPIAdapter() +_MODEL: Final = "openai/gpt-6.1-sol" # --------------------------------------------------------------------------- @@ -957,13 +958,13 @@ class TestTranslateThinkingToReasoning: """Anthropic thinking param -> Responses API reasoning param.""" def test_budget_high_effort(self): - result = _ADAPTER.translate_thinking_to_reasoning({"type": "enabled", "budget_tokens": 10000}) + result = _ADAPTER.translate_thinking_to_reasoning({"type": "enabled", "budget_tokens": 10000}, model=_MODEL) # Default (reasoning_auto_summary=False): only effort, no summary assert result == {"effort": "high"} assert result is not None and "summary" not in result def test_budget_above_threshold_high_effort(self): - result = _ADAPTER.translate_thinking_to_reasoning({"type": "enabled", "budget_tokens": 50000}) + result = _ADAPTER.translate_thinking_to_reasoning({"type": "enabled", "budget_tokens": 50000}, model=_MODEL) assert result is not None assert result["effort"] == "high" assert "summary" not in result @@ -973,7 +974,8 @@ class TestTranslateThinkingToReasoning: { "type": "enabled", "budget_tokens": DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, - } + }, + model=_MODEL, ) assert result == {"effort": "medium"} assert result is not None and "summary" not in result @@ -983,13 +985,14 @@ class TestTranslateThinkingToReasoning: { "type": "enabled", "budget_tokens": DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET, - } + }, + model=_MODEL, ) assert result == {"effort": "low"} assert result is not None and "summary" not in result def test_budget_minimal_effort(self): - result = _ADAPTER.translate_thinking_to_reasoning({"type": "enabled", "budget_tokens": 500}) + result = _ADAPTER.translate_thinking_to_reasoning({"type": "enabled", "budget_tokens": 500}, model=_MODEL) assert result == {"effort": "minimal"} assert result is not None and "summary" not in result @@ -998,7 +1001,8 @@ class TestTranslateThinkingToReasoning: { "type": "enabled", "budget_tokens": DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, - } + }, + model=_MODEL, ) assert result_high is not None assert result_high["effort"] == "high" @@ -1006,7 +1010,8 @@ class TestTranslateThinkingToReasoning: { "type": "enabled", "budget_tokens": DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, - } + }, + model=_MODEL, ) assert result_medium is not None assert result_medium["effort"] == "medium" @@ -1015,23 +1020,24 @@ class TestTranslateThinkingToReasoning: { "type": "enabled", "budget_tokens": DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET, - } + }, + model=_MODEL, ) assert result_low is not None assert result_low["effort"] == "low" assert "summary" not in result_low def test_disabled_type_returns_none(self): - result = _ADAPTER.translate_thinking_to_reasoning({"type": "disabled"}) + result = _ADAPTER.translate_thinking_to_reasoning({"type": "disabled"}, model=_MODEL) assert result is None def test_non_dict_returns_none(self): - result = _ADAPTER.translate_thinking_to_reasoning("enabled") # type: ignore + result = _ADAPTER.translate_thinking_to_reasoning("enabled", model=_MODEL) # type: ignore assert result is None def test_missing_budget_defaults_to_minimal(self): """Missing budget_tokens defaults to 0, below the low threshold -> minimal.""" - result = _ADAPTER.translate_thinking_to_reasoning({"type": "enabled"}) + result = _ADAPTER.translate_thinking_to_reasoning({"type": "enabled"}, model=_MODEL) assert result == {"effort": "minimal"} assert result is not None and "summary" not in result @@ -1042,7 +1048,7 @@ class TestTranslateThinkingToReasoning: original = litellm.reasoning_auto_summary try: litellm.reasoning_auto_summary = True - result = _ADAPTER.translate_thinking_to_reasoning({"type": "enabled", "budget_tokens": 10000}) + result = _ADAPTER.translate_thinking_to_reasoning({"type": "enabled", "budget_tokens": 10000}, model=_MODEL) assert result == {"effort": "high", "summary": "detailed"} finally: litellm.reasoning_auto_summary = original @@ -1059,7 +1065,8 @@ class TestTranslateThinkingToReasoning: { "type": "enabled", "budget_tokens": DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, - } + }, + model=_MODEL, ) assert result == {"effort": "medium", "summary": "detailed"} finally: diff --git a/tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py b/tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py index 3432c9503de..7128c04a056 100644 --- a/tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py +++ b/tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py @@ -162,7 +162,8 @@ class TestAdapterAdaptiveThinking: adapter = LiteLLMAnthropicMessagesAdapter() result = adapter.translate_anthropic_thinking_to_reasoning_effort( - {"type": "adaptive"} + {"type": "adaptive"}, + model="openai/gpt-6.1-sol", ) assert result == "medium" @@ -198,6 +199,7 @@ class TestAdapterAdaptiveThinking: result = LiteLLMAnthropicToResponsesAPIAdapter.translate_thinking_to_reasoning( thinking={"type": "adaptive"}, output_config={"effort": "xhigh"}, + model="openai/gpt-6.1-sol", ) assert result is not None assert result["effort"] == "xhigh" @@ -210,6 +212,7 @@ class TestAdapterAdaptiveThinking: result = LiteLLMAnthropicToResponsesAPIAdapter.translate_thinking_to_reasoning( thinking={"type": "adaptive"}, + model="openai/gpt-6.1-sol", ) assert result is not None assert result["effort"] == "medium" diff --git a/tests/unit/llms/anthropic/test_anthropic_common_utils.py b/tests/unit/llms/anthropic/test_anthropic_common_utils.py index 3ec72f29d34..c1f28e33969 100644 --- a/tests/unit/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/unit/llms/anthropic/test_anthropic_common_utils.py @@ -11,6 +11,7 @@ Verifies that: """ import json +import logging import os import sys import threading @@ -4530,16 +4531,25 @@ def test_shared_legacy_thinking_translation_rejects_a_non_numeric_budget(budget_ @pytest.mark.usefixtures("local_model_cost_map") @pytest.mark.parametrize("budget_tokens", _NON_NUMERIC_BUDGETS) -def test_shared_legacy_thinking_translation_drops_a_non_numeric_budget_under_drop_params(budget_tokens: object) -> None: +def test_shared_legacy_thinking_translation_drops_a_non_numeric_budget_under_drop_params( + budget_tokens: object, caplog: pytest.LogCaptureFixture +) -> None: from litellm.llms.anthropic.common_utils import AnthropicModelInfo optional_params: Final = {"thinking": {"type": "enabled", "budget_tokens": budget_tokens}} - AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model( - model="claude-opus-5", optional_params=optional_params, custom_llm_provider="azure_ai", drop_params=True - ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model( + model="claude-opus-5", optional_params=optional_params, custom_llm_provider="azure_ai", drop_params=True + ) + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] assert optional_params == {"thinking": {"type": "adaptive"}} + assert len(drop_warnings) == 1 + assert all( + fragment in drop_warnings[0] + for fragment in (f"thinking.budget_tokens={budget_tokens!r}", "claude-opus-5", "(drop_params=True)") + ) @pytest.mark.parametrize( diff --git a/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py b/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py index 4d0ebef4f2d..d95f831df3e 100644 --- a/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py +++ b/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py @@ -1,4 +1,5 @@ import json +import logging from collections.abc import Mapping from datetime import datetime from types import SimpleNamespace @@ -597,6 +598,45 @@ async def test_native_count_preserves_settings_and_requires_every_prefix( assert unexpected.call_count == 0 +@pytest.mark.usefixtures("local_model_cost_map") +def test_native_baseline_body_drops_a_non_numeric_budget_without_logging_a_drop( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + monkeypatch.setattr(litellm, "drop_params", True) + request: Final = { + "model": "claude-opus-5", + "max_tokens": 64, + "messages": [{"role": "user", "content": "hi"}], + "thinking": {"type": "enabled", "budget_tokens": "high"}, + } + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + body: Final = prepare_native_baseline_body(request, "anthropic/claude-opus-5") + + assert body is not None + assert body["thinking"] == {"type": "adaptive"} + assert [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] == [] + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_native_baseline_body_drops_a_non_string_reasoning_effort_without_logging_a_drop( + caplog: pytest.LogCaptureFixture, +) -> None: + request: Final = { + "model": "claude-opus-5", + "max_tokens": 64, + "messages": [{"role": "user", "content": "hi"}], + "reasoning_effort": 5, + } + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + body: Final = prepare_native_baseline_body(request, "anthropic/claude-opus-5") + + assert body is not None + assert "reasoning_effort" not in body + assert [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] == [] + + def test_native_baseline_body_matches_the_sent_body_when_drop_params_meets_an_auto_added_beta( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py index 145d45317c3..9f68a4fb73f 100644 --- a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py +++ b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py @@ -1,4 +1,5 @@ import json +import logging import traceback from collections.abc import Callable from types import MappingProxyType @@ -758,6 +759,66 @@ def test_drop_tool_level_extra_fields_strips_copilot_mcp_server_name(): assert result["tools"][1]["function"]["name"] == "read_file" +def test_extra_parameters_error_drops_reasoning_effort_with_one_warning(caplog: pytest.LogCaptureFixture) -> None: + import httpx + + error_text: Final = json.dumps( + { + "error": { + "message": "Extra parameters ['reasoning_effort'] are not allowed when extra-parameters is not set " + "or set to be 'error'. Set extra-parameters to 'pass-through' to pass to the model." + } + } + ) + error: Final = httpx.HTTPStatusError( + message="422", + request=httpx.Request("POST", "https://example.services.ai.azure.com/models/chat/completions"), + response=httpx.Response(422, text=error_text), + ) + request_data: Final = { + "model": "Kimi-K2.6", + "messages": [{"role": "user", "content": "Say hi."}], + "reasoning_effort": "high", + } + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + retried: Final = AzureAIStudioConfig().transform_request_on_unprocessable_entity_error(error, request_data) + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] + + assert retried == {"model": "Kimi-K2.6", "messages": [{"role": "user", "content": "Say hi."}]} + assert len(drop_warnings) == 1 + assert all(fragment in drop_warnings[0] for fragment in ("reasoning_effort='high'", "Kimi-K2.6")) + assert "drop_params" not in drop_warnings[0] + + +def test_extra_inputs_error_names_drop_params_in_the_reasoning_drop_warning(caplog: pytest.LogCaptureFixture) -> None: + import httpx + + detail: Final = [ + {"type": "extra_forbidden", "loc": ["body", "reasoning_effort"], "msg": "Extra inputs are not permitted"} + ] + error: Final = httpx.HTTPStatusError( + message="422", + request=httpx.Request("POST", "https://example.services.ai.azure.com/models/chat/completions"), + response=httpx.Response(422, text=json.dumps({"error": {"message": json.dumps({"detail": detail})}})), + ) + request_data: Final = { + "model": "Mistral-Large-3", + "messages": [{"role": "user", "content": "Say hi."}], + "reasoning_effort": "high", + } + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + retried: Final = AzureAIStudioConfig().transform_request_on_unprocessable_entity_error(error, request_data) + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] + + assert retried == {"model": "Mistral-Large-3", "messages": [{"role": "user", "content": "Say hi."}]} + assert len(drop_warnings) == 1 + assert all( + fragment in drop_warnings[0] for fragment in ("reasoning_effort='high'", "Mistral-Large-3", "(drop_params=True)") + ) + + def _find_key_anywhere(obj, key: str) -> bool: if isinstance(obj, dict): if key in obj: diff --git a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py index 3faae2e3289..f4dcff1969e 100644 --- a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py +++ b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py @@ -1,6 +1,7 @@ """Bedrock Runtime Chat Completions: the default for GPT 5.6 and newer, ``bedrock/chat_completions/`` for the rest.""" import json +import logging import re import httpx @@ -706,18 +707,30 @@ def test_with_max_completion_tokens_leaves_other_params_alone(): @pytest.mark.parametrize( - "model", - ["us.xai.grok-4.6", "bedrock/us-gov-west-1/us.xai.grok-4.6"], + ("model", "logged_model"), + [ + ("us.xai.grok-4.6", "us.xai.grok-4.6"), + ("chat_completions/us.xai.grok-4.6", "us.xai.grok-4.6"), + ("bedrock/us-gov-west-1/us.xai.grok-4.6", "us-gov-west-1/us.xai.grok-4.6"), + ], ) -def test_map_openai_params_drops_reasoning_effort_none_for_grok(model): +def test_map_openai_params_drops_reasoning_effort_none_for_grok_with_one_warning( + model: str, logged_model: str, caplog: pytest.LogCaptureFixture +) -> None: cfg = AmazonBedrockRuntimeChatCompletionsConfig() - mapped = cfg.map_openai_params( - non_default_params={"reasoning_effort": "none", "max_tokens": 64}, - optional_params={}, - model=model, - drop_params=False, - ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + mapped = cfg.map_openai_params( + non_default_params={"reasoning_effort": "none", "max_tokens": 64}, + optional_params={}, + model=model, + drop_params=False, + ) + drop_warnings = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] assert "reasoning_effort" not in mapped + assert len(drop_warnings) == 1 + assert "reasoning_effort='none'" in drop_warnings[0] + assert f"model={logged_model!r}" in drop_warnings[0] + assert "drop_params" not in drop_warnings[0] def test_map_openai_params_keeps_reasoning_effort_low_for_grok(): @@ -750,17 +763,24 @@ def test_map_openai_params_refuses_a_non_string_reasoning_effort_without_drop_pa @pytest.mark.parametrize("reasoning_effort", [["low"], {"effort": "low"}, 5], ids=["list", "object", "int"]) @pytest.mark.parametrize("drop_params_via", ["request", "litellm.drop_params"]) def test_map_openai_params_drops_a_non_string_reasoning_effort_under_drop_params( - monkeypatch, model, reasoning_effort, drop_params_via + monkeypatch, caplog, model, reasoning_effort, drop_params_via ): monkeypatch.setattr(litellm, "drop_params", drop_params_via == "litellm.drop_params") - mapped = AmazonBedrockRuntimeChatCompletionsConfig().map_openai_params( - non_default_params={"reasoning_effort": reasoning_effort, "max_tokens": 64}, - optional_params={}, - model=model, - drop_params=drop_params_via == "request", - ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + mapped = AmazonBedrockRuntimeChatCompletionsConfig().map_openai_params( + non_default_params={"reasoning_effort": reasoning_effort, "max_tokens": 64}, + optional_params={}, + model=model, + drop_params=drop_params_via == "request", + ) + drop_warnings = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] assert "reasoning_effort" not in mapped assert mapped["max_completion_tokens"] == 64 + assert len(drop_warnings) == 1 + assert all( + fragment in drop_warnings[0] + for fragment in (f"reasoning_effort={reasoning_effort!r}", repr(model), "(drop_params=True)") + ) def test_map_openai_params_keeps_reasoning_effort_none_for_gpt56(): diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 743e7e74431..524caaba7a1 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -13,6 +13,7 @@ import pytest import respx import litellm +from litellm._logging import verbose_logger from litellm import ModelResponse, Router, completion from litellm.litellm_core_utils.prompt_templates.factory import ( _bedrock_converse_messages_pt, @@ -11096,6 +11097,52 @@ def test_a_deployment_with_a_batch_control_plane_endpoint_keeps_it_out_of_chat_r assert "aws_bedrock_control_plane_endpoint" not in route.calls[0].request.content.decode() +def _warnings_naming(caplog: pytest.LogCaptureFixture, param: str) -> tuple[str, ...]: + return tuple( + r.getMessage() for r in caplog.records if r.levelno == logging.WARNING and f"Dropping {param}=" in r.getMessage() + ) + + +_DEEPSEEK_R1_PROFILE_ARN: Final = "arn:aws:bedrock:us-east-1:123456789012:inference-profile/us.deepseek.r1-v1:0" + + +@pytest.mark.parametrize( + "param, value", (("thinking", {"type": "enabled", "budget_tokens": 1024}), ("reasoning_effort", "high")) +) +def test_bedrock_deepseek_arn_reasoning_drop_warns_once( + param: str, value: object, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + optional_params: Final = AmazonConverseConfig().map_openai_params( + non_default_params={param: value, "max_tokens": 100}, + optional_params={}, + model=_DEEPSEEK_R1_PROFILE_ARN, + drop_params=False, + ) + + warnings: Final = _warnings_naming(caplog, param) + assert param not in optional_params + assert len(warnings) == 1 + assert _DEEPSEEK_R1_PROFILE_ARN in warnings[0] + + +@pytest.mark.parametrize( + "param, value", (("thinking", {"type": "enabled", "budget_tokens": 1024}), ("reasoning_effort", "high")) +) +def test_bedrock_deepseek_r1_drop_of_an_unadvertised_param_does_not_warn( + param: str, value: object, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + AmazonConverseConfig().map_openai_params( + non_default_params={param: value, "max_tokens": 100}, + optional_params={}, + model="us.deepseek.r1-v1:0", + drop_params=False, + ) + + assert _warnings_naming(caplog, param) == () + + def _converse_bound_thinking_request( headers: dict[str, str] | None, body_betas: tuple[str, ...] = (), drop_params: bool = True ) -> dict[str, object]: diff --git a/tests/unit/llms/deepseek/chat/test_deepseek_chat_transformation.py b/tests/unit/llms/deepseek/chat/test_deepseek_chat_transformation.py index 3fbb12881da..56af47aeebe 100644 --- a/tests/unit/llms/deepseek/chat/test_deepseek_chat_transformation.py +++ b/tests/unit/llms/deepseek/chat/test_deepseek_chat_transformation.py @@ -1,4 +1,5 @@ import json +import logging from collections.abc import Iterator from typing import Final @@ -6,6 +7,7 @@ import httpx import litellm import pytest import respx +from litellm._logging import verbose_logger from litellm.caching.llm_caching_handler import LLMClientCache from litellm.llms.deepseek.chat.transformation import DeepSeekChatConfig @@ -951,3 +953,42 @@ def test_get_complete_url_ignores_a_trailing_slash_on_api_base(api_base: str): ) == "https://gateway.example.com/deepseek/chat/completions" ) + + +def _budget_warnings(caplog: pytest.LogCaptureFixture) -> tuple[str, ...]: + return tuple( + r.getMessage() + for r in caplog.records + if r.levelno == logging.WARNING and "Dropping thinking.budget_tokens=" in r.getMessage() + ) + + +def test_enabled_thinking_budget_dropped_for_deepseek_warns_once(caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + result: Final = DeepSeekChatConfig().map_openai_params( + non_default_params={"thinking": {"type": "enabled", "budget_tokens": 4000}}, + optional_params={}, + model="deepseek-reasoner", + drop_params=False, + ) + + warnings: Final = _budget_warnings(caplog) + assert result["thinking"] == {"type": "enabled"} + assert len(warnings) == 1 + assert "budget_tokens=4000" in warnings[0] + assert "deepseek-reasoner" in warnings[0] + + +@pytest.mark.parametrize( + "thinking", ({"type": "enabled"}, {"type": "disabled", "budget_tokens": 4000}), ids=("no-budget", "disabled") +) +def test_deepseek_thinking_without_a_requested_budget_does_not_warn( + thinking: dict[str, object], caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + result: Final = DeepSeekChatConfig().map_openai_params( + non_default_params={"thinking": thinking}, optional_params={}, model="deepseek-reasoner", drop_params=False + ) + + assert result["thinking"] == {"type": thinking["type"]} + assert _budget_warnings(caplog) == () diff --git a/tests/unit/llms/fal_ai/chat/test_fal_ai_chat_transformation.py b/tests/unit/llms/fal_ai/chat/test_fal_ai_chat_transformation.py index b7a79f03f8d..f4a7455ff27 100644 --- a/tests/unit/llms/fal_ai/chat/test_fal_ai_chat_transformation.py +++ b/tests/unit/llms/fal_ai/chat/test_fal_ai_chat_transformation.py @@ -1,3 +1,6 @@ +import logging +from typing import Final + import httpx import pytest @@ -242,11 +245,33 @@ def test_map_openai_params_maps_reasoning_effort(effort, expected): assert mapped["reasoning"] is expected -def test_map_openai_params_drops_unknown_reasoning_effort_when_dropping(): - mapped = FalAIChatConfig().map_openai_params( - non_default_params={"reasoning_effort": "extreme"}, optional_params={}, model=MODEL, drop_params=True - ) +@pytest.mark.parametrize("drop_globally", [False, True]) +def test_map_openai_params_drops_unknown_reasoning_effort_with_one_warning( + drop_globally: bool, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + monkeypatch.setattr(litellm, "drop_params", drop_globally) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + mapped: Final = FalAIChatConfig().map_openai_params( + non_default_params={"reasoning_effort": "xhigh"}, + optional_params={}, + model=MODEL, + drop_params=not drop_globally, + ) + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] assert "reasoning" not in mapped + assert len(drop_warnings) == 1 + assert all(fragment in drop_warnings[0] for fragment in ("reasoning_effort='xhigh'", MODEL, "(drop_params=True)")) + + +def test_map_openai_params_rejects_unknown_reasoning_effort_without_drop_params( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(FalAIError) as exc_info: + FalAIChatConfig().map_openai_params( + non_default_params={"reasoning_effort": "xhigh"}, optional_params={}, model=MODEL, drop_params=False + ) + assert exc_info.value.status_code == 400 @pytest.mark.parametrize("effort", [{"level": "low"}, ["low"], 1]) diff --git a/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index 28f4fdead9e..ed166d206c8 100644 --- a/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -1,4 +1,5 @@ import json +import logging from collections.abc import Callable from typing import Final from unittest.mock import MagicMock, patch @@ -9,6 +10,7 @@ import respx from pydantic import ValidationError import litellm +from litellm._logging import verbose_logger from litellm.constants import SESSION_ID_GENERATED_METADATA_KEY from litellm.litellm_core_utils.get_supported_openai_params import get_supported_openai_params from litellm.llms.custom_httpx.http_handler import HTTPHandler @@ -2361,6 +2363,39 @@ def test_global_disable_flag_with_transform_messages_helper( assert json.loads(route.calls.last.request.content)["messages"] == [{"role": "user", "content": content}] +def _warnings_naming(caplog: pytest.LogCaptureFixture, param: str) -> tuple[str, ...]: + return tuple( + r.getMessage() for r in caplog.records if r.levelno == logging.WARNING and f"Dropping {param}=" in r.getMessage() + ) + + +def test_chat_template_kwargs_effort_dropped_for_non_reasoning_model_warns_once( + caplog: pytest.LogCaptureFixture, +) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + FireworksAIConfig().map_extra_body_params( + {"extra_body": {"chat_template_kwargs": {"reasoning_budget": 512}}}, + _NON_REASONING_MODEL, + ) + + warnings: Final = _warnings_naming(caplog, "chat_template_kwargs") + assert len(warnings) == 1 + assert "'reasoning_budget': 512" in warnings[0] + assert "llama-v3-70b-instruct" in warnings[0] + + +def test_chat_template_kwargs_thinking_off_for_non_reasoning_model_does_not_warn( + caplog: pytest.LogCaptureFixture, +) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + FireworksAIConfig().map_extra_body_params( + {"extra_body": {"chat_template_kwargs": {"enable_thinking": False}}}, + _NON_REASONING_MODEL, + ) + + assert _warnings_naming(caplog, "chat_template_kwargs") == () + + _CODEX_DEVELOPER_MESSAGE_AFTER_A_USER_TURN: Final = ( {"role": "system", "content": "You are Codex"}, {"role": "user", "content": "Hi there"}, diff --git a/tests/unit/llms/fireworks_ai/completion/test_fireworks_ai_text_completion_transformation.py b/tests/unit/llms/fireworks_ai/completion/test_fireworks_ai_text_completion_transformation.py index e5a77aa8d41..d35442d4095 100644 --- a/tests/unit/llms/fireworks_ai/completion/test_fireworks_ai_text_completion_transformation.py +++ b/tests/unit/llms/fireworks_ai/completion/test_fireworks_ai_text_completion_transformation.py @@ -1,7 +1,11 @@ +import logging +from typing import Final + import pytest import litellm +from litellm._logging import verbose_logger from litellm.llms.fireworks_ai.completion.transformation import ( @@ -204,3 +208,36 @@ def test_transform_text_completion_request_keeps_sdk_rejected_keys_in_extra_body assert "chat_template_kwargs" not in data["extra_body"] assert "best_of" not in data["extra_body"] assert "response_format" not in data + + +def _warnings_naming(caplog: pytest.LogCaptureFixture, param: str) -> tuple[str, ...]: + return tuple( + r.getMessage() for r in caplog.records if r.levelno == logging.WARNING and f"Dropping {param}=" in r.getMessage() + ) + + +def test_chat_template_kwargs_effort_dropped_for_non_reasoning_model_warns_once( + caplog: pytest.LogCaptureFixture, +) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + FireworksAITextCompletionConfig().map_extra_body_params( + {"extra_body": {"chat_template_kwargs": {"reasoning_budget": 512}}}, + _NON_REASONING_MODEL, + ) + + warnings: Final = _warnings_naming(caplog, "chat_template_kwargs") + assert len(warnings) == 1 + assert "'reasoning_budget': 512" in warnings[0] + assert "llama-v3-70b-instruct" in warnings[0] + + +def test_chat_template_kwargs_thinking_off_for_non_reasoning_model_does_not_warn( + caplog: pytest.LogCaptureFixture, +) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + FireworksAITextCompletionConfig().map_extra_body_params( + {"extra_body": {"chat_template_kwargs": {"enable_thinking": False}}}, + _NON_REASONING_MODEL, + ) + + assert _warnings_naming(caplog, "chat_template_kwargs") == () diff --git a/tests/unit/llms/mistral/test_mistral_chat_transformation.py b/tests/unit/llms/mistral/test_mistral_chat_transformation.py index c340f77c317..720a1b4f458 100644 --- a/tests/unit/llms/mistral/test_mistral_chat_transformation.py +++ b/tests/unit/llms/mistral/test_mistral_chat_transformation.py @@ -1,4 +1,5 @@ import json +import logging from collections.abc import Mapping from typing import Final, List, cast from unittest.mock import MagicMock, patch @@ -9,6 +10,7 @@ import pytest import respx from openai import AsyncOpenAI +from litellm._logging import verbose_logger from litellm.litellm_core_utils.prompt_templates.common_utils import TOOL_RESULT_IMAGE_BOUNDARY from litellm.llms.mistral.chat.transformation import ( MistralChatResponseIterator, @@ -1670,6 +1672,109 @@ def test_mistral_transform_request_drops_an_empty_tool_message_whose_call_id_is_ assert [m.get("role") for m in request["messages"]] == ["user", "assistant"] +def _warnings_naming(caplog: pytest.LogCaptureFixture, param: str) -> tuple[str, ...]: + return tuple( + r.getMessage() for r in caplog.records if r.levelno == logging.WARNING and f"Dropping {param}=" in r.getMessage() + ) + + +@pytest.mark.parametrize( + "param, value", + ( + ("reasoning_effort", "none"), + ("reasoning_effort", "xhigh"), + ("thinking", {"type": "disabled"}), + ("thinking", {"type": ["enabled"]}), + ), +) +def test_magistral_reasoning_request_replaced_by_prompt_warns_once( + param: str, value: object, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + result: Final = MistralConfig().map_openai_params( + non_default_params={param: value}, + optional_params={}, + model="magistral-medium-latest", + drop_params=False, + ) + + warnings: Final = _warnings_naming(caplog, param) + assert result["_add_reasoning_prompt"] is True + assert len(warnings) == 1 + assert "magistral-medium-latest" in warnings[0] + + +@pytest.mark.parametrize( + "param, value", + ( + ("reasoning_effort", "low"), + ("reasoning_effort", "medium"), + ("reasoning_effort", "high"), + ("thinking", {"type": "enabled", "budget_tokens": 1024}), + ), +) +def test_magistral_reasoning_request_the_prompt_honors_does_not_warn( + param: str, value: object, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + result: Final = MistralConfig().map_openai_params( + non_default_params={param: value}, + optional_params={}, + model="magistral-medium-latest", + drop_params=False, + ) + + assert result["_add_reasoning_prompt"] is True + assert _warnings_naming(caplog, param) == () + + +def test_non_magistral_reasoning_effort_is_sent_without_warning(caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + result: Final = MistralConfig().map_openai_params( + non_default_params={"reasoning_effort": "high"}, + optional_params={}, + model="mistral-medium-latest", + drop_params=False, + ) + + assert result["reasoning_effort"] == "high" + assert _warnings_naming(caplog, "reasoning_effort") == () + + +@pytest.mark.parametrize("reasoning_effort", [5, {"effort": 5}, {"effort": 5, "summary": "auto"}]) +def test_mistral_warns_once_when_drop_params_drops_a_level_it_cannot_send( + caplog: pytest.LogCaptureFixture, reasoning_effort: object +) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + result: Final = MistralConfig().map_openai_params( + non_default_params={"reasoning_effort": reasoning_effort}, + optional_params={}, + model="mistral-medium-latest", + drop_params=True, + ) + + assert "reasoning_effort" not in result + warnings: Final = _warnings_naming(caplog, "reasoning_effort") + assert len(warnings) == 1 + assert "model='mistral-medium-latest'" in warnings[0] + assert "(drop_params=True)" in warnings[0] + + +@pytest.mark.parametrize("reasoning_effort", [{"summary": "auto"}, {"effort": "high", "summary": "auto"}]) +def test_mistral_stays_quiet_under_drop_params_when_no_level_is_lost( + caplog: pytest.LogCaptureFixture, reasoning_effort: object +) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + MistralConfig().map_openai_params( + non_default_params={"reasoning_effort": reasoning_effort}, + optional_params={}, + model="mistral-medium-latest", + drop_params=True, + ) + + assert _warnings_naming(caplog, "reasoning_effort") == () + + @pytest.mark.parametrize( ("model", "custom_llm_provider"), [ diff --git a/tests/unit/llms/oci/chat/test_oci_chat_transformation.py b/tests/unit/llms/oci/chat/test_oci_chat_transformation.py index 0535279d177..f22b5e32593 100644 --- a/tests/unit/llms/oci/chat/test_oci_chat_transformation.py +++ b/tests/unit/llms/oci/chat/test_oci_chat_transformation.py @@ -1,4 +1,5 @@ import datetime +import logging from typing import Final import httpx import pytest @@ -1103,19 +1104,25 @@ class TestOCIReasoningEffort: ) assert chat_request.get("reasoningEffort") == "HIGH" - def test_reasoning_effort_unsupported_on_cohere_dropped(self): - """drop_params=True → silently drop reasoning_effort for Cohere.""" + def test_reasoning_effort_unsupported_on_cohere_dropped_with_one_warning(self, caplog): from litellm.llms.oci.chat.transformation import OCIChatConfig config = OCIChatConfig() - result = config.map_openai_params( - non_default_params={"reasoning_effort": "low"}, - optional_params={}, - model="cohere.command-latest", - drop_params=True, - ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + result = config.map_openai_params( + non_default_params={"reasoning_effort": "low"}, + optional_params={}, + model="cohere.command-latest", + drop_params=True, + ) + drop_warnings = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] assert "reasoning_effort" not in result assert "reasoningEffort" not in result + assert len(drop_warnings) == 1 + assert all( + fragment in drop_warnings[0] + for fragment in ("reasoning_effort='low'", "cohere.command-latest", "(drop_params=True)") + ) def test_reasoning_effort_unsupported_on_cohere_raises(self): """drop_params=False → raise rather than ship a payload Cohere will reject.""" @@ -2110,16 +2117,20 @@ class TestReasoningEffortGating: assert "reasoning_effort" not in OCIChatConfig().get_supported_openai_params("meta.llama-3.3-70b-instruct") @pytest.mark.parametrize("model", SERVED_GROK_4_20_MODELS) - def test_reasoning_effort_dropped_under_drop_params(self, local_model_cost_map, model): - result = OCIChatConfig().map_openai_params( - non_default_params={"reasoning_effort": "low", "temperature": 0.1}, - optional_params={}, - model=model, - drop_params=True, - ) + def test_reasoning_effort_dropped_under_drop_params_with_one_warning(self, local_model_cost_map, model, caplog): + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + result = OCIChatConfig().map_openai_params( + non_default_params={"reasoning_effort": "low", "temperature": 0.1}, + optional_params={}, + model=model, + drop_params=True, + ) + drop_warnings = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] assert "reasoningEffort" not in result assert "reasoning_effort" not in result assert result["temperature"] == 0.1 + assert len(drop_warnings) == 1 + assert all(fragment in drop_warnings[0] for fragment in ("reasoning_effort='low'", model, "(drop_params=True)")) def test_reasoning_effort_raises_without_drop_params_for_gated_grok(self, local_model_cost_map, monkeypatch): monkeypatch.setattr(litellm, "drop_params", False) diff --git a/tests/unit/llms/ollama/test_ollama_chat_transformation.py b/tests/unit/llms/ollama/test_ollama_chat_transformation.py index df9f5fd9d14..af1b85cf325 100644 --- a/tests/unit/llms/ollama/test_ollama_chat_transformation.py +++ b/tests/unit/llms/ollama/test_ollama_chat_transformation.py @@ -1,5 +1,6 @@ import asyncio import inspect +import logging import os import sys from typing import Final, cast @@ -23,6 +24,7 @@ import json from unittest.mock import ANY, MagicMock, patch import litellm +from litellm._logging import verbose_logger from litellm.types.utils import Choices, EmbeddingResponse, Message, ModelResponse, ModelResponseStream from unittest import mock @@ -1114,3 +1116,36 @@ def test_get_complete_url_ignores_a_trailing_slash_on_api_base(api_base: str): ) == "http://gateway.example.com/ollama/api/chat" ) + + +def _warnings_naming(caplog: pytest.LogCaptureFixture, param: str) -> tuple[str, ...]: + return tuple( + r.getMessage() for r in caplog.records if r.levelno == logging.WARNING and f"Dropping {param}=" in r.getMessage() + ) + + +@pytest.mark.parametrize("effort", ("minimal", "xhigh")) +def test_reasoning_effort_mapped_to_think_off_warns_once(effort: str, caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + result: Final = OllamaChatConfig().map_openai_params( + non_default_params={"reasoning_effort": effort}, optional_params={}, model="qwen3", drop_params=False + ) + + warnings: Final = _warnings_naming(caplog, "reasoning_effort") + assert result["think"] is False + assert len(warnings) == 1 + assert f"reasoning_effort={effort!r}" in warnings[0] + assert "qwen3" in warnings[0] + + +@pytest.mark.parametrize("effort, think", (("none", False), ("low", True), ("high", True))) +def test_reasoning_effort_with_a_matching_think_value_does_not_warn( + effort: str, think: bool, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + result: Final = OllamaChatConfig().map_openai_params( + non_default_params={"reasoning_effort": effort}, optional_params={}, model="qwen3", drop_params=False + ) + + assert result["think"] is think + assert _warnings_naming(caplog, "reasoning_effort") == () diff --git a/tests/unit/llms/ollama/test_ollama_completion_transformation.py b/tests/unit/llms/ollama/test_ollama_completion_transformation.py index 248e745ebff..edea39bb10c 100644 --- a/tests/unit/llms/ollama/test_ollama_completion_transformation.py +++ b/tests/unit/llms/ollama/test_ollama_completion_transformation.py @@ -1,6 +1,7 @@ import base64 import io import json +import logging import sys from typing import Final from unittest.mock import MagicMock, patch @@ -10,6 +11,7 @@ import pytest import respx import litellm +from litellm._logging import verbose_logger from litellm._uuid import uuid from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.ollama.completion.transformation import ( @@ -990,3 +992,40 @@ def test_get_complete_url_ignores_a_trailing_slash_on_api_base(api_base: str): ) == "http://gateway.example.com/ollama/api/generate" ) + +def _reasoning_effort_warnings(caplog: pytest.LogCaptureFixture) -> tuple[str, ...]: + return tuple( + r.getMessage() + for r in caplog.records + if r.levelno == logging.WARNING and "Dropping reasoning_effort=" in r.getMessage() + ) + + +@pytest.mark.parametrize("effort", ("minimal", "xhigh")) +def test_reasoning_effort_mapped_to_think_off_warns_once(effort: str, caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + result: Final = OllamaConfig().map_openai_params( + non_default_params={"reasoning_effort": effort}, optional_params={}, model="qwen3:0.6b", drop_params=False + ) + + warnings: Final = _reasoning_effort_warnings(caplog) + assert result["think"] is False + assert len(warnings) == 1 + assert f"reasoning_effort={effort!r}" in warnings[0] + assert "qwen3:0.6b" in warnings[0] + + +@pytest.mark.parametrize( + "model, effort, think", + (("qwen3:0.6b", "none", False), ("qwen3:0.6b", "high", True), ("gpt-oss:20b", "xhigh", "xhigh")), +) +def test_reasoning_effort_ollama_honors_does_not_warn( + model: str, effort: str, think: object, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + result: Final = OllamaConfig().map_openai_params( + non_default_params={"reasoning_effort": effort}, optional_params={}, model=model, drop_params=False + ) + + assert result["think"] == think + assert _reasoning_effort_warnings(caplog) == () diff --git a/tests/unit/llms/openai/test_openai_common_utils.py b/tests/unit/llms/openai/test_openai_common_utils.py index 46c14fcfd8f..f0865cca7f9 100644 --- a/tests/unit/llms/openai/test_openai_common_utils.py +++ b/tests/unit/llms/openai/test_openai_common_utils.py @@ -1,4 +1,6 @@ import asyncio +import json +import logging from typing import Final from unittest.mock import MagicMock, patch @@ -9,7 +11,11 @@ import respx import litellm from litellm.litellm_core_utils.token_counter import token_counter -from litellm.llms.openai.common_utils import BaseOpenAILLM, is_openai_backed_api_base +from litellm.llms.openai.common_utils import ( + BaseOpenAILLM, + drop_params_from_unprocessable_entity_error, + is_openai_backed_api_base, +) # Test parameters for different API functions API_FUNCTION_PARAMS = [ @@ -319,6 +325,104 @@ def _completion_kwargs(provider: str, client, **overrides) -> dict: } +_REASONING_EFFORT_422: Final = { + "error": { + "message": json.dumps( + {"detail": [{"loc": ["body", "reasoning_effort"], "msg": "Extra inputs are not permitted"}]} + ) + } +} +_CHAT_COMPLETION: Final = { + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 1, + "model": "gpt-6.1-sol", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, +} +_CHAT_COMPLETION_STREAM: Final = ( + 'data: {"id":"chatcmpl-1","object":"chat.completion.chunk","created":1,"model":"gpt-6.1-sol",' + '"choices":[{"index":0,"delta":{"role":"assistant","content":"ok"},"finish_reason":"stop"}]}\n\n' + "data: [DONE]\n\n" +) + + +def _answer_422_while_reasoning_effort_is_sent(request: httpx.Request) -> httpx.Response: + body: Final = json.loads(request.content) + if "reasoning_effort" in body: + return httpx.Response(422, json=_REASONING_EFFORT_422) + if body.get("stream"): + return httpx.Response(200, text=_CHAT_COMPLETION_STREAM, headers={"content-type": "text/event-stream"}) + return httpx.Response(200, json=_CHAT_COMPLETION) + + +async def _complete_through_the_422_retry(call_shape: str) -> str: + transport: Final = httpx.MockTransport(_answer_422_while_reasoning_effort_is_sent) + kwargs: Final = { + "model": "gpt-6.1-sol", + "messages": [{"role": "user", "content": "hi"}], + "reasoning_effort": "high", + } + if call_shape == "sync": + sync_client: Final = openai.OpenAI(api_key="sk-not-a-real-key", http_client=httpx.Client(transport=transport)) + return litellm.completion(**kwargs, client=sync_client).choices[0].message.content + async_client: Final = openai.AsyncOpenAI( + api_key="sk-not-a-real-key", http_client=httpx.AsyncClient(transport=transport) + ) + if call_shape == "async": + return (await litellm.acompletion(**kwargs, client=async_client)).choices[0].message.content + stream: Final = await litellm.acompletion(**kwargs, client=async_client, stream=True) + return "".join([chunk.choices[0].delta.content or "" async for chunk in stream]) + + +@pytest.mark.parametrize("call_shape", ["sync", "async", "async_stream"]) +def test_a_422_naming_reasoning_effort_drops_it_with_one_warning( + call_shape: str, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + monkeypatch.setattr(litellm, "drop_params", True) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + content: Final = asyncio.run(_complete_through_the_422_retry(call_shape)) + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] + + assert content == "ok" + assert len(drop_warnings) == 1 + assert all( + fragment in drop_warnings[0] + for fragment in ("reasoning_effort='high'", "gpt-6.1-sol", "(drop_params=True)", "422") + ) + + +def test_a_422_with_an_unhashable_loc_still_drops_reasoning_effort_with_one_warning( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + detail: Final = [ + {"loc": ["body", ["messages", 0]], "msg": "Invalid"}, + {"loc": ["body", "reasoning_effort"], "msg": "Extra inputs are not permitted"}, + ] + + def answer(request: httpx.Request) -> httpx.Response: + if "reasoning_effort" in json.loads(request.content): + return httpx.Response(422, json={"error": {"message": json.dumps({"detail": detail})}}) + return httpx.Response(200, json=_CHAT_COMPLETION) + + monkeypatch.setattr(litellm, "drop_params", True) + client: Final = openai.OpenAI( + api_key="sk-not-a-real-key", http_client=httpx.Client(transport=httpx.MockTransport(answer)) + ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + response: Final = litellm.completion( + model="gpt-6.1-sol", + messages=[{"role": "user", "content": "hi"}], + reasoning_effort="high", + client=client, + ) + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] + + assert response.choices[0].message.content == "ok" + assert len(drop_warnings) == 1 + assert "reasoning_effort='high'" in drop_warnings[0] + + @pytest.mark.parametrize("provider", ["openai", "azure"]) def test_sync_output_limit_400_maps_to_length_truncated_response(provider): response = litellm.completion( @@ -588,3 +692,24 @@ async def test_azure_gateway_and_image_variations_use_the_callers_async_session( ) assert response.data[0].url == "https://example.com/image.png" assert not session.is_closed + + + +def test_reasoning_drop_warning_caps_a_large_caller_value(caplog: pytest.LogCaptureFixture) -> None: + error: Final = httpx.HTTPStatusError( + message="422", + request=httpx.Request("POST", "https://example.invalid/v1/chat/completions"), + response=httpx.Response(422, json=_REASONING_EFFORT_422), + ) + data: Final[dict[str, object]] = {"model": "gpt-6.1-sol", "messages": [], "reasoning_effort": "x" * 5000} + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + retried: Final = drop_params_from_unprocessable_entity_error( + error, data, model="gpt-6.1-sol", drop_params_gated=True + ) + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] + + assert "reasoning_effort" not in retried + assert len(drop_warnings) == 1 + assert len(drop_warnings[0]) < 300 + assert all(fragment in drop_warnings[0] for fragment in ("reasoning_effort='xxx", "'gpt-6.1-sol'", "422")) diff --git a/tests/unit/llms/tencent/chat/test_tencent_chat_transformation.py b/tests/unit/llms/tencent/chat/test_tencent_chat_transformation.py index 4d6d252ae6e..1bf646fc2ad 100644 --- a/tests/unit/llms/tencent/chat/test_tencent_chat_transformation.py +++ b/tests/unit/llms/tencent/chat/test_tencent_chat_transformation.py @@ -1,5 +1,10 @@ +import logging +from typing import Final from unittest.mock import patch +import pytest + +from litellm._logging import verbose_logger from litellm.llms.tencent.chat.transformation import TencentChatConfig @@ -208,6 +213,27 @@ class TestAdaptiveThinkingCoercion: assert result["extra_body"]["thinking"] == {"type": "adaptive", "budget_tokens": 4096} + @pytest.mark.parametrize("budget, warned", (("high", 1), (4096, 0)), ids=("non-integer", "integer")) + def test_enabled_thinking_on_adaptive_only_model_warns_only_when_the_budget_is_dropped( + self, local_model_cost_map, budget: object, warned: int, caplog: pytest.LogCaptureFixture + ): + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + result: Final = TencentChatConfig().map_openai_params( + non_default_params={"thinking": {"type": "enabled", "budget_tokens": budget}}, + optional_params={}, + model="tencent/minimax-m3", + drop_params=False, + ) + + warnings: Final = tuple( + r.getMessage() + for r in caplog.records + if r.levelno == logging.WARNING and f"Dropping thinking.budget_tokens={budget!r}" in r.getMessage() + ) + assert result["extra_body"]["thinking"]["type"] == "adaptive" + assert result["extra_body"]["thinking"].get("budget_tokens") == (None if warned else budget) + assert len(warnings) == warned + def test_disabled_thinking_kept_for_adaptive_only_model(self, local_model_cost_map): config = TencentChatConfig() result = config.map_openai_params( diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 67c78b62151..98d1b8f1300 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -1693,6 +1693,78 @@ def test_vertex_ai_map_thinking_param_without_budget_tokens_for_gemini_3(): assert result["thinkingConfig"] == {"includeThoughts": True} +def _budget_drop_warnings(caplog: pytest.LogCaptureFixture) -> tuple[str, ...]: + return tuple( + r.getMessage() + for r in caplog.records + if r.levelno == logging.WARNING and "thinking.budget_tokens" in r.getMessage() + ) + + +@pytest.mark.parametrize("budget", ("high", 1024.5, {"tokens": 1024})) +def test_gemini_non_integer_thinking_budget_drop_warns_once(budget: object, caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + result: Final = VertexGeminiConfig().map_openai_params( + non_default_params={"thinking": {"type": "enabled", "budget_tokens": budget}}, + optional_params={}, + model="gemini-2.5-flash", + drop_params=False, + ) + + warnings: Final = _budget_drop_warnings(caplog) + assert "thinkingBudget" not in result["thinkingConfig"] + assert len(warnings) == 1 + assert f"thinking.budget_tokens={budget!r}" in warnings[0] + assert "gemini-2.5-flash" in warnings[0] + + +def test_gemini_integer_thinking_budget_is_sent_without_warning(caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + result: Final = VertexGeminiConfig().map_openai_params( + non_default_params={"thinking": {"type": "enabled", "budget_tokens": 2048}}, + optional_params={}, + model="gemini-2.5-flash", + drop_params=False, + ) + + assert result["thinkingConfig"]["thinkingBudget"] == 2048 + assert _budget_drop_warnings(caplog) == () + + +def test_gemini_3_thinking_budget_drop_warns_once(caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + result: Final = VertexGeminiConfig().map_openai_params( + non_default_params={"thinking": {"type": "enabled", "budget_tokens": 2048}}, + optional_params={}, + model="gemini-3-pro-preview", + drop_params=False, + ) + + warnings: Final = _budget_drop_warnings(caplog) + assert "thinkingBudget" not in result["thinkingConfig"] + assert len(warnings) == 1 + assert "thinking.budget_tokens=2048" in warnings[0] + assert "gemini-3-pro-preview" in warnings[0] + + +@pytest.mark.parametrize( + "thinking", + ({"type": "enabled"}, {"type": "enabled", "budget_tokens": 0}, {"type": "disabled", "budget_tokens": 2048}), +) +def test_gemini_3_thinking_without_a_dropped_budget_does_not_warn( + thinking: dict[str, object], caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + VertexGeminiConfig().map_openai_params( + non_default_params={"thinking": thinking}, + optional_params={}, + model="gemini-3-pro-preview", + drop_params=False, + ) + + assert _budget_drop_warnings(caplog) == () + + def test_vertex_ai_map_tools(): v = VertexGeminiConfig() optional_params = {} diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index aaec486b60f..9f2d2a9bbc5 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -10262,9 +10262,10 @@ def test_a_non_numeric_thinking_budget_on_adaptive_claude_is_rejected_with_a_400 @pytest.mark.parametrize("drop_globally", [False, True]) @pytest.mark.parametrize("model_info", [None, _OPUS_DECLARING_LOW_AND_HIGH]) @pytest.mark.parametrize("body_sent", [_anthropic_chat_body_sent, _bedrock_converse_request_fields_sent]) -def test_drop_params_drops_a_non_numeric_thinking_budget_on_adaptive_claude( +def test_drop_params_drops_a_non_numeric_thinking_budget_on_adaptive_claude_with_one_warning( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, body_sent: Callable[..., Mapping[str, object]], model_info: Mapping[str, object] | None, drop_globally: bool, @@ -10272,13 +10273,40 @@ def test_drop_params_drops_a_non_numeric_thinking_budget_on_adaptive_claude( monkeypatch.setattr(litellm, "drop_params", drop_globally) request_drop: Final = {} if drop_globally else {"drop_params": True} - dropped: Final = body_sent( - respx_mock, model_info, thinking={"type": "enabled", "budget_tokens": "lots"}, **request_drop - ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + dropped: Final = body_sent( + respx_mock, model_info, thinking={"type": "enabled", "budget_tokens": "lots"}, **request_drop + ) + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] adaptive_alone: Final = body_sent(respx_mock, model_info, thinking={"type": "adaptive"}, **request_drop) assert "output_config" not in adaptive_alone assert dropped == adaptive_alone + assert len(drop_warnings) == 1 + assert all( + fragment in drop_warnings[0] + for fragment in ("thinking.budget_tokens='lots'", "claude-opus-5-5", "(drop_params=True)") + ) + + +@pytest.mark.parametrize("model_info", [None, _KIMI_DEPLOYMENT]) +def test_drop_params_drops_a_non_numeric_thinking_budget_on_hosted_vllm_with_one_warning( + respx_mock: respx.MockRouter, caplog: pytest.LogCaptureFixture, model_info: Mapping[str, object] | None +) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + sent: Final = _chat_body_sent( + respx_mock, + "hosted_vllm/my-kimi", + None, + model_info, + thinking={"type": "enabled", "budget_tokens": "lots"}, + drop_params=True, + ) + + assert not {"reasoning_effort", "thinking"} & sent.keys() + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] + assert len(drop_warnings) == 1 + assert all(fragment in drop_warnings[0] for fragment in ("thinking.budget_tokens='lots'", "my-kimi")) @pytest.mark.parametrize("model", ["azure_ai/my-kimi", "hosted_vllm/my-kimi"]) From d18efdfe1362194edfb98d814c85307af35d2038 Mon Sep 17 00:00:00 2001 From: alvcol Date: Fri, 30 Oct 2026 16:57:48 +0100 Subject: [PATCH 08/74] fix(vllm): send the pass-through key as a bearer on both /vllm routes (#46861) * fix(vllm): send the deployment api_key on /vllm passthrough calls VLLMModelInfo.get_api_key always returned None, so /vllm/{endpoint} calls to a router model reached the vLLM server with no auth header even when the deployment had an api_key, and a server started with --api-key answered 401 (#23443). Return the deployment key (or VLLM_API_KEY) and send it as Authorization: Bearer, the header vllm serve --api-key checks. x-api-key was never actually sent before, since the key was always None. * fix(vllm): send the pass-through key as a bearer on both /vllm routes vllm serve --api-key only reads Authorization: Bearer, so a keyed vLLM answered 401 on both pass-through routes. Send the key as a bearer and keep x-api-key for gateways that read it. Router models send their own deployment key and never fall back to VLLM_API_KEY, which belongs to the /vllm route's own server. * fix(vllm): send only the bearer on router models and stop doubling /v1 on /vllm Router models now send the same Authorization: Bearer header /v1/chat/completions sends for that deployment, without the x-api-key copy that route never sent. The VLLM_API_BASE route keeps x-api-key next to the bearer, as it sent it before. A router model whose api_base already ends in /v1 no longer gets /v1/v1 on /vllm/v1 paths. * fix(vllm): send no placeholder key from keyless router models and stop doubling /v1 on the VLLM_API_BASE route * fix(vllm): send VLLM_API_KEY when a router model with no api_base is forwarded to VLLM_API_BASE * fix(vllm): keep a router model's own key on the VLLM_API_BASE server and resolve an oidc VLLM_API_KEY * fix(vllm): treat an empty key as no key so pass-through sends no empty bearer * fix(vllm): send no bearer for a Google oidc key and match VLLM_API_BASE across a trailing slash --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/llms/vllm/common_utils.py | 20 +- .../llms/vllm/passthrough/transformation.py | 35 +++- .../llm_passthrough_endpoints.py | 3 +- tests/unit/passthrough/test_main.py | 175 ++++++++++++++++++ .../test_llm_pass_through_endpoints.py | 62 +++++++ 5 files changed, 289 insertions(+), 6 deletions(-) diff --git a/litellm/llms/vllm/common_utils.py b/litellm/llms/vllm/common_utils.py index 6484cdc88e3..014fabd5187 100644 --- a/litellm/llms/vllm/common_utils.py +++ b/litellm/llms/vllm/common_utils.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from typing import Final import httpx @@ -10,6 +11,14 @@ from litellm.types.llms.openai import AllMessageValues from litellm.utils import add_path_to_api_base +def endpoint_under_api_base(endpoint: str, api_base: str | None) -> str: + return ( + endpoint.lstrip("/").removeprefix("v1/") + if api_base is not None and api_base.rstrip("/").endswith("/v1") + else endpoint + ) + + class VLLMError(BaseLLMException): def __init__( self, @@ -39,9 +48,14 @@ class VLLMModelInfo(BaseLLMModelInfo): api_key: str | None = None, api_base: str | None = None, ) -> dict: - if api_key is not None: - headers["x-api-key"] = api_key - return headers + upstream_key: Final = self.upstream_api_key(api_key, api_base) + return {**headers, **self.upstream_auth_headers(upstream_key)} if upstream_key else headers + + def upstream_api_key(self, api_key: str | None, api_base: str | None) -> str | None: + return api_key + + def upstream_auth_headers(self, api_key: str) -> Mapping[str, str]: + return {"Authorization": f"Bearer {api_key}", "x-api-key": api_key} @staticmethod def get_api_base(api_base: str | None = None) -> str | None: diff --git a/litellm/llms/vllm/passthrough/transformation.py b/litellm/llms/vllm/passthrough/transformation.py index e7da6e54410..2b253e7fa38 100644 --- a/litellm/llms/vllm/passthrough/transformation.py +++ b/litellm/llms/vllm/passthrough/transformation.py @@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Final from httpx import Response +from litellm.constants import PLACEHOLDER_API_KEY from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig from litellm.llms.openai_like.passthrough.transformation import ( @@ -12,8 +13,14 @@ from litellm.llms.openai_like.passthrough.transformation import ( logged_text_completion_stream, relays_text_completion, ) +from litellm.secret_managers.main import ( + GOOGLE_OIDC_SECRET_PREFIX, + UnresolvableSecretReference, + get_secret_str, + resolve_local_oidc_reference, +) -from ..common_utils import VLLMModelInfo +from ..common_utils import VLLMModelInfo, endpoint_under_api_base if TYPE_CHECKING: from httpx import URL @@ -22,6 +29,30 @@ if TYPE_CHECKING: class VLLMPassthroughConfig(VLLMModelInfo, BasePassthroughConfig): + @staticmethod + def get_api_key(api_key: str | None = None) -> str | None: + return None if api_key == PLACEHOLDER_API_KEY else api_key + + def upstream_api_key(self, api_key: str | None, api_base: str | None) -> str | None: + if api_key is not None or not self._is_vllm_api_base(api_base): + return api_key + vllm_api_key: Final = get_secret_str("VLLM_API_KEY") + if vllm_api_key is None or vllm_api_key.startswith(GOOGLE_OIDC_SECRET_PREFIX): + return None + match resolve_local_oidc_reference(vllm_api_key): + case UnresolvableSecretReference(): + return None + case str() as resolved: + return resolved + + @staticmethod + def _is_vllm_api_base(api_base: str | None) -> bool: + vllm_api_base: Final = get_secret_str("VLLM_API_BASE") + return api_base is not None and vllm_api_base is not None and api_base.rstrip("/") == vllm_api_base.rstrip("/") + + def upstream_auth_headers(self, api_key: str) -> Mapping[str, str]: + return {"Authorization": f"Bearer {api_key}"} + def is_streaming_request(self, endpoint: str, request_data: dict) -> bool: return "stream" in request_data @@ -40,7 +71,7 @@ class VLLMPassthroughConfig(VLLMModelInfo, BasePassthroughConfig): raise Exception("VLLM api base not found") return ( - self.format_url(endpoint, base_target_url, request_query_params), + self.format_url(endpoint_under_api_base(endpoint, base_target_url), base_target_url, request_query_params), base_target_url, ) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 21078dfc848..8de7441121f 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -72,6 +72,7 @@ from litellm.llms.openai.common_utils import is_openai_agents_api_path, openai_a from litellm.llms.openai.workload_identity import get_workload_identity_bearer_token_for_api_base from litellm.llms.oss_decision import OssDecisionProvider, oss_connection, validate_oss_request from litellm.llms.vertex_ai.vertex_llm_base import VertexBase +from litellm.llms.vllm.common_utils import endpoint_under_api_base from litellm.passthrough.main import AsyncPassthroughStreamingResponse, is_provider_error from litellm.proxy._types import * from litellm.proxy.auth.auth_checks import enforced_model_allowlists @@ -600,7 +601,7 @@ async def vllm_proxy_route( ) return await llm_passthrough_factory_proxy_route( - endpoint=endpoint, + endpoint=endpoint_under_api_base(endpoint, get_secret_str("VLLM_API_BASE")), request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, diff --git a/tests/unit/passthrough/test_main.py b/tests/unit/passthrough/test_main.py index 78fa83d126b..4997d736d0a 100644 --- a/tests/unit/passthrough/test_main.py +++ b/tests/unit/passthrough/test_main.py @@ -51,3 +51,178 @@ async def test_allm_passthrough_route_hosted_vllm_sends_normalized_model(respx_m outbound: Final = json.loads(route.calls[0].request.content) assert outbound["model"] == "my-deployment" assert outbound["messages"] == [{"role": "user", "content": "Hello"}] + + +@pytest.mark.asyncio +async def test_allm_passthrough_route_hosted_vllm_sends_deployment_api_key(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(f"{_API_BASE}/v1/chat/completions").mock( + return_value=httpx.Response(200, json={"ok": True}) + ) + response: Final = await allm_passthrough_route( + method="POST", + endpoint="v1/chat/completions", + model="hosted_vllm/my-deployment", + api_base=_API_BASE, + api_key="sk-upstream", + json={"model": "anything", "messages": [{"role": "user", "content": "Hello"}]}, + client=AsyncHTTPHandler(), + ) + assert response.status_code == 200 + assert route.calls[0].request.headers["authorization"] == "Bearer sk-upstream" + assert "x-api-key" not in route.calls[0].request.headers + + +@pytest.mark.parametrize("endpoint", ["v1/chat/completions", "chat/completions"]) +@pytest.mark.asyncio +async def test_allm_passthrough_route_hosted_vllm_reaches_v1_paths_under_an_api_base_ending_in_v1( + respx_mock: respx.MockRouter, endpoint: str +) -> None: + route: Final = respx_mock.post(f"{_API_BASE}/v1/chat/completions").mock( + return_value=httpx.Response(200, json={"ok": True}) + ) + response: Final = await allm_passthrough_route( + method="POST", + endpoint=endpoint, + model="hosted_vllm/my-deployment", + api_base=f"{_API_BASE}/v1", + api_key="sk-upstream", + json={"model": "anything", "messages": [{"role": "user", "content": "Hello"}]}, + client=AsyncHTTPHandler(), + ) + assert response.status_code == 200 + assert route.call_count == 1 + + +@pytest.mark.asyncio +async def test_allm_passthrough_route_hosted_vllm_sends_no_auth_for_a_keyless_deployment( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("VLLM_API_KEY", "sk-vllm-route-only") + monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False) + route: Final = respx_mock.post(f"{_API_BASE}/v1/chat/completions").mock( + return_value=httpx.Response(200, json={"ok": True}) + ) + response: Final = await allm_passthrough_route( + method="POST", + endpoint="v1/chat/completions", + model="hosted_vllm/my-deployment", + api_base=_API_BASE, + json={"model": "anything", "messages": [{"role": "user", "content": "Hello"}]}, + client=AsyncHTTPHandler(), + ) + assert response.status_code == 200 + assert "authorization" not in route.calls[0].request.headers + assert "x-api-key" not in route.calls[0].request.headers + + +@pytest.mark.parametrize( + ("vllm_api_key", "expected_authorization"), + [("sk-vllm-env", "Bearer sk-vllm-env"), ("oidc/env/VLLM_OIDC_TOKEN", "Bearer sk-vllm-oidc")], +) +@pytest.mark.asyncio +async def test_allm_passthrough_route_keyless_hosted_vllm_without_api_base_sends_the_vllm_api_base_key( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, vllm_api_key: str, expected_authorization: str +) -> None: + monkeypatch.setenv("VLLM_API_BASE", _API_BASE) + monkeypatch.setenv("VLLM_API_KEY", vllm_api_key) + monkeypatch.setenv("VLLM_OIDC_TOKEN", "sk-vllm-oidc") + monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False) + monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False) + route: Final = respx_mock.post(f"{_API_BASE}/v1/chat/completions").mock( + return_value=httpx.Response(200, json={"ok": True}) + ) + response: Final = await allm_passthrough_route( + method="POST", + endpoint="v1/chat/completions", + model="hosted_vllm/my-deployment", + json={"model": "anything", "messages": [{"role": "user", "content": "Hello"}]}, + client=AsyncHTTPHandler(), + ) + assert response.status_code == 200 + assert route.calls[0].request.headers["authorization"] == expected_authorization + + +@pytest.mark.parametrize( + ("model_api_base", "vllm_api_base"), [(f"{_API_BASE}/", _API_BASE), (_API_BASE, f"{_API_BASE}/")] +) +@pytest.mark.asyncio +async def test_allm_passthrough_route_keyless_hosted_vllm_matches_the_vllm_api_base_across_a_trailing_slash( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, model_api_base: str, vllm_api_base: str +) -> None: + monkeypatch.setenv("VLLM_API_BASE", vllm_api_base) + monkeypatch.setenv("VLLM_API_KEY", "sk-vllm-env") + monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False) + monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False) + route: Final = respx_mock.post(url__regex=rf"^{_API_BASE}/+v1/chat/completions$").mock( + return_value=httpx.Response(200, json={"ok": True}) + ) + response: Final = await allm_passthrough_route( + method="POST", + endpoint="v1/chat/completions", + model="hosted_vllm/my-deployment", + api_base=model_api_base, + json={"model": "anything", "messages": [{"role": "user", "content": "Hello"}]}, + client=AsyncHTTPHandler(), + ) + assert response.status_code == 200 + assert route.calls[0].request.headers["authorization"] == "Bearer sk-vllm-env" + + +@pytest.mark.parametrize("vllm_api_key", ["", "oidc/google/audience"]) +@pytest.mark.asyncio +async def test_allm_passthrough_route_unusable_vllm_api_key_sends_no_authorization_header( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, vllm_api_key: str +) -> None: + monkeypatch.setenv("VLLM_API_BASE", _API_BASE) + monkeypatch.setenv("VLLM_API_KEY", vllm_api_key) + monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False) + monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False) + route: Final = respx_mock.post(f"{_API_BASE}/v1/chat/completions").mock( + return_value=httpx.Response(200, json={"ok": True}) + ) + response: Final = await allm_passthrough_route( + method="POST", + endpoint="v1/chat/completions", + model="hosted_vllm/my-deployment", + json={"model": "anything", "messages": [{"role": "user", "content": "Hello"}]}, + client=AsyncHTTPHandler(), + ) + assert response.status_code == 200 + assert "authorization" not in route.calls[0].request.headers + + +@pytest.mark.parametrize( + ("api_base", "api_key", "expected_authorization"), + [ + (None, "sk-upstream", "Bearer sk-upstream"), + (None, "sk-hosted-env", "Bearer sk-hosted-env"), + (_API_BASE, "sk-hosted-env", "Bearer sk-hosted-env"), + (None, None, "Bearer sk-hosted-env"), + ], +) +@pytest.mark.asyncio +async def test_allm_passthrough_route_hosted_vllm_on_the_vllm_api_base_sends_its_own_resolved_key( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + api_base: str | None, + api_key: str | None, + expected_authorization: str, +) -> None: + monkeypatch.setenv("VLLM_API_BASE", _API_BASE) + monkeypatch.setenv("VLLM_API_KEY", "sk-vllm-env") + monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False) + monkeypatch.setenv("HOSTED_VLLM_API_KEY", "sk-hosted-env") + route: Final = respx_mock.post(f"{_API_BASE}/v1/chat/completions").mock( + return_value=httpx.Response(200, json={"ok": True}) + ) + response: Final = await allm_passthrough_route( + method="POST", + endpoint="v1/chat/completions", + model="hosted_vllm/my-deployment", + api_base=api_base, + api_key=api_key, + json={"model": "anything", "messages": [{"role": "user", "content": "Hello"}]}, + client=AsyncHTTPHandler(), + ) + assert response.status_code == 200 + assert route.calls[0].request.headers["authorization"] == expected_authorization diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 806d88b2467..aab227e9494 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -2759,6 +2759,68 @@ class TestVLLMProxyRoute: mock_factory_route.assert_awaited_once() +@pytest.mark.parametrize( + ("vllm_api_base", "request_path"), + [ + ("http://vllm-upstream.test:8090", "/vllm/v1/chat/completions"), + ("http://vllm-upstream.test:8090/v1", "/vllm/v1/chat/completions"), + ("http://vllm-upstream.test:8090/v1", "/vllm/chat/completions"), + ], +) +def test_vllm_provider_route_reaches_v1_paths_with_its_key_as_a_bearer( + monkeypatch: pytest.MonkeyPatch, vllm_api_base: str, request_path: str +) -> None: + from litellm.proxy import proxy_server + + monkeypatch.setenv("VLLM_API_BASE", vllm_api_base) + monkeypatch.setenv("VLLM_API_KEY", "sk-vllm-upstream") + monkeypatch.setattr(proxy_server, "llm_router", None) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + monkeypatch.setitem( + proxy_server.app.dependency_overrides, user_api_key_auth, lambda: UserAPIKeyAuth(api_key="sk-virtual") + ) + + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("http://vllm-upstream.test:8090/v1/chat/completions").mock( + return_value=httpx.Response(200, json={"id": "chatcmpl-1"}) + ) + response: Final = TestClient(proxy_server.app).post( + request_path, + json={"model": "Qwen/Qwen3-0.6B", "messages": [{"role": "user", "content": "hi"}]}, + ) + + assert (response.status_code, response.json()) == (200, {"id": "chatcmpl-1"}) + assert route.calls.last.request.headers["authorization"] == "Bearer sk-vllm-upstream" + assert route.calls.last.request.headers["x-api-key"] == "sk-vllm-upstream" + + +def test_vllm_provider_route_with_an_empty_key_sends_no_credential_headers(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy import proxy_server + + monkeypatch.setenv("VLLM_API_BASE", "http://vllm-upstream.test:8090") + monkeypatch.setenv("VLLM_API_KEY", "") + monkeypatch.setattr(proxy_server, "llm_router", None) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + monkeypatch.setitem( + proxy_server.app.dependency_overrides, user_api_key_auth, lambda: UserAPIKeyAuth(api_key="sk-virtual") + ) + + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("http://vllm-upstream.test:8090/v1/chat/completions").mock( + return_value=httpx.Response(200, json={"id": "chatcmpl-1"}) + ) + response: Final = TestClient(proxy_server.app).post( + "/vllm/v1/chat/completions", + json={"model": "Qwen/Qwen3-0.6B", "messages": [{"role": "user", "content": "hi"}]}, + ) + + assert (response.status_code, response.json()) == (200, {"id": "chatcmpl-1"}) + assert "authorization" not in route.calls.last.request.headers + assert "x-api-key" not in route.calls.last.request.headers + + class TestGigachatProxyRoute: @pytest.mark.asyncio @patch( # test-quality-ok: patching litellm internal for unit test isolation From b195d15b937ab5d09a368ed3972ba3e2cd48d19a Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 09:28:19 -0700 Subject: [PATCH 09/74] fix(vertex_ai,azure_ai)!: auto-add the compaction, thinking display, and tool change betas on Claude routes (#44747) * fix(vertex_ai): forward and auto-add the compact-2026-09-04 beta on Claude routes * test(vertex_ai): prove the compact-2026-09-04 beta reaches the wire through both Claude handlers * fix(vertex_ai): move a client-sent body compact-2026-09-04 beta to the request header * refactor(vertex_ai): build the Claude beta set without mutation and inject the client in the messages wire test * fix(vertex_ai): keep client body betas in the body and answer 400 on a malformed anthropic_beta * fix(rust): allow the compact-2026-09-04 beta on vertex_ai in the Rust beta table The beta header config now maps compact-2026-09-04 for vertex_ai, so the Rust table that mirrors it must stop rejecting that beta there, and the Rust Vertex messages route adds it for a compaction request like the Python route does. * fix(vertex_ai): keep client body betas out of the Vertex body next to litellm betas, honor drop_params, and send the compaction beta on count_tokens Litellm's own betas replace a client's body anthropic_beta again, as on main, so a body beta Vertex rejects no longer turns a 200 into a 400. A malformed beta drops under drop_params and names its field otherwise. inline-tools-2026-09-15 joins compact-2026-09-04 as header-only, since Vertex rejects both in the body. Vertex count_tokens sends compact-2026-09-04 when the request carries compaction. * fix(logging): log and bill Anthropic messages compaction responses on the Rust route The Rust route hands the served Anthropic body to the logger, which validates it against AnthropicResponse. That model had no compaction content block and no compaction stop reason, so a compaction reply raised and wrote no spend row. * fix(vertex_ai,azure_ai): forward compact-2026-09-04 to Azure AI Foundry, merge Vertex body betas, and redact compaction summaries Azure AI Foundry needs the compact-2026-09-04 header for compaction, a replayed compaction block, and count tokens, so the beta map, the native compaction providers, and the Rust beta table now allow it there. Vertex keeps a client's body betas next to the ones litellm adds, as the Anthropic route does, and turn_off_message_logging now redacts compaction summaries too * fix(vertex_ai,azure_ai): auto-add thinking display and tool change betas, redact streamed compaction summaries Folds the header changes from #44103 into this PR. Vertex AI and Azure AI Claude routes now add thinking-display-updates-2026-08-18 when thinking.display is "updates" and mid-conversation-tool-changes-2026-07-01 when a tool_addition block is present, matching the Anthropic and Bedrock routes. With turn_off_message_logging on, compaction summaries from streamed messages and streamed chat completions are now redacted in spend logs and callback payloads, like the non-streamed response already was. * fix(vertex_ai,azure_ai,rust): add tool change betas on count tokens and Rust messages, bill Rust server tool replies Vertex and Azure count tokens now add mid-conversation-tool-changes-2026-07-01 when the history carries a tool_addition or tool_removal block, through one shared helper with the compaction beta. The Rust /v1/messages route adds thinking-display-updates-2026-08-18 and mid-conversation-tool-changes-2026-07-01 when a request uses them, on the providers that accept each beta. Rust-route logging parses the served body with the Anthropic response parser, so web_search, pause_turn, mcp_tool_use and container_upload replies get a spend row. Legacy enabled thinking keeps display updates on anthropic too, and the Vertex transform reads drop_params through drops_unsupported_params. * style(rust): rewrap beta map test line with cargo fmt * test(rust): expect the display beta on Databricks, which main's beta list now accepts --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../src/providers/anthropic/beta.rs | 34 +- .../src/anthropic/messages/transformation.rs | 99 +++++- .../src/azure_ai/messages/transformation.rs | 5 + .../src/vertex_ai/messages/transformation.rs | 12 + litellm/anthropic_beta_headers_config.json | 9 +- litellm/anthropic_beta_headers_manager.py | 39 ++- litellm/litellm_core_utils/litellm_logging.py | 15 +- litellm/litellm_core_utils/redact_messages.py | 1 + litellm/llms/anthropic/common_utils.py | 41 ++- .../messages/native/transformation.py | 3 +- .../anthropic/count_tokens/handler.py | 1 + .../anthropic/count_tokens/transformation.py | 8 +- .../anthropic/messages/transformation.py | 30 +- .../anthropic/transformation.py | 82 ++++- .../count_tokens/handler.py | 25 +- litellm/types/llms/anthropic.py | 15 +- .../test_litellm_logging.py | 89 ++++++ .../test_redact_messages.py | 5 +- .../anthropic/test_anthropic_common_utils.py | 61 +++- .../count_tokens/test_token_counter.py | 53 +++ ...azure_anthropic_messages_transformation.py | 88 +++++ .../test_azure_anthropic_transformation.py | 84 +++++ ...artner_models_anthropic_messages_config.py | 195 ++++++++++- ...partner_models_anthropic_transformation.py | 302 +++++++++++++++++- .../count_tokens/test_handler.py | 60 ++++ .../test_anthropic_beta_headers_filtering.py | 51 ++- 26 files changed, 1323 insertions(+), 84 deletions(-) create mode 100644 tests/unit/llms/vertex_ai/vertex_ai_partner_models/count_tokens/test_handler.py diff --git a/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs b/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs index 80095cac36c..01ed023999d 100644 --- a/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs +++ b/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs @@ -199,7 +199,7 @@ impl AnthropicBeta { &[BetaProvider::BedrockConverse, BetaProvider::Databricks] } Self::ContextManagement20250627 => &[BetaProvider::BedrockConverse], - Self::AdvisorTool20260301 | Self::Compact20260904 => &[ + Self::AdvisorTool20260301 => &[ BetaProvider::AzureAi, BetaProvider::BedrockConverse, BetaProvider::Bedrock, @@ -207,6 +207,12 @@ impl AnthropicBeta { BetaProvider::VertexAi, BetaProvider::Databricks, ], + Self::Compact20260904 => &[ + BetaProvider::BedrockConverse, + BetaProvider::Bedrock, + BetaProvider::BedrockMantle, + BetaProvider::Databricks, + ], Self::Bash20241022 | Self::Bash20250124 | Self::McpServers20251204 @@ -256,14 +262,18 @@ impl AnthropicBeta { Self::InterleavedThinking20250514 | Self::WebSearch20250305 => { &[BetaProvider::BedrockConverse, BetaProvider::Bedrock] } - Self::MidConversationOutputConfig20260701 - | Self::MidConversationToolChanges20260701 => &[ + Self::MidConversationOutputConfig20260701 => &[ BetaProvider::AzureAi, BetaProvider::BedrockConverse, BetaProvider::BedrockMantle, BetaProvider::VertexAi, BetaProvider::Databricks, ], + Self::MidConversationToolChanges20260701 => &[ + BetaProvider::BedrockConverse, + BetaProvider::BedrockMantle, + BetaProvider::Databricks, + ], Self::Output128k20250219 | Self::TokenEfficientTools20250219 => &[ BetaProvider::AzureAi, BetaProvider::BedrockConverse, @@ -483,7 +493,7 @@ mod tests { let config = beta_headers_config(); let config_columns: Vec = config .keys() - .filter(|column| column.as_str() != "description") + .filter(|column| column.as_str() != "description" && !column.ends_with("_header_only")) .cloned() .collect(); let beta_provider_columns: Vec = BetaProvider::VARIANTS @@ -502,12 +512,16 @@ mod tests { let config = beta_headers_config(); for provider in BetaProvider::VARIANTS { for beta in &AnthropicBeta::KNOWN { - let expected = config - .get(provider.as_ref()) - .and_then(serde_json::Value::as_object) - .and_then(|column| column.get(beta.as_str())) - .and_then(serde_json::Value::as_str) - .map(str::to_string); + let cell = |column: &str| { + config + .get(column) + .and_then(serde_json::Value::as_object) + .and_then(|values| values.get(beta.as_str())) + .and_then(serde_json::Value::as_str) + .map(str::to_string) + }; + let expected = + cell(&format!("{provider}_header_only")).or_else(|| cell(provider.as_ref())); let actual = beta.on(*provider).map(|name| name.to_string()); assert_eq!(actual, expected, "provider {provider}, beta {beta}"); if matches!(beta, AnthropicBeta::AdvancedToolUse20251120) diff --git a/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs b/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs index 6657cbf262f..fb6d7259bf9 100644 --- a/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs @@ -1,7 +1,8 @@ use litellm_auth::CredentialPlacement; use litellm_llms_types::{ formats::messages::{ - ContextEdit, ContextManagement, Message, MessagesOptionalParams, MessagesRequest, Speed, + ContentBlock, ContentBlockType, ContextEdit, ContextManagement, Message, + MessagesOptionalParams, MessagesRequest, Speed, ThinkingConfig, ThinkingDisplay, }, providers::anthropic::{AnthropicBeta, BetaProvider, BetaSet, DEFAULT_HEADERS}, recognized::Recognized, @@ -203,6 +204,10 @@ fn feature_betas(request: &MessagesRequest) -> BetaSet { .then_some(AnthropicBeta::FastMode20260201), messages_carry_output_config(&request.messages) .then_some(AnthropicBeta::PerTurnControl20260701), + uses_thinking_display_updates(params) + .then_some(AnthropicBeta::ThinkingDisplayUpdates20260818), + messages_carry_tool_changes(&request.messages) + .then_some(AnthropicBeta::MidConversationToolChanges20260701), has_advisor_tool(tools).then_some(AnthropicBeta::AdvisorTool20260301), is_tool_search_used(tools).then_some(AnthropicBeta::AdvancedToolUse20251120), ] @@ -253,6 +258,36 @@ fn messages_carry_output_config(messages: &[Message]) -> bool { .any(|message| message.extra.contains_key("output_config")) } +fn uses_thinking_display_updates(params: &MessagesOptionalParams) -> bool { + let display = match params.thinking.as_ref().and_then(Recognized::known) { + Some(ThinkingConfig::Enabled(thinking)) => thinking.display.as_ref(), + Some(ThinkingConfig::Adaptive(thinking)) => thinking.display.as_ref(), + _ => None, + }; + display.and_then(Recognized::known) == Some(&ThinkingDisplay::Updates) +} + +fn is_tool_reference_change(block: &ContentBlock) -> bool { + matches!( + &block.block_type, + Some(ContentBlockType::Other(block_type)) + if block_type == "tool_addition" || block_type == "tool_removal" + ) && block + .extra + .get("tool") + .and_then(|tool| tool.get("type")) + .and_then(Value::as_str) + == Some("tool_reference") +} + +fn messages_carry_tool_changes(messages: &[Message]) -> bool { + messages + .iter() + .filter(|message| message.role == "system") + .flat_map(Message::blocks) + .any(is_tool_reference_change) +} + fn unsupported_param(model: &str, param: &str, value: &str, hint: &str) -> Error { Error::InvalidRequest(crate::ErrorDetail::UnsupportedParameter { model: model.into(), @@ -1071,6 +1106,41 @@ mod tests { json!({"safeguards": [{"type": "dangerous_tool_use", "classifier_context": {"v": 1, "permission_mode": "auto"}}]}), &[] )] + #[case::adaptive_display_updates( + json!({"thinking": {"type": "adaptive", "display": "updates"}}), + &["thinking-display-updates-2026-08-18"] + )] + #[case::enabled_display_updates( + json!({"thinking": {"type": "enabled", "budget_tokens": 2048, "display": "updates"}}), + &["thinking-display-updates-2026-08-18"] + )] + #[case::summarized_display(json!({"thinking": {"type": "adaptive", "display": "summarized"}}), &[])] + #[case::disabled_thinking(json!({"thinking": {"type": "disabled"}}), &[])] + #[case::system_tool_addition( + json!({"messages": [ + {"role": "user", "content": "Use the ping tool"}, + {"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]}, + ]}), + &["mid-conversation-tool-changes-2026-07-01"] + )] + #[case::system_tool_removal( + json!({"messages": [ + {"role": "system", "content": [{"type": "tool_removal", "tool": {"type": "tool_reference", "name": "ping"}}]}, + ]}), + &["mid-conversation-tool-changes-2026-07-01"] + )] + #[case::user_tool_addition( + json!({"messages": [ + {"role": "user", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]}, + ]}), + &[] + )] + #[case::tool_addition_without_a_tool_reference( + json!({"messages": [ + {"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_definition"}}]}, + ]}), + &[] + )] fn feature_betas_follow_the_request(#[case] fields: Value, #[case] expected: &[&str]) { assert_eq!(feature_betas(&request(fields)), betas(expected)); } @@ -1093,6 +1163,33 @@ mod tests { ); } + #[rstest] + #[case::vertex_ai( + BetaProvider::VertexAi, + &["mid-conversation-tool-changes-2026-07-01", "thinking-display-updates-2026-08-18"] + )] + #[case::azure_ai( + BetaProvider::AzureAi, + &["mid-conversation-tool-changes-2026-07-01", "thinking-display-updates-2026-08-18"] + )] + #[case::bedrock_converse(BetaProvider::BedrockConverse, &["thinking-display-updates-2026-08-18"])] + #[case::databricks(BetaProvider::Databricks, &["thinking-display-updates-2026-08-18"])] + fn tool_change_and_display_betas_reach_the_providers_that_accept_them( + #[case] provider: BetaProvider, + #[case] expected: &[&str], + ) { + let fields = json!({ + "thinking": {"type": "adaptive", "display": "updates"}, + "messages": [ + {"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]}, + ], + }); + assert_eq!( + provider_feature_betas(&request(fields), provider), + betas(expected) + ); + } + #[rstest] fn hosts_without_a_beta_map_column_get_safeguards_without_a_derived_beta() { let client = headers(&[("anthropic-beta", "claude-code-20250219")]); diff --git a/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs b/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs index bf934fb4d4c..d578acea68c 100644 --- a/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs @@ -545,6 +545,11 @@ mod tests { &[("anthropic-beta", "web-search-2025-03-05")], &[("x-api-key", "k"), ("anthropic-beta", "structured-outputs-2025-11-13,web-search-2025-03-05")] )] + #[case::compaction_needs_the_compact_2026_09_04_beta( + json!({"compaction": {"type": "summarize"}}), + &[], + &[("x-api-key", "k"), ("anthropic-beta", "compact-2026-09-04")] + )] #[case::no_feature_needs_a_beta(json!({}), &[], &[("x-api-key", "k")])] #[case::safeguards_need_the_dangerous_tool_use_beta( json!({"safeguards": [{"type": "dangerous_tool_use"}]}), diff --git a/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs b/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs index 46e5a68dcb8..e5e3a056fab 100644 --- a/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs @@ -798,6 +798,18 @@ mod tests { ]}}), &["compact-2026-01-12", "context-management-2025-06-27"] )] + #[case::compaction(json!({"compaction": {"type": "summarize"}}), &["compact-2026-09-04"])] + #[case::thinking_display_updates( + json!({"thinking": {"type": "adaptive", "display": "updates"}}), + &["thinking-display-updates-2026-08-18"] + )] + #[case::tool_addition( + json!({"messages": [ + {"role": "user", "content": "Use the ping tool"}, + {"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]} + ]}), + &["mid-conversation-tool-changes-2026-07-01"] + )] #[case::advisor_tool_is_not_a_vertex_beta( json!({"tools": [{"type": "advisor_20260301", "name": "advisor"}]}), &[] diff --git a/litellm/anthropic_beta_headers_config.json b/litellm/anthropic_beta_headers_config.json index cee5ce15469..253074b0e1f 100644 --- a/litellm/anthropic_beta_headers_config.json +++ b/litellm/anthropic_beta_headers_config.json @@ -1,5 +1,5 @@ { - "description": "Mapping of Anthropic beta headers for each provider. Keys are input header names, values are provider-specific header names (or null if unsupported). Only headers present in mapping keys with non-null values can be forwarded.", + "description": "Mapping of Anthropic beta headers for each provider. Keys are input header names, values are provider-specific header names (or null if unsupported). Only headers present in mapping keys with non-null values can be forwarded. A _header_only column lists betas that provider accepts only as the anthropic-beta HTTP header: they are forwarded there and dropped from the anthropic_beta body field. Proxies that predate these columns ignore them and read the provider column alone.", "anthropic": { "advisor-tool-2026-03-01": "advisor-tool-2026-03-01", "advanced-tool-use-2025-11-20": "advanced-tool-use-2025-11-20", @@ -46,6 +46,7 @@ "bash_20250124": null, "code-execution-2025-08-25": "code-execution-2025-08-25", "compact-2026-01-12": null, + "compact-2026-09-04": "compact-2026-09-04", "computer-use-2025-01-24": "computer-use-2025-01-24", "computer-use-2025-11-24": "computer-use-2025-11-24", "context-1m-2025-08-07": "context-1m-2025-08-07", @@ -59,6 +60,7 @@ "mcp-client-2025-11-20": "mcp-client-2025-11-20", "mcp-client-2025-04-04": "mcp-client-2025-04-04", "mcp-servers-2025-12-04": null, + "mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01", "output-128k-2025-02-19": null, "structured-output-2024-03-01": null, "per-turn-control-2026-07-01": "per-turn-control-2026-07-01", @@ -202,6 +204,7 @@ "mcp-client-2025-11-20": null, "mcp-client-2025-04-04": null, "mcp-servers-2025-12-04": null, + "mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01", "output-128k-2025-02-19": null, "structured-output-2024-03-01": null, "per-turn-control-2026-07-01": "per-turn-control-2026-07-01", @@ -218,6 +221,10 @@ "web-search-2025-03-05": "web-search-2025-03-05", "inline-tools-2026-09-15": "inline-tools-2026-09-15" }, + "vertex_ai_header_only": { + "compact-2026-09-04": "compact-2026-09-04", + "inline-tools-2026-09-15": "inline-tools-2026-09-15" + }, "databricks": { "advisor-tool-2026-03-01": null, "advanced-tool-use-2025-11-20": null, diff --git a/litellm/anthropic_beta_headers_manager.py b/litellm/anthropic_beta_headers_manager.py index 07730a0fbf8..be2ff07e088 100644 --- a/litellm/anthropic_beta_headers_manager.py +++ b/litellm/anthropic_beta_headers_manager.py @@ -30,7 +30,7 @@ from collections.abc import Mapping from concurrent.futures import ThreadPoolExecutor from functools import cache from importlib.resources import files -from typing import Final, TypeAlias +from typing import Final, Literal, TypeAlias import httpx from pydantic import TypeAdapter @@ -47,6 +47,9 @@ BetaHeadersConfig: TypeAlias = Mapping[str, str | Mapping[str, str | None]] _BETA_HEADERS_CONFIG_ADAPTER: Final = TypeAdapter(BetaHeadersConfig) _DECODED_JSON: Final = TypeAdapter(object) +BetaPlacement: TypeAlias = Literal["header", "body"] +_HEADER_ONLY_COLUMN_SUFFIX: Final = "_header_only" + class GetAnthropicBetaHeadersConfig: """ @@ -241,6 +244,31 @@ def get_provider_name(provider: str) -> str: return aliases.get(provider, provider) +def _beta_column(name: str) -> Mapping[str, str | None]: + column: Final[Mapping[str, str | None]] = _load_beta_headers_config().get(name, {}) + return column + + +def _header_only_column(canonical_provider: str) -> Mapping[str, str | None]: + return _beta_column(f"{canonical_provider}{_HEADER_ONLY_COLUMN_SUFFIX}") + + +def _provider_beta_mapping(provider: str, placement: BetaPlacement) -> Mapping[str, str | None]: + canonical_provider: Final = get_provider_name(provider) + column: Final = _beta_column(canonical_provider) + header_only: Final = _header_only_column(canonical_provider) + match placement: + case "header": + return {**column, **header_only} + case "body": + return {beta: name for beta, name in column.items() if beta not in header_only} + + +def get_header_only_betas(provider: str) -> frozenset[str]: + header_only: Final = _header_only_column(get_provider_name(provider)) + return frozenset(beta for beta, name in header_only.items() if name is not None) + + def has_beta_headers_config(provider: str) -> bool: config: Final = _bundled_beta_headers_config() aliases: Final = config.get("provider_aliases", {}) @@ -251,6 +279,7 @@ def has_beta_headers_config(provider: str) -> bool: def filter_and_transform_beta_headers( beta_headers: list[str], provider: str, + placement: BetaPlacement = "header", ) -> list[str]: """ Filter and transform beta headers based on provider's mapping configuration. @@ -263,6 +292,7 @@ def filter_and_transform_beta_headers( Args: beta_headers: List of Anthropic beta header values provider: Provider name (e.g., "anthropic", "bedrock", "vertex_ai") + placement: Whether the betas travel as the anthropic-beta header or the anthropic_beta body field Returns: List of filtered and transformed beta headers for the provider @@ -270,11 +300,7 @@ def filter_and_transform_beta_headers( if not beta_headers: return [] - config: Final = _load_beta_headers_config() - provider = get_provider_name(provider) - - # Get the header mapping for this provider - provider_mapping: Final = config.get(provider, {}) + provider_mapping: Final = _provider_beta_mapping(provider, placement) filtered_headers: Final[set[str]] = set() @@ -426,6 +452,7 @@ def update_request_with_filtered_beta( filtered_body_betas: Final = filter_and_transform_beta_headers( beta_headers=existing_body_betas, provider=provider, + placement="body", ) if filtered_body_betas: diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index cfe8c1739c6..26949d9bbfb 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -569,6 +569,7 @@ def _resolve_mantle_region_for_cost( _PER_CALL_LITELLM_PARAM_DICTS: Final = frozenset({"metadata", "litellm_metadata"}) _LITELLM_PARAMS: Final = TypeAdapter(dict[str, object]) +_SERVED_ANTHROPIC_BODY: Final = TypeAdapter(dict[str, object]) _REDACTED_RESULT: Final = TypeAdapter(object) _GUARDRAIL_INFORMATION_KEY: Final = "standard_logging_guardrail_information" @@ -5067,13 +5068,15 @@ class Logging(LiteLLMLoggingBaseClass): verbose_logger.warning("LiteLLM: the anthropic_messages stream assembled no response, logging an empty one") return litellm.ModelResponse(model=self.model) else: - from litellm.types.llms.anthropic import AnthropicResponse - - pydantic_result: Final = AnthropicResponse.model_validate({"stop_sequence": None, **(served_body or {})}) - import httpx - + anthropic_body: Final = ( + served_body + if served_body is not None + else _SERVED_ANTHROPIC_BODY.validate_python( + result.model_dump() if isinstance(result, BaseModel) else result + ) + ) result = litellm.AnthropicConfig().transform_parsed_response( - completion_response=pydantic_result.model_dump(), + completion_response={"stop_sequence": None, **anthropic_body}, raw_response=httpx.Response( status_code=200, headers={}, diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index 548d088d833..981859c620b 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -249,6 +249,7 @@ def _redacted_provider_specific_fields(fields: Mapping[str, object]) -> Mapping[ "reasoningContentBlocks", "citations", "compaction_blocks", + "compaction_start", "compaction_delta", ) if key in fields diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 6d1ac24af55..e1f9a7d540f 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -58,6 +58,7 @@ from litellm.types.llms.anthropic import ( ANTHROPIC_THINKING_BINDING_CONTROLS_BETA_HEADER, ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, ANTHROPIC_TOOL_SEARCH_TOOL_TYPES, + LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS, AllAnthropicToolsValues, AnthropicMcpServerTool, AnthropicMessagesToolChoice, @@ -126,6 +127,8 @@ _JSON_VALUE: Final = TypeAdapter(JsonValue) _CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/") +NATIVE_COMPACTION_PROVIDERS: Final = frozenset({"anthropic", "azure_ai", "vertex_ai"}) + def is_claude_tool_target(custom_llm_provider: str | None, model: str) -> bool: if custom_llm_provider == "anthropic": @@ -204,7 +207,7 @@ def requires_native_compaction_beta( optional_params: Mapping[str, object], messages: Sequence[object], ) -> bool: - return custom_llm_provider == "anthropic" and ( + return custom_llm_provider in NATIVE_COMPACTION_PROVIDERS and ( optional_params.get("compaction") is not None or carries_signed_compaction_block(messages) ) @@ -246,6 +249,27 @@ def context_management_betas(context_management: JsonValue) -> tuple[str, ...]: ) +def count_tokens_feature_betas( + custom_llm_provider: str, + optional_params: Mapping[str, object], + messages: Sequence[object], +) -> tuple[str, ...]: + return tuple( + beta + for beta, used in ( + ( + ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value, + requires_native_compaction_beta(custom_llm_provider, optional_params, messages), + ), + ( + ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER, + AnthropicModelInfo().is_mid_conversation_tool_change_used(messages), + ), + ) + if used + ) + + def supports_anthropic_cache_control(model: str, custom_llm_provider: str | None) -> bool: from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.utils import supports_prompt_caching @@ -645,10 +669,12 @@ class AnthropicModelInfo(BaseLLMModelInfo): file_ids: Final = get_file_ids_from_messages(messages) return len(file_ids) > 0 - def is_thinking_display_updates_used(self, thinking: AnthropicThinkingParam | None) -> bool: - if not isinstance(thinking, dict): - return False - return thinking.get("type") in ("adaptive", "enabled") and thinking.get("display") == "updates" + def is_thinking_display_updates_used(self, thinking: object) -> bool: + match thinking: + case {"type": "adaptive" | "enabled", "display": "updates"}: + return True + case _: + return False def thinking_betas(self, thinking: AnthropicThinkingParam | None) -> tuple[str, ...]: return ( @@ -1130,7 +1156,10 @@ class AnthropicModelInfo(BaseLLMModelInfo): budget_tokens: Final = _legacy_budget_tokens(thinking.get("budget_tokens", 0), drop_params=drop_params) display: Final = thinking.get("display") - supported_display: Final = {"display": display} if display in ("summarized", "omitted") else {} + keeps_display: Final = display in ("summarized", "omitted") or ( + display == "updates" and custom_llm_provider in LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS + ) + supported_display: Final = {"display": display} if keeps_display else {} carried_keys: Final = carried_thinking_keys( thinking, rebuilt_keys=frozenset(("type", "budget_tokens", "display")), drop_params=drop_params ) diff --git a/litellm/llms/anthropic/messages/native/transformation.py b/litellm/llms/anthropic/messages/native/transformation.py index c148a0defe8..5281ad99620 100644 --- a/litellm/llms/anthropic/messages/native/transformation.py +++ b/litellm/llms/anthropic/messages/native/transformation.py @@ -41,6 +41,7 @@ from litellm.types.llms.anthropic_tool_search import get_tool_search_beta_header from litellm.types.router import GenericLiteLLMParams from ...common_utils import ( + NATIVE_COMPACTION_PROVIDERS, OBJECT_LIST_ADAPTER, AnthropicError, AnthropicModelInfo, @@ -128,7 +129,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): "tool_choice", "thinking", "context_management", - *(("compaction",) if self._resolved_provider == "anthropic" else ()), + *(("compaction",) if self._resolved_provider in NATIVE_COMPACTION_PROVIDERS else ()), "output_format", "inference_geo", "speed", diff --git a/litellm/llms/azure_ai/anthropic/count_tokens/handler.py b/litellm/llms/azure_ai/anthropic/count_tokens/handler.py index 4b3f263201a..c324f0f0980 100644 --- a/litellm/llms/azure_ai/anthropic/count_tokens/handler.py +++ b/litellm/llms/azure_ai/anthropic/count_tokens/handler.py @@ -84,6 +84,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig): headers: Final = self.get_required_headers( api_key=api_key, litellm_params=litellm_params, + messages=messages, ) if not {"x-api-key", "Authorization"} & headers.keys(): raise AnthropicError( diff --git a/litellm/llms/azure_ai/anthropic/count_tokens/transformation.py b/litellm/llms/azure_ai/anthropic/count_tokens/transformation.py index 6e3cfce5eb4..c02c92a95d6 100644 --- a/litellm/llms/azure_ai/anthropic/count_tokens/transformation.py +++ b/litellm/llms/azure_ai/anthropic/count_tokens/transformation.py @@ -4,11 +4,13 @@ Azure AI Anthropic CountTokens API transformation logic. Extends the base Anthropic CountTokens transformation with Azure authentication. """ +from collections.abc import Sequence from typing import Final from urllib.parse import urlsplit from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.anthropic.common_utils import count_tokens_feature_betas, merge_anthropic_beta_headers from litellm.llms.anthropic.count_tokens.transformation import ( AnthropicCountTokensConfig, ) @@ -28,6 +30,7 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig): self, api_key: str | None, litellm_params: dict[str, object] | None = None, + messages: Sequence[object] = (), ) -> dict[str, str]: """ Get the required headers for the Azure AI Anthropic CountTokens API. @@ -38,6 +41,7 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig): Args: api_key: The Azure AI API key litellm_params: Optional LiteLLM parameters for additional auth config + messages: The request messages, read for betas the counted features need Returns: Dictionary of required headers with both x-api-key and Azure authentication @@ -46,7 +50,9 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig): headers: Final = { "Content-Type": "application/json", "anthropic-version": "2023-06-01", - "anthropic-beta": ANTHROPIC_TOKEN_COUNTING_BETA_VERSION, + "anthropic-beta": merge_anthropic_beta_headers( + ANTHROPIC_TOKEN_COUNTING_BETA_VERSION, count_tokens_feature_betas("azure_ai", {}, messages) + ), **({"x-api-key": api_key} if api_key else {}), } diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py index da82766c2b9..98fa8ed2b77 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py @@ -1,9 +1,15 @@ from collections.abc import Mapping, Sequence from typing import Final +from pydantic import TypeAdapter + from litellm.exceptions import UnsupportedParamsError from litellm.litellm_core_utils.core_helpers import drops_unsupported_params -from litellm.llms.anthropic.common_utils import AnthropicModelInfo, context_management_betas +from litellm.llms.anthropic.common_utils import ( + AnthropicModelInfo, + context_management_betas, + requires_native_compaction_beta, +) from litellm.llms.anthropic.messages.native.transformation import ( AnthropicMessagesConfig, messages_carry_output_config, @@ -11,6 +17,7 @@ from litellm.llms.anthropic.messages.native.transformation import ( from litellm.types.llms.anthropic import ( ANTHROPIC_BETA_HEADER_VALUES, ANTHROPIC_HOSTED_TOOLS, + ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER, ) from litellm.types.llms.anthropic_tool_search import get_tool_search_beta_header from litellm.types.llms.vertex_ai import VertexPartnerProvider @@ -19,6 +26,8 @@ from litellm.types.router import GenericLiteLLMParams from ....vertex_llm_base import VertexBase from ..output_params_utils import sanitize_vertex_anthropic_output_params +_OPTIONAL_PARAMS: Final = TypeAdapter(Mapping[str, object]) + _VERSION_HEADER_VERTEX_READS_FROM_BODY: Final = "anthropic-version" _VERTEX_DEFAULT_VERSION: Final = "vertex-2023-10-16" _VERSIONS_SERVED_AS_VERTEX_DEFAULT: Final = frozenset({"", "2023-01-01", "2023-06-01"}) @@ -190,14 +199,23 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert if messages_carry_output_config(messages): beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value) - beta_values.update( + typed_params: Final = _OPTIONAL_PARAMS.validate_python(optional_params) + if requires_native_compaction_beta("vertex_ai", typed_params, messages): + beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value) + + tool_change_betas: Final = ( + (ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER,) + if anthropic_model_info.is_mid_conversation_tool_change_used(messages) + else () + ) + all_beta_values: Final = beta_values.union( anthropic_model_info.thinking_betas( optional_params.get("thinking"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # optional_params is the base config's bare dict - ) + ), + tool_change_betas, ) - - if beta_values: - request_headers["anthropic-beta"] = ",".join(beta_values) + if all_beta_values: + request_headers["anthropic-beta"] = ",".join(sorted(all_beta_values)) return request_headers, api_base diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py index 79fd9f9e8b1..3d1690fca33 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py @@ -2,19 +2,23 @@ ## Handler file for calling claude-3 on vertex ai import base64 import json -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Final import httpx -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError import litellm +from litellm.anthropic_beta_headers_manager import get_header_only_betas +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.prompt_templates.image_handling import RemoteMedia, inline_remote_image_urls from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj +from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse from ....anthropic.chat.transformation import AnthropicConfig +from ....anthropic.common_utils import AnthropicError, requires_native_compaction_beta from .output_params_utils import sanitize_vertex_anthropic_output_params if TYPE_CHECKING: @@ -36,6 +40,32 @@ def _vertex_labels_header(labels: object, header_names: Mapping[str, object]) -> return base64.b64encode(json.dumps(string_labels).encode()).decode() if string_labels else None +_BETA_VALUE: Final[TypeAdapter[str | tuple[str, ...] | None]] = TypeAdapter(str | tuple[str, ...] | None) + +_REQUEST_MESSAGES: Final = TypeAdapter(Sequence[object]) + + +def _beta_names(value: object) -> frozenset[str] | None: + try: + parsed: Final = _BETA_VALUE.validate_python(value) + except ValidationError: + return None + names: Final = parsed.split(",") if isinstance(parsed, str) else parsed or () + return frozenset(name.strip() for name in names if name.strip()) + + +def _client_betas(value: object, field: str, drop_params: bool) -> frozenset[str]: + names: Final = _beta_names(value) + if names is not None: + return names + if drop_params: + return frozenset() + raise AnthropicError( + status_code=400, + message=f"{field} must be a string or a list of strings, got {type(value).__name__}", + ) + + class VertexAIError(Exception): def __init__(self, status_code, message): self.status_code = status_code @@ -157,21 +187,43 @@ class VertexAIAnthropicConfig(AnthropicConfig): if context_management: self._add_context_management_beta_headers(beta_set, context_management) - extra_headers: Final = optional_params.get("extra_headers") or {} - anthropic_beta_value: Final = extra_headers.get("anthropic-beta", "") - if isinstance(anthropic_beta_value, str) and anthropic_beta_value: - for beta in anthropic_beta_value.split(","): - beta = beta.strip() - if beta: - beta_set.add(beta) - elif isinstance(anthropic_beta_value, list): - beta_set.update(anthropic_beta_value) - + request_params: Final = _NON_DEFAULT_PARAMS.validate_python(optional_params) + compaction_betas: Final = ( + frozenset({ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value}) + if requires_native_compaction_beta( + "vertex_ai", request_params, _REQUEST_MESSAGES.validate_python(data["messages"]) + ) + else frozenset[str]() + ) + drop_params: Final = drops_unsupported_params(litellm_params.get("drop_params")) + extra_headers: Final = _NON_DEFAULT_PARAMS.validate_python(request_params.get("extra_headers") or {}) data.pop("extra_headers", None) + injected_betas: Final = frozenset(beta_set) | _client_betas( + extra_headers.get("anthropic-beta"), "The anthropic-beta header", drop_params + ) + client_body_betas: Final = _client_betas( + _NON_DEFAULT_PARAMS.validate_python(data).get("anthropic_beta"), "anthropic_beta", drop_params + ) - if beta_set: - data["anthropic_beta"] = list(beta_set) - headers["anthropic-beta"] = ",".join(beta_set) + requested_betas: Final = injected_betas | client_body_betas + header_only_betas: Final = get_header_only_betas("vertex_ai") if requested_betas else frozenset[str]() + body_betas: Final = requested_betas - header_only_betas + if body_betas: + data["anthropic_beta"] = sorted(body_betas) + elif "anthropic_beta" in data: + del data["anthropic_beta"] + header_betas: Final = ( + injected_betas + | compaction_betas + | (client_body_betas & header_only_betas) + | _client_betas( + _NON_DEFAULT_PARAMS.validate_python(headers).get("anthropic-beta"), + "The anthropic-beta header", + drop_params, + ) + ) + if header_betas: + headers["anthropic-beta"] = ",".join(sorted(header_betas)) return data diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py index 180761f15d9..e341c8f5f48 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py @@ -6,12 +6,19 @@ Unlike Gemini models which use Google's token counting API, partner models use their respective publisher-specific count-tokens endpoints. """ -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from typing import Any, Final +from pydantic import TypeAdapter + from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.url_utils import join_api_base -from litellm.llms.anthropic.common_utils import OBJECT_LIST_ADAPTER, is_claude_model_target, tools_without_custom_field +from litellm.llms.anthropic.common_utils import ( + OBJECT_LIST_ADAPTER, + count_tokens_feature_betas, + is_claude_model_target, + tools_without_custom_field, +) from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.llms.vertex_ai.common_utils import get_vertex_base_url from litellm.llms.vertex_ai.vertex_llm_base import VertexBase @@ -23,6 +30,9 @@ _COUNT_TOKENS_PUBLISHERS: Final = ( ) +_REQUEST_MESSAGES: Final = TypeAdapter(Sequence[object]) + + def count_tokens_request_body( request_data: Mapping[str, object], model: str, drop_params: object ) -> Mapping[str, object]: @@ -43,6 +53,14 @@ def count_tokens_publisher(model: str) -> str | None: return next((publisher for marker, publisher in _COUNT_TOKENS_PUBLISHERS if marker in model), None) +def count_tokens_headers(access_token: str, request_data: Mapping[str, object]) -> Mapping[str, str]: + authorization: Final = {"Authorization": f"Bearer {access_token}"} + betas: Final = count_tokens_feature_betas( + "vertex_ai", request_data, _REQUEST_MESSAGES.validate_python(request_data["messages"]) + ) + return {**authorization, "anthropic-beta": ",".join(betas)} if betas else authorization + + class VertexAIPartnerModelsTokenCounter(VertexBase): """ Token counter for Vertex AI Partner Models. @@ -175,8 +193,7 @@ class VertexAIPartnerModelsTokenCounter(VertexBase): api_base=litellm_params.get("api_base"), ) - # Prepare headers - headers: Final = {"Authorization": f"Bearer {access_token}"} + headers: Final = dict(count_tokens_headers(access_token, request_data)) # Get async HTTP client from litellm import LlmProviders diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index a648376c542..152614e4385 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -713,6 +713,14 @@ class AnthropicResponseContentBlockRedactedThinking(LiteLLMBaseModel): data: str +class AnthropicResponseContentBlockCompaction(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + type: Literal["compaction"] + content: str | None = None + signature: str | None = None + + class AnthropicResponseContentBlockServerToolUse(LiteLLMBaseModel): model_config = ConfigDict(frozen=True) @@ -743,7 +751,7 @@ class AnthropicOutputTokensDetails(LiteLLMBaseModel): thinking_tokens: int | None = None -AnthropicFinishReason = Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "refusal"] +AnthropicFinishReason = Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "refusal", "compaction"] class AnthropicResponse(LiteLLMBaseModel): @@ -761,6 +769,7 @@ class AnthropicResponse(LiteLLMBaseModel): | AnthropicResponseContentBlockToolUse | AnthropicResponseContentBlockThinking | AnthropicResponseContentBlockRedactedThinking + | AnthropicResponseContentBlockCompaction | AnthropicResponseContentBlockServerToolUse | AnthropicResponseContentBlockWebSearchToolResult ] @@ -808,6 +817,9 @@ class AnthropicThinkingParam(TypedDict, total=False): block_binding: ReadOnly[AnthropicThinkingBlockBinding] +LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS: Final = frozenset(("anthropic", "azure_ai", "vertex_ai")) + + class ANTHROPIC_HOSTED_TOOLS(str, Enum): WEB_SEARCH = "web_search" BASH = "bash" @@ -828,6 +840,7 @@ class ANTHROPIC_BETA_HEADER_VALUES(str, Enum): CONTEXT_MANAGEMENT_2025_06_27 = "context-management-2025-06-27" COMPACT_2026_01_12 = "compact-2026-01-12" COMPACT_2026_09_04 = "compact-2026-09-04" + INLINE_TOOLS_2026_09_15 = "inline-tools-2026-09-15" STRUCTURED_OUTPUT_2025_09_25 = "structured-outputs-2025-11-13" ADVANCED_TOOL_USE_2025_11_20 = "advanced-tool-use-2025-11-20" FAST_MODE_2026_02_01 = "fast-mode-2026-02-01" diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 336e9f7d24a..d0e31cdd57a 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -6956,6 +6956,95 @@ def test_handle_anthropic_messages_response_logging_reparses_the_served_body_wit assert logged.usage.completion_tokens == 7 # type: ignore[attr-defined] +@pytest.mark.parametrize( + "compaction_block", + [ + {"type": "compaction", "content": "Summary of the chat so far.", "signature": "sig-abc"}, + {"type": "compaction", "signature": "sig-abc"}, + ], + ids=["with_summary", "without_content"], +) +def test_handle_anthropic_messages_response_logging_reparses_a_served_compaction_response( + compaction_block: dict[str, str], +): + logging_obj = _anthropic_messages_logging_obj() + logging_obj.optional_params = {} + + logged = logging_obj._handle_anthropic_messages_response_logging( + result={ + "id": "msg_compacted", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [compaction_block], + "stop_reason": "compaction", + "stop_sequence": None, + "usage": {"input_tokens": 410, "output_tokens": 95}, + } + ) + + assert isinstance(logged, ModelResponse) + assert logged.id == "msg_compacted" + assert logged.usage.prompt_tokens == 410 + assert logged.usage.completion_tokens == 95 + assert logged.choices[0].message.provider_specific_fields["compaction_blocks"] == [compaction_block] + + +_WEB_SEARCH_REPLY: Final = ( + {"type": "server_tool_use", "id": "srvtoolu_1", "name": "web_search", "input": {"query": "weather"}}, + { + "type": "web_search_tool_result", + "tool_use_id": "srvtoolu_1", + "content": [{"type": "web_search_result", "url": "https://example.com", "title": "Weather"}], + }, + {"type": "text", "text": "It is sunny."}, +) + + +@pytest.mark.parametrize( + ("content", "stop_reason"), + [ + (list(_WEB_SEARCH_REPLY), "end_turn"), + (list(_WEB_SEARCH_REPLY), "pause_turn"), + ([{"type": "text", "text": "The context window is full."}], "model_context_window_exceeded"), + ( + [ + {"type": "mcp_tool_use", "id": "mcptoolu_1", "name": "lookup", "server_name": "docs", "input": {}}, + {"type": "mcp_tool_result", "tool_use_id": "mcptoolu_1", "is_error": False, "content": [{"type": "text", "text": "ok"}]}, + {"type": "text", "text": "Found it."}, + ], + "end_turn", + ), + ([{"type": "container_upload", "file_id": "file_1"}, {"type": "text", "text": "Uploaded."}], "end_turn"), + ], + ids=["web_search", "pause_turn", "model_context_window_exceeded", "mcp_tool_use", "container_upload"], +) +def test_handle_anthropic_messages_response_logging_bills_a_served_server_tool_response( + content: list[dict[str, object]], stop_reason: str +): + logging_obj = _anthropic_messages_logging_obj() + logging_obj.optional_params = {} + + logged = logging_obj._handle_anthropic_messages_response_logging( + result={ + "id": "msg_server_tools", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": content, + "stop_reason": stop_reason, + "stop_sequence": None, + "usage": {"input_tokens": 1200, "output_tokens": 80}, + } + ) + + assert isinstance(logged, ModelResponse) + assert logged.id == "msg_server_tools" + assert logged.usage.prompt_tokens == 1200 + assert logged.usage.completion_tokens == 80 + assert logged.choices[0].message.content == content[-1]["text"] + + def test_handle_anthropic_messages_response_logging_passes_model_response_through(): """Anthropic-native path already yields a ModelResponse; it must be returned unchanged.""" logging_obj = _anthropic_messages_logging_obj() diff --git a/tests/unit/litellm_core_utils/test_redact_messages.py b/tests/unit/litellm_core_utils/test_redact_messages.py index 5b64e42d8a8..9e385c588ae 100644 --- a/tests/unit/litellm_core_utils/test_redact_messages.py +++ b/tests/unit/litellm_core_utils/test_redact_messages.py @@ -57,9 +57,10 @@ def _provider_fields() -> dict[str, object]: "thinking_blocks": [{"type": "thinking", "thinking": SECRET}], "reasoningContentBlocks": [{"reasoningText": {"text": SECRET}}], "citations": [{"cited_text": SECRET}], - "thought_signatures": [SIGNATURE], - "compaction_blocks": [{"type": "compaction", "content": SECRET}], + "compaction_blocks": [{"type": "compaction", "content": SECRET, "signature": "compaction-signature"}], + "compaction_start": {"type": "compaction", "content": SECRET}, "compaction_delta": {"type": "compaction_delta", "content": SECRET}, + "thought_signatures": [SIGNATURE], } diff --git a/tests/unit/llms/anthropic/test_anthropic_common_utils.py b/tests/unit/llms/anthropic/test_anthropic_common_utils.py index c1f28e33969..0808ce0ec3c 100644 --- a/tests/unit/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/unit/llms/anthropic/test_anthropic_common_utils.py @@ -3544,12 +3544,11 @@ class TestWifProviderAllowlist: monkeypatch.setenv("WIF_TEST_JWT", "jwt-assertion-value") def test_vertex_anthropic_never_mints_or_sends_the_assertion(self, monkeypatch, wif_engine): + import litellm from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import ( VertexAIAnthropicConfig, ) - import litellm - poster, calls = wif_engine self._env_only_wif(monkeypatch) @@ -3775,7 +3774,7 @@ class TestModelDiscovery: ) monkeypatch.setattr("litellm.module_level_client", client) - with pytest.raises(Exception, match="invalid x-api-key") as exc_info: # noqa: B017, PT011 # the callee raises a bare Exception; match pins the sanitized text + with pytest.raises(Exception, match="invalid x-api-key") as exc_info: # noqa: B017 # the callee raises a bare Exception; match pins the sanitized text AnthropicModelInfo().get_models(api_base="https://api.anthropic.com") assert "invalid x-api-key" in str(exc_info.value) @@ -4238,15 +4237,18 @@ def test_thinking_display_beta_requires_active_thinking(thinking: object, expect @pytest.mark.usefixtures("local_model_cost_map") @pytest.mark.parametrize( - ("display", "expected_thinking"), + ("provider", "display", "expected_thinking"), ( - ("summarized", {"type": "adaptive", "display": "summarized"}), - ("omitted", {"type": "adaptive", "display": "omitted"}), - ("updates", {"type": "adaptive"}), + ("azure_ai", "summarized", {"type": "adaptive", "display": "summarized"}), + ("azure_ai", "omitted", {"type": "adaptive", "display": "omitted"}), + ("azure_ai", "updates", {"type": "adaptive", "display": "updates"}), + ("vertex_ai", "updates", {"type": "adaptive", "display": "updates"}), + ("anthropic", "updates", {"type": "adaptive", "display": "updates"}), + ("databricks", "updates", {"type": "adaptive"}), ), ) def test_shared_legacy_thinking_translation_preserves_supported_display( - display: str, expected_thinking: dict[str, str] + provider: str, display: str, expected_thinking: dict[str, str] ) -> None: from litellm.llms.anthropic.common_utils import AnthropicModelInfo @@ -4257,7 +4259,7 @@ def test_shared_legacy_thinking_translation_preserves_supported_display( AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model( model="claude-opus-5", optional_params=optional_params, - custom_llm_provider="azure_ai", + custom_llm_provider=provider, drop_params=False, ) @@ -4552,6 +4554,47 @@ def test_shared_legacy_thinking_translation_drops_a_non_numeric_budget_under_dro ) +_SIGNED_COMPACTION_BLOCK: Final = {"type": "compaction", "content": "summary so far", "signature": "sig"} + + +@pytest.mark.parametrize( + "provider,expected", + [("anthropic", True), ("vertex_ai", True), ("azure_ai", True), ("bedrock", False), ("bedrock_converse", False)], +) +@pytest.mark.parametrize( + "optional_params,messages", + [ + ({"compaction": {"type": "summarize"}}, [{"role": "user", "content": "hi"}]), + ({}, [{"role": "assistant", "content": [_SIGNED_COMPACTION_BLOCK]}, {"role": "user", "content": "go on"}]), + ], + ids=["compaction_param", "signed_compaction_block"], +) +def test_requires_native_compaction_beta_per_provider( + provider: str, expected: bool, optional_params: dict[str, object], messages: list[object] +) -> None: + from litellm.llms.anthropic.common_utils import requires_native_compaction_beta + + assert requires_native_compaction_beta(provider, optional_params, messages) is expected + + +@pytest.mark.parametrize("provider", ["anthropic", "azure_ai", "vertex_ai"]) +@pytest.mark.parametrize( + "optional_params,messages", + [ + ({"max_tokens": 64}, [{"role": "user", "content": "hi"}]), + ({}, [{"role": "assistant", "content": [{"type": "compaction", "content": "unsigned"}]}]), + ({"compaction": None}, [{"role": "user", "content": "hi"}]), + ], + ids=["no_signal", "unsigned_block", "compaction_none"], +) +def test_requires_native_compaction_beta_is_false_without_a_signal( + provider: str, optional_params: dict[str, object], messages: list[object] +) -> None: + from litellm.llms.anthropic.common_utils import requires_native_compaction_beta + + assert requires_native_compaction_beta(provider, optional_params, messages) is False + + @pytest.mark.parametrize( ("custom_llm_provider", "model", "expected"), [ diff --git a/tests/unit/llms/azure_ai/anthropic/count_tokens/test_token_counter.py b/tests/unit/llms/azure_ai/anthropic/count_tokens/test_token_counter.py index 4913d2b0d1d..7807975be24 100644 --- a/tests/unit/llms/azure_ai/anthropic/count_tokens/test_token_counter.py +++ b/tests/unit/llms/azure_ai/anthropic/count_tokens/test_token_counter.py @@ -215,3 +215,56 @@ async def test_count_uses_the_shared_api_base_chat_would_use(monkeypatch: pytest assert result is not None assert result.total_tokens == 6 assert route.called + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("assistant_content", "expected_beta"), + [ + ( + [{"type": "compaction", "content": "Summary of the chat so far.", "signature": "sig-abc"}], + "compact-2026-09-04,token-counting-2024-11-01", + ), + ("Use Recipe and Step tables.", "token-counting-2024-11-01"), + ], + ids=("signed_compaction_block", "plain_turn"), +) +async def test_count_sends_the_compact_beta_only_when_the_history_replays_a_compaction_block( + monkeypatch: pytest.MonkeyPatch, assistant_content: object, expected_beta: str +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock: + route: Final = respx.post(COUNT_URL).mock(return_value=httpx.Response(200, json={"input_tokens": 376})) + result: Final = await AzureAIAnthropicTokenCounter().count_tokens( + model_to_use="claude-sonnet-5-5", + messages=[ + {"role": "assistant", "content": assistant_content}, + {"role": "user", "content": "What tables did we pick?"}, + ], + contents=None, + deployment={"litellm_params": {"api_key": "azure-key", "api_base": API_BASE}}, + request_model="azure_ai/claude-sonnet-5-5", + ) + assert result is not None + assert result.total_tokens == 376 + assert route.calls.last.request.headers["anthropic-beta"] == expected_beta + + +@pytest.mark.asyncio +async def test_count_sends_the_tool_change_beta_when_the_history_adds_a_tool(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock: + route: Final = respx.post(COUNT_URL).mock(return_value=httpx.Response(200, json={"input_tokens": 541})) + result: Final = await AzureAIAnthropicTokenCounter().count_tokens( + model_to_use="claude-sonnet-5-5", + messages=[ + {"role": "user", "content": "Use the ping tool once."}, + {"role": "system", "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}]}, + ], + contents=None, + deployment={"litellm_params": {"api_key": "azure-key", "api_base": API_BASE}}, + request_model="azure_ai/claude-sonnet-5-5", + ) + assert result is not None + assert result.total_tokens == 541 + assert route.calls.last.request.headers["anthropic-beta"] == "mid-conversation-tool-changes-2026-07-01,token-counting-2024-11-01" diff --git a/tests/unit/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py b/tests/unit/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py index 5e0395f84dc..1c4097ad7e0 100644 --- a/tests/unit/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py +++ b/tests/unit/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py @@ -607,3 +607,91 @@ async def test_outgoing_azure_request_keeps_an_object_output_format_under_drop_p body: Final = await _azure_request_body_for_output_format(output_format, drop_params=True) assert body["output_format"] == output_format + + +_SIGNED_COMPACTION_BLOCK: Final = {"type": "compaction", "content": "Summary of the chat so far.", "signature": "sig"} + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("local_beta_headers_config") +@pytest.mark.parametrize( + ("messages", "params"), + [ + ([{"role": "user", "content": "Summarize the chat so far."}], {"compaction": {"type": "summarize"}}), + ( + [{"role": "assistant", "content": [_SIGNED_COMPACTION_BLOCK]}, {"role": "user", "content": "Go on"}], + {}, + ), + ], + ids=["compaction_param", "signed_compaction_block"], +) +async def test_outgoing_azure_compaction_request_carries_the_compact_2026_09_04_beta( + messages: list[object], params: dict[str, object] +) -> None: + """Foundry answers 400 "compaction: this parameter requires anthropic-beta: compact-2026-09-04" for the param + and 400 "Input tag 'compaction' ... does not match" for a replayed block without the header, and 200 with it + (live, claude-sonnet-5-5 on Azure AI Foundry, 2026-10-20).""" + received: Final[list[httpx.Request]] = [] + + def foundry_answers(request: httpx.Request) -> httpx.Response: + received.append(request) + return _foundry_answer(request) + + upstream: Final = AsyncHTTPHandler(transport=httpx.MockTransport(foundry_answers)) + try: + await litellm.anthropic.messages.acreate( + max_tokens=256, + messages=messages, + model="azure_ai/claude-sonnet-5-5", + api_key="test-api-key", + api_base="https://test-resource.services.ai.azure.com", + client=upstream, + **params, + ) + finally: + await upstream.close() + + assert len(received) == 1, received + assert received[0].headers["anthropic-beta"].split(",").count("compact-2026-09-04") == 1, dict(received[0].headers) + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize("feature", ("thinking", "output_config", "tool_addition", "tool_removal", "none")) +@pytest.mark.parametrize("explicit_beta", (False, True)) +def test_messages_feature_beta_headers(feature: str, explicit_beta: bool) -> None: + from typing import Final + + from litellm.llms.azure_ai.anthropic.messages_transformation import AzureAnthropicMessagesConfig + from litellm.types.llms.anthropic import ( + ANTHROPIC_BETA_HEADER_VALUES, + ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER, + ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, + ) + + beta: Final = ( + ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER + if feature == "thinking" + else ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value + if feature == "output_config" + else ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER + ) + message: Final = ( + {"role": "system", "content": [], "output_config": {"effort": "high"}} + if feature == "output_config" + else {"role": "system", "content": [{"type": feature, "tool": {"type": "tool_reference", "name": "ping"}}]} + if feature in ("tool_addition", "tool_removal") + else {"role": "user", "content": "Reply with OK"} + ) + headers, _ = AzureAnthropicMessagesConfig().validate_anthropic_messages_environment( + headers={"anthropic-beta": f"existing-beta,{beta}" if explicit_beta else "existing-beta"}, + model="claude-fable-5-1", + messages=[{"role": "user", "content": "Hello"}, message], + optional_params={"thinking": {"type": "adaptive", "display": "updates"}} if feature == "thinking" else {}, + litellm_params={"vertex_ai_project": "test-project", "vertex_ai_location": "us-east5"}, + api_key="test-key", + api_base="https://example.com/anthropic", + ) + + betas: Final = headers["anthropic-beta"].split(",") + assert betas.count(beta) == int(feature != "none" or explicit_beta) + assert betas.count("existing-beta") == 1 diff --git a/tests/unit/llms/azure_ai/claude/test_azure_anthropic_transformation.py b/tests/unit/llms/azure_ai/claude/test_azure_anthropic_transformation.py index 1179b0dd5da..6ff2a6973a3 100644 --- a/tests/unit/llms/azure_ai/claude/test_azure_anthropic_transformation.py +++ b/tests/unit/llms/azure_ai/claude/test_azure_anthropic_transformation.py @@ -5,10 +5,12 @@ sys.path.insert( 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../..")) ) +from typing import Final from unittest.mock import MagicMock, patch import pytest +from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta from litellm.llms.azure_ai.anthropic.transformation import AzureAnthropicConfig from litellm.types.router import GenericLiteLLMParams @@ -484,3 +486,85 @@ def test_chat_flagged_model_keeps_mid_conversation_system_role_in_place(local_mo "role": "system", "content": [{"type": "text", "text": "Answer with exactly one word."}], } + + +@pytest.mark.parametrize( + ("optional_params", "messages", "expected"), + [ + ({"compaction": {"type": "summarize"}}, [{"role": "user", "content": "Hello"}], True), + ( + {}, + [ + {"role": "assistant", "content": [{"type": "compaction", "content": "Summary.", "signature": "sig"}]}, + {"role": "user", "content": "Go on"}, + ], + True, + ), + ({}, [{"role": "user", "content": "Hello"}], False), + ], + ids=["compaction_param", "signed_compaction_block", "no_compaction"], +) +def test_azure_chat_compaction_sends_the_compact_2026_09_04_beta( + local_beta_headers_config: None, optional_params: dict[str, object], messages: list[object], expected: bool +) -> None: + headers: Final = AzureAnthropicConfig().update_headers_with_optional_anthropic_beta( + headers={}, optional_params=optional_params, messages=messages + ) + + filtered: Final = update_headers_with_filtered_beta(headers=headers, provider="azure_ai") + + assert ("compact-2026-09-04" in filtered.get("anthropic-beta", "").split(",")) is expected + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize("thinking_type", ("adaptive", "enabled")) +@pytest.mark.parametrize("display", ("summarized", "omitted", "updates")) +def test_chat_preserves_thinking_display_and_beta(thinking_type: str, display: str) -> None: + from typing import Final + + from litellm.llms.azure_ai.anthropic.transformation import AzureAnthropicConfig + from litellm.types.llms.anthropic import ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER + + config: Final = AzureAnthropicConfig() + optional_params: Final = config.map_openai_params( + non_default_params={"thinking": {"type": thinking_type, "display": display, "budget_tokens": 2048}}, + optional_params={}, + model="claude-fable-5-1", + drop_params=False, + ) + headers: Final = config.validate_environment( + headers={"anthropic-beta": "existing-beta"}, + model="claude-fable-5-1", + messages=[{"role": "user", "content": "Reply with OK"}], + optional_params=optional_params, + litellm_params={}, + api_key="test-key", + ) + result: Final = config.transform_request( + model="claude-fable-5-1", + messages=[{"role": "user", "content": "Reply with OK"}], + optional_params=optional_params, + litellm_params={}, + headers=headers, + ) + + assert result["thinking"]["type"] == "adaptive" + assert result["thinking"]["display"] == display + assert (ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER in headers["anthropic-beta"].split(",")) == (display == "updates") + assert "existing-beta" in headers["anthropic-beta"].split(",") + + +@pytest.mark.parametrize("thinking", ("enabled", {"type": "bogus"}, {"type": "adaptive", "display": "future-mode"})) +def test_beta_detection_preserves_unrecognized_thinking(thinking: object) -> None: + from typing import Final + + from litellm.types.llms.anthropic import ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER + + params: Final = {"thinking": thinking} + headers: Final = AzureAnthropicConfig().validate_environment( + headers={}, model="claude-fable-5-1", messages=[], optional_params=params, + litellm_params={}, api_key="test-key", + ) + + assert params["thinking"] == thinking + assert ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER not in headers.get("anthropic-beta", "").split(",") diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py index 7f16957ddc4..fc57dd13302 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py @@ -8,11 +8,13 @@ from threading import get_ident from typing import Final from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest import requests import responses import litellm +from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.messages.transformation import ( VertexAIPartnerModelsAnthropicMessagesConfig, @@ -115,7 +117,11 @@ def _vertex_client() -> MagicMock: async def _send_to_vertex(client: MagicMock, caller_headers: Mapping[str, str], drop_params: bool) -> None: with responses.RequestsMock(assert_all_requests_are_fired=False) as oauth: - oauth.add(responses.POST, "https://oauth2.googleapis.com/token", json={"access_token": "vertex-token", "expires_in": 3600}) + oauth.add( + responses.POST, + "https://oauth2.googleapis.com/token", + json={"access_token": "vertex-token", "expires_in": 3600}, + ) await litellm.anthropic.messages.acreate( model="vertex_ai/claude-sonnet-5", messages=[{"role": "user", "content": "hi"}], @@ -289,7 +295,12 @@ def test_no_safeguards_leaves_dangerous_tool_use_beta_header_out(): assert "dangerous-tool-use-2026-09-03" not in updated_headers.get("anthropic-beta", "") -def _validate_vertex_headers(client_headers, messages, thinking: Mapping[str, object] | None = None): +def _validate_vertex_headers( + client_headers: dict[str, str], + messages: list[object], + optional_params: dict[str, object] | None = None, + thinking: Mapping[str, object] | None = None, +) -> dict[str, str]: config = VertexAIPartnerModelsAnthropicMessagesConfig() litellm_params = { "vertex_ai_project": "test-project", @@ -305,7 +316,10 @@ def _validate_vertex_headers(client_headers, messages, thinking: Mapping[str, ob headers=client_headers, model="claude-opus-5-5", messages=messages, - optional_params={"max_tokens": 64, **({"thinking": thinking} if thinking is not None else {})}, + optional_params={ + **(optional_params or {"max_tokens": 64}), + **({"thinking": thinking} if thinking is not None else {}), + }, litellm_params=litellm_params, api_base=None, ) @@ -1040,3 +1054,178 @@ def test_vertex_claude_4_8_plus_cost_map_entries_carry_mid_conversation_system_f and info.get("supports_mid_conversation_system") is not True ] assert missing == [] + + +_SIGNED_COMPACTION_BLOCK: Final = {"type": "compaction", "content": "summary so far", "signature": "sig"} + + +@pytest.mark.parametrize( + "client_headers", + [{"anthropic-beta": "compact-2026-09-04"}, {}], + ids=["client_sends_beta", "client_omits_beta"], +) +@pytest.mark.parametrize( + "optional_params,messages", + [ + ({"max_tokens": 64, "compaction": {"type": "summarize"}}, [{"role": "user", "content": "Hello"}]), + ( + {"max_tokens": 64}, + [{"role": "assistant", "content": [_SIGNED_COMPACTION_BLOCK]}, {"role": "user", "content": "Go on"}], + ), + ], + ids=["compaction_param", "signed_compaction_block"], +) +def test_native_compaction_reaches_vertex_with_compact_2026_09_04_beta( + local_beta_headers_config: None, + client_headers: dict[str, str], + optional_params: dict[str, object], + messages: list[object], +) -> None: + """Vertex serves on-demand compaction only behind compact-2026-09-04 (Google Cloud: beta on + https://platform.claude.com/docs/en/build-with-claude/compaction-on-demand, read 2026-10-05) + and answers 400 "compaction: Extra inputs are not permitted" without it, so the beta rides + along with the `compaction` param or a signed compaction block exactly once, whether or not + the client sent it, and survives the Vertex beta filter.""" + headers: Final = _validate_vertex_headers(client_headers, messages, optional_params) + + filtered: Final = update_headers_with_filtered_beta(headers=headers, provider="vertex_ai") + + assert filtered["anthropic-beta"].split(",").count("compact-2026-09-04") == 1 + + +def test_no_compaction_signal_leaves_compact_2026_09_04_beta_out() -> None: + headers: Final = _validate_vertex_headers({}, [{"role": "user", "content": "Hello"}]) + + assert "compact-2026-09-04" not in headers.get("anthropic-beta", "") + + +_VERTEX_MESSAGE_RESPONSE: Final = { + "id": "msg_vrtx_test", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-6", + "content": [{"type": "text", "text": "pong"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 5, "output_tokens": 1}, +} + + +@pytest.mark.asyncio +async def test_vertex_messages_request_sends_compact_2026_09_04_beta_on_the_wire( + local_beta_headers_config: None, +) -> None: + """A /v1/messages compaction request to a Vertex Claude deployment reaches Vertex with the beta in the + anthropic-beta request header and never in the body, the shape Vertex accepts (live, 2026-10-05).""" + mock_response: Final = MagicMock() + mock_response.status_code = 200 + mock_response.text = json.dumps(_VERTEX_MESSAGE_RESPONSE) + mock_response.headers = httpx.Headers({}) + mock_response.json.return_value = _VERTEX_MESSAGE_RESPONSE + + client: Final = MagicMock(spec=AsyncHTTPHandler) + client.post = AsyncMock(return_value=mock_response) + + with patch( + "litellm.llms.vertex_ai.vertex_llm_base.VertexBase._ensure_access_token_async", + AsyncMock(return_value=("token", "test-project")), + ): + await litellm.anthropic.messages.acreate( + model="vertex_ai/claude-sonnet-4-6", + max_tokens=64, + messages=[{"role": "user", "content": "Reply with the single word: pong"}], + compaction={"type": "summarize"}, + vertex_project="test-project", + vertex_location="us-east5", + client=client, + ) + + sent: Final = client.post.call_args.kwargs + sent_body: Final = sent["json"] if "json" in sent else json.loads(sent["data"]) + assert sent["headers"]["anthropic-beta"].split(",").count("compact-2026-09-04") == 1 + assert sent_body["compaction"] == {"type": "summarize"} + assert "anthropic_beta" not in sent_body + + +@pytest.mark.usefixtures("local_model_cost_map", "local_beta_headers_config") +@pytest.mark.parametrize("feature", ("thinking", "output_config", "tool_addition", "tool_removal", "none")) +@pytest.mark.parametrize("explicit_beta", (False, True)) +def test_messages_feature_beta_headers(feature: str, explicit_beta: bool) -> None: + from typing import Final + from unittest.mock import patch + + from google.oauth2.credentials import Credentials + + from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta + from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.messages.transformation import ( + VertexAIPartnerModelsAnthropicMessagesConfig, + ) + from litellm.types.llms.anthropic import ( + ANTHROPIC_BETA_HEADER_VALUES, + ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER, + ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER, + ) + + beta: Final = ( + ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER + if feature == "thinking" + else ANTHROPIC_BETA_HEADER_VALUES.PER_TURN_CONTROL_2026_07_01.value + if feature == "output_config" + else ANTHROPIC_MID_CONVERSATION_TOOL_CHANGES_BETA_HEADER + ) + message: Final = ( + {"role": "system", "content": [], "output_config": {"effort": "high"}} + if feature == "output_config" + else {"role": "system", "content": [{"type": feature, "tool": {"type": "tool_reference", "name": "ping"}}]} + if feature in ("tool_addition", "tool_removal") + else {"role": "user", "content": "Reply with OK"} + ) + with patch( + "google.auth.default", + return_value=(MagicMock(spec=Credentials, token="test-token", expired=False), "test-project"), + ): + headers, _ = VertexAIPartnerModelsAnthropicMessagesConfig().validate_anthropic_messages_environment( + headers={"anthropic-beta": f"existing-beta,{beta}" if explicit_beta else "existing-beta"}, + model="claude-fable-5-1", + messages=[{"role": "user", "content": "Hello"}, message], + optional_params={"thinking": {"type": "adaptive", "display": "updates"}} if feature == "thinking" else {}, + litellm_params={"vertex_ai_project": "test-project", "vertex_ai_location": "us-east5"}, + api_key="test-key", + api_base="https://example.com/anthropic", + ) + + betas: Final = headers["anthropic-beta"].split(",") + assert betas.count(beta) == int(feature != "none" or explicit_beta) + assert betas.count("existing-beta") == 1 + + if feature in ("tool_addition", "tool_removal"): + filtered_headers: Final = update_headers_with_filtered_beta(headers=headers, provider="vertex_ai") + assert filtered_headers.get("anthropic-beta", "").split(",").count(beta) == 1 + + +@pytest.mark.parametrize("thinking", ("enabled", {"type": "bogus"}, {"type": "adaptive", "display": "future-mode"})) +def test_beta_detection_preserves_unrecognized_thinking(thinking: object) -> None: + from typing import Final + from unittest.mock import patch + + from google.oauth2.credentials import Credentials + + from litellm.types.llms.anthropic import ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER + + params: Final = {"thinking": thinking} + with patch( + "google.auth.default", + return_value=(MagicMock(spec=Credentials, token="test-token", expired=False), "test-project"), + ): + headers, _ = VertexAIPartnerModelsAnthropicMessagesConfig().validate_anthropic_messages_environment( + headers={}, + model="claude-fable-5-1", + messages=[], + optional_params=params, + litellm_params={"vertex_ai_project": "test-project", "vertex_ai_location": "us-east5"}, + api_key="test-key", + api_base="https://example.com/anthropic", + ) + + assert params["thinking"] == thinking + assert ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER not in headers.get("anthropic-beta", "").split(",") diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py index 3ae208bb3a8..0a582c95749 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py @@ -4,7 +4,7 @@ import copy import json from collections.abc import Mapping from typing import Final -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch import httpx import pytest @@ -12,10 +12,12 @@ import respx from google.oauth2.credentials import Credentials import litellm +from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.anthropic_beta_headers_manager import ( update_headers_with_filtered_beta, update_request_with_filtered_beta, ) +from litellm.llms.anthropic.common_utils import AnthropicError from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import ( VertexAIAnthropicConfig, ) @@ -475,8 +477,10 @@ def test_vertex_ai_anthropic_extra_headers_beta_propagation(): def test_vertex_ai_anthropic_inline_tools_beta_survives_the_chat_beta_filter(local_beta_headers_config: None) -> None: """The chat path runs the Vertex beta filter over both the header and the `anthropic_beta` body field right - before the request goes out, so inline-tools-2026-09-15 must survive both (source and date in - tests/unit/test_anthropic_beta_headers_filtering.py).""" + before the request goes out, so inline-tools-2026-09-15 must survive it as the header (source and date in + tests/unit/test_anthropic_beta_headers_filtering.py) and stay out of the body field, which Vertex rejects + with 400 "Unexpected value(s) `inline-tools-2026-09-15` for the `anthropic_beta` parameter" (live, global, + claude-sonnet-5-5, 2026-10-20).""" config: Final = VertexAIAnthropicConfig() headers: Final[dict[str, str]] = {} optional_params: Final[dict[str, object]] = { @@ -497,7 +501,7 @@ def test_vertex_ai_anthropic_inline_tools_beta_survives_the_chat_beta_filter(loc ) assert filtered_headers["anthropic-beta"].split(",").count("inline-tools-2026-09-15") == 1 - assert "inline-tools-2026-09-15" in filtered_request["anthropic_beta"] + assert "inline-tools-2026-09-15" not in filtered_request.get("anthropic_beta", []) def test_vertex_ai_anthropic_extra_headers_beta_merged_with_auto_betas(): @@ -1232,3 +1236,293 @@ def test_refusal_stop_details_kept_through_vertex_anthropic_transform(): provider_fields: Final = result.choices[0].message.provider_specific_fields assert provider_fields["stop_reason"] == "refusal" assert provider_fields["stop_details"] == ANTHROPIC_DOCS_REFUSAL_MESSAGE["stop_details"] + + +_SIGNED_COMPACTION_BLOCK: Final = {"type": "compaction", "content": "summary so far", "signature": "sig"} +_CLEAR_TOOL_USES: Final = {"edits": [{"type": "clear_tool_uses_20250919"}]} + + +def _vertex_request(optional_params: dict[str, object], headers: dict[str, str], litellm_params: dict[str, object] | None = None) -> dict: + return VertexAIAnthropicConfig().transform_request( + model="claude-opus-4-6", + messages=[{"role": "user", "content": "Hello"}], + optional_params={"max_tokens": 100, "is_vertex_request": True, **optional_params}, + litellm_params=litellm_params or {}, + headers=headers, + ) + + +@pytest.mark.parametrize( + "optional_params,messages", + [ + ( + {"max_tokens": 100, "is_vertex_request": True, "compaction": {"type": "summarize"}}, + [{"role": "user", "content": "Hello"}], + ), + ( + {"max_tokens": 100, "is_vertex_request": True}, + [ + { + "role": "assistant", + "content": "", + "provider_specific_fields": {"compaction_blocks": [_SIGNED_COMPACTION_BLOCK]}, + }, + {"role": "user", "content": "Go on"}, + ], + ), + ], + ids=["compaction_param", "signed_compaction_block_sent_back"], +) +def test_vertex_ai_anthropic_native_compaction_adds_compact_2026_09_04_beta( + local_beta_headers_config: None, optional_params: dict[str, object], messages: list[dict[str, object]] +) -> None: + """The chat path skips the Anthropic beta injection for Vertex requests, so the Vertex config + adds compact-2026-09-04 itself (Google Cloud: beta on + https://platform.claude.com/docs/en/build-with-claude/compaction-on-demand, read 2026-10-05) + as the HTTP header only, since Vertex answered 400 "Unexpected value(s) `compact-2026-09-04` + for the `anthropic_beta` parameter" with the name in the body field (us-east5, + claude-sonnet-4-6, 2026-10-05), and the header must survive the Vertex beta filter.""" + headers: Final[dict[str, str]] = {} + + result: Final = VertexAIAnthropicConfig().transform_request( + model="claude-opus-4-6", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers=headers, + ) + filtered_headers, filtered_body = update_request_with_filtered_beta( + headers=headers, request_data=result, provider="vertex_ai" + ) + + assert filtered_headers["anthropic-beta"].split(",").count("compact-2026-09-04") == 1 + assert "compact-2026-09-04" not in filtered_body.get("anthropic_beta", []) + + +def test_vertex_ai_anthropic_no_compaction_signal_leaves_compact_2026_09_04_beta_out() -> None: + headers: Final[dict[str, str]] = {} + + result: Final = _vertex_request({}, headers) + + assert "compact-2026-09-04" not in result.get("anthropic_beta", []) + assert "compact-2026-09-04" not in headers.get("anthropic-beta", "") + + +def test_vertex_ai_anthropic_client_sent_compact_2026_09_04_stays_out_of_the_body_field( + local_beta_headers_config: None, +) -> None: + headers: Final[dict[str, str]] = {} + + result: Final = _vertex_request( + { + "compaction": {"type": "summarize"}, + "extra_headers": {"anthropic-beta": "compact-2026-09-04,interleaved-thinking-2025-05-14"}, + }, + headers, + ) + + assert headers["anthropic-beta"].split(",").count("compact-2026-09-04") == 1 + assert result["anthropic_beta"] == ["interleaved-thinking-2025-05-14"] + + +def test_vertex_ai_anthropic_compaction_keeps_the_betas_the_client_sent_as_headers( + local_beta_headers_config: None, +) -> None: + headers: Final = {"anthropic-beta": "context-1m-2025-08-07"} + + result: Final = _vertex_request({"compaction": {"type": "summarize"}}, headers) + + assert sorted(headers["anthropic-beta"].split(",")) == ["compact-2026-09-04", "context-1m-2025-08-07"] + assert "anthropic_beta" not in result + + +@pytest.mark.parametrize("header_only_beta", ["compact-2026-09-04", "inline-tools-2026-09-15"]) +def test_vertex_ai_anthropic_client_sent_body_header_only_beta_moves_to_the_header( + local_beta_headers_config: None, header_only_beta: str +) -> None: + """Vertex rejects these betas inside the body `anthropic_beta` field with 400 "Unexpected value(s)" and + accepts them as the header (live, global, claude-sonnet-5-5, 2026-10-20), so a client that opts in through that field still + gets the beta as a header and keeps its other betas in the body.""" + headers: Final[dict[str, str]] = {} + + result: Final = _vertex_request( + {"anthropic_beta": [header_only_beta, "interleaved-thinking-2025-05-14"]}, headers + ) + + assert headers["anthropic-beta"] == header_only_beta + assert result["anthropic_beta"] == ["interleaved-thinking-2025-05-14"] + + +@pytest.mark.parametrize( + "client_beta,body,header", + [ + ( + "context-1m-2025-08-07", + ["context-1m-2025-08-07", "context-management-2025-06-27"], + "context-management-2025-06-27", + ), + ( + "inline-tools-2026-09-15", + ["context-management-2025-06-27"], + "context-management-2025-06-27,inline-tools-2026-09-15", + ), + ], + ids=["body_beta", "header_only_beta"], +) +def test_vertex_ai_anthropic_litellm_betas_merge_with_the_client_body_betas( + local_beta_headers_config: None, client_beta: str, body: list[str], header: str +) -> None: + headers: Final[dict[str, str]] = {} + + result: Final = _vertex_request({"context_management": _CLEAR_TOOL_USES, "anthropic_beta": [client_beta]}, headers) + + assert result["anthropic_beta"] == body + assert headers["anthropic-beta"] == header + + +@pytest.mark.parametrize( + "anthropic_beta", + [["interleaved-thinking-2025-05-14"], ("interleaved-thinking-2025-05-14",)], + ids=["list", "sdk_tuple"], +) +def test_vertex_ai_anthropic_client_sent_body_betas_stay_out_of_the_header(anthropic_beta: object) -> None: + headers: Final[dict[str, str]] = {} + + result: Final = _vertex_request({"anthropic_beta": anthropic_beta}, headers) + + assert result["anthropic_beta"] == ["interleaved-thinking-2025-05-14"] + assert "anthropic-beta" not in headers + + +_MALFORMED_BETAS: Final = pytest.mark.parametrize( + "malformed", [5, [1, 2], {"beta": "x"}], ids=["int", "list_of_ints", "dict"] +) +_MALFORMED_BETA_SOURCES: Final = pytest.mark.parametrize( + "source,field", + [("anthropic_beta", "anthropic_beta"), ("extra_headers", "The anthropic-beta header")], + ids=["body_field", "extra_header"], +) + + +def _malformed_beta_params(source: str, malformed: object) -> dict[str, object]: + if source == "extra_headers": + return {"extra_headers": {"anthropic-beta": malformed}} + return {"anthropic_beta": malformed} + + +@_MALFORMED_BETAS +@_MALFORMED_BETA_SOURCES +def test_vertex_ai_anthropic_malformed_anthropic_beta_answers_400_naming_its_field( + malformed: object, source: str, field: str +) -> None: + with pytest.raises(AnthropicError) as excinfo: + _vertex_request(_malformed_beta_params(source, malformed), {}) + + assert excinfo.value.status_code == 400 + assert excinfo.value.message.startswith(f"{field} must be") + + +@_MALFORMED_BETAS +@_MALFORMED_BETA_SOURCES +@pytest.mark.parametrize("auto_beta", [False, True], ids=["alone", "next_to_litellm_beta"]) +def test_vertex_ai_anthropic_malformed_anthropic_beta_is_dropped_under_drop_params( + local_beta_headers_config: None, malformed: object, source: str, field: str, auto_beta: bool +) -> None: + headers: Final[dict[str, str]] = {} + params: Final = {**_malformed_beta_params(source, malformed), **({"context_management": _CLEAR_TOOL_USES} if auto_beta else {})} + + result: Final = _vertex_request(params, headers, litellm_params={"drop_params": True}) + + assert result.get("anthropic_beta") == (["context-management-2025-06-27"] if auto_beta else None) + assert headers.get("anthropic-beta") == ("context-management-2025-06-27" if auto_beta else None) + + +_VERTEX_MESSAGE_RESPONSE: Final = { + "id": "msg_vrtx_test", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-6", + "content": [{"type": "text", "text": "pong"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 5, "output_tokens": 1}, +} + + +def test_vertex_ai_anthropic_completion_sends_compact_2026_09_04_beta_on_the_wire(local_beta_headers_config: None) -> None: + """A chat completion with `compaction` on a Vertex Claude deployment reaches Vertex with the beta in the + anthropic-beta request header and never in the body field, the shape Vertex accepts (live, 2026-10-05).""" + mock_response: Final = MagicMock() + mock_response.status_code = 200 + mock_response.text = json.dumps(_VERTEX_MESSAGE_RESPONSE) + mock_response.headers = {} + mock_response.json.return_value = _VERTEX_MESSAGE_RESPONSE + client: Final = MagicMock(spec=HTTPHandler) + client.post.return_value = mock_response + + with patch( + "litellm.llms.vertex_ai.vertex_llm_base.VertexBase._ensure_access_token", + return_value=("token", "test-project"), + ): + litellm.completion( + model="vertex_ai/claude-sonnet-4-6", + messages=[{"role": "user", "content": "Reply with the single word: pong"}], + compaction={"type": "summarize"}, + vertex_project="test-project", + vertex_location="us-east5", + client=client, + ) + + sent: Final = client.post.call_args.kwargs + sent_body: Final = sent["json"] if "json" in sent else json.loads(sent["data"]) + assert sent["headers"]["anthropic-beta"].split(",").count("compact-2026-09-04") == 1 + assert sent_body["compaction"] == {"type": "summarize"} + assert "anthropic_beta" not in sent_body + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize("thinking_type", ("adaptive", "enabled")) +@pytest.mark.parametrize("display", ("summarized", "omitted", "updates")) +def test_chat_preserves_thinking_display_and_beta(thinking_type: str, display: str) -> None: + from typing import Final + + from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import VertexAIAnthropicConfig + from litellm.types.llms.anthropic import ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER + + config: Final = VertexAIAnthropicConfig() + optional_params: Final = config.map_openai_params( + non_default_params={"thinking": {"type": thinking_type, "display": display, "budget_tokens": 2048}}, + optional_params={}, + model="claude-fable-5-1", + drop_params=False, + ) + optional_params["extra_headers"] = {"anthropic-beta": "existing-beta"} + headers: Final = {} + result: Final = config.transform_request( + model="claude-fable-5-1", + messages=[{"role": "user", "content": "Reply with OK"}], + optional_params=optional_params, + litellm_params={}, + headers=headers, + ) + + assert result["thinking"]["type"] == "adaptive" + assert result["thinking"]["display"] == display + assert (ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER in headers["anthropic-beta"].split(",")) == (display == "updates") + assert "existing-beta" in headers["anthropic-beta"].split(",") + + +@pytest.mark.parametrize("thinking", ("enabled", {"type": "bogus"}, {"type": "adaptive", "display": "future-mode"})) +def test_beta_detection_preserves_unrecognized_thinking(thinking: object) -> None: + from typing import Final + + from litellm.types.llms.anthropic import ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER + + headers: Final = {} + result: Final = VertexAIAnthropicConfig().transform_request( + model="claude-fable-5-1", messages=[{"role": "user", "content": "Hello"}], + optional_params={"thinking": thinking}, litellm_params={}, headers=headers, + ) + + assert result["thinking"] == thinking + assert ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER not in headers.get("anthropic-beta", "").split(",") diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/count_tokens/test_handler.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/count_tokens/test_handler.py new file mode 100644 index 00000000000..c03f08ab610 --- /dev/null +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/count_tokens/test_handler.py @@ -0,0 +1,60 @@ +from typing import Final + +import pytest + +from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens.handler import count_tokens_headers + +_HISTORY: Final = ({"role": "user", "content": "Hello"},) + + +@pytest.mark.parametrize( + "request_data", + [ + {"messages": [{"role": "assistant", "content": [{"type": "compaction", "content": "so far", "signature": "sig"}]}]}, + {"messages": list(_HISTORY), "compaction": {"type": "summarize"}}, + ], + ids=["signed_compaction_block", "compaction_param"], +) +def test_count_tokens_on_a_compaction_request_sends_the_compact_beta(request_data: dict[str, object]) -> None: + assert count_tokens_headers("token", request_data) == { + "Authorization": "Bearer token", + "anthropic-beta": "compact-2026-09-04", + } + + +@pytest.mark.parametrize( + "request_data", + [ + {"messages": list(_HISTORY)}, + {"messages": [{"role": "assistant", "content": [{"type": "compaction", "content": "so far", "signature": ""}]}]}, + ], + ids=["plain", "unsigned_compaction_block"], +) +def test_count_tokens_without_compaction_sends_no_beta(request_data: dict[str, object]) -> None: + assert count_tokens_headers("token", request_data) == {"Authorization": "Bearer token"} + + +_TOOL_ADDITION: Final = { + "role": "system", + "content": [{"type": "tool_addition", "tool": {"type": "tool_reference", "name": "ping"}}], +} + + +@pytest.mark.parametrize( + ("request_data", "expected_beta"), + [ + ({"messages": [*_HISTORY, _TOOL_ADDITION]}, "mid-conversation-tool-changes-2026-07-01"), + ( + {"messages": [*_HISTORY, _TOOL_ADDITION], "compaction": {"type": "summarize"}}, + "compact-2026-09-04,mid-conversation-tool-changes-2026-07-01", + ), + ], + ids=["tool_addition", "tool_addition_with_compaction"], +) +def test_count_tokens_on_a_tool_change_request_sends_the_tool_change_beta( + request_data: dict[str, object], expected_beta: str +) -> None: + assert count_tokens_headers("token", request_data) == { + "Authorization": "Bearer token", + "anthropic-beta": expected_beta, + } diff --git a/tests/unit/test_anthropic_beta_headers_filtering.py b/tests/unit/test_anthropic_beta_headers_filtering.py index deba7fe9587..c8fef95cf98 100644 --- a/tests/unit/test_anthropic_beta_headers_filtering.py +++ b/tests/unit/test_anthropic_beta_headers_filtering.py @@ -85,7 +85,8 @@ class TestAnthropicBetaHeadersFiltering: filtered = filter_and_transform_beta_headers( beta_headers=all_headers, provider=provider ) - assert ("compact-2026-09-04" in filtered) is (provider == "anthropic") + assert ("compact-2026-09-04" in filtered) is (provider in ("anthropic", "azure_ai", "vertex_ai")) + assert ("mid-conversation-tool-changes-2026-07-01" in filtered) is (provider != "bedrock_converse") for header in unsupported_headers: assert ( @@ -406,15 +407,49 @@ class TestAnthropicBetaHeadersFiltering: test_case["expected"] in filtered ), f"Header '{test_case['input']}' should be mapped to '{test_case['expected']}' for {test_case['provider']}, but got: {filtered}" - def test_filter_and_transform_beta_headers_vertex_ai_keeps_compact(self): - """Vertex AI supports compact context edits, so the compact beta header - must be forwarded instead of stripped (it was previously mapped to null, - which broke compact_20260112 context edits over /v1/messages).""" - filtered = filter_and_transform_beta_headers( - beta_headers=["compact-2026-01-12"], provider="vertex_ai" + @pytest.mark.parametrize("beta", ["compact-2026-09-04", "inline-tools-2026-09-15"]) + def test_vertex_ai_header_only_beta_is_forwarded_as_the_header_and_dropped_from_the_body( + self, beta: str + ) -> None: + headers, body = update_request_with_filtered_beta( + headers={"anthropic-beta": f"{beta},context-1m-2025-08-07"}, + request_data={"anthropic_beta": [beta, "context-1m-2025-08-07"]}, + provider="vertex_ai", ) - assert filtered == ["compact-2026-01-12"] + assert set(headers["anthropic-beta"].split(",")) == {beta, "context-1m-2025-08-07"} + assert body["anthropic_beta"] == ["context-1m-2025-08-07"] + + def test_header_only_betas_stay_out_of_the_provider_columns_older_proxies_read(self) -> None: + """Proxies released before header-only columns existed fetch this map from main, read only the + provider column, and copy its betas into the body field too, where Vertex answers 400. So a + header-only beta may sit in its provider column only if older proxies already forwarded it.""" + header_only_columns: Final = { + column.removesuffix("_header_only"): betas + for column, betas in self.config.items() + if column.endswith("_header_only") + } + forwarded_by_older_proxies: Final = { + provider: {beta for beta in betas if self.config[provider].get(beta) is not None} + for provider, betas in header_only_columns.items() + } + + assert "vertex_ai" in header_only_columns + assert {provider: betas for provider, betas in forwarded_by_older_proxies.items() if betas} == { + "vertex_ai": {"inline-tools-2026-09-15"} + } + + def test_filter_and_transform_beta_headers_vertex_ai_keeps_compact(self): + """Vertex AI serves both compaction betas (threshold compaction behind compact-2026-01-12 + and on-demand compaction behind compact-2026-09-04, Google Cloud: beta on + https://platform.claude.com/docs/en/build-with-claude/compaction-on-demand, read 2026-10-05), + so the Vertex filter forwards both instead of stripping them: a dropped header makes + Vertex answer 400 "compaction: Extra inputs are not permitted".""" + filtered = filter_and_transform_beta_headers( + beta_headers=["compact-2026-01-12", "compact-2026-09-04"], provider="vertex_ai" + ) + + assert filtered == ["compact-2026-01-12", "compact-2026-09-04"] @pytest.mark.parametrize("provider", ["bedrock_converse", "bedrock"]) def test_fine_grained_tool_streaming_forwarded_for_bedrock(self, provider): From a084d136358a2375c0999b56afadf64ebaa4d107 Mon Sep 17 00:00:00 2001 From: tessellark <349031000+tessellark@users.noreply.github.com> Date: Fri, 30 Oct 2026 09:40:58 -0700 Subject: [PATCH 10/74] fix(passthrough): log the URL model and declared provider on Gemini and Vertex failures (#49075) * fix(passthrough): log the URL model and declared provider on Gemini and Vertex failures Pass-through failures on Gemini and Vertex routes carry the model only in the URL, so the failure spend row and the Prometheus failure counter logged an empty model. The failure payload now falls back to the model parsed from the URL on those routes, and the Prometheus provider label honors the provider the request declares before inferring one from the model name * fix(passthrough): keep model and provider on mid-stream and Vertex failure logs Build the failure payload from the URL at both streamed relay sites so a 200 stream that dies mid-body still logs the model, derive vertex_ai for Vertex routes that never declare a provider, and have the Vertex logging handler reuse get_vertex_model_id_from_url instead of its own regex * fix(passthrough): bound the Prometheus provider label and name Vertex failures like the success handler Declared providers only reach the api_provider label when they name a registered provider, otherwise the label falls back to inference. Vertex failure rows now use the success handler's model naming (vertex_ai/ prefix on rawPredict and streamRawPredict, vertex_ai/search_api on :search) and its host-based provider detection, and the detected-stream failure sites are covered by tests. * fix(passthrough): name Google failures from the URL on untracked methods too A Gemini or Vertex pass-through failure on a method the success handler does not track (countTokens, for one) still logged model "" and no provider, although the model sits in the URL. The failure payload now falls back to the Google AI host for the provider and reads /models/ from the URL, so countTokens 400s name their model and provider like generateContent ones do. Non-Google hosts keep the tracked-route rule and still log nothing extra. * fix(passthrough): trust the declared Google provider and Vertex-shaped paths on custom bases The failure path only named the Google provider from the URL when the method was one the success handler tracks or the host was a Google API host, so a Gemini countTokens failure behind a custom GEMINI_API_BASE (or a Vertex one behind a custom base) still logged an empty model. A route that declares gemini or vertex_ai now wins outright, and a URL whose path carries /projects/

/locations// counts as Vertex on any host, while a tracked method on a non-Google host no longer passes as Vertex. * fix(pass_through): stop the URL model parser at the query string so a configured key never lands in a spend row's model * test(pass_through): run the URL-model failure tests through the real ProxyLogging with a recording callback * fix(passthrough): scope declared provider to pass-through failures and match Google hosts only Prometheus trusts the resolved provider as before and reads the request-declared provider only on pass-through failures. Vertex attribution matches Google hosts, strips operation poll suffixes, prefixes vertex_ai/ only for rawPredict, reads the URL path only, covers Vertex Live WebSocket failures, and labels a router alias failure with the alias the client requested * fix(passthrough): attribute failures from the URL only on built-in Google routes and keep other routes' metric labels A user-defined pass-through route to a Google host keeps its empty model and provider, since its URL names an upstream model rather than a LiteLLM-managed one. Prometheus failure labels read the route's provider and model_group only for gemini and vertex_ai, so Azure, OpenRouter, Bedrock and other pass-through routes keep their earlier labels. The Gemini URL parser drops the query string, Google host matching needs a dot or dash boundary, and the Vertex Claude prefix follows alt=sse like the success handler. * fix(passthrough): log the URL model on guardrail blocks and the named model on Vertex Live failures A pre-call guardrail block on a pass-through route now builds its failure payload the same way upstream failures do, so the model and provider read from the URL reach spend logs and metrics. Vertex Live websocket failures (connect errors, upstream rejections, and the credential failure) now carry the model from the ?model= query parameter under the vertex_ai provider * fix(passthrough): mark guardrail-block and Vertex Live credential failures as pass-through calls Prometheus reads a failure's declared provider only on pass-through calls, so without call_type it fell back to guessing from the model name. That labeled a guardrail block on the Gemini route as vertex_ai and the Live credential failure as None * fix(passthrough): let the route name a failure's provider and give guardrail blocks their own spend row * fix(pass-through): keep URL model attribution off routes that name another provider A built-in route that names a non-Google provider, like /openai pointed at Gemini's OpenAI-compatible host, no longer takes its failure model from the Google URL. A guardrail block response now returns the x-litellm-call-id of its spend row. --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/integrations/prometheus.py | 53 +- litellm/llms/vertex_ai/common_utils.py | 2 +- .../llm_passthrough_endpoints.py | 12 +- .../gemini_passthrough_logging_handler.py | 8 +- .../vertex_passthrough_logging_handler.py | 63 +- .../pass_through_endpoints.py | 136 ++- .../integrations/test_prometheus_labels.py | 97 +++ .../vertex_ai/test_vertex_ai_common_utils.py | 3 + .../test_llm_pass_through_endpoints.py | 53 +- .../test_pass_through_endpoints.py | 772 +++++++++++++++--- 10 files changed, 1025 insertions(+), 174 deletions(-) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index e6318525b88..5c2f0f553a8 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -82,6 +82,7 @@ from litellm.types.proxy.carried_budget_state import ( UserBudgetSnapshot, ) from litellm.types.utils import ( + CallTypes, StandardLoggingGuardrailInformation, StandardLoggingPayload, StandardLoggingZeroCostDiagnostic, @@ -279,6 +280,11 @@ def _logged_fields(value: object) -> Mapping[str, object]: return _NO_LOGGED_FIELDS +_GOOGLE_PASS_THROUGH_PROVIDERS: Final = frozenset( + (litellm.LlmProviders.GEMINI.value, litellm.LlmProviders.VERTEX_AI.value) +) + + def _is_llm_provider(name: str) -> bool: routable: Final = ( name in litellm.provider_list or JSONProviderRegistry.exists(name) or is_registered_custom_provider(name) @@ -2942,6 +2948,22 @@ class PrometheusLogger(CustomLogger): return False + @staticmethod + def _google_pass_through_provider(request_data: Mapping[str, object]) -> str | None: + declared_provider: Final = request_data.get("custom_llm_provider") + if request_data.get("call_type") != CallTypes.pass_through.value or not isinstance(declared_provider, str): + return None + return declared_provider if declared_provider in _GOOGLE_PASS_THROUGH_PROVIDERS else None + + @staticmethod + def _failed_request_requested_model(request_data: Mapping[str, object]) -> object: + requested_model: Final = request_data.get("model", "") + if PrometheusLogger._google_pass_through_provider(request_data) is None: + return requested_model + request_metadata: Final = _logged_fields(_logged_fields(request_data.get("litellm_params")).get("metadata")) + model_group: Final = request_metadata.get("model_group") + return model_group if isinstance(model_group, str) and model_group else requested_model + @staticmethod def _extract_api_provider_from_request_data(request_data: dict) -> str | None: """ @@ -2950,22 +2972,27 @@ class PrometheusLogger(CustomLogger): A request can fail before a deployment is resolved, so the provider is not always known. Prefer the resolved ``custom_llm_provider`` on ``litellm_params``, then any provider recovered onto a partial - ``standard_logging_object`` (e.g. a stream that broke mid-flight), and - finally infer it from the requested model name (e.g. ``gpt-4o-mini`` -> - ``openai``) since the proxy's failure ``request_data`` usually carries - only the client-supplied model. Return ``None`` when it cannot be - determined so the label emits empty rather than a guess. + ``standard_logging_object`` (e.g. a stream that broke mid-flight). On a + Gemini or Vertex pass-through failure only, next take the provider the + failure payload names (``gemini`` or ``vertex_ai``), so every other + pass-through route keeps its earlier label. Finally infer it from the + requested model name (e.g. ``gpt-4o-mini`` -> ``openai``) since the + proxy's failure ``request_data`` usually carries only the client-supplied + model. Return ``None`` when it cannot be determined so the label emits + empty rather than a guess. """ litellm_params: Final = request_data.get("litellm_params") or {} - provider = litellm_params.get("custom_llm_provider") - if provider: - return provider standard_logging_object: Final = request_data.get("standard_logging_object") or {} - provider = standard_logging_object.get("custom_llm_provider") - if provider: - return provider + resolved_provider: Final = litellm_params.get("custom_llm_provider") or standard_logging_object.get( + "custom_llm_provider" + ) + if resolved_provider: + return resolved_provider + google_provider: Final = PrometheusLogger._google_pass_through_provider(request_data) + if google_provider is not None: + return google_provider model: Final = litellm_params.get("model") or request_data.get("model") - if not model: + if not isinstance(model, str) or not model: return None try: return litellm.get_llm_provider(model=model)[1] or None @@ -3052,7 +3079,7 @@ class PrometheusLogger(CustomLogger): team_alias=user_api_key_dict.team_alias, org_id=user_api_key_dict.org_id, org_alias=user_api_key_dict.organization_alias, - requested_model=_bounded_requested_model_label(request_data.get("model", "")), + requested_model=_bounded_requested_model_label(self._failed_request_requested_model(request_data)), status_code=str(status_code), exception_status=str(status_code), exception_class=self._get_exception_class_name(original_exception), diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index 0fa80cb2cad..ad64d852ef9 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -1257,7 +1257,7 @@ def get_vertex_model_id_from_url(url: str) -> str | None: `https://${LOCATION}-aiplatform.googleapis.com/v1/projects/${PROJECT_ID}/locations/${LOCATION}/publishers/google/models/${MODEL_ID}:streamGenerateContent` """ - match: Final = re.search(r"/models/([^:]+)", url) + match: Final = re.search(r"/models/([^:?#]+)", url) return match.group(1) if match else None diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 8de7441121f..33b18c3d939 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -32,6 +32,7 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm import get_llm_provider from litellm._logging import verbose_proxy_logger +from litellm._uuid import uuid from litellm.constants import ( ALLOWED_VERTEX_AI_PASSTHROUGH_HEADERS, AWS_ERROR_TYPE_HEADER, @@ -149,7 +150,7 @@ from litellm.types.passthrough_endpoints.tinyfish import ( ) from litellm.types.passthrough_endpoints.vertex_ai import VertexPassThroughCredentials from litellm.types.router import LiteLLMParamsTypedDict -from litellm.types.utils import LlmProviders +from litellm.types.utils import CallTypes, LlmProviders from litellm.types.vector_stores import LiteLLM_ManagedVectorStore from litellm.utils import ProviderConfigManager @@ -4105,7 +4106,12 @@ async def vertex_ai_live_websocket_passthrough( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, - request_data={}, + request_data={ + "model": model or "", + "custom_llm_provider": litellm.LlmProviders.VERTEX_AI.value, + "call_type": CallTypes.pass_through.value, + "litellm_call_id": str(uuid.uuid4()), + }, ) if websocket.client_state != WebSocketState.DISCONNECTED: await websocket.close(code=1011, reason=VERTEX_LIVE_UNCONFIGURED_CLOSE_REASON) @@ -4140,6 +4146,8 @@ async def vertex_ai_live_websocket_passthrough( forward_headers=False, endpoint="/vertex_ai/live", accept_websocket=False, + builtin_provider_route=True, + model=model, setup_model_rewriter=_build_vertex_live_setup_model_rewriter( vertex_project=resolved_project, vertex_location=resolved_location, diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/gemini_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/gemini_passthrough_logging_handler.py index 7a5d5a1e259..d449f62af2c 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/gemini_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/gemini_passthrough_logging_handler.py @@ -1,4 +1,3 @@ -import re from collections.abc import Sequence from datetime import datetime from typing import TYPE_CHECKING, Any, Final @@ -9,6 +8,7 @@ import litellm from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.gemini.videos.transformation import GeminiVideoConfig +from litellm.llms.vertex_ai.common_utils import get_vertex_model_id_from_url from litellm.proxy._types import PassThroughEndpointLoggingTypedDict from litellm.proxy.pass_through_endpoints.llm_provider_handlers.vertex_passthrough_logging_handler import ( GeminiSseChunkParser, @@ -207,11 +207,7 @@ class GeminiPassthroughLoggingHandler: @staticmethod def extract_model_from_url(url: str) -> str: - pattern: Final = r"/models/([^:]+)" - match: Final = re.search(pattern, url) - if match: - return match.group(1) - return "unknown" + return get_vertex_model_id_from_url(url) or "unknown" @staticmethod def _create_gemini_response_logging_payload_for_generate_content( diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py index b3cbad91624..d5c162296d3 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py @@ -3,7 +3,7 @@ import re from collections.abc import Mapping, Sequence from datetime import datetime from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, cast -from urllib.parse import urlparse +from urllib.parse import parse_qs, urlparse import httpx from pydantic import ConfigDict, TypeAdapter @@ -19,6 +19,7 @@ from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import from litellm.llms.vertex_ai.common_utils import ( get_vertex_ai_lyria_generation_cost, get_vertex_location_from_url, + get_vertex_model_id_from_url, ) from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( ModelResponseIterator as VertexModelResponseIterator, @@ -47,6 +48,19 @@ from litellm.types.utils import ( ) vertex_search_api_config: Final = VertexSearchAPIVectorStoreConfig() +GEMINI_API_HOST_SUFFIX: Final = "generativelanguage.googleapis.com" +GOOGLE_AI_HOST_SUFFIXES: Final = ( + GEMINI_API_HOST_SUFFIX, + "aiplatform.googleapis.com", + "discoveryengine.googleapis.com", +) +_VERTEX_MULTI_REGION_HOST: Final = re.compile(r"aiplatform\.[a-z0-9-]+\.rep\.googleapis\.com") + + +def _is_host_or_subdomain(hostname: str, suffix: str) -> bool: + return hostname == suffix or hostname.endswith((f".{suffix}", f"-{suffix}")) + + if TYPE_CHECKING: from litellm.types.utils import LiteLLMBatch @@ -684,11 +698,18 @@ class VertexPassthroughLoggingHandler: @staticmethod def extract_model_from_url(url: str) -> str: - pattern: Final = r"/models/([^:]+)" - match: Final = re.search(pattern, url) - if match: - return match.group(1) - return "unknown" + return get_vertex_model_id_from_url(url) or "unknown" + + @staticmethod + def model_from_url_route(url_route: str) -> str | None: + parsed_url: Final = urlparse(url_route) + model: Final = get_vertex_model_id_from_url(parsed_url.path) + if model is None: + return "vertex_ai/search_api" if ":search" in parsed_url.path else None + streamed: Final = "sse" in parse_qs(parsed_url.query).get("alt", ()) + if parsed_url.path.endswith((":rawPredict", ":streamRawPredict")) and not streamed: + return f"vertex_ai/{model}" + return model @staticmethod def extract_model_name_from_vertex_path(vertex_model_path: str) -> str: @@ -734,18 +755,32 @@ class VertexPassthroughLoggingHandler: return None @staticmethod - def _get_custom_llm_provider(url: str, logging_obj: LiteLLMLoggingObj) -> str: + def is_google_ai_url(url: str) -> bool: hostname: Final = urlparse(url).hostname - if logging_obj.model_call_details.get("custom_llm_provider") == litellm.LlmProviders.GEMINI.value or ( - hostname is not None - and ( - hostname == "generativelanguage.googleapis.com" - or hostname.endswith(".generativelanguage.googleapis.com") - ) - ): + return hostname is not None and ( + any(_is_host_or_subdomain(hostname, suffix) for suffix in GOOGLE_AI_HOST_SUFFIXES) + or _VERTEX_MULTI_REGION_HOST.fullmatch(hostname) is not None + ) + + @staticmethod + def is_gemini_api_host(url: str) -> bool: + hostname: Final = urlparse(url).hostname + return hostname is not None and ( + hostname == GEMINI_API_HOST_SUFFIX or hostname.endswith(f".{GEMINI_API_HOST_SUFFIX}") + ) + + @staticmethod + def custom_llm_provider_from_url(url: str) -> str: + if VertexPassthroughLoggingHandler.is_gemini_api_host(url): return litellm.LlmProviders.GEMINI.value return litellm.LlmProviders.VERTEX_AI.value + @staticmethod + def _get_custom_llm_provider(url: str, logging_obj: LiteLLMLoggingObj) -> str: + if logging_obj.model_call_details.get("custom_llm_provider") == litellm.LlmProviders.GEMINI.value: + return litellm.LlmProviders.GEMINI.value + return VertexPassthroughLoggingHandler.custom_llm_provider_from_url(url) + @staticmethod def _is_multimodal_embedding_response(json_response: dict) -> bool: """ diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index bda2bfb9e04..cfac28db1a9 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -3,6 +3,7 @@ import asyncio import copy import json import posixpath +import re import traceback from base64 import b64encode from collections.abc import AsyncGenerator, AsyncIterator, Callable, Iterable, Mapping, Sequence @@ -77,6 +78,7 @@ from litellm.llms.base_llm.managed_resources.utils import ( from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.llms.eu_jev.common_utils import eu_jev_scoped_model, eu_jev_upstream_body from litellm.llms.oss_decision import validate_oss_request +from litellm.llms.vertex_ai.common_utils import get_vertex_model_id_from_url from litellm.passthrough import BasePassthroughUtils from litellm.proxy._lazy_features import lazy_owned_routes from litellm.proxy._types import ( @@ -147,11 +149,12 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( EndpointType, PassthroughStandardLoggingPayload, ) -from litellm.types.utils import TRUSTED_CALLBACK_VARS_FIELD, Usage +from litellm.types.utils import TRUSTED_CALLBACK_VARS_FIELD, CallTypes, Usage from .llm_provider_handlers.tinyfish_passthrough_logging_handler import ( is_tinyfish_agent_url, ) +from .llm_provider_handlers.vertex_passthrough_logging_handler import VertexPassthroughLoggingHandler from .streaming_handler import PassThroughStreamingHandler from .success_handler import PassThroughEndpointLogging from .upstream_usage_headers import ( @@ -874,13 +877,45 @@ def _carry_guardrail_logging_info(request_data: dict, guardrail_data: dict | Non metadata.setdefault("standard_logging_guardrail_information", list(entries)) +_GOOGLE_PASSTHROUGH_PROVIDERS: Final = frozenset( + (litellm.LlmProviders.GEMINI.value, litellm.LlmProviders.VERTEX_AI.value) +) + + +def _passthrough_google_provider(url_route: str, custom_llm_provider: str | None) -> str | None: + if custom_llm_provider is not None: + return custom_llm_provider if custom_llm_provider in _GOOGLE_PASSTHROUGH_PROVIDERS else None + if VertexPassthroughLoggingHandler.is_google_ai_url(url_route): + return VertexPassthroughLoggingHandler.custom_llm_provider_from_url(url_route) + return None + + +_OPERATION_POLL_SUFFIX: Final = re.compile(r"/operations/[^/]+$") + + +def _without_operation_poll_suffix(model: str | None) -> str | None: + return _OPERATION_POLL_SUFFIX.sub("", model) if model else model + + +def _passthrough_google_model(url_route: str, google_provider: str | None) -> str | None: + match google_provider: + case "gemini": + return _without_operation_poll_suffix(get_vertex_model_id_from_url(urlparse(url_route).path)) + case "vertex_ai": + return _without_operation_poll_suffix(VertexPassthroughLoggingHandler.model_from_url_route(url_route)) + case _: + return None + + def _build_passthrough_failure_request_payload( parsed_body: dict | None, kwargs: dict | None, logging_obj: LiteLLMLoggingObj | None, custom_llm_provider: str | None, upstream_usage: UpstreamReportedUsage | None = None, + url: httpx.URL | None = None, upstream_model: str | None = None, + builtin_provider_route: bool = False, ) -> dict: """Build the ``request_data`` dict passed to ``post_call_failure_hook``. @@ -893,16 +928,25 @@ def _build_passthrough_failure_request_payload( comes paired with a usage object, so both keys are written together. ``upstream_model`` is the model name the upstream reported, which replaces the body's ``model`` on the failure row just as it does on a success row. + ``builtin_provider_route`` lets a Gemini or Vertex URL name the failure's + model and provider. A user-defined route never gets that, since its URL + names an upstream model rather than a LiteLLM-managed one. """ request_payload: Final[dict] = dict(cast(Mapping[str, object], parsed_body or {})) # cast-ok: json body if kwargs: request_payload.update(kwargs) if logging_obj is not None: request_payload["litellm_logging_obj"] = logging_obj - if "model" not in request_payload and parsed_body and isinstance(parsed_body, dict): - request_payload["model"] = parsed_body.get("model", "") - if "custom_llm_provider" not in request_payload and custom_llm_provider: - request_payload["custom_llm_provider"] = custom_llm_provider + url_route: Final = str(url) if url is not None else "" + google_provider: Final = ( + _passthrough_google_provider(url_route, custom_llm_provider) if builtin_provider_route else None + ) + if "model" not in request_payload: + request_payload["model"] = _passthrough_google_model(url_route, google_provider) or "" + resolved_provider: Final = custom_llm_provider or google_provider + request_payload.pop("custom_llm_provider", None) + if resolved_provider: + request_payload["custom_llm_provider"] = resolved_provider if upstream_usage is not None: request_payload["response_cost"] = upstream_usage.response_cost or 0.0 request_payload["combined_usage_object"] = Usage(total_tokens=upstream_usage.total_tokens or 0) @@ -1203,6 +1247,7 @@ async def pass_through_request( # Initialize variables ######################################################### litellm_call_id: Final = str(uuid.uuid4()) + builtin_provider_route: Final = not request_dispatched_to_pass_through_endpoint(request) url: httpx.URL | None = None # parsed request body @@ -1612,6 +1657,8 @@ async def pass_through_request( logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, upstream_usage=upstream_usage, + url=url, + builtin_provider_route=builtin_provider_route, upstream_model=upstream_model, ), logging_obj=logging_obj, @@ -1651,6 +1698,8 @@ async def pass_through_request( kwargs=kwargs, logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, + url=url, + builtin_provider_route=builtin_provider_route, ), ), managed_id_provider=_managed_id_provider, @@ -1710,6 +1759,8 @@ async def pass_through_request( logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, upstream_usage=upstream_usage, + url=url, + builtin_provider_route=builtin_provider_route, upstream_model=upstream_model, ), logging_obj=logging_obj, @@ -1749,6 +1800,8 @@ async def pass_through_request( kwargs=kwargs, logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, + url=url, + builtin_provider_route=builtin_provider_route, ), ), managed_id_provider=_managed_id_provider, @@ -1811,6 +1864,8 @@ async def pass_through_request( logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, upstream_usage=upstream_usage, + url=url, + builtin_provider_route=builtin_provider_route, upstream_model=upstream_model, ) failure_request_payload["response_body"] = response_body @@ -1991,7 +2046,14 @@ async def pass_through_request( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, - request_data=e.request_data, + request_data=_build_passthrough_failure_request_payload( + parsed_body=e.request_data, + kwargs={"call_type": CallTypes.pass_through.value, "litellm_call_id": litellm_call_id}, + logging_obj=None, + custom_llm_provider=custom_llm_provider, + url=url, + builtin_provider_route=builtin_provider_route, + ), ) except Exception: verbose_proxy_logger.warning( @@ -2010,6 +2072,7 @@ async def pass_through_request( content=json.dumps(error_body), status_code=200, media_type="application/json", + headers={"x-litellm-call-id": litellm_call_id}, ) except Exception as e: custom_headers = ProxyBaseLLMRequestProcessing.get_custom_headers( @@ -2033,19 +2096,14 @@ async def pass_through_request( # Monitoring: Trigger post_call_failure_hook # for pass through endpoint failure ######################################################### - request_payload: Final[dict] = _parsed_body or {} - # add user_api_key_dict, litellm_call_id, passthrough_logging_payloa for logging - if kwargs: - for key, value in kwargs.items(): - request_payload[key] = value - if logging_obj is not None: - request_payload["litellm_logging_obj"] = logging_obj - - if "model" not in request_payload and _parsed_body and isinstance(_parsed_body, dict): - request_payload["model"] = _parsed_body.get("model", "") - if "custom_llm_provider" not in request_payload and custom_llm_provider: - request_payload["custom_llm_provider"] = custom_llm_provider - + request_payload: Final = _build_passthrough_failure_request_payload( + parsed_body=_parsed_body, + kwargs=kwargs, + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + url=url, + builtin_provider_route=builtin_provider_route, + ) _carry_guardrail_logging_info(request_payload, post_call_guardrail_data) await proxy_logging_obj.post_call_failure_hook( @@ -2547,6 +2605,8 @@ async def websocket_passthrough_request( cost_per_request: float | None = None, accept_websocket: bool = True, setup_model_rewriter: Callable[[str], str] | None = None, + builtin_provider_route: bool = False, + model: str | None = None, ): """ WebSocket passthrough request handler. @@ -2560,6 +2620,8 @@ async def websocket_passthrough_request( endpoint: The endpoint path (for logging purposes) cost_per_request: Optional field - cost per request to the target endpoint setup_model_rewriter: Optional rewrite of the setup frame's model before it reaches the upstream + builtin_provider_route: Whether a built-in provider route serves the connection, so its URL names the provider + model: The model the caller named before connecting, logged on a failure before the setup frame arrives """ from litellm.litellm_core_utils.litellm_logging import Logging from litellm.proxy.proxy_server import proxy_config, proxy_logging_obj @@ -2912,19 +2974,17 @@ async def websocket_passthrough_request( except InvalidStatus as exc: verbose_proxy_logger.exception("WebSocket passthrough (%s): upstream rejected WebSocket connection", endpoint) - # Prepare request payload for logging - request_payload = {} - if kwargs: - for key, value in kwargs.items(): - request_payload[key] = value - if logging_obj is not None: - request_payload["litellm_logging_obj"] = logging_obj - - # Log the connection failure using the same pattern as HTTP await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=exc, - request_data=request_payload, + request_data=_build_passthrough_failure_request_payload( + parsed_body=None if model is None else {"model": model}, + kwargs=kwargs, + logging_obj=logging_obj, + custom_llm_provider=None, + url=httpx.URL(target), + builtin_provider_route=builtin_provider_route, + ), traceback_str=traceback.format_exc( limit=MAXIMUM_TRACEBACK_LINES_TO_LOG, ), @@ -2940,19 +3000,17 @@ async def websocket_passthrough_request( "WebSocket passthrough (%s): unexpected error while proxying WebSocket", endpoint ) - # Prepare request payload for logging - request_payload = {} - if kwargs: - for key, value in kwargs.items(): - request_payload[key] = value - if logging_obj is not None: - request_payload["litellm_logging_obj"] = logging_obj - - # Log the unexpected error using the same pattern as HTTP await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, - request_data=request_payload, + request_data=_build_passthrough_failure_request_payload( + parsed_body=None if model is None else {"model": model}, + kwargs=kwargs, + logging_obj=logging_obj, + custom_llm_provider=None, + url=httpx.URL(target), + builtin_provider_route=builtin_provider_route, + ), traceback_str=traceback.format_exc( limit=MAXIMUM_TRACEBACK_LINES_TO_LOG, ), diff --git a/tests/unit/integrations/test_prometheus_labels.py b/tests/unit/integrations/test_prometheus_labels.py index bff89b6cbf1..878f19c66f8 100644 --- a/tests/unit/integrations/test_prometheus_labels.py +++ b/tests/unit/integrations/test_prometheus_labels.py @@ -238,6 +238,53 @@ def test_extract_api_provider_from_request_data_failure_path(): assert extract({"model": "some-unknown-model-xyz"}) is None +_PASS_THROUGH_CALL_TYPE: Final = "pass_through_endpoint" + + +@pytest.mark.parametrize( + "request_data, expected_provider", + [ + ({"model": "gpt-4o-mini", "call_type": "acompletion", "custom_llm_provider": "anthropic"}, "openai"), + ({"model": "gpt-4o-mini", "litellm_params": {"custom_llm_provider": "humanloop"}}, "humanloop"), + ({"model": "gemini-3.8-flash", "call_type": _PASS_THROUGH_CALL_TYPE}, "vertex_ai"), + ( + {"model": "gemini-3.8-flash", "call_type": _PASS_THROUGH_CALL_TYPE, "custom_llm_provider": "gemini"}, + "gemini", + ), + ( + { + "call_type": _PASS_THROUGH_CALL_TYPE, + "custom_llm_provider": "gemini", + "standard_logging_object": {"custom_llm_provider": "vertex_ai"}, + }, + "vertex_ai", + ), + ], +) +def test_extract_api_provider_reads_the_request_declared_provider_only_on_pass_through_failures( + request_data: dict[str, object], expected_provider: str +): + from litellm.integrations.prometheus import PrometheusLogger + + assert PrometheusLogger._extract_api_provider_from_request_data(request_data) == expected_provider + + +@pytest.mark.parametrize( + "declared_provider", + ["azure", "bedrock", "openrouter", "client-supplied-value", "", {"nested": "object"}, 42], +) +def test_extract_api_provider_ignores_pass_through_declared_values_outside_gemini_and_vertex( + declared_provider: object, +): + from litellm.integrations.prometheus import PrometheusLogger + + extract = PrometheusLogger._extract_api_provider_from_request_data + declared = {"call_type": _PASS_THROUGH_CALL_TYPE, "custom_llm_provider": declared_provider} + + assert extract({"model": "gpt-4o-mini", **declared}) == "openai" + assert extract({"model": "no-such-model", **declared}) is None + + def test_extract_api_provider_swallows_unknown_model_but_logs_unexpected_errors(): """ get_llm_provider raises BadRequestError for a model that maps to no @@ -849,6 +896,56 @@ async def test_failure_hook_prefers_request_data_provider_over_exception_provide ) == ["azure"] +@pytest.mark.asyncio +@pytest.mark.parametrize( + "call_type, custom_llm_provider, expected_requested_model", + [ + (_PASS_THROUGH_CALL_TYPE, "vertex_ai", "gcp/google/gemini-2.5-flash"), + ("acompletion", "vertex_ai", "other"), + (_PASS_THROUGH_CALL_TYPE, "bedrock", "other"), + ], +) +async def test_failure_hook_labels_a_pass_through_failure_with_the_router_model_the_client_requested( + call_type: str, custom_llm_provider: str, expected_requested_model: str +): + from unittest.mock import patch + + from litellm.integrations.prometheus import PrometheusLogger + from litellm.proxy._types import UserAPIKeyAuth + + router = litellm.Router( + model_list=[ + { + "model_name": "gcp/google/gemini-2.5-flash", + "litellm_params": { + "model": "vertex_ai/gemini-2.5-flash", + "vertex_project": "p", + "vertex_location": "us-central1", + }, + } + ] + ) + _clear_prometheus_registry() + try: + with patch("litellm.proxy.proxy_server.llm_router", router, create=True): # test-quality-ok: production reads proxy_server.llm_router lazily, no injection seam + await PrometheusLogger().async_post_call_failure_hook( + request_data={ + "model": "gemini-2.5-flash", + "call_type": call_type, + "custom_llm_provider": custom_llm_provider, + "metadata": {}, + "litellm_params": {"metadata": {"model_group": "gcp/google/gemini-2.5-flash"}}, + }, + original_exception=Exception("upstream 400"), + user_api_key_dict=UserAPIKeyAuth(token="tok"), + ) + assert { + s.labels.get("requested_model") for s in _collected_samples("litellm_proxy_failed_requests_metric_total") + } == {expected_requested_model} + finally: + _clear_prometheus_registry() + + def test_model_group_in_deployment_metrics(): """ Test that model_group label is present on the deployment-scoped metrics diff --git a/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py b/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py index 152ab2ffc13..3b222be6e5e 100644 --- a/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py +++ b/tests/unit/llms/vertex_ai/test_vertex_ai_common_utils.py @@ -1293,6 +1293,9 @@ def test_get_vertex_model_id_from_url(): model_id = get_vertex_model_id_from_url(url) assert model_id == "gemini-pro" + url = "https://generativelanguage.googleapis.com/v1beta/models/gemini-pro?key=provider-secret#frag" + assert get_vertex_model_id_from_url(url) == "gemini-pro" + # Test with invalid URL url = "https://invalid-url.com" model_id = get_vertex_model_id_from_url(url) diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index aab227e9494..ff27ac54a5b 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -25,12 +25,14 @@ from starlette.datastructures import FormData import litellm from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.prometheus import PrometheusLogger from tests._master_key import MASTER_KEY as SHARED_MASTER_KEY from litellm.caching.caching import DualCache -from litellm.types.utils import CallTypesLiteral +from litellm.types.utils import CallTypes, CallTypesLiteral from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException, ModifyResponseException from litellm.proxy._types import AwsErrorTypeProxyException, ProxyException +from litellm.proxy.spend_tracking.spend_tracking_utils import get_spend_logs_id from litellm.proxy.utils import ProxyLogging from litellm.passthrough.main import raise_for_provider_status from litellm.llms.bedrock.common_utils import BedrockError @@ -6490,6 +6492,55 @@ class TestVertexAILiveWebsocketPassthrough: assert "default_vertex_config" in close_kwargs["reason"] assert len(close_kwargs["reason"].encode("utf-8")) <= 123 + @pytest.mark.asyncio + @pytest.mark.parametrize("model, expected_model", [("gemini-3.8-live", "gemini-3.8-live"), (None, "")]) + async def test_credential_failure_logs_the_named_model_under_vertex_ai( + self, monkeypatch: pytest.MonkeyPatch, model: str | None, expected_model: str + ): + from litellm.proxy.pass_through_endpoints import ( + llm_passthrough_endpoints as passthrough_module, + ) + + class _FailureRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.request_data: tuple[dict[str, object], ...] = () + + async def async_post_call_failure_hook( + self, + request_data: dict[str, object], + original_exception: Exception, + user_api_key_dict: UserAPIKeyAuth, + traceback_str: str | None = None, + ) -> None: + self.request_data = (*self.request_data, request_data) + + recorder: Final = _FailureRecorder() + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None) + monkeypatch.setattr( + "litellm.proxy.proxy_server.proxy_logging_obj", ProxyLogging(user_api_key_cache=DualCache()) + ) + monkeypatch.setattr(litellm, "callbacks", [recorder]) + monkeypatch.setattr(passthrough_module.passthrough_endpoint_router, "default_vertex_config", None) + self._clear_vertex_env(monkeypatch) + ensure_token: Final = AsyncMock(side_effect=Exception("Unable to find your credentials")) + + with patch.object(passthrough_module.vertex_llm_base, "ensure_access_token_async", ensure_token): + await passthrough_module.vertex_ai_live_websocket_passthrough( + websocket=self._websocket(), + model=model, + user_api_key_dict=UserAPIKeyAuth(), + ) + + assert [ + ( + d["model"], + PrometheusLogger._extract_api_provider_from_request_data(d), + get_spend_logs_id(CallTypes.pass_through.value, {}, d) is not None, + ) + for d in recorder.request_data + ] == [(expected_model, "vertex_ai", True)] + class TestAnthropicProxyRoute: """The /anthropic passthrough route: custom auth headers must not clobber the diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 7b62853b0e9..a9098ac3d59 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -30,20 +30,26 @@ from starlette.datastructures import UploadFile as StarletteUploadFile import litellm from litellm._logging import verbose_proxy_logger +from litellm.caching.dual_cache import DualCache from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.prometheus import PrometheusLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy._lazy_features import LazyFeature, attach_lazy_features from litellm.proxy._types import ProxyException, UserAPIKeyAuth from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import router as llm_passthrough_router +from litellm.proxy.pass_through_endpoints.llm_provider_handlers.vertex_passthrough_logging_handler import ( + VertexPassthroughLoggingHandler, +) from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( DEFAULT_PASS_THROUGH_REQUEST_TIMEOUT_SECONDS, LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY, HttpPassThroughEndpointHelpers, InitPassThroughEndpointHelpers, SafeRouteAdder, + _build_passthrough_failure_request_payload, _registered_pass_through_routes, _truncate_upstream_error_body, _update_metadata_with_tags_in_header, @@ -61,8 +67,11 @@ from litellm.proxy.pass_through_endpoints.success_handler import ( PassThroughEndpointLogging, ) from litellm.proxy.route_llm_request import ProxyModelNotFoundError +from litellm.proxy.spend_tracking.spend_tracking_utils import get_spend_logs_id +from litellm.proxy.utils import ProxyLogging from litellm.types import utils as types_utils from litellm.types.guardrails import GuardrailEventHooks +from litellm.types.utils import CallTypes from litellm.types.passthrough_endpoints.pass_through_endpoints import ( LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, @@ -140,9 +149,7 @@ def test_is_multipart(): ), ], ) -def test_get_endpoint_type_classifies_anthropic_messages_routes( - url: str, expected_endpoint_type: EndpointType -) -> None: +def test_get_endpoint_type_classifies_anthropic_messages_routes(url: str, expected_endpoint_type: EndpointType) -> None: assert HttpPassThroughEndpointHelpers.get_endpoint_type(url) == expected_endpoint_type @@ -430,53 +437,18 @@ async def test_pass_through_request_failure_handler(): Critical Test: When a users pass through endpoint request fails, we must log the failure code, exception in litellm spend logs. """ - with patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_proxy_logging: - with patch("litellm.llms.custom_httpx.http_handler.get_async_httpx_client") as mock_get_client: - with patch( - "litellm.proxy.pass_through_endpoints.pass_through_endpoints.ProxyBaseLLMRequestProcessing" - ) as mock_processing: - # Setup mock for post_call_failure_hook and pre_call_hook - mock_proxy_logging.post_call_failure_hook = AsyncMock() - mock_proxy_logging.pre_call_hook = AsyncMock() + request_failure = httpx.HTTPError("Request failed") + user_api_key_dict = UserAPIKeyAuth(api_key="sk-test") - # Setup mock for httpx client - mock_client = MagicMock() - mock_client.client = MagicMock() - mock_client.client.request = AsyncMock(side_effect=httpx.HTTPError("Request failed")) - mock_get_client.return_value = mock_client + failure: Final = await _run_passthrough_failure( + "http://test.com", None, request_failure, user_api_key_dict=user_api_key_dict + ) - # Mock headers for custom headers - mock_processing.get_custom_headers.return_value = {} - - # Create mock request - mock_request = MagicMock(spec=Request) - mock_request.method = "POST" - mock_request.body = AsyncMock(return_value=b'{"test": "data"}') - mock_request.headers = Headers({}) - - # Create a simple empty QueryParams - mock_request.query_params = QueryParams({}) - - # Create mock user API key dict - mock_user_api_key_dict = MagicMock() - - # Call the function with a target that will trigger an HTTPError - with pytest.raises(ProxyException): - await pass_through_request( - request=mock_request, - target="http://test.com", - custom_headers={}, - user_api_key_dict=mock_user_api_key_dict, - ) - - # Assert post_call_failure_hook was called - mock_proxy_logging.post_call_failure_hook.assert_called_once() - - # Verify the arguments to post_call_failure_hook - call_args = mock_proxy_logging.post_call_failure_hook.call_args[1] - assert call_args["user_api_key_dict"] == mock_user_api_key_dict - assert isinstance(call_args["original_exception"], TypeError) # Now expecting TypeError - assert "traceback_str" in call_args + assert failure.user_api_key_dict is user_api_key_dict + assert failure.original_exception is request_failure + assert "Request failed" in (failure.traceback_str or "") + assert failure.request_data["contents"] == "not a list" + assert failure.request_data["model"] == "" @pytest.mark.asyncio @@ -3656,9 +3628,7 @@ async def test_pass_through_request_preserves_target_query_without_client_query( @pytest.mark.asyncio -async def test_config_pass_through_without_configured_headers_preserves_no_authorization( - tmp_path, monkeypatch -): +async def test_config_pass_through_without_configured_headers_preserves_no_authorization(tmp_path, monkeypatch): proxy = await _boot_db_backed_proxy( tmp_path, monkeypatch, @@ -3725,9 +3695,7 @@ async def test_pass_through_rpm_limit_is_sequential_and_scoped_per_key(tmp_path, proxy = await _boot_db_backed_proxy( tmp_path, monkeypatch, - config_pass_through_endpoints=[ - {"path": "/rpm-rerank", "target": "http://upstream.test/rerank", "auth": True} - ], + config_pass_through_endpoints=[{"path": "/rpm-rerank", "target": "http://upstream.test/rerank", "auth": True}], db_pass_through_endpoints=[], master_key=MASTER_KEY, ) @@ -4902,8 +4870,10 @@ from fastapi import HTTPException as _FastAPIHTTPException from litellm.exceptions import ( BlockedPiiEntityError, GuardrailRaisedException, + ModifyResponseException, ) from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER + _PT_MODULE = "litellm.proxy.pass_through_endpoints.pass_through_endpoints" @@ -8182,8 +8152,10 @@ async def test_user_defined_passthrough_is_neither_tracked_nor_enforced(metadata } ) body: Final = { - "model": "upstream-only-model", metadata_slot: { - "model_group": "managed-model", "customer_label": "retained", + "model": "upstream-only-model", + metadata_slot: { + "model_group": "managed-model", + "customer_label": "retained", "user_api_key_team_model_max_budget": budget, }, } @@ -8191,48 +8163,84 @@ async def test_user_defined_passthrough_is_neither_tracked_nor_enforced(metadata assert await limiter.is_team_within_model_budget("shared-team", budget, None, "managed-model") start: Final = datetime.now() logging_obj: Final = LiteLLMLoggingObj( - model="upstream-only-model", messages=[], stream=False, call_type="pass_through_endpoint", - start_time=start, litellm_call_id="custom-budget", function_id="custom-budget", kwargs={}, + model="upstream-only-model", + messages=[], + stream=False, + call_type="pass_through_endpoint", + start_time=start, + litellm_call_id="custom-budget", + function_id="custom-budget", + kwargs={}, dynamic_async_success_callbacks=[limiter], ) payload: Final = { - "url": "https://upstream.test/echo", "request_body": body, "request_method": "POST", "cost_per_request": 0.25, + "url": "https://upstream.test/echo", + "request_body": body, + "request_method": "POST", + "cost_per_request": 0.25, } kwargs: Final = HttpPassThroughEndpointHelpers._init_kwargs_for_pass_through_endpoint( - request=request, user_api_key_dict=auth, passthrough_logging_payload=payload, logging_obj=logging_obj, - _parsed_body=body, litellm_call_id="custom-budget", + request=request, + user_api_key_dict=auth, + passthrough_logging_payload=payload, + logging_obj=logging_obj, + _parsed_body=body, + litellm_call_id="custom-budget", ) logging_obj.update_environment_variables( - model="upstream-only-model", user="unknown", optional_params={}, - litellm_params=kwargs["litellm_params"], call_type="pass_through_endpoint", + model="upstream-only-model", + user="unknown", + optional_params={}, + litellm_params=kwargs["litellm_params"], + call_type="pass_through_endpoint", ) response: Final = httpx.Response( - 200, request=httpx.Request("POST", "https://upstream.test/echo"), json={"ok": True}, + 200, + request=httpx.Request("POST", "https://upstream.test/echo"), + json={"ok": True}, ) await PassThroughEndpointLogging().pass_through_async_success_handler( - httpx_response=response, response_body={"ok": True}, request_body=body, logging_obj=logging_obj, - url_route="https://upstream.test/echo", result=response.text, start_time=start, end_time=datetime.now(), - cache_hit=False, **kwargs, + httpx_response=response, + response_body={"ok": True}, + request_body=body, + logging_obj=logging_obj, + url_route="https://upstream.test/echo", + result=response.text, + start_time=start, + end_time=datetime.now(), + cache_hit=False, + **kwargs, ) assert logging_obj.model_call_details["response_cost"] == 0.25 assert await limiter.is_team_within_model_budget("shared-team", budget, None, "managed-model") metadata: Final = kwargs["litellm_params"]["metadata"] assert (metadata["model_group"], metadata["customer_label"]) == ("managed-model", "retained") - assert metadata.keys().isdisjoint({ - "user_api_key_model_max_budget", "user_api_key_team_model_max_budget", - "user_api_key_user_model_max_budget", "user_api_key_end_user_model_max_budget", - }) + assert metadata.keys().isdisjoint( + { + "user_api_key_model_max_budget", + "user_api_key_team_model_max_budget", + "user_api_key_user_model_max_budget", + "user_api_key_end_user_model_max_budget", + } + ) @pytest.mark.parametrize("metadata_slot", ["metadata", "litellm_metadata"]) def test_builtin_passthrough_pins_model_group_to_the_resolved_model(metadata_slot: str) -> None: - request: Final = Request({ - "type": "http", "method": "POST", "path": "/gemini/v1beta/models/gemini-2.5-flash:generateContent", - "headers": [], "query_string": b"", - }) + request: Final = Request( + { + "type": "http", + "method": "POST", + "path": "/gemini/v1beta/models/gemini-2.5-flash:generateContent", + "headers": [], + "query_string": b"", + } + ) kwargs: Final = HttpPassThroughEndpointHelpers._init_kwargs_for_pass_through_endpoint( - request=request, user_api_key_dict=UserAPIKeyAuth(token="hash", user_id="u-1"), - passthrough_logging_payload=MagicMock(), logging_obj=MagicMock(), + request=request, + user_api_key_dict=UserAPIKeyAuth(token="hash", user_id="u-1"), + passthrough_logging_payload=MagicMock(), + logging_obj=MagicMock(), _parsed_body={"contents": [], metadata_slot: {"model_group": "unbounded-client-choice"}}, ) assert kwargs["litellm_params"]["metadata"]["model_group"] == "gemini-2.5-flash" @@ -8591,6 +8599,581 @@ def test_passthrough_client_cannot_forge_session_id_omission(client_metadata_key ) +_GEMINI_FLASH_TARGET: Final = "https://generativelanguage.googleapis.com/v1beta/models/gemini-3.8-flash:generateContent" +_VERTEX_FLASH_TARGET: Final = ( + "https://us-east5-aiplatform.googleapis.com/v1/projects/p/locations/us-east5" + "/publishers/google/models/gemini-3.8-flash:generateContent" +) +_VERTEX_CLAUDE_RAW_PREDICT_TARGET: Final = ( + "https://us-east5-aiplatform.googleapis.com/v1/projects/p/locations/us-east5" + "/publishers/anthropic/models/claude-opus-5:rawPredict" +) +_VERTEX_SEARCH_TARGET: Final = ( + "https://discoveryengine.googleapis.com/v1/projects/p/locations/global" + "/dataStores/default/servingConfigs/default:search" +) +_RELAYED_GEMINI_COUNT_TOKENS_TARGET: Final = "http://127.0.0.1:30732/v1beta/models/gemini-3.8-flash:countTokens" +_TRANSLATE_TARGET: Final = "https://translate.googleapis.com/v3/projects/p/locations/global:translateText" +_NON_GOOGLE_VERTEX_SHAPED_TARGET: Final = "https://llm.example.com/v1/projects/a/locations/b/models/my-model:predict" +_VERTEX_LIVE_TARGET: Final = ( + "wss://us-central1-aiplatform.googleapis.com/ws/google.cloud.aiplatform.v1.LlmBidiService/BidiGenerateContent" +) +_VEO_OPERATION_SUFFIX: Final = "/models/veo-3.0-generate-001/operations/op-8f2c1" +_RELAYED_VERTEX_COUNT_TOKENS_TARGET: Final = ( + "http://127.0.0.1:30732/v1/projects/p/locations/us-east5/publishers/google/models/gemini-3.8-flash:countTokens" +) + + +@dataclass(frozen=True, slots=True) +class _RecordedFailure: + request_data: dict[str, object] + original_exception: Exception + user_api_key_dict: UserAPIKeyAuth + traceback_str: str | None + + +class _FailureHookRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.failures: tuple[_RecordedFailure, ...] = () + + async def async_post_call_failure_hook( + self, + request_data: dict[str, object], + original_exception: Exception, + user_api_key_dict: UserAPIKeyAuth, + traceback_str: str | None = None, + ) -> None: + self.failures = ( + *self.failures, + _RecordedFailure(request_data, original_exception, user_api_key_dict, traceback_str), + ) + + +@dataclass(frozen=True, slots=True) +class _PassthroughRun: + failure: _RecordedFailure + response: Response | None + + +async def _drained_pass_through_response( + request: Request, + target: str, + custom_llm_provider: str | None, + user_api_key_dict: UserAPIKeyAuth, + stream: bool, +) -> Response | None: + try: + received: Final = await pass_through_request( + request=request, + target=target, + custom_headers={}, + user_api_key_dict=user_api_key_dict, + custom_llm_provider=custom_llm_provider, + stream=stream, + ) + if isinstance(received, StreamingResponse): + async for _ in received.body_iterator: + pass + except (ProxyException, httpx.HTTPError): + return None + return received if isinstance(received, Response) else None + + +async def _run_passthrough_failure( + target: str, + custom_llm_provider: str | None, + upstream: httpx.Response | Exception, + stream: bool = False, + user_api_key_dict: UserAPIKeyAuth | None = None, + dispatched_endpoint: Callable[..., object] | None = None, + guardrail: CustomGuardrail | None = None, + body: bytes = b'{"contents": "not a list"}', +) -> _RecordedFailure: + run: Final = await _run_passthrough( + target, custom_llm_provider, upstream, stream, user_api_key_dict, dispatched_endpoint, guardrail, body + ) + return run.failure + + +async def _run_passthrough( + target: str, + custom_llm_provider: str | None, + upstream: httpx.Response | Exception, + stream: bool = False, + user_api_key_dict: UserAPIKeyAuth | None = None, + dispatched_endpoint: Callable[..., object] | None = None, + guardrail: CustomGuardrail | None = None, + body: bytes = b'{"contents": "not a list"}', +) -> _PassthroughRun: + recorder: Final = _FailureHookRecorder() + with ( + pytest.MonkeyPatch.context() as litellm_patch, + patch( # test-quality-ok: proxy_logging_obj is a proxy_server module global read inside pass_through_request; there is no injection seam + "litellm.proxy.proxy_server.proxy_logging_obj", ProxyLogging(user_api_key_cache=DualCache()) + ), + respx.mock(assert_all_called=False) as upstream_mock, + ): + litellm_patch.setattr(litellm, "disable_aiohttp_transport", True) + litellm_patch.setattr(litellm, "callbacks", [recorder] if guardrail is None else [guardrail, recorder]) + litellm.in_memory_llm_clients_cache.flush_cache() + upstream_route: Final = upstream_mock.route(method="POST") + if isinstance(upstream, Exception): + upstream_route.mock(side_effect=upstream) + else: + upstream_route.mock(return_value=upstream) + mock_request: Final = MagicMock(spec=Request) + mock_request.method = "POST" + proxy_url: Final = httpx.URL(target.replace("https://", "http://test-proxy.com/")) + mock_request.url = proxy_url + mock_request.scope = { + "type": "http", + "method": "POST", + "path": proxy_url.path, + **({"endpoint": dispatched_endpoint} if dispatched_endpoint is not None else {}), + } + mock_request.body = AsyncMock(return_value=body) + mock_request.headers = Headers({"content-type": "application/json"}) + mock_request.query_params = QueryParams({}) + response: Final = await _drained_pass_through_response( + mock_request, + target, + custom_llm_provider, + user_api_key_dict or UserAPIKeyAuth(api_key="sk-test"), + stream, + ) + await asyncio.sleep(0) + litellm.in_memory_llm_clients_cache.flush_cache() + assert len(recorder.failures) == 1 + return _PassthroughRun(recorder.failures[0], response) + + +def _upstream_400(target: str) -> httpx.Response: + return httpx.Response( + status_code=400, + headers={"content-type": "application/json"}, + content=json.dumps({"error": {"code": 400, "status": "INVALID_ARGUMENT"}}).encode(), + request=httpx.Request("POST", target), + ) + + +def _upstream_400_as_event_stream(target: str) -> httpx.Response: + return httpx.Response( + status_code=400, + headers={"content-type": "text/event-stream"}, + content=b'data: {"error": {"code": 400, "status": "INVALID_ARGUMENT"}}\n\n', + request=httpx.Request("POST", target), + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "target, custom_llm_provider, stream, expected_provider", + [ + (_GEMINI_FLASH_TARGET, "gemini", False, "gemini"), + (_GEMINI_FLASH_TARGET, "gemini", True, "gemini"), + (_VERTEX_FLASH_TARGET, None, False, "vertex_ai"), + (_VERTEX_FLASH_TARGET, None, True, "vertex_ai"), + (_GEMINI_FLASH_TARGET.replace(":generateContent", ":countTokens"), "gemini", False, "gemini"), + (_VERTEX_FLASH_TARGET.replace(":generateContent", ":countTokens"), None, False, "vertex_ai"), + (_RELAYED_GEMINI_COUNT_TOKENS_TARGET, "gemini", False, "gemini"), + ], +) +async def test_passthrough_upstream_error_logs_model_from_url( + target: str, custom_llm_provider: str | None, stream: bool, expected_provider: str +): + failure: Final = await _run_passthrough_failure(target, custom_llm_provider, _upstream_400(target), stream=stream) + assert failure.request_data["model"] == "gemini-3.8-flash" + assert failure.request_data["custom_llm_provider"] == expected_provider + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "target", + [ + _RELAYED_VERTEX_COUNT_TOKENS_TARGET, + _TRANSLATE_TARGET, + _NON_GOOGLE_VERTEX_SHAPED_TARGET, + ], +) +async def test_passthrough_upstream_error_to_a_host_outside_gemini_and_vertex_gets_no_url_attribution(target: str): + failure: Final = await _run_passthrough_failure(target, None, _upstream_400(target)) + assert failure.request_data["model"] == "" + assert failure.request_data.get("custom_llm_provider") is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("target", [_GEMINI_FLASH_TARGET, _VERTEX_FLASH_TARGET]) +async def test_passthrough_upstream_error_on_a_user_defined_route_to_google_gets_no_url_attribution(target: str): + failure: Final = await _run_passthrough_failure( + target, None, _upstream_400(target), dispatched_endpoint=_marked_pass_through_endpoint() + ) + assert failure.request_data["model"] == "" + assert failure.request_data.get("custom_llm_provider") is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "target, custom_llm_provider, expected_provider", + [ + (_GEMINI_FLASH_TARGET, "gemini", "gemini"), + (_VERTEX_FLASH_TARGET, None, "vertex_ai"), + ], +) +async def test_passthrough_detected_stream_upstream_error_logs_model_from_url( + target: str, custom_llm_provider: str | None, expected_provider: str +): + failure: Final = await _run_passthrough_failure( + target, custom_llm_provider, _upstream_400_as_event_stream(target), stream=False + ) + assert failure.request_data["model"] == "gemini-3.8-flash" + assert failure.request_data["custom_llm_provider"] == expected_provider + + +@pytest.mark.asyncio +async def test_passthrough_internal_failure_logs_model_from_url(): + failure: Final = await _run_passthrough_failure( + _GEMINI_FLASH_TARGET, "gemini", httpx.ReadTimeout("upstream timed out") + ) + assert failure.request_data["model"] == "gemini-3.8-flash" + assert failure.request_data["custom_llm_provider"] == "gemini" + + +def _upstream_dropping_stream(target: str) -> httpx.Response: + return httpx.Response( + status_code=200, + headers={"content-type": "text/event-stream"}, + stream=_UpstreamDroppingMidStream(), + request=httpx.Request("POST", target), + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [True, False]) +@pytest.mark.parametrize( + "target, custom_llm_provider, expected_provider", + [ + (_GEMINI_FLASH_TARGET.replace(":generateContent", ":streamGenerateContent?alt=sse"), "gemini", "gemini"), + (_VERTEX_FLASH_TARGET.replace(":generateContent", ":streamGenerateContent?alt=sse"), None, "vertex_ai"), + ], +) +async def test_passthrough_mid_stream_drop_logs_model_from_url( + target: str, custom_llm_provider: str | None, expected_provider: str, stream: bool +): + failure: Final = await _run_passthrough_failure( + target, custom_llm_provider, _upstream_dropping_stream(target), stream=stream + ) + assert isinstance(failure.original_exception, httpx.ReadError) + assert failure.request_data["model"] == "gemini-3.8-flash" + assert failure.request_data["custom_llm_provider"] == expected_provider + + +class _ModifyResponseOnPreCallGuardrail(CustomGuardrail): + async def async_pre_call_hook( + self, + user_api_key_dict: UserAPIKeyAuth, + cache: DualCache, + data: dict[str, object], + call_type: str, + ) -> None: + self.raise_passthrough_exception(violation_message="blocked by policy", request_data=data, detection_info=None) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "target, custom_llm_provider, body, expected_model, expected_provider", + [ + (_GEMINI_FLASH_TARGET, "gemini", b'{"contents": "not a list"}', "gemini-3.8-flash", "gemini"), + (_VERTEX_FLASH_TARGET, None, b'{"contents": "not a list"}', "gemini-3.8-flash", "vertex_ai"), + (_VERTEX_FLASH_TARGET, None, b'{"custom_llm_provider": "gemini"}', "gemini-3.8-flash", "vertex_ai"), + ], +) +async def test_passthrough_guardrail_block_logs_model_from_url( + target: str, custom_llm_provider: str | None, body: bytes, expected_model: str, expected_provider: str +): + guardrail: Final = _ModifyResponseOnPreCallGuardrail( + guardrail_name="block-policy", event_hook=GuardrailEventHooks.pre_call, default_on=True + ) + failure: Final = await _run_passthrough_failure( + target, custom_llm_provider, _upstream_400(target), guardrail=guardrail, body=body + ) + assert isinstance(failure.original_exception, ModifyResponseException) + assert failure.request_data["model"] == expected_model + assert failure.request_data["custom_llm_provider"] == expected_provider + assert PrometheusLogger._extract_api_provider_from_request_data(failure.request_data) == expected_provider + assert get_spend_logs_id(CallTypes.pass_through.value, {}, failure.request_data) is not None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("body", [b'{"contents": "not a list"}', b'{"litellm_call_id": "client-chosen-id"}']) +async def test_passthrough_guardrail_block_response_names_the_call_id_of_its_spend_row(body: bytes): + guardrail: Final = _ModifyResponseOnPreCallGuardrail( + guardrail_name="block-policy", event_hook=GuardrailEventHooks.pre_call, default_on=True + ) + run: Final = await _run_passthrough( + _GEMINI_FLASH_TARGET, "gemini", _upstream_400(_GEMINI_FLASH_TARGET), guardrail=guardrail, body=body + ) + assert isinstance(run.failure.original_exception, ModifyResponseException) + assert run.response is not None + assert run.response.headers["x-litellm-call-id"] == get_spend_logs_id( + CallTypes.pass_through.value, {}, run.failure.request_data + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("body", [b'{"contents": "not a list"}', b'{"custom_llm_provider": "vertex_ai"}']) +async def test_passthrough_guardrail_block_on_a_user_defined_route_gets_no_url_attribution(body: bytes): + guardrail: Final = _ModifyResponseOnPreCallGuardrail( + guardrail_name="block-policy", event_hook=GuardrailEventHooks.pre_call, default_on=True + ) + failure: Final = await _run_passthrough_failure( + _GEMINI_FLASH_TARGET, + None, + _upstream_400(_GEMINI_FLASH_TARGET), + dispatched_endpoint=_marked_pass_through_endpoint(), + guardrail=guardrail, + body=body, + ) + assert isinstance(failure.original_exception, ModifyResponseException) + assert failure.request_data["model"] == "" + assert "custom_llm_provider" not in failure.request_data + + +def _websocket_connect_failure(kind: str) -> Exception: + from websockets.datastructures import Headers as WebSocketHeaders + from websockets.exceptions import InvalidStatus + from websockets.http11 import Response as WebSocketHandshakeResponse + + if kind == "rejected_handshake": + return InvalidStatus(WebSocketHandshakeResponse(403, "Forbidden", WebSocketHeaders())) + return OSError("connection refused") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure_kind", ["rejected_handshake", "unreachable_host"]) +@pytest.mark.parametrize( + "target, builtin_provider_route, expected_provider", + [ + (_VERTEX_LIVE_TARGET, True, "vertex_ai"), + (_VERTEX_LIVE_TARGET, False, None), + ("wss://api.openai.com/v1/realtime?model=gpt-realtime", True, None), + ], +) +async def test_websocket_passthrough_connect_failure_logs_the_provider_of_the_upstream_url( + failure_kind: str, target: str, builtin_provider_route: bool, expected_provider: str | None +): + failure: Final = await _run_websocket_connect_failure(failure_kind, target, builtin_provider_route) + assert failure.request_data.get("custom_llm_provider") == expected_provider + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure_kind", ["rejected_handshake", "unreachable_host"]) +@pytest.mark.parametrize("model, expected_model", [("gemini-3.8-live", "gemini-3.8-live"), (None, "")]) +async def test_websocket_passthrough_connect_failure_logs_the_model_the_caller_named( + failure_kind: str, model: str | None, expected_model: str +): + failure: Final = await _run_websocket_connect_failure(failure_kind, _VERTEX_LIVE_TARGET, True, model=model) + assert failure.request_data["model"] == expected_model + + +async def _run_websocket_connect_failure( + failure_kind: str, target: str, builtin_provider_route: bool, model: str | None = None +) -> _RecordedFailure: + recorder: Final = _FailureHookRecorder() + with ( + pytest.MonkeyPatch.context() as litellm_patch, + patch( # test-quality-ok: proxy_logging_obj is a proxy_server module global read inside websocket_passthrough_request; there is no injection seam + "litellm.proxy.proxy_server.proxy_logging_obj", ProxyLogging(user_api_key_cache=DualCache()) + ), + patch( + "litellm.proxy.pass_through_endpoints.pass_through_endpoints.connect", + side_effect=_websocket_connect_failure(failure_kind), + ), + ): + litellm_patch.setattr(litellm, "callbacks", [recorder]) + await websocket_passthrough_request( + websocket=_client_websocket(_pending_receive), + target=target, + custom_headers={}, + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), + endpoint="/vertex_ai/live", + accept_websocket=False, + builtin_provider_route=builtin_provider_route, + model=model, + ) + assert len(recorder.failures) == 1 + return recorder.failures[0] + + +@pytest.mark.parametrize( + "parsed_body, url, custom_llm_provider, expected_model, expected_provider", + [ + ({"model": "from-body"}, _GEMINI_FLASH_TARGET, "gemini", "from-body", "gemini"), + ({"contents": []}, _GEMINI_FLASH_TARGET, "gemini", "gemini-3.8-flash", "gemini"), + ({"contents": []}, _VERTEX_FLASH_TARGET, None, "gemini-3.8-flash", "vertex_ai"), + ({"contents": []}, _VERTEX_FLASH_TARGET, "anthropic", "", "anthropic"), + ({}, "https://generativelanguage.googleapis.com/v1beta/openai/models/gemini-3.8-flash", "openai", "", "openai"), + ({"custom_llm_provider": "from-body"}, _VERTEX_FLASH_TARGET, None, "gemini-3.8-flash", "vertex_ai"), + ({"custom_llm_provider": "gemini"}, _VERTEX_FLASH_TARGET, "anthropic", "", "anthropic"), + ({"contents": []}, _GEMINI_FLASH_TARGET, None, "gemini-3.8-flash", "gemini"), + ( + {}, + "https://generativelanguage.googleapis.com/v1beta/models/gemini-3.8-flash?key=provider-secret", + "gemini", + "gemini-3.8-flash", + "gemini", + ), + ( + {}, + "https://us-east5-aiplatform.googleapis.com/v1/projects/p/locations/us-east5" + "/publishers/google/models/gemini-3.8-flash?key=provider-secret", + None, + "gemini-3.8-flash", + "vertex_ai", + ), + ({}, _VERTEX_CLAUDE_RAW_PREDICT_TARGET, None, "vertex_ai/claude-opus-5", "vertex_ai"), + ( + {}, + _VERTEX_CLAUDE_RAW_PREDICT_TARGET.replace(":rawPredict", ":streamRawPredict?alt=sse"), + None, + "claude-opus-5", + "vertex_ai", + ), + ({}, "https://generativelanguage.googleapis.com/v1beta/models?pageToken=/models/x", "gemini", "", "gemini"), + ( + {"contents": []}, + "https://evilaiplatform.googleapis.com/v1/projects/p/locations/l" + "/publishers/google/models/gemini-3.8-flash:generateContent", + None, + "", + None, + ), + ({}, _VERTEX_SEARCH_TARGET, None, "vertex_ai/search_api", "vertex_ai"), + ({}, f"{_VERTEX_FLASH_TARGET}?tag=rawPredict", None, "gemini-3.8-flash", "vertex_ai"), + ( + {}, + "https://us-east5-aiplatform.googleapis.com/v1/projects/p/locations/us-east5/batchPredictionJobs" + "?filter=:search", + None, + "", + "vertex_ai", + ), + ( + {}, + f"https://generativelanguage.googleapis.com/v1beta{_VEO_OPERATION_SUFFIX}", + "gemini", + "veo-3.0-generate-001", + "gemini", + ), + ( + {}, + "https://us-central1-aiplatform.googleapis.com/v1/projects/p/locations/us-central1" + f"/publishers/google{_VEO_OPERATION_SUFFIX}", + None, + "veo-3.0-generate-001", + "vertex_ai", + ), + ( + {"contents": []}, + "https://aiplatform.us.rep.googleapis.com/v1/projects/p/locations/us" + "/publishers/google/models/gemini-3.8-flash:generateContent", + None, + "gemini-3.8-flash", + "vertex_ai", + ), + ( + {}, + "https://us-east5-aiplatform.googleapis.com/v1/projects/p/locations/us-east5/batchPredictionJobs", + None, + "", + "vertex_ai", + ), + ( + {"contents": []}, + _GEMINI_FLASH_TARGET.replace(":generateContent", ":countTokens"), + "gemini", + "gemini-3.8-flash", + "gemini", + ), + ( + {"contents": []}, + _VERTEX_FLASH_TARGET.replace(":generateContent", ":countTokens"), + None, + "gemini-3.8-flash", + "vertex_ai", + ), + ( + {"contents": []}, + "https://aiplatform.googleapis.com/v1/projects/p/locations/global" + "/publishers/google/models/gemini-3.8-flash:countTokens", + None, + "gemini-3.8-flash", + "vertex_ai", + ), + ( + {}, + "https://us-central1-aiplatform.googleapis.com/v1/projects/p/locations/us-central1/cachedContents", + None, + "", + "vertex_ai", + ), + ({"contents": []}, _RELAYED_GEMINI_COUNT_TOKENS_TARGET, "gemini", "gemini-3.8-flash", "gemini"), + ({"contents": []}, _RELAYED_VERTEX_COUNT_TOKENS_TARGET, None, "", None), + ({"contents": []}, _TRANSLATE_TARGET, None, "", None), + ({"contents": []}, _NON_GOOGLE_VERTEX_SHAPED_TARGET, None, "", None), + ({"contents": []}, "https://api.example.com/v1/models/gpt-x:countTokens", None, "", None), + ({"contents": []}, "https://api.example.com/v1/models/gpt-x:predict", None, "", None), + ({"contents": []}, "https://api.example.com/v1/models/gpt-x:thing", None, "", None), + (None, None, None, "", None), + ], +) +def test_build_passthrough_failure_request_payload_model_and_provider_precedence( + parsed_body: dict | None, + url: str | None, + custom_llm_provider: str | None, + expected_model: str, + expected_provider: str | None, +): + request_payload: Final = _build_passthrough_failure_request_payload( + parsed_body=parsed_body, + kwargs=None, + logging_obj=None, + custom_llm_provider=custom_llm_provider, + url=httpx.URL(url) if url else None, + builtin_provider_route=True, + ) + assert request_payload["model"] == expected_model + assert request_payload.get("custom_llm_provider") == expected_provider + + +@pytest.mark.parametrize( + "url_route, expected_model", + [ + (_VERTEX_FLASH_TARGET, "gemini-3.8-flash"), + (_VERTEX_FLASH_TARGET.replace(":generateContent", ":streamGenerateContent?alt=sse"), "gemini-3.8-flash"), + (_VERTEX_FLASH_TARGET.replace(":generateContent", ":predict"), "gemini-3.8-flash"), + (_VERTEX_FLASH_TARGET.replace(":generateContent", ":embedContent"), "gemini-3.8-flash"), + (_VERTEX_CLAUDE_RAW_PREDICT_TARGET, "vertex_ai/claude-opus-5"), + (_VERTEX_CLAUDE_RAW_PREDICT_TARGET.replace(":rawPredict", ":streamRawPredict"), "vertex_ai/claude-opus-5"), + ( + _VERTEX_CLAUDE_RAW_PREDICT_TARGET.replace(":rawPredict", ":streamRawPredict?alt=sse"), + "claude-opus-5", + ), + (_VERTEX_CLAUDE_RAW_PREDICT_TARGET.replace(":rawPredict", ":rawPredict?alt=sse"), "claude-opus-5"), + (f"{_VERTEX_FLASH_TARGET}?tag=rawPredict", "gemini-3.8-flash"), + (_VERTEX_SEARCH_TARGET, "vertex_ai/search_api"), + ("https://us-east5-aiplatform.googleapis.com/v1/projects/p/locations/us-east5/batchPredictionJobs", None), + ( + "https://us-east5-aiplatform.googleapis.com/v1/projects/p/locations/us-east5/batchPredictionJobs" + "?filter=:search", + None, + ), + ], +) +def test_vertex_failure_model_name_matches_success_handler_naming(url_route: str, expected_model: str | None): + assert VertexPassthroughLoggingHandler.model_from_url_route(url_route) == expected_model + + @pytest.mark.parametrize("client_metadata_key", ["litellm_metadata", "metadata"]) def test_passthrough_logs_the_resolved_deployment_model_info_over_the_request_body(client_metadata_key: str): """A provider route that resolved a router deployment stashes its model_info on request.state. That @@ -8598,7 +9181,9 @@ def test_passthrough_logs_the_resolved_deployment_model_info_over_the_request_bo the call to (LIT-1761: passthrough successes carried model_id="").""" mock_request = MagicMock(spec=Request) mock_request.method = "POST" - mock_request.url = httpx.URL("http://0.0.0.0:4000/vertex_ai/v1/projects/p/locations/global/publishers/google/models/gemini-3.8-flash:generateContent") + mock_request.url = httpx.URL( + "http://0.0.0.0:4000/vertex_ai/v1/projects/p/locations/global/publishers/google/models/gemini-3.8-flash:generateContent" + ) mock_request.headers = Headers({}) mock_request.scope = {} mock_request.state = SimpleNamespace( @@ -9668,12 +10253,14 @@ async def _drain_logging_worker(): await GLOBAL_LOGGING_WORKER.stop() yield + @pytest.fixture() def _vcr_outcome_gate(request, vcr): install_live_call_probe(request, vcr) yield record_vcr_outcome(request, vcr) + @pytest.mark.usefixtures("_drain_logging_worker", "_vcr_outcome_gate") def test_update_pass_through_route_updates_registry(): """ @@ -9763,9 +10350,7 @@ def mock_request(): return self._dict.values() class MockRequest: - def __init__( - self, headers=None, method="POST", request_body: Optional[dict] = None - ): + def __init__(self, headers=None, method="POST", request_body: Optional[dict] = None): self.headers = headers or {} self.query_params = QueryParams() self.method = method @@ -9873,9 +10458,9 @@ def test_pass_through_routes_support_all_methods(): path = route.path methods = set(route.methods) allowed = PROTOCOL_CONSTRAINED_PASS_THROUGH_ROUTES.get(path, expected_methods) - assert ( - methods == allowed - ), f"Route {path} does not support all methods. Supported: {methods}, Expected: {allowed}" + assert methods == allowed, ( + f"Route {path} does not support all methods. Supported: {methods}, Expected: {allowed}" + ) check_router_methods(llm_router) @@ -9905,19 +10490,11 @@ def test_is_bedrock_agent_runtime_route(): # Test agent runtime endpoints (should return True) assert _is_bedrock_agent_runtime_route("/knowledgebases/kb-123/retrieve") is True - assert ( - _is_bedrock_agent_runtime_route("/agents/knowledgebases/kb-123/retrieve") - is True - ) + assert _is_bedrock_agent_runtime_route("/agents/knowledgebases/kb-123/retrieve") is True # Test regular bedrock runtime endpoints (should return False) - assert ( - _is_bedrock_agent_runtime_route("/guardrail/test-id/version/1/apply") is False - ) - assert ( - _is_bedrock_agent_runtime_route("/model/cohere.command-r-v1:0/converse") - is False - ) + assert _is_bedrock_agent_runtime_route("/guardrail/test-id/version/1/apply") is False + assert _is_bedrock_agent_runtime_route("/model/cohere.command-r-v1:0/converse") is False assert _is_bedrock_agent_runtime_route("/some/random/endpoint") is False @@ -9984,9 +10561,7 @@ def test_custom_pricing_used_in_cost_calculation(): # Verify custom cost is different from standard cost (unless prices happen to match) # This confirms custom pricing is actually being applied - assert ( - custom_cost != standard_cost - ), "Custom pricing should produce different cost than standard pricing" + assert custom_cost != standard_cost, "Custom pricing should produce different cost than standard pricing" # Test 3: Custom pricing with cache_read_input_token_cost and input_cost_per_token_batches # This specifically tests the parameters that were causing the original issue @@ -10562,6 +11137,7 @@ def test_init_kwargs_client_metadata_cannot_mark_a_pass_through_call_as_a_silent assert "is_silent_experiment" not in json.loads(row["metadata"]) assert result["litellm_params"]["metadata"]["client_key"] == "kept" + @pytest.mark.asyncio async def test_direct_passthrough_keeps_mapped_baggage_tags_outside_upstream_body(): from litellm.proxy.common_utils.http_parsing_utils import read_request_body From 8ac9b757370a27f51de80a8dcc84b26f743fe583 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 09:43:01 -0700 Subject: [PATCH 11/74] fix(model_prices): rolling registry audit, add Gemini batch and Azure Astra flex tiers (#49114) * fix(model_prices): add EmpirioLabs model rows from the provider catalog Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(model_prices): align EmpirioLabs rows with the registry endpoint vocabulary and schema Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(model_prices): absorb Gemini long-prompt batch rates and Azure GPT-6 Astra long-context flex rates Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(model_prices): drop EmpirioLabs rows from the registry audit --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 36 ++++++++++++++----- model_prices_and_context_window.json | 36 ++++++++++++++----- 2 files changed, 54 insertions(+), 18 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 0799a298942..28a8fe83fa2 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -8715,7 +8715,7 @@ "search_context_size_low": 0.014, "search_context_size_medium": 0.014 }, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -8742,7 +8742,11 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05, + "cache_read_input_token_cost_above_272k_tokens_flex": 1e-06, + "input_cost_per_token_above_272k_tokens_flex": 1e-05, + "output_cost_per_token_above_272k_tokens_flex": 3.75e-05 }, "azure/gpt-6-astra-2026-09-03": { "cache_creation_input_token_cost": 1.25e-05, @@ -8763,7 +8767,7 @@ "search_context_size_low": 0.014, "search_context_size_medium": 0.014 }, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -8790,7 +8794,11 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05, + "cache_read_input_token_cost_above_272k_tokens_flex": 1e-06, + "input_cost_per_token_above_272k_tokens_flex": 1e-05, + "output_cost_per_token_above_272k_tokens_flex": 3.75e-05 }, "azure/gpt-6-luna": { "cache_creation_input_token_cost": 1.25e-07, @@ -30856,8 +30864,9 @@ "output_cost_per_token": 0, "output_vector_size": 3072, "rpm": 10000, - "source": "https://ai.google.dev/gemini-api/docs/embeddings#model-versions", - "tpm": 10000000 + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "tpm": 10000000, + "input_cost_per_token_batches": 7.5e-08 }, "gemini/gemini-embedding-2": { "input_cost_per_audio_token": 6.5e-06, @@ -31713,7 +31722,10 @@ "input_cost_per_token_flex": 6.25e-07, "output_cost_per_token_batches": 5e-06, "output_cost_per_token_flex": 5e-06, - "supports_url_context": true + "supports_url_context": true, + "cache_read_input_token_cost_above_200k_tokens_batches": 2.5e-07, + "input_cost_per_token_above_200k_tokens_batches": 1.25e-06, + "output_cost_per_token_above_200k_tokens_batches": 7.5e-06 }, "gemini/gemini-2.5-computer-use-preview-10-2025": { "deprecation_date": "2026-07-28", @@ -32280,7 +32292,10 @@ "web_search_billing_unit": "per_query", "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_token_flex": 1e-06, - "output_cost_per_token_flex": 6e-06 + "output_cost_per_token_flex": 6e-06, + "cache_read_input_token_cost_above_200k_tokens_batches": 4e-07, + "input_cost_per_token_above_200k_tokens_batches": 2e-06, + "output_cost_per_token_above_200k_tokens_batches": 9e-06 }, "gemini/gemini-3.1-pro-preview-customtools": { "prompt_cache_min_tokens": 4096, @@ -32343,7 +32358,10 @@ "web_search_billing_unit": "per_query", "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_token_flex": 1e-06, - "output_cost_per_token_flex": 6e-06 + "output_cost_per_token_flex": 6e-06, + "cache_read_input_token_cost_above_200k_tokens_batches": 4e-07, + "input_cost_per_token_above_200k_tokens_batches": 2e-06, + "output_cost_per_token_above_200k_tokens_batches": 9e-06 }, "gemini-3-flash-preview": { "cache_read_input_audio_token_cost": 1e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 0799a298942..28a8fe83fa2 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -8715,7 +8715,7 @@ "search_context_size_low": 0.014, "search_context_size_medium": 0.014 }, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -8742,7 +8742,11 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05, + "cache_read_input_token_cost_above_272k_tokens_flex": 1e-06, + "input_cost_per_token_above_272k_tokens_flex": 1e-05, + "output_cost_per_token_above_272k_tokens_flex": 3.75e-05 }, "azure/gpt-6-astra-2026-09-03": { "cache_creation_input_token_cost": 1.25e-05, @@ -8763,7 +8767,7 @@ "search_context_size_low": 0.014, "search_context_size_medium": 0.014 }, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -8790,7 +8794,11 @@ "supports_tool_choice": true, "supports_vision": true, "supports_web_search": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05, + "cache_read_input_token_cost_above_272k_tokens_flex": 1e-06, + "input_cost_per_token_above_272k_tokens_flex": 1e-05, + "output_cost_per_token_above_272k_tokens_flex": 3.75e-05 }, "azure/gpt-6-luna": { "cache_creation_input_token_cost": 1.25e-07, @@ -30856,8 +30864,9 @@ "output_cost_per_token": 0, "output_vector_size": 3072, "rpm": 10000, - "source": "https://ai.google.dev/gemini-api/docs/embeddings#model-versions", - "tpm": 10000000 + "source": "https://ai.google.dev/gemini-api/docs/pricing", + "tpm": 10000000, + "input_cost_per_token_batches": 7.5e-08 }, "gemini/gemini-embedding-2": { "input_cost_per_audio_token": 6.5e-06, @@ -31713,7 +31722,10 @@ "input_cost_per_token_flex": 6.25e-07, "output_cost_per_token_batches": 5e-06, "output_cost_per_token_flex": 5e-06, - "supports_url_context": true + "supports_url_context": true, + "cache_read_input_token_cost_above_200k_tokens_batches": 2.5e-07, + "input_cost_per_token_above_200k_tokens_batches": 1.25e-06, + "output_cost_per_token_above_200k_tokens_batches": 7.5e-06 }, "gemini/gemini-2.5-computer-use-preview-10-2025": { "deprecation_date": "2026-07-28", @@ -32280,7 +32292,10 @@ "web_search_billing_unit": "per_query", "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_token_flex": 1e-06, - "output_cost_per_token_flex": 6e-06 + "output_cost_per_token_flex": 6e-06, + "cache_read_input_token_cost_above_200k_tokens_batches": 4e-07, + "input_cost_per_token_above_200k_tokens_batches": 2e-06, + "output_cost_per_token_above_200k_tokens_batches": 9e-06 }, "gemini/gemini-3.1-pro-preview-customtools": { "prompt_cache_min_tokens": 4096, @@ -32343,7 +32358,10 @@ "web_search_billing_unit": "per_query", "google_maps_grounding_cost_per_query": 0.014, "input_cost_per_token_flex": 1e-06, - "output_cost_per_token_flex": 6e-06 + "output_cost_per_token_flex": 6e-06, + "cache_read_input_token_cost_above_200k_tokens_batches": 4e-07, + "input_cost_per_token_above_200k_tokens_batches": 2e-06, + "output_cost_per_token_above_200k_tokens_batches": 9e-06 }, "gemini-3-flash-preview": { "cache_read_input_audio_token_cost": 1e-07, From 9432bf6b8ee686d0fbd06863e5f75889d383c8a2 Mon Sep 17 00:00:00 2001 From: tavren27 <348969968+tavren27@users.noreply.github.com> Date: Fri, 30 Oct 2026 10:01:44 -0700 Subject: [PATCH 12/74] perf(streaming): stop raising caught AttributeErrors inside pydantic on every streamed chunk (#49166) * perf(streaming): stop raising AttributeErrors inside pydantic on every streamed chunk Every later plain chunk on openai and azure streams raised and caught 12 AttributeErrors inside pydantic. Assigning an undeclared extra goes through a class lookup that raises in the model metaclass and is never memoized, and the hidden_params getter went through the private attribute fallback set_extra_attribute writes the extra and fields_set directly when a cached per-class check says pydantic would do the same, and falls back to setattr otherwise. The hidden_params getter reads the private storage directly. Emitted chunk JSON is unchanged * fix(streaming): match pydantic's own extra-field bookkeeping on every supported version Take the direct extra write only when the installed pydantic itself adds an assigned extra to model_fields_set (2.13 and later) and the class defines no attribute of that name, so pydantic 2.11, 2.12 and 2.14 see the same model state as a plain setattr. A deleted _hidden_params raises AttributeError again, and the mutable test models carry frozen-ok reasons. * fix(streaming): keep the normal write for models that do not allow extra fields openai's construct gives every model an extras dict, so a forbid model built that way still needs setattr's ValueError instead of a silent extra write. * test(streaming): cover a ClassVar declared without a value in the extra-field parity test * fix(streaming): type the streamed choice's logprobs as the parsed value main now stores --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/litellm_core_utils/core_helpers.py | 7 +- litellm/litellm_core_utils/hidden_params.py | 35 ++- .../litellm_core_utils/streaming_handler.py | 17 +- litellm/types/utils.py | 32 ++- .../litellm_core_utils/test_hidden_params.py | 139 +++++++++++- .../test_streaming_handler.py | 211 +++++++++++++++--- 6 files changed, 388 insertions(+), 53 deletions(-) diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 2f81154ab83..20e10f63f83 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -9,12 +9,13 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeVar, cast, runtime_checkable import httpx -from pydantic import TypeAdapter, ValidationError +from pydantic import BaseModel, TypeAdapter, ValidationError from typing_extensions import TypeIs # noqa: TID251 # narrows a duck-typed value to the Protocol its caller reads import litellm from litellm._logging import verbose_logger from litellm.constants import KUBERNETES_POD_ROUTING_KEY +from litellm.litellm_core_utils.hidden_params import set_extra_attribute from litellm.litellm_core_utils.llm_response_utils.get_headers import get_provider_request_id from litellm.types.llms.openai import AllMessageValues, OpenAIChatCompletionFinishReason @@ -710,9 +711,9 @@ def preserve_upstream_non_openai_attributes( _assign_attributes(model_response, original_chunk.model_dump(include=frozenset(candidate_keys)).items()) -def _assign_attributes(target: object, values: Iterable[tuple[str, object]]) -> None: +def _assign_attributes(target: BaseModel, values: Iterable[tuple[str, object]]) -> None: for key, value in values: - setattr(target, key, value) + set_extra_attribute(target, key, value) def safe_deep_copy(data): diff --git a/litellm/litellm_core_utils/hidden_params.py b/litellm/litellm_core_utils/hidden_params.py index a74337d2549..c04d7614b90 100644 --- a/litellm/litellm_core_utils/hidden_params.py +++ b/litellm/litellm_core_utils/hidden_params.py @@ -1,8 +1,9 @@ from collections.abc import Iterator, Mapping, MutableMapping +from functools import cache, lru_cache from types import MappingProxyType from typing import Final, Protocol, cast -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict, create_model from litellm.types.llms.base import HiddenParams @@ -60,6 +61,38 @@ def stored_attribute(obj: object, name: str) -> object: return extras.get(name) +@cache +def _assigned_extras_join_fields_set() -> bool: + probe: Final = create_model("_ExtraAssignmentProbe", __config__=ConfigDict(extra="allow"))() + setattr(probe, "assigned", None) + return "assigned" in probe.model_fields_set + + +@lru_cache(maxsize=1024) +def _writes_plain_extra(model_class: type[BaseModel], name: str) -> bool: + config: Final = model_class.model_config + return ( + model_class.__setattr__ is BaseModel.__setattr__ + and config.get("extra") == "allow" + and not config.get("frozen") + and not config.get("validate_assignment") + and not name.startswith("_") + and name not in model_class.__pydantic_fields__ + and name not in model_class.__class_vars__ + and not hasattr(model_class, name) + and _assigned_extras_join_fields_set() + ) + + +def set_extra_attribute(model: BaseModel, name: str, value: object) -> None: + extras: Final = model.__pydantic_extra__ + if extras is None or not _writes_plain_extra(type(model), name): + setattr(model, name, value) + return + extras[name] = value + model.__pydantic_fields_set__.add(name) + + def _get_hidden_params_storage(obj: object) -> object | None: if isinstance(obj, dict): return obj.get(_HIDDEN_PARAMS_ATTR) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 098a404ea14..b1b3e20f218 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -28,6 +28,7 @@ from litellm.litellm_core_utils.core_helpers import chunk_usage, provider_report from litellm.litellm_core_utils.hidden_params import ( HIDDEN_PARAMS_ATTR, get_hidden_params, + set_extra_attribute, set_hidden_param, stored_attribute, ) @@ -1035,7 +1036,7 @@ class CustomStreamWrapper: _initial_delta: Final = model_response.choices[0].delta.model_dump() _initial_delta.pop("role", None) - model_response.choices[0].delta = Delta(**_initial_delta) + set_extra_attribute(model_response.choices[0], "delta", Delta(**_initial_delta)) return model_response def _prebuilt_plain_delta(self, chunk: object) -> Delta | None: @@ -1061,13 +1062,13 @@ class CustomStreamWrapper: def _set_role_on_fresh_delta(self, model_response: ModelResponseStream) -> None: delta: Final = model_response.choices[0].delta if self.sent_first_chunk is False: - delta.role = "assistant" + set_extra_attribute(delta, "role", "assistant") self.sent_first_chunk = True return if not _holds_only_scalars(delta): self.strip_role_from_delta(model_response) return - delta.role = None + set_extra_attribute(delta, "role", None) if "provider_specific_fields" not in delta.model_fields_set: delta.provider_specific_fields = None @@ -1186,8 +1187,8 @@ class CustomStreamWrapper: and _holds_only_scalars(upstream_delta) ) if can_reuse_choice and upstream_choice is not None: - model_response.choices[0].finish_reason = None # rebind-ok: keep reused choice - model_response.choices[0].index = upstream_choice.index # rebind-ok: keep reused choice + set_extra_attribute(model_response.choices[0], "finish_reason", None) + set_extra_attribute(model_response.choices[0], "index", upstream_choice.index) _prepend_upstream_choice_extras( model_response.choices[0], upstream_choice, @@ -1212,11 +1213,7 @@ class CustomStreamWrapper: else: return model_response.system_fingerprint = upstream_chunk.system_fingerprint - setattr( - model_response, - "citations", - stored_attribute(upstream_chunk, "citations"), - ) + set_extra_attribute(model_response, "citations", stored_attribute(upstream_chunk, "citations")) preserve_upstream_non_openai_attributes( model_response=model_response, original_chunk=original_chunk, diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 55aaf606175..f8115571dfd 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -75,6 +75,7 @@ from litellm.types.workload_identity import ( ) from ..litellm_core_utils.core_helpers import map_finish_reason, process_response_headers +from ..litellm_core_utils.hidden_params import set_extra_attribute from . import litellm_params as _litellm_params from .agents import LiteLLMSendMessageResponse from .guardrails import GuardrailEventHooks @@ -2149,6 +2150,13 @@ class UnreportedUsage(Usage): class StreamingChoices(OpenAIObject): + if TYPE_CHECKING: + finish_reason: str | None + index: int + delta: Delta + enhancements: object + logprobs: ChoiceLogprobs | JsonValue + def __init__( self, finish_reason=None, @@ -2162,25 +2170,22 @@ class StreamingChoices(OpenAIObject): # https://github.com/BerriAI/litellm/issues/8455 params.pop("message", None) super().__init__(**params) - if finish_reason: - self.finish_reason = map_finish_reason(finish_reason) - else: - self.finish_reason = None - self.index = index + set_extra_attribute(self, "finish_reason", map_finish_reason(finish_reason) if finish_reason else None) + set_extra_attribute(self, "index", index) if delta is not None: if isinstance(delta, Delta): - self.delta = delta + set_extra_attribute(self, "delta", delta) elif isinstance(delta, dict): - self.delta = Delta(**delta) + set_extra_attribute(self, "delta", Delta(**delta)) else: - self.delta = Delta() + set_extra_attribute(self, "delta", Delta()) if enhancements is not None: - self.enhancements = enhancements + set_extra_attribute(self, "enhancements", enhancements) if isinstance(logprobs, (dict, BaseModel)): - self.logprobs = CHOICE_LOGPROBS_OR_JSON.validate_python(logprobs) + set_extra_attribute(self, "logprobs", CHOICE_LOGPROBS_OR_JSON.validate_python(logprobs)) else: - self.logprobs = logprobs + set_extra_attribute(self, "logprobs", logprobs) def __contains__(self, key) -> bool: # Define custom behavior for the 'in' operator @@ -2234,7 +2239,10 @@ class ModelResponseBase(OpenAIObject): @property def hidden_params(self) -> dict[str, object]: # mutable-ok: API requires mutation - return self._hidden_params + private: Final = self.__pydantic_private__ + if private is None or "_hidden_params" not in private: + return self._hidden_params + return private["_hidden_params"] @hidden_params.setter def hidden_params(self, hidden_params: dict[str, object]) -> None: # mutable-ok: API requires mutation diff --git a/tests/unit/litellm_core_utils/test_hidden_params.py b/tests/unit/litellm_core_utils/test_hidden_params.py index c0485a07696..b32692c67ba 100644 --- a/tests/unit/litellm_core_utils/test_hidden_params.py +++ b/tests/unit/litellm_core_utils/test_hidden_params.py @@ -1,7 +1,11 @@ -from typing import Final +import pickle +from collections.abc import Callable +from typing import ClassVar, Final import pytest +from openai import BaseModel as OpenAIBaseModel from openai.types.chat import ChatCompletionChunk +from pydantic import BaseModel, ConfigDict, PrivateAttr from litellm.litellm_core_utils.hidden_params import ( HIDDEN_PARAMS_ATTR, @@ -9,13 +13,14 @@ from litellm.litellm_core_utils.hidden_params import ( get_hidden_params_storage, get_or_create_hidden_params, served_from_cache, + set_extra_attribute, set_hidden_param, set_hidden_params, ) from litellm.llms.openai.completion.transformation import OpenAITextCompletionConfig from litellm.types.decisions import DecisionsResponse from litellm.types.llms.base import HiddenParams -from litellm.types.utils import ModelResponse, TextChoices, TextCompletionResponse, Usage +from litellm.types.utils import ModelResponse, ModelResponseStream, TextChoices, TextCompletionResponse, Usage def test_get_and_set_hidden_params_on_plain_object() -> None: @@ -309,3 +314,133 @@ def test_served_from_cache_is_true_only_for_a_response_the_cache_stamped( assert served_from_cache(model_response) is expected assert served_from_cache({HIDDEN_PARAMS_ATTR: dict(hidden_params)}) is expected assert served_from_cache(object()) is False + + +class _ExtraModel(BaseModel): # frozen-ok: the assignment tests need a model that accepts attribute writes + model_config = ConfigDict(extra="allow") + + declared: str | None = None + marker: ClassVar[str | None] = None + unset_marker: ClassVar[int] + _secret: str = PrivateAttr(default="hidden") + + @property + def doubled(self) -> str | None: + return self.declared + + @doubled.setter + def doubled(self, value: str) -> None: + self.declared = value * 2 + + +class _FrozenExtraModel(BaseModel): + model_config = ConfigDict(extra="allow", frozen=True) + + +class _ValidatedExtraModel(BaseModel): # frozen-ok: the assignment tests need a model that accepts attribute writes + model_config = ConfigDict(extra="allow", validate_assignment=True) + + __pydantic_extra__: dict[str, int] + + +class _IgnoredExtraModel(BaseModel): # frozen-ok: the assignment tests need a model that accepts attribute writes + model_config = ConfigDict(extra="ignore") + + +class _MirroringModel(BaseModel): # frozen-ok: the assignment tests need a model that accepts attribute writes + model_config = ConfigDict(extra="allow") + + def __setattr__(self, name: str, value: object) -> None: + super().__setattr__(name, value) + super().__setattr__(f"{name}_copy", value) + + +class _NoneClassAttribute: + inherited = None + + +class _InheritingExtraModel(_NoneClassAttribute, BaseModel): # frozen-ok: assignment tests need attribute writes + model_config = ConfigDict(extra="allow") + + +class _ForbiddingOpenAIModel(OpenAIBaseModel): # frozen-ok: assignment tests need attribute writes + model_config = ConfigDict(extra="forbid") + + +def _assignment_outcome(model: BaseModel, assign: Callable[[], None], name: str) -> tuple[object, ...]: + try: + assign() + except (AttributeError, ValueError) as error: + return type(error), str(error) + return model.model_dump(), getattr(model, name), model.model_fields_set, model.model_dump_json() + + +@pytest.mark.parametrize( + "build, name, value", + [ + pytest.param(lambda: _ExtraModel(existing=1, later=3), "fresh", 1, id="new-extra"), + pytest.param(lambda: _ExtraModel(existing=1, later=3), "existing", 2, id="existing-extra-keeps-order"), + pytest.param(lambda: _ExtraModel(existing=1, later=3), "declared", "x", id="declared-field"), + pytest.param(lambda: _ExtraModel(existing=1, later=3), "doubled", "ab", id="property"), + pytest.param(lambda: _ExtraModel(existing=1, later=3), "_free", 1, id="undeclared-underscore-name"), + pytest.param(lambda: _ExtraModel(existing=1, later=3), "_secret", "x", id="private-attribute"), + pytest.param(lambda: _ExtraModel(existing=1, later=3), "marker", 1, id="class-var"), + pytest.param(lambda: _ExtraModel(existing=1, later=3), "unset_marker", 1, id="class-var-without-value"), + pytest.param(lambda: _ExtraModel(existing=1, later=3), "model_config", 1, id="class-attribute"), + pytest.param(_FrozenExtraModel, "fresh", 1, id="frozen"), + pytest.param(_ValidatedExtraModel, "fresh", "5", id="validate-assignment"), + pytest.param(_IgnoredExtraModel, "fresh", 1, id="extra-ignore"), + pytest.param(_MirroringModel, "fresh", 1, id="setattr-override"), + pytest.param(_InheritingExtraModel, "inherited", 1, id="class-attribute-set-to-none"), + pytest.param( + lambda: _ForbiddingOpenAIModel.construct(existing=1), "fresh", 1, id="extra-forbid-built-by-construct" + ), + ], +) +def test_set_extra_attribute_matches_setattr(build: Callable[[], BaseModel], name: str, value: object) -> None: + through_setattr: Final = build() + through_helper: Final = build() + + expected: Final = _assignment_outcome(through_setattr, lambda: setattr(through_setattr, name, value), name) + actual: Final = _assignment_outcome(through_helper, lambda: set_extra_attribute(through_helper, name, value), name) + + assert actual == expected + + +@pytest.mark.parametrize("response_type", [ModelResponse, ModelResponseStream]) +def test_hidden_params_raises_attribute_error_once_deleted( + response_type: type[ModelResponse | ModelResponseStream], +) -> None: + response: Final = response_type() + del response._hidden_params + + assert getattr(response, "hidden_params", "fallback") == "fallback" + + +def _round_trip(response: ModelResponse | ModelResponseStream) -> ModelResponse | ModelResponseStream: + return pickle.loads(pickle.dumps(response)) + + +@pytest.mark.parametrize("response_type", [ModelResponse, ModelResponseStream]) +@pytest.mark.parametrize( + "rebuild", + [ + pytest.param(lambda response: response, id="init"), + pytest.param(lambda response: response.model_copy(), id="model-copy"), + pytest.param(lambda response: response.model_copy(deep=True), id="deep-copy"), + pytest.param(_round_trip, id="pickle"), + ], +) +def test_hidden_params_is_the_stored_dict_on_every_construction_path( + response_type: type[ModelResponse | ModelResponseStream], + rebuild: Callable[[ModelResponse | ModelResponseStream], ModelResponse | ModelResponseStream], +) -> None: + original: Final = response_type() + original.hidden_params["model_id"] = "deployment-1" + response: Final = rebuild(original) + + response.hidden_params["api_base"] = "https://example.com" + + assert response.hidden_params is response._hidden_params + assert get_hidden_params(response) == response._hidden_params + assert response._hidden_params["api_base"] == "https://example.com" diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index b9c8feb223d..148b7193ef6 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -11,7 +11,7 @@ import time from collections.abc import Mapping, Sequence from dataclasses import dataclass from pathlib import Path -from types import CodeType, FrameType, MappingProxyType +from types import CodeType, FrameType, MappingProxyType, TracebackType from typing import AsyncIterator, Callable, Final, Iterator, Literal, NoReturn, Optional, cast from unittest.mock import AsyncMock, MagicMock, Mock, patch @@ -6292,53 +6292,76 @@ def test_openai_chunk_creator_skips_full_chunk_dump_and_delta_rebuild( assert finish_output.choices[0].delta.content is None -def _openai_chunk(chunk_id: str, delta: Mapping[str, object]) -> ChatCompletionChunk: +def _openai_chunk( + chunk_id: str, + delta: Mapping[str, object], + choice_extras: Mapping[str, object] = MappingProxyType({}), + finish_reason: str | None = None, + chunk_extras: Mapping[str, object] = MappingProxyType({}), +) -> ChatCompletionChunk: return ChatCompletionChunk( id=chunk_id, created=1, model="gpt-4o", object="chat.completion.chunk", - choices=[{"index": 0, "delta": delta, "finish_reason": None}], + choices=[{"index": 0, "delta": delta, "finish_reason": finish_reason, **choice_extras}], + **chunk_extras, ) -def _missed_attributes(wrapper: CustomStreamWrapper, chunks: Sequence[ChatCompletionChunk]) -> tuple[str, ...]: - attribute_lookup: Final = BaseModel.__getattr__.__code__ - misses: Final[list[str]] = [] +def _raised_attribute_errors(wrapper: CustomStreamWrapper, chunks: Sequence[ChatCompletionChunk]) -> tuple[str, ...]: + raised: Final[list[str]] = [] - def record_miss(frame: FrameType, event: str, arg: object) -> Callable[[FrameType, str, object], object]: - if event == "exception" and isinstance(arg, tuple) and isinstance(arg[1], AttributeError): - misses.append(str(frame.f_locals["item"])) - return record_miss - - def trace_lookups(frame: FrameType, event: str, _arg: object) -> Callable[[FrameType, str, object], object] | None: - return record_miss if event == "call" and frame.f_code is attribute_lookup else None + def record_raise(frame: FrameType, event: str, arg: object) -> Callable[[FrameType, str, object], object]: + if event == "exception" and isinstance(arg, tuple): + _, error, traceback = arg + if isinstance(error, AttributeError) and isinstance(traceback, TracebackType) and traceback.tb_next is None: + raised.append(f"{frame.f_code.co_qualname}: {error}") + return record_raise previous_tracer: Final = sys.gettrace() - sys.settrace(trace_lookups) + sys.settrace(record_raise) try: tuple(wrapper.chunk_creator(chunk=chunk) for chunk in chunks) finally: sys.settrace(previous_tracer) - return tuple(misses) + return tuple(raised) -def test_openai_chunk_creator_builds_one_delta_and_misses_no_attribute_per_later_chunk( - logging_obj: Logging, +_AZURE_CHOICE_EXTRAS: Final = MappingProxyType( + {"content_filter_results": {"hate": {"filtered": False, "severity": "safe"}}} +) +_OPENAI_AND_AZURE_CHOICE_EXTRAS: Final = ( + pytest.param("openai", MappingProxyType({}), id="openai"), + pytest.param("azure", _AZURE_CHOICE_EXTRAS, id="azure"), +) + + +@pytest.mark.parametrize(("custom_llm_provider", "choice_extras"), _OPENAI_AND_AZURE_CHOICE_EXTRAS) +def test_openai_chunk_creator_builds_one_delta_and_raises_no_attribute_error_per_later_chunk( + logging_obj: Logging, custom_llm_provider: str, choice_extras: Mapping[str, object] ) -> None: wrapper: Final = CustomStreamWrapper( completion_stream=iter(()), model="gpt-4o", - custom_llm_provider="openai", + custom_llm_provider=custom_llm_provider, logging_obj=logging_obj, ) - wrapper.chunk_creator(chunk=_openai_chunk("chunk-1", {"role": "assistant", "content": ""})) - plain_chunks: Final = ( - _openai_chunk("chunk-2", {"content": "hello"}), - _openai_chunk("chunk-3", {"content": " world"}), + + def later_chunk(chunk_id: str, delta: Mapping[str, object]) -> ChatCompletionChunk: + return _openai_chunk( + chunk_id, delta, choice_extras, chunk_extras={"service_tier": "default", "obfuscation": chunk_id} + ) + + nested_delta: Final = {"content": "!", "reasoning_details": [{"type": "reasoning.text", "text": "thought"}]} + first_chunk_of_each_shape: Final = ( + later_chunk("chunk-1", {"role": "assistant", "content": ""}), + later_chunk("chunk-2", nested_delta), ) - nested_chunk: Final = _openai_chunk( - "chunk-4", {"content": "!", "reasoning_details": [{"type": "reasoning.text", "text": "thought"}]} + tuple(wrapper.chunk_creator(chunk=chunk) for chunk in first_chunk_of_each_shape) + plain_chunks: Final = ( + later_chunk("chunk-3", {"content": "hello"}), + later_chunk("chunk-4", {"content": " world"}), ) delta_init: Final = Delta.__init__.__code__ streaming_choices_init: Final = StreamingChoices.__init__.__code__ @@ -6348,7 +6371,145 @@ def test_openai_chunk_creator_builds_one_delta_and_misses_no_attribute_per_later assert sum(call.code is delta_init for call in calls) == len(plain_chunks) assert sum(call.code is streaming_choices_init for call in calls) == len(plain_chunks) assert [output.choices[0].delta.content if output else None for output in outputs] == ["hello", " world"] - assert _missed_attributes(wrapper, (*plain_chunks, nested_chunk)) == () + assert _raised_attribute_errors(wrapper, (*plain_chunks, later_chunk("chunk-5", nested_delta))) == () + + +_OPENAI_STREAM_JSON: Final = ( + ( + '{"id":"chatcmpl-1","created":1,"model":"gpt-4o","object":"chat.completion.chunk",' + '"system_fingerprint":null,"choices":[{"finish_reason":null,"index":0,' + '"delta":{"provider_specific_fields":null,"refusal":null,"content":"","role":"assistant",' + '"function_call":null,"tool_calls":null,"audio":null},"logprobs":null}],' + '"provider_specific_fields":null,"citations":null,"service_tier":"default","obfuscation":"o0"}' + ), + ( + '{"id":"chatcmpl-1","created":1,"model":"gpt-4o","object":"chat.completion.chunk",' + '"system_fingerprint":null,"choices":[{"finish_reason":null,"index":0,' + '"delta":{"provider_specific_fields":null,"refusal":null,"content":"one","role":null,' + '"function_call":null,"tool_calls":null,"audio":null},"logprobs":null}],' + '"provider_specific_fields":null,"citations":null,"service_tier":"default","obfuscation":"o1"}' + ), + ( + '{"id":"chatcmpl-1","created":1,"model":"gpt-4o","object":"chat.completion.chunk",' + '"system_fingerprint":null,"choices":[{"finish_reason":null,"index":0,' + '"delta":{"provider_specific_fields":null,"refusal":null,"content":" two","role":null,' + '"function_call":null,"tool_calls":null,"audio":null},"logprobs":null}],' + '"provider_specific_fields":null,"citations":null,"service_tier":"default","obfuscation":"o2"}' + ), + ( + '{"id":"chatcmpl-1","created":1,"model":"gpt-4o","object":"chat.completion.chunk",' + '"system_fingerprint":null,"choices":[{"finish_reason":"stop","index":0,' + '"delta":{"provider_specific_fields":null,"content":null,"role":null,"function_call":null,' + '"tool_calls":null,"audio":null},"logprobs":null}],"provider_specific_fields":null,' + '"service_tier":"default","obfuscation":"o3"}' + ), + ( + '{"id":"chatcmpl-1","created":1,"model":"gpt-4o","object":"chat.completion.chunk",' + '"system_fingerprint":null,"choices":[],"provider_specific_fields":null,' + '"usage":{"completion_tokens":3,"prompt_tokens":9,"total_tokens":12,"completion_tokens_details":null,' + '"prompt_tokens_details":null}}' + ), +) +_AZURE_STREAM_JSON: Final = ( + ( + '{"id":"chatcmpl-1","created":1,"model":"gpt-4o","object":"chat.completion.chunk",' + '"system_fingerprint":null,"choices":[{"content_filter_results":{"hate":{"filtered":false,' + '"severity":"safe"}},"finish_reason":null,"index":0,"delta":{"provider_specific_fields":null,' + '"refusal":null,"content":"","role":"assistant","function_call":null,"tool_calls":null,"audio":null},' + '"logprobs":null}],"provider_specific_fields":null,"citations":null,"service_tier":"default",' + '"obfuscation":"o0"}' + ), + ( + '{"id":"chatcmpl-1","created":1,"model":"gpt-4o","object":"chat.completion.chunk",' + '"system_fingerprint":null,"choices":[{"content_filter_results":{"hate":{"filtered":false,' + '"severity":"safe"}},"finish_reason":null,"index":0,"delta":{"provider_specific_fields":null,' + '"refusal":null,"content":"one","role":null,"function_call":null,"tool_calls":null,"audio":null},' + '"logprobs":null}],"provider_specific_fields":null,"citations":null,"service_tier":"default",' + '"obfuscation":"o1"}' + ), + ( + '{"id":"chatcmpl-1","created":1,"model":"gpt-4o","object":"chat.completion.chunk",' + '"system_fingerprint":null,"choices":[{"content_filter_results":{"hate":{"filtered":false,' + '"severity":"safe"}},"finish_reason":null,"index":0,"delta":{"provider_specific_fields":null,' + '"refusal":null,"content":" two","role":null,"function_call":null,"tool_calls":null,"audio":null},' + '"logprobs":null}],"provider_specific_fields":null,"citations":null,"service_tier":"default",' + '"obfuscation":"o2"}' + ), + ( + '{"id":"chatcmpl-1","created":1,"model":"gpt-4o","object":"chat.completion.chunk",' + '"system_fingerprint":null,"choices":[{"finish_reason":"stop","index":0,' + '"delta":{"provider_specific_fields":null,"content":null,"role":null,"function_call":null,' + '"tool_calls":null,"audio":null},"logprobs":null}],"provider_specific_fields":null,' + '"service_tier":"default","obfuscation":"o3"}' + ), + ( + '{"id":"chatcmpl-1","created":1,"model":"gpt-4o","object":"chat.completion.chunk",' + '"system_fingerprint":null,"choices":[],"provider_specific_fields":null,' + '"usage":{"completion_tokens":3,"prompt_tokens":9,"total_tokens":12,"completion_tokens_details":null,' + '"prompt_tokens_details":null}}' + ), +) + + +@pytest.mark.parametrize( + ("custom_llm_provider", "choice_extras", "expected_json"), + [ + pytest.param("openai", MappingProxyType({}), _OPENAI_STREAM_JSON, id="openai"), + pytest.param("azure", _AZURE_CHOICE_EXTRAS, _AZURE_STREAM_JSON, id="azure"), + ], +) +def test_openai_chunk_creator_emits_each_chunk_as_the_same_json( + logging_obj: Logging, + custom_llm_provider: str, + choice_extras: Mapping[str, object], + expected_json: tuple[str | None, ...], +) -> None: + wrapper: Final = CustomStreamWrapper( + completion_stream=iter(()), + model="gpt-4o", + custom_llm_provider=custom_llm_provider, + logging_obj=logging_obj, + stream_options={"include_usage": True}, + ) + chunks: Final = ( + _openai_chunk( + "chatcmpl-1", + {"role": "assistant", "content": ""}, + choice_extras, + chunk_extras={"service_tier": "default", "obfuscation": "o0"}, + ), + _openai_chunk( + "chatcmpl-1", + {"content": "one"}, + choice_extras, + chunk_extras={"service_tier": "default", "obfuscation": "o1"}, + ), + _openai_chunk( + "chatcmpl-1", + {"content": " two"}, + choice_extras, + chunk_extras={"service_tier": "default", "obfuscation": "o2"}, + ), + _openai_chunk( + "chatcmpl-1", {}, choice_extras, "stop", chunk_extras={"service_tier": "default", "obfuscation": "o3"} + ), + ChatCompletionChunk( + id="chatcmpl-1", + created=1, + model="gpt-4o", + object="chat.completion.chunk", + choices=[], + service_tier="default", + obfuscation="o4", + usage={"prompt_tokens": 9, "completion_tokens": 3, "total_tokens": 12}, + ), + ) + + outputs: Final = tuple(wrapper.chunk_creator(chunk=chunk) for chunk in chunks) + + assert tuple( + output.model_copy(update={"created": 1}).model_dump_json() if output else None for output in outputs + ) == expected_json def test_chunk_creator_skips_full_dump_of_model_response_stream_chunks( From ab3f6caddcd096b820359a7ae2829e9440b77c4f Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 10:18:29 -0700 Subject: [PATCH 13/74] fix(gemini): flag computer use support on the four models the Gemini computer use docs list (#49252) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 10 +++++++--- model_prices_and_context_window.json | 10 +++++++--- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 28a8fe83fa2..e1131117fcf 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -31883,7 +31883,8 @@ }, "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "supports_computer_use": true }, "gemini/gemini-3-flash-preview": { "cache_read_input_audio_token_cost": 1e-07, @@ -31949,7 +31950,8 @@ "input_cost_per_token_flex": 2.5e-07, "output_cost_per_token_batches": 1.5e-06, "output_cost_per_token_flex": 1.5e-06, - "supports_audio_input": true + "supports_audio_input": true, + "supports_computer_use": true }, "gemini/gemini-3.5-flash": { "prompt_cache_min_tokens": 4096, @@ -32196,7 +32198,8 @@ }, "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "supports_computer_use": true }, "gemini/gemini-omni-flash-preview": { "input_cost_per_audio_token": 1.5e-06, @@ -76558,6 +76561,7 @@ ], "supports_audio_input": true, "supports_audio_output": false, + "supports_computer_use": true, "supports_function_calling": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 28a8fe83fa2..e1131117fcf 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -31883,7 +31883,8 @@ }, "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "supports_computer_use": true }, "gemini/gemini-3-flash-preview": { "cache_read_input_audio_token_cost": 1e-07, @@ -31949,7 +31950,8 @@ "input_cost_per_token_flex": 2.5e-07, "output_cost_per_token_batches": 1.5e-06, "output_cost_per_token_flex": 1.5e-06, - "supports_audio_input": true + "supports_audio_input": true, + "supports_computer_use": true }, "gemini/gemini-3.5-flash": { "prompt_cache_min_tokens": 4096, @@ -32196,7 +32198,8 @@ }, "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "web_search_billing_unit": "per_query", - "google_maps_grounding_cost_per_query": 0.014 + "google_maps_grounding_cost_per_query": 0.014, + "supports_computer_use": true }, "gemini/gemini-omni-flash-preview": { "input_cost_per_audio_token": 1.5e-06, @@ -76558,6 +76561,7 @@ ], "supports_audio_input": true, "supports_audio_output": false, + "supports_computer_use": true, "supports_function_calling": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, From 372c096388f3102419e0177f501f42126c7170c7 Mon Sep 17 00:00:00 2001 From: Etienne Chabert Date: Fri, 30 Oct 2026 18:20:00 +0100 Subject: [PATCH 14/74] fix(proxy): evict a deleted key from every worker's auth cache (#47609) * fix(proxy): evict a deleted key from every worker's auth cache /key/delete dropped the key object only from the in-memory cache of the worker that served the call. Auth serves key objects cache-first, so every other worker, and the Redis copy when enable_redis_auth_cache is on, kept authenticating the deleted key until user_api_key_cache_ttl expired. /key/update already evicts and broadcasts through evict_and_broadcast, delete now does the same for the tokens it deletes * fix(proxy): evict only the deleted rows and broadcast them in one batch Scope the cross-worker eviction on /key/delete to the rows the caller was allowed to delete and send it through delete_cache_key_objects, so a caller-supplied name can no longer drop another team's cached entries on every worker * refactor(proxy): drop the comment above the delete eviction call * fix(proxy): run the usage cache eviction beside the auth eviction so a hung Redis costs one wait * test(proxy): wait on the broadcast instead of a sleep and type the peer eviction tests * test(proxy): wait for the peer to apply the broadcast instead of polling its cache * test(proxy): cover hashed keys and aliases when deleting a key warmed on a peer * fix(key_management): evict every requested key on delete, including keys whose row is already gone A batch delete that names a live key and a key whose row was removed outside the API evicted only the live key, so the gone key kept authenticating from every worker's cache. Every requested key hash is evicted now, while raw non-key cache keys are still left alone * test(key_management): split the gone-row eviction assertion into one comprehension per worker --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/proxy/auth/auth_checks.py | 37 +-- .../key_management_endpoints.py | 15 +- .../management/test_key_delete_revocation.py | 15 +- tests/unit/proxy/auth/test_auth_checks.py | 50 +++- .../test_key_management_endpoints.py | 231 +++++++++++++++++- 5 files changed, 317 insertions(+), 31 deletions(-) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index f80296f4bd7..79a46ef355e 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -3457,28 +3457,35 @@ async def delete_cache_key_objects( proxy_logging_obj: ProxyLogging | None, ) -> None: """ - Evict a batch of key objects, for callers that delete keys in bulk rather than through - `/key/delete`. Auth resolves a cached key object without re-reading its team, so a key left - cached after its row is gone keeps buying access until its TTL expires. - - Evicting locally only reaches this worker; `_delete_cache_key_object` itself broadcasts each - token, so a deleted key left in a peer worker's in-memory cache still authenticates there until - its TTL expires. + Evict a batch of deleted key objects here and on every other worker. Auth resolves a cached key + object without re-reading its team, so a key left cached after its row is gone keeps buying + access until its TTL expires. Best-effort per key: the rows are already deleted by the time this runs, so an unreachable cache backend must not abort the caller partway through its own cascade. - Every token is dropped from this worker's memory first, then from the auth cache and the - internal usage cache in one batch each, so a deleted key stops authenticating here the instant - its rows are gone instead of refilling from Redis on the next miss. Every other worker then - gets the eviction in one pipelined broadcast: a publish per key would pass the publisher's - pending cap on a bulk delete, and the dropped keys would keep authenticating there until their - TTL. + Every token is dropped from this worker's memory first, then from the auth cache's Redis layer, + so a deleted key stops authenticating here the instant its rows are gone instead of refilling + from Redis on the next miss. Every other worker then gets the eviction in one pipelined + broadcast: a publish per key would pass the publisher's pending cap on a bulk delete, and the + dropped keys would keep authenticating there until their TTL. The internal usage cache is not + an auth refill source, so its batch delete runs beside that chain and a hung Redis costs one + bounded wait there instead of another in line. """ evict_local(cache_keys=hashed_tokens, user_api_key_cache=user_api_key_cache) + usage_eviction: Final = ( + () + if proxy_logging_obj is None + else (evict_shared(cache_keys=hashed_tokens, cache=proxy_logging_obj.internal_usage_cache.dual_cache),) + ) + _ = await asyncio.gather( + _evict_auth_cache_then_broadcast(hashed_tokens=hashed_tokens, user_api_key_cache=user_api_key_cache), + *usage_eviction, + ) + + +async def _evict_auth_cache_then_broadcast(hashed_tokens: Sequence[str], user_api_key_cache: UserApiKeyCache) -> None: await evict_shared(cache_keys=hashed_tokens, cache=user_api_key_cache) - if proxy_logging_obj is not None: - await evict_shared(cache_keys=hashed_tokens, cache=proxy_logging_obj.internal_usage_cache.dual_cache) await broadcast_evictions(cache_keys=hashed_tokens) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index a6814b593ca..3021b21de51 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -50,6 +50,7 @@ from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module expo _delete_cache_key_object, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export can_team_access_model, delete_cache_key_object, + delete_cache_key_objects, get_jwt_key_mapping_cache_keys_for_token, get_key_end_user_budget_id, get_org_object, @@ -78,7 +79,7 @@ from litellm.proxy.common_utils.config_sync_pubsub import ( from litellm.proxy.common_utils.openai_error_payload import openai_error_param, proxy_exception_for from litellm.proxy.common_utils.rbac_utils import check_org_admin_can_generate_keys from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time -from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET, UserApiKeyCache +from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET, UserApiKeyCache, is_user_key_cache_key from litellm.proxy.hooks.key_management_event_hooks import KeyManagementEventHooks from litellm.proxy.hooks.model_max_budget_limiter import build_model_max_budget_usage from litellm.proxy.management.teams.authz import TEAM_ADMIN_ONLY, TEAM_OR_ORG_ADMIN, is_team_admin @@ -5483,7 +5484,7 @@ async def delete_verification_tokens( - List of keys being deleted, this contains information about the key_alias, token, and user_id being deleted, this is passed down to the KeyManagementEventHooks to delete the keys from the secret manager and handle audit logs """ - from litellm.proxy.proxy_server import prisma_client + from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj failed_tokens: list = [] try: @@ -5567,11 +5568,11 @@ async def delete_verification_tokens( verbose_proxy_logger.debug(traceback.format_exc()) raise e - for key in tokens: - user_api_key_cache.delete_cache(key) - # remove hash token from cache - hashed_token = hash_token(cast(str, key)) - user_api_key_cache.delete_cache(hashed_token) + await delete_cache_key_objects( + hashed_tokens=tuple(token for token in hashed_tokens if is_user_key_cache_key(token)), + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) # After credential invalidation, so a failure here can never keep a deleted key alive. for deleted_key in authorized_keys: diff --git a/tests/integration/management/test_key_delete_revocation.py b/tests/integration/management/test_key_delete_revocation.py index 5b3527a52ae..bdc91dfe5f6 100644 --- a/tests/integration/management/test_key_delete_revocation.py +++ b/tests/integration/management/test_key_delete_revocation.py @@ -171,11 +171,8 @@ def test_deleting_a_key_revokes_it(gateway: Gateway, spelling: str) -> None: assert deleted_rows == (_DeletedKeyRow(token=key_hash),) -def test_deleting_a_key_revokes_it_on_a_warmed_peer(gateway: Gateway, peer: Gateway) -> None: - pytest.skip( - "BUG: POST /key/delete does not broadcast the key eviction, so a warmed peer keeps answering 200 for about " - "60 seconds until its in-memory cache TTL expires" - ) +@pytest.mark.parametrize("spelling", ("keys", "hashed_keys", "key_aliases")) +def test_deleting_a_key_revokes_it_on_a_warmed_peer(gateway: Gateway, peer: Gateway, spelling: str) -> None: backend: Final = "integration-keys-auth-delete" prompts: Final = ( "delete warm primary " + uuid.uuid4().hex, @@ -208,9 +205,13 @@ def test_deleting_a_key_revokes_it_on_a_warmed_peer(gateway: Gateway, peer: Gate ) assert len(warm_responses) == 2 assert tuple((request.method, request.target) for request in wire.drain()) == (_EXPECTED_CHAT,) * 2 - deleted: Final = gateway.request("POST", "/key/delete", {"keys": [key]}) + identifier: Final = {"keys": key, "hashed_keys": sha256(key.encode()).hexdigest(), "key_aliases": alias}[ + spelling + ] + delete_body: Final = {"key_aliases" if spelling == "key_aliases" else "keys": [identifier]} + deleted: Final = gateway.request("POST", "/key/delete", delete_body) assert deleted.status_code == 200, deleted.text - assert _DeleteResponse.model_validate_json(deleted.content).deleted_keys == (key,), deleted.text + assert _DeleteResponse.model_validate_json(deleted.content).deleted_keys == (identifier,), deleted.text peer_refusal: Final = eventually( lambda: _chat(peer, model, key, prompts[2]), lambda response: response.status_code == 401, diff --git a/tests/unit/proxy/auth/test_auth_checks.py b/tests/unit/proxy/auth/test_auth_checks.py index e783f01769e..9d4fe67614e 100644 --- a/tests/unit/proxy/auth/test_auth_checks.py +++ b/tests/unit/proxy/auth/test_auth_checks.py @@ -46,7 +46,7 @@ from litellm.proxy.auth.auth_checks import ( _team_soft_budget_check, common_checks, ) -from litellm.proxy.utils import ProxyLogging +from litellm.proxy.utils import InternalUsageCache, ProxyLogging from litellm.proxy.utils import CallInfo from litellm.types.passthrough_endpoints.pass_through_endpoints import LITELLM_PASS_THROUGH_ENDPOINT_MARKER @@ -2425,3 +2425,51 @@ async def test_bulk_eviction_clears_the_redis_copies_before_waiting_on_a_wedged_ assert refilled == [] assert redis.entries == {} + + +class _SignallingPublishRedisClient(_SlowPublishRedisClient): + def __init__(self) -> None: + super().__init__() + self.sent = asyncio.Event() + + async def _send(self, messages: Sequence[str]) -> None: + await super()._send(messages) + self.sent.set() + + +class _DeleteAfterBroadcastRedisCache(_AuthRedisCache): + def __init__(self, broadcast_sent: asyncio.Event) -> None: + super().__init__({}) + self._broadcast_sent = broadcast_sent + self.finished = False + + async def delete_cache_keys(self, keys: Sequence[str]) -> None: + await self._broadcast_sent.wait() + self.finished = True + + +@pytest.mark.asyncio +async def test_bulk_eviction_broadcasts_while_the_usage_cache_redis_is_wedged( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(pubsub_module, "_in_flight_publishes", asyncio.Semaphore(16)) + monkeypatch.setattr(pubsub_module, "_pending_publishes", set()) + client: Final = _SignallingPublishRedisClient() + usage_redis: Final = _DeleteAfterBroadcastRedisCache(broadcast_sent=client.sent) + proxy_logging_obj: Final = ProxyLogging(user_api_key_cache=None) + proxy_logging_obj.internal_usage_cache = InternalUsageCache( + dual_cache=DualCache(redis_cache=usage_redis) # pyright: ignore[reportArgumentType] # a dict-backed fake of the Redis layer + ) + hashed_tokens: Final = ("token-a", "token-b") + + with patch( + "litellm.proxy.common_utils.auth_cache_invalidation_pubsub.coordination_redis_cache", + return_value=_PubSubRedisCache(client), + ): + await delete_cache_key_objects( + hashed_tokens=hashed_tokens, user_api_key_cache=UserApiKeyCache(), proxy_logging_obj=proxy_logging_obj + ) + await _drain_publishes() + + assert usage_redis.finished + assert {json.loads(message)["cache_key"] for message in client.published} == set(hashed_tokens) diff --git a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py index 62dd367b8e5..c26a4bf19b8 100644 --- a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py @@ -7,7 +7,7 @@ from datetime import datetime, timedelta, timezone from types import SimpleNamespace from typing import Final from unittest.mock import AsyncMock, MagicMock, patch -from uuid import UUID +from uuid import UUID, uuid4 import pytest import yaml @@ -16,6 +16,7 @@ from fastapi.testclient import TestClient from pydantic import JsonValue, ValidationError import litellm +from litellm.caching.in_memory_cache import InMemoryCache from litellm.constants import PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS from litellm.litellm_core_utils.duration_parser import duration_in_seconds from litellm.models.object_permission import LiteLLM_ObjectPermissionTable @@ -41,12 +42,15 @@ from litellm.proxy._types import ( RegenerateKeyRequest, ResetSpendRequest, UpdateKeyRequest, + hash_token, ) from litellm.proxy.auth.auth_checks import ( delete_cache_key_object, + get_key_object, jwt_key_mapping_cache_key, ) from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth +from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import AuthCacheInvalidationSubscriber from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache, project_cache_key from litellm.proxy.db.db_lookup_gate import db_lookup_stall_tracker from litellm.proxy.management_endpoints.key_management_endpoints import ( @@ -5755,6 +5759,231 @@ async def test_delete_verification_tokens_evicts_jwt_key_mapping_cache(monkeypat ) +class _ChannelSubscription: + """Marks a message done when the subscriber comes back for the next one, so ``messages.join()`` + returns once every delivered message has been applied""" + + def __init__(self) -> None: + self.messages: asyncio.Queue[dict[str, object]] = asyncio.Queue() + self._holding_message = False + + async def subscribe(self, *channels: str) -> None: + return None + + async def get_message(self, *, ignore_subscribe_messages: bool, timeout: float) -> dict[str, object] | None: + if self._holding_message: + self._holding_message = False + self.messages.task_done() + try: + message: Final = await asyncio.wait_for(self.messages.get(), timeout) + except asyncio.TimeoutError: + return None + self._holding_message = True + return message + + async def aclose(self) -> None: + return None + + +class _ChannelPipeline: + def __init__(self, redis: "_SharedCoordinationRedis") -> None: + self._redis = redis + self._messages: tuple[str, ...] = () + + def publish(self, channel: str, message: str) -> "_ChannelPipeline": + self._messages = (*self._messages, message) + return self + + async def execute(self) -> None: + for message in self._messages: + self._redis.deliver(message) + + +class _SharedCoordinationRedis: + """The coordination Redis two workers share: whatever one worker publishes reaches the other's subscriber""" + + namespace = None + + def __init__(self) -> None: + self.published: tuple[str, ...] = () + self._subscriptions: tuple[_ChannelSubscription, ...] = () + + def init_pubsub_client(self) -> "_SharedCoordinationRedis": + return self + + def pubsub(self) -> _ChannelSubscription: + subscription: Final = _ChannelSubscription() + self._subscriptions = (*self._subscriptions, subscription) + return subscription + + def pipeline(self, transaction: bool = True) -> _ChannelPipeline: + return _ChannelPipeline(self) + + async def publish(self, channel: str, message: str) -> int: + self.deliver(message) + return 1 + + def deliver(self, message: str) -> None: + self.published = (*self.published, message) + for subscription in self._subscriptions: + subscription.messages.put_nowait({"type": "message", "data": message.encode()}) + + async def until_every_subscriber_applied_its_messages(self) -> None: + _ = await asyncio.gather(*(subscription.messages.join() for subscription in self._subscriptions)) + + +def _owned_key_row(hashed_token: str) -> LiteLLM_VerificationToken: + return LiteLLM_VerificationToken( + token=hashed_token, + user_id="user-123", + team_id=None, + key_alias="doomed-key", + spend=0.0, + max_budget=None, + models=[], + aliases={}, + config={}, + permissions={}, + metadata={}, + model_max_budget={}, + model_spend={}, + soft_budget_cooldown=False, + allowed_routes=[], + ) + + +def _prisma_with_key_row(key_row: LiteLLM_VerificationToken) -> AsyncMock: + prisma_client: Final = AsyncMock() + prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[key_row]) + prisma_client.db.litellm_jwtkeymapping = _CascadingJWTMappingTable([]) + prisma_client.db.litellm_deletedverificationtoken.create_many = AsyncMock() + prisma_client.delete_data = AsyncMock(return_value=[key_row.token]) + prisma_client.get_data = AsyncMock(return_value=None) + return prisma_client + + +@pytest.mark.asyncio +async def test_delete_verification_tokens_stops_the_key_on_a_peer_worker(monkeypatch: pytest.MonkeyPatch) -> None: + """Auth serves key objects cache-first, so a peer worker that is not told about the delete keeps + authenticating the deleted key from its own copy until user_api_key_cache_ttl expires""" + raw_key: Final = "sk-" + uuid4().hex + hashed_token: Final = hash_token(raw_key) + deleting_worker_cache: Final = UserApiKeyCache() + peer_worker_cache: Final = UserApiKeyCache() + for cache in (deleting_worker_cache, peer_worker_cache): + cache.set_cache(key=hashed_token, value=UserAPIKeyAuth(token=hashed_token, user_id="user-123")) + coordination_redis: Final = _SharedCoordinationRedis() + peer_subscriber: Final = AuthCacheInvalidationSubscriber( + redis_cache=coordination_redis, user_api_key_cache=peer_worker_cache + ) + peer_subscriber.start() + prisma_client: Final = _prisma_with_key_row(_owned_key_row(hashed_token)) + monkeypatch.setattr("litellm.proxy.proxy_server.redis_usage_cache", coordination_redis) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma_client) + + try: + await delete_verification_tokens( + tokens=[raw_key], + user_api_key_cache=deleting_worker_cache, + user_api_key_dict=UserAPIKeyAuth( + user_id="admin-user", api_key="sk-admin", user_role=LitellmUserRoles.PROXY_ADMIN.value + ), + ) + await coordination_redis.until_every_subscriber_applied_its_messages() + finally: + await peer_subscriber.stop() + + assert deleting_worker_cache.get_cache(key=hashed_token) is None + with pytest.raises(ProxyException) as denied: + await get_key_object( + hashed_token=hashed_token, prisma_client=prisma_client, user_api_key_cache=peer_worker_cache + ) + assert denied.value.code == "401" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "deleter", + [ + UserAPIKeyAuth(user_id="admin-user", api_key="sk-admin", user_role=LitellmUserRoles.PROXY_ADMIN.value), + UserAPIKeyAuth(user_id="user-123", api_key="sk-owner", user_role=LitellmUserRoles.INTERNAL_USER.value), + ], + ids=["proxy_admin", "key_owner"], +) +async def test_delete_verification_tokens_evicts_a_requested_key_whose_row_is_already_gone( + monkeypatch: pytest.MonkeyPatch, deleter: UserAPIKeyAuth +) -> None: + """A key whose row was removed outside the API stays cached on every worker, so deleting it in a + batch beside a live key must still stop it everywhere instead of only evicting the rows found""" + live_key: Final = "sk-" + uuid4().hex + gone_key: Final = "sk-" + uuid4().hex + live_token: Final = hash_token(live_key) + gone_token: Final = hash_token(gone_key) + deleting_worker_cache: Final = UserApiKeyCache() + peer_worker_cache: Final = UserApiKeyCache() + for cache in (deleting_worker_cache, peer_worker_cache): + for token in (live_token, gone_token): + cache.set_cache(key=token, value=UserAPIKeyAuth(token=token, user_id="user-123")) + coordination_redis: Final = _SharedCoordinationRedis() + peer_subscriber: Final = AuthCacheInvalidationSubscriber( + redis_cache=coordination_redis, user_api_key_cache=peer_worker_cache + ) + peer_subscriber.start() + monkeypatch.setattr("litellm.proxy.proxy_server.redis_usage_cache", coordination_redis) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", _prisma_with_key_row(_owned_key_row(live_token))) + + try: + await delete_verification_tokens( + tokens=[live_key, gone_key], user_api_key_cache=deleting_worker_cache, user_api_key_dict=deleter + ) + await coordination_redis.until_every_subscriber_applied_its_messages() + finally: + await peer_subscriber.stop() + + assert [deleting_worker_cache.get_cache(key=token) for token in (live_token, gone_token)] == [None, None] + assert [peer_worker_cache.get_cache(key=token) for token in (live_token, gone_token)] == [None, None] + + +@pytest.mark.asyncio +async def test_delete_verification_tokens_leaves_unowned_cache_entries_alone_on_every_worker(monkeypatch: pytest.MonkeyPatch) -> None: + """/key/delete accepts any string, and a name that is not a key row passes through unhashed. A key + owner padding the request with another team's spend counter name must not wipe that counter from + every worker's memory: only the rows the caller was allowed to delete are evicted and broadcast""" + raw_key: Final = "sk-" + uuid4().hex + hashed_token: Final = hash_token(raw_key) + other_team_counter: Final = "spend:team:other-team" + deleting_worker_cache: Final = UserApiKeyCache() + peer_worker_cache: Final = UserApiKeyCache() + peer_spend_counters: Final = InMemoryCache() + peer_spend_counters.set_cache(other_team_counter, 42.0) + peer_worker_cache.set_cache(key=hashed_token, value=UserAPIKeyAuth(token=hashed_token, user_id="user-123")) + coordination_redis: Final = _SharedCoordinationRedis() + peer_subscriber: Final = AuthCacheInvalidationSubscriber( + redis_cache=coordination_redis, + user_api_key_cache=peer_worker_cache, + additional_in_memory_caches=(peer_spend_counters,), + ) + peer_subscriber.start() + monkeypatch.setattr("litellm.proxy.proxy_server.redis_usage_cache", coordination_redis) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", _prisma_with_key_row(_owned_key_row(hashed_token))) + + try: + await delete_verification_tokens( + tokens=[other_team_counter, raw_key], + user_api_key_cache=deleting_worker_cache, + user_api_key_dict=UserAPIKeyAuth( + user_id="user-123", api_key="sk-owner", user_role=LitellmUserRoles.INTERNAL_USER.value + ), + ) + await coordination_redis.until_every_subscriber_applied_its_messages() + finally: + await peer_subscriber.stop() + + assert peer_worker_cache.get_cache(key=hashed_token) is None + assert peer_spend_counters.get_cache(other_team_counter) == 42.0 + assert coordination_redis.published == (json.dumps({"cache_key": hashed_token}),) + + @pytest.mark.asyncio async def test_delete_key_fn_persists_deleted_keys(monkeypatch): from litellm.proxy._types import KeyRequest From 17d1f5833689739003abf379950137d7e8eeb2f5 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Fri, 30 Oct 2026 10:31:57 -0700 Subject: [PATCH 15/74] fix(ui): preserve team model grants on unrelated settings saves (#49258) --- .../ModelSelect/ModelSelect.test.tsx | 33 +++++++++++ .../components/ModelSelect/ModelSelect.tsx | 5 +- .../team/TeamInfo.integration.test.tsx | 59 ++++++++++++++++++- .../src/components/team/TeamInfo.test.tsx | 1 - .../src/components/team/TeamInfo.tsx | 5 +- 5 files changed, 97 insertions(+), 6 deletions(-) diff --git a/ui/litellm-dashboard/src/components/ModelSelect/ModelSelect.test.tsx b/ui/litellm-dashboard/src/components/ModelSelect/ModelSelect.test.tsx index 7ab757cfed9..d7394b1f949 100644 --- a/ui/litellm-dashboard/src/components/ModelSelect/ModelSelect.test.tsx +++ b/ui/litellm-dashboard/src/components/ModelSelect/ModelSelect.test.tsx @@ -775,6 +775,39 @@ describe("ModelSelect", () => { expect(mockOnChange).toHaveBeenCalledWith(["gpt-4"]); }); + it.each([ + { value: "all-proxy-models", label: "All Proxy Models", includeSpecialOptions: true }, + { value: "no-default-models", label: "No Default Models", includeSpecialOptions: false }, + ])( + "should label retained $value without offering it again after removal", + async ({ value, label, includeSpecialOptions }) => { + const user = userEvent.setup(); + mockUseOrganization.mockReturnValue({ + data: createMockOrganization(["gpt-4"]), + isLoading: false, + } as unknown as ReturnType); + const props = { + onChange: mockOnChange, + context: "team" as const, + organizationID: "org-1", + options: { includeSpecialOptions }, + }; + const { rerender } = renderWithProviders(); + + expect(screen.getByLabelText(label)).toBeInTheDocument(); + await openModelList(user); + const retainedGroup = within(screen.getByRole("listbox")).getAllByRole("group")[0]; + expect(within(retainedGroup).getByText("Unavailable")).toBeInTheDocument(); + await user.click(within(retainedGroup).getByRole("option", { name: label })); + expect(mockOnChange).toHaveBeenCalledWith([]); + + rerender(); + expectNotOffered(label); + expectNotOffered("claude-3"); + expectOffered("gpt-4"); + }, + ); + it("should keep the other selections when removing one unavailable model alongside a special option", async () => { const user = userEvent.setup(); renderWithProviders( diff --git a/ui/litellm-dashboard/src/components/ModelSelect/ModelSelect.tsx b/ui/litellm-dashboard/src/components/ModelSelect/ModelSelect.tsx index f169382b4ed..c7c428f2ccf 100644 --- a/ui/litellm-dashboard/src/components/ModelSelect/ModelSelect.tsx +++ b/ui/litellm-dashboard/src/components/ModelSelect/ModelSelect.tsx @@ -240,7 +240,10 @@ export const ModelSelect = (props: ModelSelectProps) => { ]; const offeredByValue = new Map(offeredGroups.flatMap((group) => group.items).map((option) => [option.value, option])); - const selectedOptions = value.map((v) => offeredByValue.get(v) ?? { label: v, value: v }); + const selectedOptions = value.map( + (v) => + offeredByValue.get(v) ?? MODEL_SENTINEL_OPTIONS.find((option) => option.value === v) ?? { label: v, value: v }, + ); const groups: ModelOptionGroup[] = [ ...unavailableGroups( selectedOptions, diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.integration.test.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.integration.test.tsx index fc054e51e5b..545c504281b 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.integration.test.tsx @@ -1,6 +1,6 @@ import { useTeamMetadataSchema } from "@/app/(dashboard)/hooks/teams/useTeamMetadataSchema"; import * as networking from "@/components/networking"; -import { fireEvent, screen, waitFor } from "@testing-library/react"; +import { fireEvent, screen, waitFor, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { renderWithProviders } from "../../../tests/test-utils"; @@ -817,7 +817,7 @@ describe("TeamInfoView - member budget reset prompt", () => { }); }); -describe("TeamInfoView - model rate limit picker", () => { +describe("TeamInfoView - model settings", () => { const props = { teamId: "123", onUpdate: vi.fn(), @@ -836,6 +836,61 @@ describe("TeamInfoView - model rate limit picker", () => { vi.clearAllMocks(); }); + it.each([ + { name: "unrestricted", models: [], organizationId: null }, + { name: "explicit all models", models: ["all-proxy-models"], organizationId: null }, + { name: "access groups only", models: ["no-default-models"], organizationId: null }, + { name: "retained explicit models", models: ["retired-model", "openai/*"], organizationId: null }, + { name: "unrestricted within an organization", models: [], organizationId: "org-1" }, + ])("preserves $name model grants when saving only a budget", async ({ models, organizationId }) => { + const user = userEvent.setup({ delay: null }); + vi.mocked(networking.teamInfoCall).mockResolvedValue( + createMockTeamData({ models, organization_id: organizationId }), + ); + mockUseTeam.mockReturnValue({ + data: { team_id: "123", organization_models: organizationId ? ["gpt-4"] : undefined }, + isLoading: false, + } as unknown as ReturnType); + + renderWithProviders(); + await user.click(await screen.findByRole("tab", { name: "Settings" })); + await user.click(await screen.findByRole("button", { name: /edit settings/i })); + if (models.length === 0) { + expect(await screen.findByLabelText("All Proxy Models")).toBeInTheDocument(); + } + fireEvent.change(await screen.findByLabelText("Max Budget (USD)"), { target: { value: "100" } }); + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => expect(networking.teamUpdateCall).toHaveBeenCalledTimes(1)); + const payload = vi.mocked(networking.teamUpdateCall).mock.calls[0][1]; + expect(payload).toMatchObject({ team_id: "123", max_budget: 100 }); + expect(payload).not.toHaveProperty("models"); + }); + + it("discards a cancelled model clear but saves an intentional clear of unrestricted access", async () => { + const user = userEvent.setup({ delay: null }); + vi.mocked(networking.teamInfoCall).mockResolvedValue(createMockTeamData({ models: [] })); + + renderWithProviders(); + await user.click(await screen.findByRole("tab", { name: "Settings" })); + await user.click(await screen.findByRole("button", { name: /edit settings/i })); + await user.click(within(await screen.findByLabelText("All Proxy Models")).getByRole("button")); + await user.click(screen.getByRole("button", { name: "Cancel" })); + await user.click(await screen.findByRole("button", { name: /edit settings/i })); + expect(await screen.findByLabelText("All Proxy Models")).toBeInTheDocument(); + fireEvent.change(screen.getByLabelText("Max Budget (USD)"), { target: { value: "100" } }); + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => expect(networking.teamUpdateCall).toHaveBeenCalledTimes(1)); + expect(vi.mocked(networking.teamUpdateCall).mock.calls[0][1]).not.toHaveProperty("models"); + await user.click(await screen.findByRole("button", { name: /edit settings/i })); + await user.click(within(await screen.findByLabelText("All Proxy Models")).getByRole("button")); + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => expect(networking.teamUpdateCall).toHaveBeenCalledTimes(2)); + expect(vi.mocked(networking.teamUpdateCall).mock.calls[1][1].models).toEqual(["no-default-models"]); + }); + it("offers access-group models, not no-default-models, in the model rate limit picker", async () => { const user = userEvent.setup({ delay: null }); vi.mocked(networking.teamInfoCall).mockResolvedValue( diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx index e4bf67ab72f..46081ae96f5 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx @@ -2421,7 +2421,6 @@ describe("TeamInfoView - the exact bytes the update call sends", () => { const alwaysSent = { team_id: "123", team_alias: "Test Team", - models: ["gpt-4"], tpm_limit: 1000, rpm_limit: 1000, tpd_limit: null, diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx index 0d20d160283..329552d735a 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx @@ -72,6 +72,7 @@ import { } from "../key_team_helpers/ModelMaxBudgetEditor"; import { modelMaxBudgetUpdate, StoredModelMaxBudget } from "../key_team_helpers/modelMaxBudgetPayload"; import { + ALL_PROXY_MODELS, computeTeamModelBadges, normalizeTeamModelSelection, TeamAccessGroupModelGrant, @@ -515,7 +516,7 @@ const computeEffectiveGuardrails = (info: TeamInfoRecord, globalGuardrailNames: const toTeamFormValues = (info: TeamInfoRecord, effectiveGuardrails: string[]): TeamUpdateFormValues => ({ team_alias: info.team_alias, - models: info.models, + models: info.models.length === 0 ? [ALL_PROXY_MODELS] : info.models, max_budget: info.max_budget, soft_budget: info.soft_budget, soft_budget_alerting_emails: Array.isArray(info.metadata?.soft_budget_alerting_emails) @@ -1064,7 +1065,7 @@ const TeamInfoView: React.FC = ({ const updateData: any = { team_id: teamId, team_alias: values.team_alias, - models: normalizeTeamModelSelection(values.models), + ...(form.getFieldState("models").isDirty ? { models: normalizeTeamModelSelection(values.models) } : {}), tpm_limit: numberOrNull(values.tpm_limit), rpm_limit: numberOrNull(values.rpm_limit), tpd_limit: numberOrNull(values.tpd_limit), From 707ac1ffee620e2906ef0fb5f9ad699cfd65a16d Mon Sep 17 00:00:00 2001 From: islomjon-aiand Date: Sat, 31 Oct 2026 02:55:26 +0900 Subject: [PATCH 16/74] fix(aiand)!: show ai&'s name and logo, and sync ai&'s model list (#49158) * feat(ui): show ai&'s name and logo in the dashboard The Add Model form already lists ai& from its provider metadata, but the dashboard had no entry for it, so the provider showed no logo and model tables printed the raw aiand slug. Adds ai& to the dashboard provider list with its official mark, plated in dark mode so the red and black mark keep their colours, and the aiand/openai/gpt-oss-120b model placeholder * fix(aiand): retire deepseek-v4-flash, glm-5.2 and kimi-k2.7-code ai& retired these three models on 2026-10-27 and https://api.aiand.com/v1/models now lists ten. The rows keep their prices with a deprecation_date, so earlier spend still prices, and aiand/* stops listing them, the same way baseten handles retired rows. The other ten rows already match the live prices, context windows and output limits * feat(aiand): add deepseek-v4.1-flash-1017 ai& launches deepseek-ai/deepseek-v4.1-flash-1017 on 2026-11-02. It is served on https://staging-api.aiand.com/v1/models with the same price, context window, output limit and capabilities as deepseek-v4.1-flash, so its row copies that one * fix(aiand): retire deepseek-v4.1-flash ai& retires deepseek-ai/deepseek-v4.1-flash on 2026-10-30, now that deepseek-v4.1-flash-1017 is live in production on https://api.aiand.com/v1/models with the same price, limits and capabilities. Its row keeps its prices with a deprecation_date, like the other three retired ai& rows * fix(aiand): retire deepseek-v4.1-flash on 2026-11-06 https://api.aiand.com/v1/models now gives deepseek-v4.1-flash a deprecation_date of 2026-11-05T15:00Z, which is 2026-11-06 00:00 in Japan, so its row uses 2026-11-06 instead of 2026-10-30 and aiand/* keeps listing it until then. The wildcard test now leaves a model out only once its retirement date has passed * test(aiand): derive retired models from the rows instead of pinning dates The wildcard test now reads each row's deprecation_date, so it holds whichever dates ai& sets, and the test that only restated the JSON dates is gone * fix(aiand): retire models on their UTC date while the proxy runs, and add muse-spark-open The wildcard listing now checks each row's deprecation_date when it expands, against the UTC date the field documents, so a model drops on its day without a restart. deepseek-v4.1-flash now carries 2026-11-05, the UTC day ai& retires it, and meta/muse-spark-open gets its price row from ai&'s live /v1/models * fix(aiand): hide retired models only from bundled listings, including the wildcard health check * fix(aiand): share the UTC retirement check with Together AI and the health check sort --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/__init__.py | 21 ++-- .../litellm_core_utils/llm_request_utils.py | 21 ++-- .../litellm_core_utils/model_retirement.py | 19 ++++ ...odel_prices_and_context_window_backup.json | 34 ++++++ litellm/utils.py | 10 +- model_prices_and_context_window.json | 34 ++++++ .../test_llm_request_utils.py | 38 +++++++ .../baseten/chat/test_baseten_completions.py | 33 +++++- .../llms/openai_like/test_aiand_provider.py | 100 +++++++++++++++++- .../public/assets/logos/aiand.svg | 4 + .../components/provider_info_helpers.test.tsx | 12 +++ .../src/components/provider_info_helpers.tsx | 5 + .../src/lib/logoTreatments.ts | 1 + 13 files changed, 296 insertions(+), 36 deletions(-) create mode 100644 litellm/litellm_core_utils/model_retirement.py create mode 100644 ui/litellm-dashboard/public/assets/logos/aiand.svg diff --git a/litellm/__init__.py b/litellm/__init__.py index 5f14c605cd0..f64531f86fb 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -7,7 +7,6 @@ _apply_process_defaults() ### INIT VARIABLES ######################### import threading import os -import datetime from collections.abc import Mapping, Sequence from typing import ( @@ -550,6 +549,7 @@ _key_management_system: Optional["KeyManagementSystem"] = None output_parse_pii: bool = False ############################################# from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map, mark_litellm_import_complete +from litellm.litellm_core_utils.model_retirement import is_hidden_as_retired model_cost = get_model_cost_map(url=model_cost_map_url) cost_discount_config: Dict[str, float] = {} # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount @@ -756,16 +756,6 @@ def is_openai_finetune_model(key: str) -> bool: return key.startswith("ft:") and not key.count(":") > 1 -def _is_past_deprecation_date(value: Mapping[str, object]) -> bool: - deprecation_date: Final = value.get("deprecation_date") - if not isinstance(deprecation_date, str): - return False - try: - return datetime.date.fromisoformat(deprecation_date) <= datetime.date.today() - except ValueError: - return False - - def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, object]]) -> None: for key, value in tuple(model_cost_map.items()): if value.get("litellm_provider") == "openai" and not is_openai_finetune_model(key): @@ -873,12 +863,12 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje if "-to-" not in key and "fireworks-ai-default" not in key: fireworks_ai_models.add(key) elif value.get("litellm_provider") == "together_ai": - if _is_past_deprecation_date(value): + if is_hidden_as_retired(value): together_ai_models.discard(key) elif not key.startswith("together-ai-"): together_ai_models.add(key) elif value.get("litellm_provider") == "baseten": - if _is_past_deprecation_date(value): + if is_hidden_as_retired(value): baseten_models.discard(key) else: baseten_models.add(key) @@ -1051,7 +1041,10 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje elif value.get("litellm_provider") == "coralbricks": coralbricks_models.add(key) elif value.get("litellm_provider") == "aiand": - aiand_models.add(key) + if is_hidden_as_retired(value): + aiand_models.discard(key) + else: + aiand_models.add(key) elif value.get("litellm_provider") == "reducto": reducto_models.add(key) elif value.get("litellm_provider") == "bedrock_mantle": diff --git a/litellm/litellm_core_utils/llm_request_utils.py b/litellm/litellm_core_utils/llm_request_utils.py index 9c2ec0b8984..84f161e0c62 100644 --- a/litellm/litellm_core_utils/llm_request_utils.py +++ b/litellm/litellm_core_utils/llm_request_utils.py @@ -1,4 +1,3 @@ -import datetime from collections.abc import Iterable, Iterator, Mapping from dataclasses import dataclass from typing import Final, cast @@ -9,6 +8,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider +from litellm.litellm_core_utils.model_retirement import is_past_deprecation_date def _form_field_value(value: object) -> str: @@ -168,7 +168,9 @@ class RankedChatModel: def rank_chat_models_from_llm_provider(custom_llm_provider: str) -> tuple[RankedChatModel, ...]: - known_models: Final[Iterable[str]] = litellm.models_by_provider.get(custom_llm_provider, ()) + from litellm.utils import bundled_provider_models + + known_models: Final = bundled_provider_models(custom_llm_provider) ranked_models: Final = ( RankedChatModel( model=model, route_rank=route_rank, cost=cost, is_past_deprecation_date=is_past_deprecation_date @@ -187,18 +189,11 @@ _COST_MAP_DEPRECATION: Final = TypeAdapter(_CostMapDeprecation) def _is_past_deprecation_date(cost_map_key: str) -> bool: - deprecation_date: Final = _COST_MAP_DEPRECATION.validate_python( - litellm.model_cost.get(cost_map_key) or {} # pyright: ignore[reportUnknownMemberType] # untyped cost map JSON - ).get("deprecation_date") - if not isinstance(deprecation_date, str): - return False - try: - return ( - datetime.date.fromisoformat(deprecation_date) - <= datetime.datetime.now(datetime.timezone.utc).astimezone().date() + return is_past_deprecation_date( + _COST_MAP_DEPRECATION.validate_python( + litellm.model_cost.get(cost_map_key) or {} # pyright: ignore[reportUnknownMemberType] # untyped cost map JSON ) - except ValueError: - return False + ) def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator[tuple[str, float, bool]]: diff --git a/litellm/litellm_core_utils/model_retirement.py b/litellm/litellm_core_utils/model_retirement.py new file mode 100644 index 00000000000..3bfb834733d --- /dev/null +++ b/litellm/litellm_core_utils/model_retirement.py @@ -0,0 +1,19 @@ +import datetime +from collections.abc import Mapping +from typing import Final + +PROVIDERS_HIDING_RETIRED_MODELS: Final = frozenset({"aiand", "baseten", "together_ai"}) + + +def is_past_deprecation_date(value: Mapping[str, object]) -> bool: + deprecation_date: Final = value.get("deprecation_date") + if not isinstance(deprecation_date, str): + return False + try: + return datetime.date.fromisoformat(deprecation_date) <= datetime.datetime.now(datetime.timezone.utc).date() + except ValueError: + return False + + +def is_hidden_as_retired(value: Mapping[str, object]) -> bool: + return value.get("litellm_provider") in PROVIDERS_HIDING_RETIRED_MODELS and is_past_deprecation_date(value) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index e1131117fcf..ea9cacbdabe 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -72249,6 +72249,7 @@ }, "aiand/deepseek-ai/deepseek-v4-flash": { "cache_read_input_token_cost": 8e-08, + "deprecation_date": "2026-10-27", "input_cost_per_token": 1.5e-07, "litellm_provider": "aiand", "max_input_tokens": 1048576, @@ -72278,6 +72279,22 @@ "supports_vision": false }, "aiand/deepseek-ai/deepseek-v4.1-flash": { + "cache_read_input_token_cost": 2e-08, + "deprecation_date": "2026-11-05", + "input_cost_per_token": 3e-07, + "litellm_provider": "aiand", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 6e-07, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "aiand/deepseek-ai/deepseek-v4.1-flash-1017": { "cache_read_input_token_cost": 2e-08, "input_cost_per_token": 3e-07, "litellm_provider": "aiand", @@ -72307,8 +72324,24 @@ "supports_tool_choice": true, "supports_vision": true }, + "aiand/meta/muse-spark-open": { + "cache_read_input_token_cost": 1.5e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "aiand", + "max_input_tokens": 1048576, + "max_output_tokens": 943718, + "mode": "chat", + "output_cost_per_token": 4.25e-06, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, "aiand/moonshotai/kimi-k2.7-code": { "cache_read_input_token_cost": 2e-07, + "deprecation_date": "2026-10-27", "input_cost_per_token": 7.5e-07, "litellm_provider": "aiand", "max_input_tokens": 262144, @@ -72399,6 +72432,7 @@ }, "aiand/zai-org/glm-5.2": { "cache_read_input_token_cost": 3e-07, + "deprecation_date": "2026-10-27", "input_cost_per_token": 1e-06, "litellm_provider": "aiand", "max_input_tokens": 1048576, diff --git a/litellm/utils.py b/litellm/utils.py index 6f56cc65f45..2ca8a49f968 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -94,6 +94,7 @@ from litellm.litellm_core_utils.fallback_generalizations import ( ) from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry, GetModelCostMap from litellm.litellm_core_utils.model_param_helper import peek_reasoning_summary_aliases +from litellm.litellm_core_utils.model_retirement import is_hidden_as_retired from litellm.litellm_core_utils.offline_mode import is_offline_mode from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload from litellm.litellm_core_utils.tokenizer import strip_special_tokens @@ -8806,6 +8807,12 @@ def _get_valid_models_from_provider_api( return [] +def bundled_provider_models(provider: str) -> tuple[str, ...]: + bundled: Final[tuple[str, ...]] = tuple(litellm.models_by_provider.get(provider, ())) + cost_map: Final = _builtin_cost_map() + return tuple(model for model in bundled if not is_hidden_as_retired(cost_map.get(model, {}))) + + def get_valid_models( check_provider_endpoint: bool | None = None, custom_llm_provider: str | None = None, @@ -8871,8 +8878,7 @@ def get_valid_models( ) ) else: - models_for_provider = copy.deepcopy(litellm.models_by_provider.get(provider, [])) - valid_models.extend(models_for_provider) + valid_models.extend(bundled_provider_models(provider)) return valid_models except Exception as e: diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e1131117fcf..ea9cacbdabe 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -72249,6 +72249,7 @@ }, "aiand/deepseek-ai/deepseek-v4-flash": { "cache_read_input_token_cost": 8e-08, + "deprecation_date": "2026-10-27", "input_cost_per_token": 1.5e-07, "litellm_provider": "aiand", "max_input_tokens": 1048576, @@ -72278,6 +72279,22 @@ "supports_vision": false }, "aiand/deepseek-ai/deepseek-v4.1-flash": { + "cache_read_input_token_cost": 2e-08, + "deprecation_date": "2026-11-05", + "input_cost_per_token": 3e-07, + "litellm_provider": "aiand", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 6e-07, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "aiand/deepseek-ai/deepseek-v4.1-flash-1017": { "cache_read_input_token_cost": 2e-08, "input_cost_per_token": 3e-07, "litellm_provider": "aiand", @@ -72307,8 +72324,24 @@ "supports_tool_choice": true, "supports_vision": true }, + "aiand/meta/muse-spark-open": { + "cache_read_input_token_cost": 1.5e-07, + "input_cost_per_token": 1.25e-06, + "litellm_provider": "aiand", + "max_input_tokens": 1048576, + "max_output_tokens": 943718, + "mode": "chat", + "output_cost_per_token": 4.25e-06, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, "aiand/moonshotai/kimi-k2.7-code": { "cache_read_input_token_cost": 2e-07, + "deprecation_date": "2026-10-27", "input_cost_per_token": 7.5e-07, "litellm_provider": "aiand", "max_input_tokens": 262144, @@ -72399,6 +72432,7 @@ }, "aiand/zai-org/glm-5.2": { "cache_read_input_token_cost": 3e-07, + "deprecation_date": "2026-10-27", "input_cost_per_token": 1e-06, "litellm_provider": "aiand", "max_input_tokens": 1048576, diff --git a/tests/unit/litellm_core_utils/test_llm_request_utils.py b/tests/unit/litellm_core_utils/test_llm_request_utils.py index e7350fae395..5243d308543 100644 --- a/tests/unit/litellm_core_utils/test_llm_request_utils.py +++ b/tests/unit/litellm_core_utils/test_llm_request_utils.py @@ -1,3 +1,5 @@ +import datetime +import time from typing import Final import httpx @@ -245,3 +247,39 @@ def test_rank_chat_models_flags_a_retired_model_the_provider_lists_without_its_c (ranked_model.model, ranked_model.is_past_deprecation_date) for ranked_model in rank_chat_models_from_llm_provider("vertex_ai") ] == [("fake-retired-vertex-chat", True), ("fake-live-vertex-chat", False)] + + +@pytest.mark.parametrize("host_zone", ["Etc/GMT+12", "Etc/GMT-14"]) +def test_rank_chat_models_flags_retirement_by_the_utc_date_whatever_the_host_zone( + monkeypatch: pytest.MonkeyPatch, host_zone: str +) -> None: + utc_today: Final = datetime.datetime.now(datetime.timezone.utc).date() + retirement_dates: Final = { + "deepseek/fake-retires-today-chat": utc_today, + "deepseek/fake-retires-tomorrow-chat": utc_today + datetime.timedelta(days=1), + } + for name, retirement_date in retirement_dates.items(): + monkeypatch.setitem( + litellm.model_cost, + name, + { + "input_cost_per_token": 1e-12, + "output_cost_per_token": 1e-12, + "mode": "chat", + "litellm_provider": PICKER_PROVIDER, + "deprecation_date": retirement_date.isoformat(), + }, + ) + monkeypatch.setitem(litellm.models_by_provider, PICKER_PROVIDER, tuple(retirement_dates)) + monkeypatch.setenv("TZ", host_zone) + time.tzset() + try: + flags: Final = { + ranked_model.model: ranked_model.is_past_deprecation_date + for ranked_model in rank_chat_models_from_llm_provider(PICKER_PROVIDER) + } + finally: + monkeypatch.undo() + time.tzset() + + assert flags == {"deepseek/fake-retires-today-chat": True, "deepseek/fake-retires-tomorrow-chat": False} diff --git a/tests/unit/llms/baseten/chat/test_baseten_completions.py b/tests/unit/llms/baseten/chat/test_baseten_completions.py index 4f135a1a106..407b39319e9 100644 --- a/tests/unit/llms/baseten/chat/test_baseten_completions.py +++ b/tests/unit/llms/baseten/chat/test_baseten_completions.py @@ -7,8 +7,9 @@ import pytest import respx import litellm +from litellm.litellm_core_utils.model_retirement import is_past_deprecation_date from litellm.llms.baseten.chat import BasetenConfig -from litellm.utils import get_valid_models +from litellm.utils import _model_cache, get_valid_models class TestBasetenRouting: @@ -65,7 +66,7 @@ class TestBasetenModelDiscovery: servable_registry_models: Final = { key for key, value in litellm.model_cost.items() - if value.get("litellm_provider") == "baseten" and not litellm._is_past_deprecation_date(value) + if value.get("litellm_provider") == "baseten" and not is_past_deprecation_date(value) } assert servable_registry_models @@ -92,6 +93,20 @@ class TestBasetenModelDiscovery: litellm.baseten_models.discard(model) litellm.add_known_models({}) + def test_wildcard_drops_a_model_whose_retirement_date_passes_while_running(self, monkeypatch: pytest.MonkeyPatch): + model: Final = "baseten/acme/retires-while-running" + monkeypatch.setitem(litellm.model_cost, model, {"litellm_provider": "baseten", "deprecation_date": "2999-01-01"}) + try: + litellm.add_known_models({model: litellm.model_cost[model]}) + assert model in get_valid_models(custom_llm_provider="baseten") + + monkeypatch.setitem(litellm.model_cost, model, {"litellm_provider": "baseten", "deprecation_date": "2000-01-01"}) + + assert model not in get_valid_models(custom_llm_provider="baseten") + finally: + litellm.baseten_models.discard(model) + litellm.add_known_models({}) + @pytest.mark.parametrize( "value, expected", [ @@ -102,7 +117,7 @@ class TestBasetenModelDiscovery: ], ) def test_is_past_deprecation_date(self, value, expected): - assert litellm._is_past_deprecation_date(value) is expected + assert is_past_deprecation_date(value) is expected def test_get_models_calls_model_api(self, respx_mock: respx.MockRouter) -> None: route: Final = respx_mock.get("https://inference.baseten.co/v1/models").mock( @@ -156,6 +171,18 @@ class TestBasetenModelDiscovery: assert models == ["baseten/brand-new/model"] + @patch.dict(os.environ, {"BASETEN_API_KEY": "test-key"}) + def test_live_catalog_keeps_a_model_the_cost_map_dates_as_retired(self, respx_mock: respx.MockRouter) -> None: + assert is_past_deprecation_date(litellm.model_cost["baseten/zai-org/GLM-5"]) + _model_cache.flush_cache() + respx_mock.get("https://inference.baseten.co/v1/models").mock( + return_value=httpx.Response(200, json={"data": [{"id": "zai-org/GLM-5"}]}) + ) + + models: Final = get_valid_models(check_provider_endpoint=True, custom_llm_provider="baseten") + + assert models == ["baseten/zai-org/GLM-5"] + class TestBasetenReasoningEffort: def test_reasoning_effort_passthrough_with_drop_params(self): diff --git a/tests/unit/llms/openai_like/test_aiand_provider.py b/tests/unit/llms/openai_like/test_aiand_provider.py index 804f3a3b6eb..0274b58f19a 100644 --- a/tests/unit/llms/openai_like/test_aiand_provider.py +++ b/tests/unit/llms/openai_like/test_aiand_provider.py @@ -1,4 +1,6 @@ +import datetime import json +import time from typing import Final import httpx @@ -8,6 +10,8 @@ import respx import litellm from litellm.caching.llm_caching_handler import LLMClientCache from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.litellm_core_utils.health_check_helpers import wildcard_health_check_models +from litellm.litellm_core_utils.model_retirement import is_past_deprecation_date from litellm.llms.openai_like.json_loader import JSONProviderRegistry from litellm.proxy.auth.model_checks import get_known_models_from_wildcard from litellm.utils import ProviderConfigManager @@ -89,11 +93,99 @@ def _aiand_catalog() -> frozenset[str]: return frozenset(name for name, info in litellm.model_cost.items() if info.get("litellm_provider") == "aiand") -def test_aiand_wildcard_lists_its_priced_models(): - catalog: Final = _aiand_catalog() +def _utc_today() -> datetime.date: + return datetime.datetime.now(datetime.timezone.utc).date() - assert catalog - assert frozenset(get_known_models_from_wildcard("aiand/*")) == catalog + +def test_aiand_wildcard_lists_only_models_not_yet_retired(): + catalog: Final = _aiand_catalog() + retired: Final = frozenset( + name + for name in catalog + if (retired_on := litellm.model_cost[name].get("deprecation_date")) is not None + and datetime.date.fromisoformat(retired_on) <= _utc_today() + ) + + assert catalog - retired + assert frozenset(get_known_models_from_wildcard("aiand/*")) == catalog - retired + + +def test_aiand_wildcard_drops_a_model_whose_retirement_date_passes_while_running(monkeypatch: pytest.MonkeyPatch): + model: Final = "aiand/acme/retires-while-running" + monkeypatch.setitem(litellm.model_cost, model, {"litellm_provider": "aiand", "deprecation_date": "2999-01-01"}) + try: + litellm.add_known_models({model: litellm.model_cost[model]}) + assert model in get_known_models_from_wildcard("aiand/*") + + monkeypatch.setitem(litellm.model_cost, model, {"litellm_provider": "aiand", "deprecation_date": "2000-01-01"}) + + assert model not in get_known_models_from_wildcard("aiand/*") + finally: + litellm.aiand_models.discard(model) + litellm.add_known_models({}) + + +def test_aiand_models_by_provider_leaves_out_models_retired_at_load(): + retired: Final = frozenset( + name + for name, row in litellm.model_cost.items() + if row.get("litellm_provider") == "aiand" and is_past_deprecation_date(row) + ) + + assert retired + assert retired.isdisjoint(litellm.models_by_provider["aiand"]) + + +def test_aiand_wildcard_health_check_skips_a_model_whose_retirement_date_passes_while_running( + monkeypatch: pytest.MonkeyPatch, +): + model: Final = "aiand/acme/free-and-retiring" + live_row: Final = { + "litellm_provider": "aiand", + "mode": "chat", + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "deprecation_date": "2999-01-01", + } + monkeypatch.setitem(litellm.model_cost, model, live_row) + try: + litellm.add_known_models({model: live_row}) + assert model in wildcard_health_check_models(wildcard="aiand/*", custom_llm_provider="aiand") + + monkeypatch.setitem(litellm.model_cost, model, {**live_row, "deprecation_date": "2000-01-01"}) + + assert model not in wildcard_health_check_models(wildcard="aiand/*", custom_llm_provider="aiand") + finally: + litellm.aiand_models.discard(model) + litellm.add_known_models({}) + + +@pytest.mark.parametrize("host_zone", ["Etc/GMT+12", "Etc/GMT-14"]) +def test_aiand_models_retire_on_the_utc_date_whatever_the_host_zone(monkeypatch: pytest.MonkeyPatch, host_zone: str): + today: Final = _utc_today() + rows: Final = { + "aiand/acme/retires-today": {"litellm_provider": "aiand", "deprecation_date": today.isoformat()}, + "aiand/acme/retires-tomorrow": { + "litellm_provider": "aiand", + "deprecation_date": (today + datetime.timedelta(days=1)).isoformat(), + }, + } + for name, row in rows.items(): + monkeypatch.setitem(litellm.model_cost, name, row) + monkeypatch.setenv("TZ", host_zone) + time.tzset() + try: + litellm.add_known_models(rows) + listed: Final = frozenset(get_known_models_from_wildcard("aiand/*")) + + assert "aiand/acme/retires-today" not in listed + assert "aiand/acme/retires-tomorrow" in listed + finally: + monkeypatch.undo() + time.tzset() + for name in rows: + litellm.aiand_models.discard(name) + litellm.add_known_models({}) def test_aiand_calls_are_billed_from_their_cost_rows(): diff --git a/ui/litellm-dashboard/public/assets/logos/aiand.svg b/ui/litellm-dashboard/public/assets/logos/aiand.svg new file mode 100644 index 00000000000..486c73b8dbc --- /dev/null +++ b/ui/litellm-dashboard/public/assets/logos/aiand.svg @@ -0,0 +1,4 @@ + + + + diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx index 810b010f25e..02c31ecc082 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx @@ -32,6 +32,18 @@ describe("provider_info_helpers", () => { expect(getPlaceholder(Providers.BIGMODEL)).toMatch(/^bigmodel\//); }); + it("resolves ai& metadata, name, logo and model placeholder", () => { + const metadata = PROVIDER_CREATE_FIELDS.find((provider) => provider.provider === "AIAND"); + expect(provider_map.AIAND).toBe(metadata?.litellm_provider); + expect(getProviderLogoAndName("aiand")).toEqual({ + displayName: "ai&", + logo: providerLogoMap[Providers.AIAND], + }); + expect(getProviderLogoAndName("AIAND")).toEqual(getProviderLogoAndName("aiand")); + expect(providerLogoMap[Providers.AIAND]).toContain("aiand"); + expect(getPlaceholder(Providers.AIAND)).toBe("aiand/openai/gpt-oss-120b"); + }); + it("maps Microsoft 365 Copilot to its chat model provider and placeholder", () => { expect(provider_map.MICROSOFT_365_COPILOT).toBe("microsoft_365_copilot"); expect(getPlaceholder(Providers.MICROSOFT_365_COPILOT)).toBe("microsoft_365_copilot/chat"); diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx index 1ed32c160e6..6448bd936cc 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx @@ -1,6 +1,7 @@ import { resolveLogoSrc } from "@/lib/assetPaths"; import a2aAgentLogo from "../../public/assets/logos/a2a_agent.png"; import ai21Logo from "../../public/assets/logos/ai21.svg"; +import aiandLogo from "../../public/assets/logos/aiand.svg"; import aimlApiLogo from "../../public/assets/logos/aiml_api.svg"; import anthropicLogo from "../../public/assets/logos/anthropic.svg"; import assemblyaiSmallLogo from "../../public/assets/logos/assemblyai_small.png"; @@ -76,6 +77,7 @@ export enum Providers { A2A_Agent = "A2A Agent", AI21 = "Ai21", AI21_CHAT = "Ai21 Chat", + AIAND = "ai&", AIML = "AI/ML API", AIOHTTP_OPENAI = "Aiohttp Openai", Anthropic = "Anthropic", @@ -207,6 +209,7 @@ export const provider_map: Record = { A2A_Agent: "a2a_agent", AI21: "ai21", AI21_CHAT: "ai21_chat", + AIAND: "aiand", AIML: "aiml", AIOHTTP_OPENAI: "aiohttp_openai", Anthropic: "anthropic", @@ -344,6 +347,7 @@ export const providerLogoMap: Partial> = { [Providers.A2A_Agent]: a2aAgentLogo.src, [Providers.AI21]: ai21Logo.src, [Providers.AI21_CHAT]: ai21Logo.src, + [Providers.AIAND]: aiandLogo.src, [Providers.AIML]: aimlApiLogo.src, [Providers.AIOHTTP_OPENAI]: openaiSmallLogo.src, [Providers.Anthropic]: anthropicLogo.src, @@ -472,6 +476,7 @@ export const getProviderLogoAndName = (providerValue: string): { logo: string; d const providerPlaceholderMap: Partial> = { [Providers.AIML]: "aiml/flux-pro/v1.1", + [Providers.AIAND]: "aiand/openai/gpt-oss-120b", [Providers.Anthropic]: "claude-3-opus", [Providers.Azure]: "my-deployment", [Providers.Azure_AI_Studio]: "azure_ai/command-r-plus", diff --git a/ui/litellm-dashboard/src/lib/logoTreatments.ts b/ui/litellm-dashboard/src/lib/logoTreatments.ts index fc19c5c364d..4cd6f65a613 100644 --- a/ui/litellm-dashboard/src/lib/logoTreatments.ts +++ b/ui/litellm-dashboard/src/lib/logoTreatments.ts @@ -32,6 +32,7 @@ const TREATMENT_BY_ASSET: Readonly> = { "v0.svg": "invert", "vercel.svg": "invert", "watsonx.svg": "invert", + "aiand.svg": "plate", "aiml_api.svg": "plate", "akto.svg": "plate", "aws.svg": "plate", From ccdc8c7d10f8b042f576b1adfb670a02011cddcd Mon Sep 17 00:00:00 2001 From: mrinal-berri Date: Fri, 30 Oct 2026 10:59:28 -0700 Subject: [PATCH 17/74] docs(daily-activity): describe guardrail spend fields and reporting key IDs in OpenAPI (#49236) * docs(daily-activity): describe guardrail spend and reporting key IDs Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * docs(daily-activity): describe reporting key IDs on /user/daily/activity Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/_lazy_openapi_snapshot.json | 17 +++++++-- .../daily_activity_scopes.py | 6 ++- .../internal_user_endpoints.py | 3 +- .../common_daily_activity.py | 27 +++++++++----- ui/litellm-dashboard/src/lib/http/schema.d.ts | 37 +++++++++++++------ 5 files changed, 64 insertions(+), 26 deletions(-) diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index c76b9be7c33..dc8b9415020 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -3491,6 +3491,7 @@ "additionalProperties": { "$ref": "#/components/schemas/GuardrailMetric" }, + "description": "Spend by guardrail ID, already included in key and model spend. Populated on user, team and organization aggregated activity only, without a model filter", "title": "Guardrails", "type": "object" }, @@ -3812,10 +3813,12 @@ "GuardrailKeyMetric": { "properties": { "metadata": { - "$ref": "#/components/schemas/KeyMetadata" + "$ref": "#/components/schemas/KeyMetadata", + "description": "API key alias and existence information" }, "metrics": { - "$ref": "#/components/schemas/GuardrailSpendMetrics" + "$ref": "#/components/schemas/GuardrailSpendMetrics", + "description": "Guardrail spend and unpriced units for this API key" } }, "required": [ @@ -3828,6 +3831,7 @@ "GuardrailMetadata": { "properties": { "guardrail_name": { + "description": "Guardrail display name, falling back to the guardrail ID", "title": "Guardrail Name", "type": "string" } @@ -3844,14 +3848,17 @@ "additionalProperties": { "$ref": "#/components/schemas/GuardrailKeyMetric" }, + "description": "Per-key guardrail spend and unpriced units, limited to keys in the day's API key breakdown", "title": "Api Key Breakdown", "type": "object" }, "metadata": { - "$ref": "#/components/schemas/GuardrailMetadata" + "$ref": "#/components/schemas/GuardrailMetadata", + "description": "Guardrail name information" }, "metrics": { - "$ref": "#/components/schemas/GuardrailSpendMetrics" + "$ref": "#/components/schemas/GuardrailSpendMetrics", + "description": "Guardrail spend and unpriced units across all matching API keys" } }, "required": [ @@ -3865,11 +3872,13 @@ "properties": { "spend": { "default": 0.0, + "description": "Guardrail spend in USD, already included in key and model spend", "title": "Spend", "type": "number" }, "untracked_units": { "default": 0, + "description": "Guardrail units with no known price, excluded from spend", "title": "Untracked Units", "type": "integer" } diff --git a/litellm/proxy/management_endpoints/daily_activity_scopes.py b/litellm/proxy/management_endpoints/daily_activity_scopes.py index ca1a96f9d84..7ff73ac6e2e 100644 --- a/litellm/proxy/management_endpoints/daily_activity_scopes.py +++ b/litellm/proxy/management_endpoints/daily_activity_scopes.py @@ -216,7 +216,11 @@ def _user_query( start_date: str | None = Query(default=None, description="Start date in YYYY-MM-DD format"), end_date: str | None = Query(default=None, description="End date in YYYY-MM-DD format"), model: str | None = Query(default=None, description="Filter by specific model"), - api_key: str | None = Query(default=None, description="Filter by specific API key"), + api_key: str | None = Query( + default=None, + description="Filter by reporting API key ID. Missing keys are reported as unattributed. " + "For a literal key unattributed or a key starting with raw:, add an extra raw: prefix", + ), user_id: str | None = Query( default=None, description="Filter by specific user ID. Admins can filter by any user or omit for global view. " diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index 6950cf08a25..54a2d4062d2 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -2922,7 +2922,8 @@ async def get_user_daily_activity( ), api_key: str | None = fastapi.Query( default=None, - description="Filter by specific API key", + description="Filter by reporting API key ID. Missing keys are reported as unattributed. " + "For a literal key unattributed or a key starting with raw:, add an extra raw: prefix", ), user_id: str | None = fastapi.Query( default=None, diff --git a/litellm/types/proxy/management_endpoints/common_daily_activity.py b/litellm/types/proxy/management_endpoints/common_daily_activity.py index 3f1a7980b27..37f59711050 100644 --- a/litellm/types/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/types/proxy/management_endpoints/common_daily_activity.py @@ -68,29 +68,34 @@ class MetricWithMetadata(MetricBase): class GuardrailSpendMetrics(LiteLLMBaseModel): model_config = ConfigDict(frozen=True) - spend: float = 0.0 - untracked_units: int = 0 + spend: float = Field(default=0.0, description="Guardrail spend in USD, already included in key and model spend") + untracked_units: int = Field(default=0, description="Guardrail units with no known price, excluded from spend") class GuardrailKeyMetric(LiteLLMBaseModel): model_config = ConfigDict(frozen=True) - metrics: GuardrailSpendMetrics - metadata: KeyMetadata + metrics: GuardrailSpendMetrics = Field(description="Guardrail spend and unpriced units for this API key") + metadata: KeyMetadata = Field(description="API key alias and existence information") class GuardrailMetadata(LiteLLMBaseModel): model_config = ConfigDict(frozen=True) - guardrail_name: str + guardrail_name: str = Field(description="Guardrail display name, falling back to the guardrail ID") class GuardrailMetric(LiteLLMBaseModel): model_config = ConfigDict(frozen=True) - metrics: GuardrailSpendMetrics - metadata: GuardrailMetadata - api_key_breakdown: Mapping[str, GuardrailKeyMetric] = Field(default_factory=dict) + metrics: GuardrailSpendMetrics = Field( + description="Guardrail spend and unpriced units across all matching API keys" + ) + metadata: GuardrailMetadata = Field(description="Guardrail name information") + api_key_breakdown: Mapping[str, GuardrailKeyMetric] = Field( + default_factory=dict, + description="Per-key guardrail spend and unpriced units, limited to keys in the day's API key breakdown", + ) class BreakdownMetrics(LiteLLMBaseModel): @@ -103,7 +108,11 @@ class BreakdownMetrics(LiteLLMBaseModel): endpoints: dict[str, MetricWithMetadata] = Field(default_factory=dict) # endpoint -> {metrics, metadata} api_keys: dict[str, KeyMetricWithMetadata] = Field(default_factory=dict) # api_key -> {metrics, metadata} entities: dict[str, MetricWithMetadata] = Field(default_factory=dict) # entity -> {metrics, metadata} - guardrails: Mapping[str, GuardrailMetric] = Field(default_factory=dict) # guardrail_id -> spend + guardrails: Mapping[str, GuardrailMetric] = Field( + default_factory=dict, + description="Spend by guardrail ID, already included in key and model spend. " + "Populated on user, team and organization aggregated activity only, without a model filter", + ) class DailySpendData(LiteLLMBaseModel): diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 960616e2631..aafa732276b 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -27870,7 +27870,10 @@ export interface components { entities?: { [key: string]: components["schemas"]["MetricWithMetadata"]; }; - /** Guardrails */ + /** + * Guardrails + * @description Spend by guardrail ID, already included in key and model spend. Populated on user, team and organization aggregated activity only, without a model filter + */ guardrails?: { [key: string]: components["schemas"]["GuardrailMetric"]; }; @@ -33448,32 +33451,44 @@ export interface components { }; /** GuardrailKeyMetric */ GuardrailKeyMetric: { + /** @description API key alias and existence information */ metadata: components["schemas"]["KeyMetadata"]; + /** @description Guardrail spend and unpriced units for this API key */ metrics: components["schemas"]["GuardrailSpendMetrics"]; }; /** GuardrailMetadata */ GuardrailMetadata: { - /** Guardrail Name */ + /** + * Guardrail Name + * @description Guardrail display name, falling back to the guardrail ID + */ guardrail_name: string; }; /** GuardrailMetric */ GuardrailMetric: { - /** Api Key Breakdown */ + /** + * Api Key Breakdown + * @description Per-key guardrail spend and unpriced units, limited to keys in the day's API key breakdown + */ api_key_breakdown?: { [key: string]: components["schemas"]["GuardrailKeyMetric"]; }; + /** @description Guardrail name information */ metadata: components["schemas"]["GuardrailMetadata"]; + /** @description Guardrail spend and unpriced units across all matching API keys */ metrics: components["schemas"]["GuardrailSpendMetrics"]; }; /** GuardrailSpendMetrics */ GuardrailSpendMetrics: { /** * Spend + * @description Guardrail spend in USD, already included in key and model spend * @default 0 */ spend: number; /** * Untracked Units + * @description Guardrail units with no known price, excluded from spend * @default 0 */ untracked_units: number; @@ -76633,7 +76648,7 @@ export interface operations { end_date?: string | null; /** @description Filter by specific model */ model?: string | null; - /** @description Filter by specific API key */ + /** @description Filter by reporting API key ID. Missing keys are reported as unattributed. For a literal key unattributed or a key starting with raw:, add an extra raw: prefix */ api_key?: string | null; /** @description Filter by specific user ID. Admins can filter by any user or omit for global view. Non-admins must provide their own user_id. */ user_id?: string | null; @@ -76682,7 +76697,7 @@ export interface operations { end_date?: string | null; /** @description Filter by specific model */ model?: string | null; - /** @description Filter by specific API key */ + /** @description Filter by reporting API key ID. Missing keys are reported as unattributed. For a literal key unattributed or a key starting with raw:, add an extra raw: prefix */ api_key?: string | null; /** @description Filter by specific user ID. Admins can filter by any user or omit for global view. Non-admins must provide their own user_id. */ user_id?: string | null; @@ -76727,7 +76742,7 @@ export interface operations { end_date?: string | null; /** @description Filter by specific model */ model?: string | null; - /** @description Filter by specific API key */ + /** @description Filter by reporting API key ID. Missing keys are reported as unattributed. For a literal key unattributed or a key starting with raw:, add an extra raw: prefix */ api_key?: string | null; /** @description Filter by specific user ID. Admins can filter by any user or omit for global view. Non-admins must provide their own user_id. */ user_id?: string | null; @@ -76773,7 +76788,7 @@ export interface operations { end_date?: string | null; /** @description Filter by specific model */ model?: string | null; - /** @description Filter by specific API key */ + /** @description Filter by reporting API key ID. Missing keys are reported as unattributed. For a literal key unattributed or a key starting with raw:, add an extra raw: prefix */ api_key?: string | null; /** @description Filter by specific user ID. Admins can filter by any user or omit for global view. Non-admins must provide their own user_id. */ user_id?: string | null; @@ -76820,7 +76835,7 @@ export interface operations { end_date?: string | null; /** @description Filter by specific model */ model?: string | null; - /** @description Filter by specific API key */ + /** @description Filter by reporting API key ID. Missing keys are reported as unattributed. For a literal key unattributed or a key starting with raw:, add an extra raw: prefix */ api_key?: string | null; /** @description Filter by specific user ID. Admins can filter by any user or omit for global view. Non-admins must provide their own user_id. */ user_id?: string | null; @@ -76866,7 +76881,7 @@ export interface operations { end_date?: string | null; /** @description Filter by specific model */ model?: string | null; - /** @description Filter by specific API key */ + /** @description Filter by reporting API key ID. Missing keys are reported as unattributed. For a literal key unattributed or a key starting with raw:, add an extra raw: prefix */ api_key?: string | null; /** @description Filter by specific user ID. Admins can filter by any user or omit for global view. Non-admins must provide their own user_id. */ user_id?: string | null; @@ -76912,7 +76927,7 @@ export interface operations { end_date?: string | null; /** @description Filter by specific model */ model?: string | null; - /** @description Filter by specific API key */ + /** @description Filter by reporting API key ID. Missing keys are reported as unattributed. For a literal key unattributed or a key starting with raw:, add an extra raw: prefix */ api_key?: string | null; /** @description Filter by specific user ID. Admins can filter by any user or omit for global view. Non-admins must provide their own user_id. */ user_id?: string | null; @@ -76958,7 +76973,7 @@ export interface operations { end_date?: string | null; /** @description Filter by specific model */ model?: string | null; - /** @description Filter by specific API key */ + /** @description Filter by reporting API key ID. Missing keys are reported as unattributed. For a literal key unattributed or a key starting with raw:, add an extra raw: prefix */ api_key?: string | null; /** @description Filter by specific user ID. Admins can filter by any user or omit for global view. Non-admins must provide their own user_id. */ user_id?: string | null; From f7f87e6193e151db09ce3db40f26dcff5333a59b Mon Sep 17 00:00:00 2001 From: Ahmet Selim Fedakar <122402435+selimfedakar@users.noreply.github.com> Date: Fri, 30 Oct 2026 21:01:00 +0300 Subject: [PATCH 18/74] fix(responses): normalize empty function_call_output to an empty tool result (#48845) * fix(responses): normalize empty function_call_output to an empty tool result * test(responses): preserve empty tool outputs through guardrails --------- Co-authored-by: Tin --- .../transformation.py | 10 ++++--- ...test_openai_responses_guardrail_handler.py | 14 +++++++--- .../test_anthropic_responses_bridge.py | 26 +++++++++++++++++ ...test_function_call_output_normalization.py | 17 ++++++++++- .../test_google_ai_studio_responses_wire.py | 28 +++++++++++++++++++ 5 files changed, 86 insertions(+), 9 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index ce8841c9190..8f0a85d67de 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -398,23 +398,25 @@ def tool_output_to_chat_content( if single_media is not None: return [single_media] if isinstance(output, list): + parts: Final = cast(Sequence[object], output) # cast-ok: checked list shape, elements remain unvalidated normalized_blocks: Final[list[ToolMessageContentPart]] = [] text_acc: Final[list[str]] = [] - for part in output: + for part in parts: if not isinstance(part, dict): continue part_type = part.get("type") if part_type in ("input_text", "output_text", "text"): txt = part.get("text") - if isinstance(txt, str) and txt: + if isinstance(txt, str): text_acc.append(txt) - normalized_blocks.append({"type": "text", "text": txt}) + if txt: + normalized_blocks.append({"type": "text", "text": txt}) elif (media := decode_responses_media(part)) is not None: normalized_blocks.append(media) if any(b.get("type") != "text" for b in normalized_blocks): return normalized_blocks - if text_acc: + if any(text_acc) or len(text_acc) == len(parts): return "".join(text_acc) try: return json.dumps(output) diff --git a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py index 673e880a1c0..d732d922375 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -724,16 +724,23 @@ class TestToolResultItemsReachTextsGuardrails: assert blocked.value.status_code == 400 @pytest.mark.asyncio - @pytest.mark.parametrize("empty_output", ["", None], ids=["empty_string", "null"]) - async def test_empty_tool_output_sends_no_text_and_stays_as_sent(self, empty_output: str | None) -> None: + @pytest.mark.parametrize(("call_item", "output_type"), _TOOL_TURNS, ids=["function", "custom"]) + @pytest.mark.parametrize( + "empty_output", + ["", None, [], *[[{"type": kind, "text": ""}] for kind in ("input_text", "output_text", "text")]], + ids=["empty_string", "null", "empty_list", "input_text", "output_text", "text"], + ) + async def test_empty_tool_output_sends_no_text_and_stays_as_sent( + self, call_item: dict[str, str], output_type: str, empty_output: object + ) -> None: guardrail: Final = RecordingMaskingGuardrail(guardrail_name="test") - call_item, output_type = _TOOL_TURNS[0] result: Final = await OpenAIResponsesHandler().process_input_messages( _tool_result_request(call_item, output_type, empty_output), guardrail ) assert guardrail.seen_texts == [["Weather?"]] + assert guardrail.seen_message_contents == [["Weather?", None]] assert result["input"][2] == {"type": output_type, "call_id": "call_1", "output": empty_output} @pytest.mark.asyncio @@ -1103,7 +1110,6 @@ class TestToolResultItemsReachTextsGuardrails: ["visible"], id="non_dict_part", ), - pytest.param([], ["[]"], id="empty"), pytest.param({"result": "SECRET forecast"}, ['{"result": "SECRET forecast"}'], id="dict"), pytest.param( [{"type": "input_image", "file_id": "file-map"}], diff --git a/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py b/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py index ef198b487b9..0341511496c 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py +++ b/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py @@ -23,6 +23,7 @@ from litellm.responses.litellm_completion_transformation.transformation import ( ) from litellm.types.llms.openai import ResponsesAPIStreamEvents from litellm.types.utils import Message, ModelResponse +from tests.unit.proxy.conftest import httpx_transport def test_response_api_handler_merges_metadata_and_service_tier_without_error(): @@ -593,6 +594,31 @@ def _anthropic_message(*content: Mapping[str, object]) -> httpx.Response: ) +@pytest.mark.usefixtures(httpx_transport.__name__) +@pytest.mark.parametrize("output", ["", [], [{"type": "input_text", "text": ""}]]) +def test_empty_responses_tool_output_reaches_anthropic_without_json_text( + respx_mock: respx.MockRouter, output: str | list[dict[str, str]] +) -> None: + route: Final = respx_mock.post(_ANTHROPIC_MESSAGES_URL).mock( + return_value=_anthropic_message({"type": "text", "text": "ok"}) + ) + response: Final = litellm.responses( + model=_MODEL, + api_key="mock_api_key", + input=[ + {"role": "user", "content": "Check the weather"}, + {"type": "function_call", "call_id": "call_empty", "name": "get_weather", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "call_empty", "output": output}, + ], + ) + sent: Final = json.loads(route.calls.last.request.content) + assert response.output_text == "ok" + assert sent["messages"][-1] == { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": "call_empty", "content": ""}], + } + + def _replay(output: Sequence[BaseModel]) -> list[object]: return [*_FIRST_INPUT, *(item.model_dump(exclude_none=True) for item in output), *_NEXT_INPUT] diff --git a/tests/unit/responses/litellm_completion_transformation/test_function_call_output_normalization.py b/tests/unit/responses/litellm_completion_transformation/test_function_call_output_normalization.py index 65800e42124..fce06598d29 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_function_call_output_normalization.py +++ b/tests/unit/responses/litellm_completion_transformation/test_function_call_output_normalization.py @@ -50,9 +50,10 @@ def test_image_detail_mime_and_adjacent_text_survive_normalization(kind: str, de else "data:application/octet-stream;base64,AAAA", **({} if nested_url else metadata), } - parts: Final = [{"type": "input_text", "text": "Screenshot follows"}, image] + parts: Final = [{"type": "input_text", "text": ""}, {"type": "input_text", "text": "Screenshot follows"}, image] original: Final = deepcopy(parts) assert _normalize_media_content(kind, parts) == [ + *([{"type": "text", "text": ""}] if kind == "direct" else []), {"type": "text", "text": "Screenshot follows"}, {"type": "image_url", "image_url": {"url": "data:application/octet-stream;base64,AAAA", **metadata}}, ] @@ -97,6 +98,12 @@ def test_media_normalization_preserves_formats_references_and_metadata(kind: str ("output", "expected"), [ (None, ""), + ("", ""), + ([], ""), + ([{"type": "input_text", "text": ""}], ""), + ([{"type": kind, "text": ""} for kind in ("input_text", "output_text", "text")], ""), + ("[]", "[]"), + ([{"type": "input_text", "text": " \n"}], " \n"), ('{"ok":true}', '{"ok":true}'), ([{"type": "input_text", "text": "hello"}, {"type": "input_text", "text": " world"}], "hello world"), ({"type": "file", "status": "ready"}, '{"type": "file", "status": "ready"}'), @@ -106,6 +113,14 @@ def test_media_normalization_preserves_formats_references_and_metadata(kind: str ), ({"type": "input_image", "format": 17}, '{"type": "input_image", "format": 17}'), ([{"type": "unknown", "value": 17}], '[{"type": "unknown", "value": 17}]'), + ([{"type": "input_text", "text": 17}], '[{"type": "input_text", "text": 17}]'), + ([{"type": "input_text"}], '[{"type": "input_text"}]'), + ([{"type": "input_text", "text": None}], '[{"type": "input_text", "text": null}]'), + ( + [{"type": "input_text", "text": ""}, {"type": "unknown", "value": 17}], + '[{"type": "input_text", "text": ""}, {"type": "unknown", "value": 17}]', + ), + ([{"type": "input_text", "text": ""}, 17], '[{"type": "input_text", "text": ""}, 17]'), ( [{"type": "input_text", "text": "keep"}, {"type": "input_image", "file_id": "file-only"}], "keep", diff --git a/tests/unit/responses/litellm_completion_transformation/test_google_ai_studio_responses_wire.py b/tests/unit/responses/litellm_completion_transformation/test_google_ai_studio_responses_wire.py index f63088a606f..c7acc70ba95 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_google_ai_studio_responses_wire.py +++ b/tests/unit/responses/litellm_completion_transformation/test_google_ai_studio_responses_wire.py @@ -84,6 +84,34 @@ async def test_web_search_preview_is_sent_as_google_search() -> None: assert request_body["tools"] == [{"googleSearch": {}}] +@pytest.mark.asyncio +@pytest.mark.parametrize("output", ["", [], [{"type": "input_text", "text": ""}]]) +async def test_empty_responses_tool_output_reaches_gemini_without_json_text( + output: str | list[dict[str, str]], +) -> None: + with respx.mock() as mock_router: + route: Final = mock_router.post(url__regex=_GEMINI_URL).mock( + return_value=httpx.Response(status_code=200, json=_function_call_response("signature")) + ) + response: Final = await litellm.aresponses( + model="gemini/gemini-2.5-flash", + api_key="test-key", + input=[ + {"role": "user", "content": "Check the weather"}, + {"type": "function_call", "call_id": "call_empty", "name": "get_weather", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "call_empty", "output": output}, + ], + ) + request_body: Final = _JSON_OBJECT.validate_json(route.calls.last.request.content) + + assert isinstance(response, ResponsesAPIResponse) + assert request_body["contents"] == [ + {"role": "user", "parts": [{"text": "Check the weather"}]}, + {"role": "model", "parts": [{"function_call": {"name": "get_weather", "args": {}}}]}, + {"role": "user", "parts": [{"function_response": {"name": "get_weather", "response": {"content": ""}}}]}, + ] + + @pytest.mark.asyncio @pytest.mark.parametrize( ("assistant_content", "assistant_parts"), From 840c83b4034a266cdd9e0858b9a568e6c348cf22 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Fri, 30 Oct 2026 11:19:08 -0700 Subject: [PATCH 19/74] test: bedrock claude prompt caching on /v1/messages reports and charges cache tokens (#49265) * test: bedrock claude prompt caching on /v1/messages reports and charges cache tokens * ci: re-run after ci-coverage and assert-shard-coverage were cancelled by the GitHub Actions incident --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...st_bedrock_messages_prompt_caching_wire.py | 191 ++++++++++++++++++ 1 file changed, 191 insertions(+) create mode 100644 tests/integration/messages_endpoint/providers/bedrock/test_bedrock_messages_prompt_caching_wire.py diff --git a/tests/integration/messages_endpoint/providers/bedrock/test_bedrock_messages_prompt_caching_wire.py b/tests/integration/messages_endpoint/providers/bedrock/test_bedrock_messages_prompt_caching_wire.py new file mode 100644 index 00000000000..f2622da1bf6 --- /dev/null +++ b/tests/integration/messages_endpoint/providers/bedrock/test_bedrock_messages_prompt_caching_wire.py @@ -0,0 +1,191 @@ +import base64 +import json +import uuid +from typing import Final + +import anthropic +import httpx +import pytest +from integration._support import claude_code as cc +from integration._support.client import Gateway, eventually +from integration._support.database import read_rows +from integration._support.upstream import _aws_event_frame +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue + +_MODEL_ID: Final = "us.anthropic.claude-sonnet-4-5-20250929-v1:0" +_TOKEN: Final = "synthetic-bedrock-bearer" +_PROMPT: Final = "Reply with exactly the word PONG" +_INPUT_TOKENS: Final = 10 +_OUTPUT_TOKENS: Final = 4 +_CACHE_READ_TOKENS: Final = 900 +_CACHE_WRITE_TOKENS: Final = 300 +_INPUT_RATE: Final = 0.001 +_OUTPUT_RATE: Final = 0.002 +_CACHE_READ_RATE: Final = 0.0001 +_CACHE_WRITE_RATE: Final = 0.0015 +_EXPECTED_SPEND: Final = ( + _INPUT_TOKENS * _INPUT_RATE + + _CACHE_READ_TOKENS * _CACHE_READ_RATE + + _CACHE_WRITE_TOKENS * _CACHE_WRITE_RATE + + _OUTPUT_TOKENS * _OUTPUT_RATE +) +_PROMPT_TOKENS: Final = _INPUT_TOKENS + _CACHE_READ_TOKENS + _CACHE_WRITE_TOKENS +_USAGE: Final = { + "input_tokens": _INPUT_TOKENS, + "output_tokens": _OUTPUT_TOKENS, + "cache_read_input_tokens": _CACHE_READ_TOKENS, + "cache_creation_input_tokens": _CACHE_WRITE_TOKENS, +} +_USER_CONTENT: Final = [{"type": "text", "text": _PROMPT, "cache_control": cc.CACHE}] +_CACHED_SYSTEM_BLOCKS: Final = [block for block in cc.system_blocks() if "cache_control" in block] + + +def _assert_claude_code_breakpoints_reach_bedrock(request: Request, target: str) -> dict[str, JsonValue]: + assert request.method == "POST" and request.target == target, request.target + assert request.headers["authorization"] == f"Bearer {_TOKEN}" + body: Final = cc.JSON_OBJECT.validate_json(request.body) + assert body["anthropic_version"] == "bedrock-2023-05-31", body + assert body["system"] == _CACHED_SYSTEM_BLOCKS, f"billing-header block dropped, breakpoints kept: {body}" + assert body["messages"] == [{"role": "user", "content": _USER_CONTENT}], body + return body + + +def _invoke_chunk(event: dict[str, JsonValue]) -> bytes: + return _aws_event_frame("chunk", {"bytes": base64.b64encode(json.dumps(event).encode()).decode()}, "sc", "u") + + +def _stream_frames(message_id: str) -> tuple[bytes, ...]: + return ( + _invoke_chunk( + { + "type": "message_start", + "message": { + "id": message_id, + "type": "message", + "role": "assistant", + "content": [], + "model": "claude-sonnet-4-5-20250929", + "stop_reason": None, + "stop_sequence": None, + "usage": {**_USAGE, "output_tokens": 1}, + }, + } + ), + _invoke_chunk({"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}), + _invoke_chunk({"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "PONG"}}), + _invoke_chunk({"type": "content_block_stop", "index": 0}), + _invoke_chunk( + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": _OUTPUT_TOKENS}, + } + ), + _invoke_chunk({"type": "message_stop"}), + ) + + +def _spend_row(message_id: str) -> dict[str, object]: + rows: Final = eventually( + lambda: read_rows( + 'SELECT spend, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs" WHERE request_id=%s', + (message_id,), + ), + lambda values: len(values) == 1, + seconds=70, + ) + return rows[0] + + +def _client(gateway: Gateway, transport: httpx.Client) -> anthropic.Anthropic: + return anthropic.Anthropic( + api_key=gateway.key, base_url=str(gateway.client.base_url), max_retries=0, http_client=transport + ) + + +def test_bedrock_claude_cache_control_reaches_invoke_and_cache_tokens_are_reported_and_charged( + gateway: Gateway, +) -> None: + message_id: Final = f"msg_{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + _assert_claude_code_breakpoints_reach_bedrock(request, f"/model/{_MODEL_ID}/invoke") + return Reply( + body=cc.message_reply( + message_id, "claude-sonnet-4-5-20250929", ({"type": "text", "text": "PONG"},), dict(_USAGE) + ) + ) + + with ( + wire_server(respond) as wire, + gateway.scenario() as scenario, + httpx.Client(timeout=15, trust_env=False) as transport, + ): + model: Final = scenario.model( + model=f"bedrock/invoke/{_MODEL_ID}", + api_key=_TOKEN, + aws_region_name="us-east-1", + api_base=wire.url, + input_cost_per_token=_INPUT_RATE, + output_cost_per_token=_OUTPUT_RATE, + cache_read_input_token_cost=_CACHE_READ_RATE, + cache_creation_input_token_cost=_CACHE_WRITE_RATE, + ) + final: Final = _client(gateway, transport).messages.create( + model=model, + max_tokens=32, + system=list(cc.system_blocks()), + messages=[{"role": "user", "content": _USER_CONTENT}], + ) + assert len(wire.drain()) == 1 + assert final.id == message_id, final + assert [(block.type, block.text) for block in final.content] == [("text", "PONG")], final + assert final.usage.model_dump(exclude_none=True, include=set(_USAGE)) == _USAGE, final.usage + row: Final = _spend_row(message_id) + + assert float(str(row["spend"])) == pytest.approx(_EXPECTED_SPEND), row + assert (row["prompt_tokens"], row["completion_tokens"]) == (_PROMPT_TOKENS, _OUTPUT_TOKENS), row + + +def test_bedrock_claude_stream_cache_tokens_from_message_start_are_reported_and_charged(gateway: Gateway) -> None: + message_id: Final = f"msg_{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + body: Final = _assert_claude_code_breakpoints_reach_bedrock( + request, f"/model/{_MODEL_ID}/invoke-with-response-stream" + ) + assert "stream" not in body, body + return Reply(chunks=_stream_frames(message_id), content_type="application/vnd.amazon.eventstream") + + with ( + wire_server(respond) as wire, + gateway.scenario() as scenario, + httpx.Client(timeout=15, trust_env=False) as transport, + ): + model: Final = scenario.model( + model=f"bedrock/invoke/{_MODEL_ID}", + api_key=_TOKEN, + aws_region_name="us-east-1", + api_base=wire.url, + input_cost_per_token=_INPUT_RATE, + output_cost_per_token=_OUTPUT_RATE, + cache_read_input_token_cost=_CACHE_READ_RATE, + cache_creation_input_token_cost=_CACHE_WRITE_RATE, + ) + with _client(gateway, transport).messages.stream( + model=model, + max_tokens=32, + system=list(cc.system_blocks()), + messages=[{"role": "user", "content": _USER_CONTENT}], + ) as stream: + deltas: Final = tuple(stream.text_stream) + final: Final = stream.get_final_message() + assert len(wire.drain()) == 1 + assert deltas == ("PONG",) + assert final.id == message_id, final + assert final.usage.model_dump(exclude_none=True, include=set(_USAGE)) == _USAGE, final.usage + row: Final = _spend_row(message_id) + + assert float(str(row["spend"])) == pytest.approx(_EXPECTED_SPEND), row + assert (row["prompt_tokens"], row["completion_tokens"]) == (_PROMPT_TOKENS, _OUTPUT_TOKENS), row From d23af8bdd08943fe1a0e1d9e528fcb12000b287a Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Fri, 30 Oct 2026 11:24:49 -0700 Subject: [PATCH 20/74] fix(cost): bill the margin once on rebuilt streams and streamed Responses through the chat bridge (#49254) * fix(cost): bill the margin once on rebuilt streams and streamed Responses through the chat bridge The last usage chunk and the Responses terminal event carry litellm's bill in usage.cost, and two readers took that bill for the upstream's own charge and margined it again: completion_cost on a stream_chunk_builder result, and stamp_responses_usage_cost on the chat bridge. The stamp now keeps the upstream's usage.cost under the hidden upstream_usage_cost key, the chunk builder carries it onto the rebuilt response, and the Responses stamp prices that saved figure * fix(cost): hand the Responses stamp the upstream charge through upstream_usage_cost only, never the response-cost header response_cost_calculator returns the response-cost header as the final bill without the margin, so a priced copy carrying the upstream's raw usage.cost there billed the client the unmargined figure. The hidden upstream_usage_cost key already feeds billed_usage_cost, which margins it once. The Responses test now prices through a real Logging object instead of a mocked calculator * fix(cost): leave stamp_responses_usage_cost as on main The Responses chat bridge copies the inner stream's final usage chunk, so the wrapper stamp keeping upstream_usage_cost already prices it once. Changing the Responses stamp was not needed for that and re-priced xai MCP rounds whose usage.cost is already litellm's bill (0.00512 for 0.00312 in test_xai_stream_auto_execute_bills_each_round_at_its_own_price_with_one_fixed_margin) --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../streaming_chunk_builder_utils.py | 18 ++++++++++- .../litellm_core_utils/streaming_handler.py | 18 ++++++++--- .../test_streaming_chunk_builder_utils.py | 24 ++++++++++++++ .../test_streaming_handler.py | 31 +++++++++++++++++++ 4 files changed, 86 insertions(+), 5 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 574f1103c1a..0ddfa1210c2 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -14,12 +14,18 @@ import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.core_helpers import ( PROVIDERS_BILLED_FROM_THEIR_OWN_USAGE_FIELDS, + UPSTREAM_USAGE_COST, chunk_usage, finite_cost, provider_reported_usage_cost, stamp_litellm_bill_on_usage, ) -from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params +from litellm.litellm_core_utils.hidden_params import ( + HIDDEN_PARAMS_ATTR, + get_hidden_params, + set_hidden_param, + set_hidden_params, +) from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import BEDROCK_TRACE_FIELDS, object_items, object_mapping from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import partition_annotations from litellm.litellm_core_utils.prompt_templates.common_utils import ( @@ -1444,6 +1450,14 @@ def _set_stream_builder_response_cost( _set_hidden_response_cost(response, response_cost) +def _carry_upstream_usage_cost(response: ModelResponse, chunks: Sequence[object]) -> None: + chunk_hidden_params: Final = (get_hidden_params(chunk) for chunk in reversed(chunks)) + stamped: Final = (hidden for hidden in chunk_hidden_params if hidden is not None and UPSTREAM_USAGE_COST in hidden) + for hidden in stamped: + set_hidden_param(response, UPSTREAM_USAGE_COST, hidden[UPSTREAM_USAGE_COST]) + return + + def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_obj: Optional["Logging"]) -> None: if logging_obj is None: return @@ -1794,6 +1808,7 @@ def stream_chunk_builder( response, logging_obj, ) + _carry_upstream_usage_cost(response, chunks) # pyright: ignore[reportUnknownArgumentType] # chunks is an untyped list param _stamp_streaming_usage_cost(usage, response, logging_obj) _set_stream_builder_response_cost( response, @@ -1836,6 +1851,7 @@ def stream_chunk_builder( ) break + _carry_upstream_usage_cost(response, chunks) # pyright: ignore[reportUnknownArgumentType] # chunks is an untyped list param _stamp_streaming_usage_cost(usage, response, logging_obj) _set_stream_builder_response_cost( response, diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index b1b3e20f218..9710956c97e 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -24,7 +24,12 @@ import litellm from litellm import verbose_logger from litellm._uuid import uuid from litellm.litellm_core_utils.asyncify import asyncify -from litellm.litellm_core_utils.core_helpers import chunk_usage, provider_reported_usage_cost +from litellm.litellm_core_utils.core_helpers import ( + UPSTREAM_USAGE_COST, + chunk_usage, + provider_reported_usage_cost, + stamp_litellm_bill_on_usage, +) from litellm.litellm_core_utils.hidden_params import ( HIDDEN_PARAMS_ATTR, get_hidden_params, @@ -98,12 +103,17 @@ def _stamp_usage_cost( usage_chunk: ModelResponse | TextCompletionResponse | ModelResponseStream, complete_response: ModelResponse | TextCompletionResponse | ModelResponseStream, ) -> None: - bill: Final = (get_hidden_params(complete_response) or {}).get("response_cost") + complete_hidden: Final = get_hidden_params(complete_response) or {} + bill: Final = complete_hidden.get("response_cost") usage: Final[object] = getattr(usage_chunk, "usage", None) if usage is None or isinstance(bill, bool) or not isinstance(bill, (int, float)): return - if bill > 0 or isinstance(getattr(usage, "cost", None), (int, float)): - setattr(usage, "cost", float(bill)) # noqa: B010 # cost is an optional extra on the usage model + if bill <= 0 and not isinstance(getattr(usage, "cost", None), (int, float)): + return + chunk_hidden: Final = get_hidden_params(usage_chunk) + if chunk_hidden is not None and UPSTREAM_USAGE_COST in complete_hidden: + chunk_hidden[UPSTREAM_USAGE_COST] = complete_hidden[UPSTREAM_USAGE_COST] + stamp_litellm_bill_on_usage(usage_chunk, usage, float(bill)) def _stamp_served_service_tier(response: ModelResponseStream, complete_streaming_response: ModelResponse) -> None: diff --git a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py index c8fa9c94c17..9df45b92686 100644 --- a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -1077,6 +1077,30 @@ def test_cost_field_in_usage_chunks(): assert usage.completion_tokens == 5 +def test_the_rebuilt_response_keeps_the_upstream_cost_a_stamped_usage_chunk_saved(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "cost_margin_config", {"global": 0.5}) + text_chunk = ModelResponseStream( + id="chatcmpl-1", + model="gpt-4.1", + choices=[StreamingChoices(finish_reason="stop", index=0, delta=Delta(content="Hi"))], + ) + usage_chunk = ModelResponseStream( + id="chatcmpl-1", + model="gpt-4.1", + choices=[], + usage=Usage(completion_tokens=5, prompt_tokens=10, total_tokens=15, cost=0.0015), + ) + usage_chunk._hidden_params = {"response_cost": 0.0015, "upstream_usage_cost": 0.001} + + rebuilt = stream_chunk_builder([text_chunk, usage_chunk], messages=[{"role": "user", "content": "hi"}]) + + assert rebuilt is not None + assert rebuilt._hidden_params["upstream_usage_cost"] == 0.001 + assert litellm.completion_cost(rebuilt, custom_llm_provider="openai") == pytest.approx(0.0015), ( + "the upstream's 0.001 gets the 50% margin once, litellm's 0.0015 bill is not margined again" + ) + + def _perplexity_usage_chunk(cost: Mapping[str, float], finish_reason: str | None) -> ModelResponseStream: return ModelResponseStream( diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index 148b7193ef6..6df4731e335 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -6847,6 +6847,37 @@ def test_the_usage_chunk_carries_the_bill_the_builder_priced_for_the_spend_row() assert usage_chunk.usage is not None and usage_chunk.usage.cost == 0.00032, "the client sees what the row bills" +@pytest.mark.parametrize("upstream_cost", [None, 0.001]) +def test_rebuilding_a_priced_stream_bills_the_margin_once(monkeypatch: pytest.MonkeyPatch, upstream_cost: float | None) -> None: + from litellm.litellm_core_utils.core_helpers import stamp_litellm_bill_on_usage + from litellm.litellm_core_utils.streaming_handler import ( + _stamp_usage_cost, # pyright: ignore[reportPrivateUsage] # the helper under test + ) + + monkeypatch.setattr(litellm, "cost_margin_config", {"global": 0.5}) + model: Final = "anthropic/claude-sonnet-4-5" + shared_usage: Final = Usage(prompt_tokens=9, completion_tokens=16, total_tokens=25) + if upstream_cost is not None: + setattr(shared_usage, "cost", upstream_cost) + reference: Final = ModelResponse(model=model, usage=shared_usage.model_copy()) + billed_once: Final = litellm.completion_cost(reference, custom_llm_provider="anthropic") + complete: Final = ModelResponse(model=model, usage=shared_usage) + stamp_litellm_bill_on_usage(complete, shared_usage, billed_once) + text_chunk: Final = ModelResponseStream( + id="chunk-1", model=model, choices=[StreamingChoices(index=0, delta=Delta(content="hi"), finish_reason="stop")] + ) + usage_chunk: Final = ModelResponseStream(id="chunk-1", model=model, choices=[], usage=shared_usage) + + _stamp_usage_cost(usage_chunk, complete) + rebuilt: Final = litellm.stream_chunk_builder([text_chunk, usage_chunk], messages=[{"role": "user", "content": "hi"}]) + + assert rebuilt is not None and rebuilt.usage is not None # pyright: ignore[reportAttributeAccessIssue] # ModelResponse carries usage at runtime + assert usage_chunk.usage is not None and usage_chunk.usage.cost == pytest.approx(billed_once) + assert litellm.completion_cost(rebuilt, custom_llm_provider="anthropic") == pytest.approx(billed_once), ( + "pricing the rebuilt stream must not margin litellm's own bill again" + ) + + def test_chunk_creator_keeps_provider_model_private_across_stream(): from litellm.router_utils.add_retry_fallback_headers import ( get_hidden_params_dict, From a1c238ff4a1e1e8fc9cf2e9520c3132418e400f6 Mon Sep 17 00:00:00 2001 From: yujonglee Date: Fri, 30 Oct 2026 11:31:04 -0700 Subject: [PATCH 21/74] refactor(rust): extract SecretValue and use it for Redis credentials (#49186) * refactor(rust): move SecretValue into litellm-secret-value * docs(rust): add rust-redaction skill * refactor(redis-client): carry credentials as SecretValue --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../.agents/skills/rust-redaction/SKILL.md | 43 +++++++++ litellm-rust/AGENTS.md | 2 + litellm-rust/Cargo.lock | 50 ++++++++--- litellm-rust/Cargo.toml | 1 + litellm-rust/crates/auth-azure/Cargo.toml | 1 + litellm-rust/crates/auth-azure/src/native.rs | 6 +- litellm-rust/crates/auth-azure/src/resolve.rs | 9 +- litellm-rust/crates/auth-azure/src/types.rs | 4 +- litellm-rust/crates/auth-gcp/Cargo.toml | 1 + litellm-rust/crates/auth-gcp/src/auth.rs | 2 +- litellm-rust/crates/auth-gcp/src/config.rs | 4 +- litellm-rust/crates/auth-types/Cargo.toml | 2 +- .../crates/auth-types/src/credential.rs | 5 +- litellm-rust/crates/auth-types/src/lib.rs | 2 - litellm-rust/crates/auth-types/src/policy.rs | 3 +- litellm-rust/crates/auth-types/src/token.rs | 3 +- litellm-rust/crates/auth-types/tests/token.rs | 6 +- litellm-rust/crates/auth/Cargo.toml | 3 + litellm-rust/crates/auth/tests/facade.rs | 3 +- litellm-rust/crates/cache-gcs/Cargo.toml | 1 + litellm-rust/crates/cache-gcs/src/token.rs | 4 +- .../crates/callbacks-legacy-python/Cargo.toml | 2 +- .../callbacks-legacy-python/src/adapter.rs | 2 +- litellm-rust/crates/config/Cargo.toml | 2 +- litellm-rust/crates/config/src/mcp.rs | 2 +- litellm-rust/crates/config/src/redis.rs | 2 +- litellm-rust/crates/config/src/settings.rs | 2 +- litellm-rust/crates/gateway-auth/Cargo.toml | 2 +- .../crates/gateway-auth/src/authentication.rs | 2 +- litellm-rust/crates/gateway-auth/src/http.rs | 2 +- litellm-rust/crates/gateway-auth/src/keys.rs | 2 +- litellm-rust/crates/gateway-auth/src/ui.rs | 2 +- .../crates/gateway-auth/tests/auth.rs | 2 +- .../crates/gateway-auth/tests/keys.rs | 2 +- litellm-rust/crates/gateway-auth/tests/ui.rs | 2 +- .../crates/gateway-inference/Cargo.toml | 1 + .../crates/gateway-inference/src/ocr.rs | 2 +- .../gateway-inference/tests/support/mod.rs | 4 +- .../crates/gateway-management/Cargo.toml | 2 +- .../crates/gateway-management/src/keys.rs | 2 +- .../crates/gateway-management/tests/keys.rs | 2 +- litellm-rust/crates/gateway-mcp/Cargo.toml | 2 +- .../crates/gateway-mcp/src/configured.rs | 2 +- litellm-rust/crates/gateway-ui/Cargo.toml | 2 +- .../crates/gateway-ui/tests/routes.rs | 2 +- litellm-rust/crates/gateway/Cargo.toml | 2 +- litellm-rust/crates/gateway/src/redis.rs | 10 +-- litellm-rust/crates/gateway/src/secrets.rs | 2 +- litellm-rust/crates/gateway/src/settings.rs | 2 +- litellm-rust/crates/gateway/tests/mcp.rs | 2 +- litellm-rust/crates/gateway/tests/server.rs | 2 +- litellm-rust/crates/host/Cargo.toml | 2 +- litellm-rust/crates/host/src/interceptors.rs | 2 +- litellm-rust/crates/inference-chat/Cargo.toml | 1 + .../crates/inference-chat/src/handler.rs | 2 +- .../crates/inference-chat/tests/caching.rs | 4 +- .../crates/inference-messages/Cargo.toml | 1 + .../crates/inference-messages/src/handler.rs | 4 +- .../inference-messages/tests/caching.rs | 4 +- litellm-rust/crates/inference-ocr/Cargo.toml | 1 + .../crates/inference-ocr/src/handler.rs | 2 +- .../inference-ocr/src/provider_config.rs | 16 ++-- .../crates/inference-ocr/src/types.rs | 4 +- litellm-rust/crates/inference-ocr/src/wire.rs | 4 +- .../inference-ocr/tests/ocr/azure_ai.rs | 6 +- .../inference-ocr/tests/ocr/lifecycle.rs | 2 +- .../crates/inference-ocr/tests/ocr/main.rs | 2 +- .../crates/inference-responses/Cargo.toml | 1 + .../inference-responses/tests/caching.rs | 4 +- .../crates/inference-testing/Cargo.toml | 1 + .../crates/inference-testing/src/lib.rs | 3 +- litellm-rust/crates/llms/Cargo.toml | 1 + .../llms/src/anthropic/chat/transformation.rs | 2 +- .../crates/llms/src/anthropic/common_utils.rs | 4 +- .../src/azure_ai/messages/transformation.rs | 2 +- .../document_intelligence/transformation.rs | 2 +- .../llms/src/azure_ai/ocr/transformation.rs | 4 +- litellm-rust/crates/llms/src/base_llm/auth.rs | 4 +- .../llms/src/base_llm/ocr/transformation.rs | 4 +- .../bedrock/chat/converse_transformation.rs | 4 +- .../messages/transformation.rs | 4 +- .../anthropic_claude_transformation.rs | 4 +- .../bedrock_mantle/messages/transformation.rs | 4 +- .../llms/src/cohere/ocr/transformation.rs | 2 +- .../src/deepseek/messages/transformation.rs | 4 +- .../src/edenai/messages/transformation.rs | 4 +- .../src/minimax/messages/transformation.rs | 4 +- .../llms/src/mistral/ocr/transformation.rs | 2 +- .../src/openai/responses/transformation.rs | 3 +- .../src/openai_like/chat/transformation.rs | 4 +- .../src/openrouter/messages/transformation.rs | 4 +- .../openrouter/responses/transformation.rs | 4 +- .../llms/src/reducto/ocr/transformation.rs | 4 +- .../src/tencent/messages/transformation.rs | 4 +- .../llms/src/vertex_ai/ocr/transformation.rs | 2 +- litellm-rust/crates/python-bridge/Cargo.toml | 1 + .../crates/python-bridge/src/credentials.rs | 4 +- .../python-bridge/src/routes/ocr/project.rs | 2 +- .../python-bridge/src/secrets/callback.rs | 4 +- .../python-bridge/src/secrets/config.rs | 4 +- .../python-bridge/src/secrets/operations.rs | 8 +- .../python-bridge/src/secrets/provider.rs | 4 +- .../python-bridge/src/secrets/python.rs | 4 +- .../python-bridge/src/secrets/resolved.rs | 5 +- .../python-bridge/src/secrets/runtime.rs | 4 +- .../src/secrets/vault/operation.rs | 7 +- litellm-rust/crates/redis-client/Cargo.toml | 1 + .../crates/redis-client/src/client.rs | 11 ++- .../crates/redis-client/src/config.rs | 90 ++++++------------- .../crates/redis-client/src/connection.rs | 4 +- .../crates/redis-client/src/test_support.rs | 3 +- litellm-rust/crates/router-types/Cargo.toml | 1 + .../crates/router-types/src/litellm_params.rs | 4 +- .../router-types/tests/litellm_params.rs | 4 +- litellm-rust/crates/secret-value/Cargo.toml | 14 +++ litellm-rust/crates/secret-value/src/lib.rs | 5 ++ .../src/secret_value.rs} | 6 +- litellm-rust/crates/secrets-aws/Cargo.toml | 1 + .../crates/secrets-aws/src/secret_manager.rs | 5 +- .../secrets-aws/tests/secret_manager.rs | 4 +- litellm-rust/crates/secrets-azure/Cargo.toml | 1 + .../crates/secrets-azure/src/key_vault.rs | 4 +- .../crates/secrets-azure/tests/key_vault.rs | 4 +- .../crates/secrets-cyberark/Cargo.toml | 1 + .../secrets-cyberark/src/secret_manager.rs | 5 +- .../secrets-cyberark/tests/secret_manager.rs | 4 +- litellm-rust/crates/secrets-google/Cargo.toml | 1 + .../crates/secrets-google/src/auth.rs | 2 +- litellm-rust/crates/secrets-google/src/kms.rs | 2 +- .../secrets-google/src/secret_manager.rs | 6 +- .../crates/secrets-hashicorp/Cargo.toml | 1 + .../crates/secrets-hashicorp/src/config.rs | 2 +- .../secrets-hashicorp/src/secret_manager.rs | 5 +- .../secrets-hashicorp/tests/secret_manager.rs | 6 +- litellm-rust/crates/secrets-types/Cargo.toml | 2 +- .../secrets-types/src/base_secret_manager.rs | 3 +- .../crates/secrets-types/src/config.rs | 2 +- .../crates/secrets-types/src/context.rs | 3 +- litellm-rust/crates/secrets-types/src/lib.rs | 1 - .../crates/secrets-types/src/value.rs | 2 +- .../crates/secrets-types/tests/config.rs | 6 +- .../crates/secrets-types/tests/context.rs | 5 +- .../crates/secrets-types/tests/rotation.rs | 6 +- litellm-rust/crates/secrets/Cargo.toml | 1 + .../crates/secrets/src/compatibility.rs | 3 +- litellm-rust/crates/secrets/src/handler.rs | 2 +- litellm-rust/crates/secrets/src/lib.rs | 4 +- litellm-rust/crates/secrets/src/oidc.rs | 3 +- litellm-rust/crates/secrets/src/resolver.rs | 3 +- litellm-rust/crates/secrets/src/source.rs | 3 +- litellm-rust/crates/secrets/tests/aws.rs | 4 +- litellm-rust/crates/secrets/tests/azure.rs | 5 +- .../secrets/tests/common_read_contract.rs | 4 +- litellm-rust/crates/secrets/tests/cyberark.rs | 4 +- litellm-rust/crates/secrets/tests/google.rs | 4 +- litellm-rust/crates/secrets/tests/oidc.rs | 6 +- .../crates/secrets/tests/resolution.rs | 3 +- 157 files changed, 439 insertions(+), 254 deletions(-) create mode 100644 litellm-rust/.agents/skills/rust-redaction/SKILL.md create mode 100644 litellm-rust/crates/secret-value/Cargo.toml create mode 100644 litellm-rust/crates/secret-value/src/lib.rs rename litellm-rust/crates/{auth-types/src/secret.rs => secret-value/src/secret_value.rs} (95%) diff --git a/litellm-rust/.agents/skills/rust-redaction/SKILL.md b/litellm-rust/.agents/skills/rust-redaction/SKILL.md new file mode 100644 index 00000000000..2e906e10607 --- /dev/null +++ b/litellm-rust/.agents/skills/rust-redaction/SKILL.md @@ -0,0 +1,43 @@ +--- +name: rust-redaction +description: Keep secrets out of Debug output in litellm-rust with SecretValue and veil. Use when a struct or enum holds a password, token, key, credential URL or credential provider, when writing or reviewing a Debug impl, or when a type with secrets gets logged or traced +metadata: + internal: true +--- + +# Rust redaction + +Every secret in a Debug output is hidden the same way, so a reviewer can tell at a glance that a field is covered and a new field cannot slip through a hand-written impl + +## Where SecretValue lives + +`SecretValue` lives in `crates/secret-value` and is imported as `litellm_secret_value::SecretValue` everywhere. No crate re-exports it, so there is one import path. Depend on `litellm-secret-value` directly. Do not pull in `litellm-auth-types` or `litellm-auth` just to get it + +`crates/secret-value` holds only the type. Credential acquisition (`TokenProvider`, `CredentialPlan`, provider param tables) stays in `litellm-auth-types`, and generic clients such as `litellm-redis-client` must not depend on it + +## Rules + +- Never hand-write `impl Debug` to hide a value, and never invent markers like `""`, `""` or `.map(|_| "...")`. A hand-written impl silently prints any field added later +- A secret that crosses a crate boundary or lives in config is typed as `SecretValue`. It redacts itself, compares in constant time and only gives up the plaintext through `expose()`. A type whose only secrets are `SecretValue` fields derives plain `Debug` + ```rust + #[derive(Clone, Debug, PartialEq)] + pub struct HostEndpoint { + pub host: String, + pub port: u16, + pub username: Option, + pub password: Option, + pub tls: bool, + pub db: i64, + } + ``` +- Keep a secret as `SecretValue` until the last conversion into the third-party type (redis-rs, reqwest, an SDK) and call `expose()` there. Calling `expose()` early spreads plaintext `String`s through our own types +- A secret that has to stay a raw type, because a serde wire shape or a third-party struct owns the field, is covered by deriving `veil::Redact` on the holder and marking the field `#[redact(with = "[REDACTED]")]`. `Redact` replaces `Debug`, so do not derive both +- A URL that can carry userinfo is a secret as a whole. Do not parse it to show the host. Keep it as a `SecretValue` +- A bag of unparsed settings that may hold credentials (`additional_fields`, raw kwargs) is redacted as a whole with `#[redact(with = "[REDACTED]")]` +- An opaque handle over `Arc` whose trait has no `Debug` gets a newtype whose `Debug` prints only the type name with `finish_non_exhaustive()`. That impl shows nothing, so it is not a redaction impl and the first rule does not apply. A type holding such a handle can derive `Debug` +- Never put a secret in a tracing field or an error message. Log the type with `?value`, or log only non-secret fields. See the rust-tracing and rust-error skills + +## Tests + +- One test per type that holds a secret, built the way production builds it, asserting the plaintext is absent from `format!("{value:?}")` and `[REDACTED]` is present +- Use a distinctive plaintext (`"credential-value"`) so the absence check cannot pass by accident diff --git a/litellm-rust/AGENTS.md b/litellm-rust/AGENTS.md index 9e7a55eb232..5eaf876e3b2 100644 --- a/litellm-rust/AGENTS.md +++ b/litellm-rust/AGENTS.md @@ -1,5 +1,7 @@ # Rust workspace rules +For keeping secrets out of Debug output, follow [.agents/skills/rust-redaction/SKILL.md](.agents/skills/rust-redaction/SKILL.md) + For diagnostic tracing changes, follow [.agents/skills/rust-tracing/SKILL.md](.agents/skills/rust-tracing/SKILL.md) For string-valued enums and their Serde conversions, follow [.agents/skills/rust-string-enums/SKILL.md](.agents/skills/rust-string-enums/SKILL.md) diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index e959f77917b..f91deaaa401 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3380,6 +3380,7 @@ dependencies = [ "litellm-auth-azure", "litellm-auth-gcp", "litellm-auth-types", + "litellm-secret-value", ] [[package]] @@ -3416,6 +3417,7 @@ dependencies = [ "azure_identity", "litellm-auth-types", "litellm-error", + "litellm-secret-value", "moka", "rstest", "serde_json", @@ -3435,6 +3437,7 @@ dependencies = [ "http 1.4.2", "litellm-auth-types", "litellm-error", + "litellm-secret-value", "moka", "rstest", "serde", @@ -3450,10 +3453,10 @@ name = "litellm-auth-types" version = "0.1.0" dependencies = [ "litellm-error", + "litellm-secret-value", "rstest", "serde", "serde_json", - "subtle", "thiserror 2.0.19", "tokio", "veil", @@ -3523,6 +3526,7 @@ dependencies = [ "litellm-cache-testing", "litellm-error", "litellm-http", + "litellm-secret-value", "percent-encoding", "reqwest", "rstest", @@ -3668,10 +3672,10 @@ dependencies = [ name = "litellm-callbacks-legacy-python" version = "0.1.0" dependencies = [ - "litellm-auth", "litellm-error", "litellm-host", "litellm-host-python", + "litellm-secret-value", "proptest", "pyo3", "rstest", @@ -3692,9 +3696,9 @@ dependencies = [ name = "litellm-config" version = "0.1.0" dependencies = [ - "litellm-auth-types", "litellm-error", "litellm-router-types", + "litellm-secret-value", "rstest", "serde", "serde_json", @@ -3804,7 +3808,6 @@ dependencies = [ "envy", "futures-util", "http-body-util", - "litellm-auth-types", "litellm-circuit-breaker", "litellm-config", "litellm-error", @@ -3817,6 +3820,7 @@ dependencies = [ "litellm-llms", "litellm-redis-client", "litellm-redis-testing", + "litellm-secret-value", "litellm-secrets", "litellm-tracing", "rstest", @@ -3839,9 +3843,9 @@ dependencies = [ "axum", "axum-login", "futures-util", - "litellm-auth-types", "litellm-config", "litellm-error", + "litellm-secret-value", "litellm-secrets", "rstest", "serde", @@ -3879,6 +3883,7 @@ dependencies = [ "litellm-llms", "litellm-llms-types", "litellm-router", + "litellm-secret-value", "litellm-secrets", "rstest", "serde", @@ -3895,9 +3900,9 @@ name = "litellm-gateway-management" version = "0.1.0" dependencies = [ "base64 0.22.1", - "litellm-auth-types", "litellm-error", "litellm-gateway-auth", + "litellm-secret-value", "rand 0.8.7", "rstest", "thiserror 2.0.19", @@ -3912,9 +3917,9 @@ dependencies = [ "base64 0.22.1", "futures-util", "http 1.4.2", - "litellm-auth-types", "litellm-config", "litellm-error", + "litellm-secret-value", "litellm-secrets", "moka", "rmcp", @@ -3942,9 +3947,9 @@ dependencies = [ "futures-util", "governor", "jsonwebtoken", - "litellm-auth-types", "litellm-error", "litellm-gateway-auth", + "litellm-secret-value", "rand 0.8.7", "rstest", "serde", @@ -3964,8 +3969,8 @@ name = "litellm-host" version = "0.1.0" dependencies = [ "futures-util", - "litellm-auth", "litellm-coroutine", + "litellm-secret-value", "rstest", "serde_json", "thiserror 2.0.19", @@ -4094,6 +4099,7 @@ dependencies = [ "litellm-inference-testing", "litellm-llms", "litellm-llms-types", + "litellm-secret-value", "litellm-secrets", "litellm-tracing", "rstest", @@ -4124,6 +4130,7 @@ dependencies = [ "litellm-llms", "litellm-llms-types", "litellm-router-types", + "litellm-secret-value", "litellm-secrets", "litellm-tracing", "reqwest", @@ -4154,6 +4161,7 @@ dependencies = [ "litellm-inference-testing", "litellm-llms", "litellm-llms-types", + "litellm-secret-value", "litellm-secrets", "litellm-tracing", "mime_guess", @@ -4188,6 +4196,7 @@ dependencies = [ "litellm-inference-testing", "litellm-llms", "litellm-llms-types", + "litellm-secret-value", "litellm-secrets", "litellm-tracing", "reqwest", @@ -4207,6 +4216,7 @@ dependencies = [ "litellm-auth", "litellm-http", "litellm-inference", + "litellm-secret-value", "litellm-secrets", "tokio", ] @@ -4254,6 +4264,7 @@ dependencies = [ "litellm-llms-types", "litellm-python-compat", "litellm-router-types", + "litellm-secret-value", "litellm-secrets", "reqwest", "rstest", @@ -4347,6 +4358,7 @@ dependencies = [ "litellm-llms-types", "litellm-model-catalog", "litellm-router-types", + "litellm-secret-value", "litellm-secrets", "litellm-secrets-aws", "litellm-secrets-types", @@ -4396,6 +4408,7 @@ dependencies = [ "futures-util", "litellm-circuit-breaker", "litellm-redis-testing", + "litellm-secret-value", "redis", "rstest", "thiserror 2.0.19", @@ -4432,12 +4445,23 @@ name = "litellm-router-types" version = "0.1.0" dependencies = [ "litellm-auth-types", + "litellm-secret-value", "rstest", "serde", "serde_json", "serde_with", ] +[[package]] +name = "litellm-secret-value" +version = "0.1.0" +dependencies = [ + "rstest", + "serde", + "subtle", + "veil", +] + [[package]] name = "litellm-secrets" version = "0.1.0" @@ -4451,6 +4475,7 @@ dependencies = [ "litellm-error", "litellm-http", "litellm-python-compat", + "litellm-secret-value", "litellm-secrets-aws", "litellm-secrets-azure", "litellm-secrets-cyberark", @@ -4479,6 +4504,7 @@ dependencies = [ "base64 0.22.1", "litellm-auth-aws", "litellm-core-utils", + "litellm-secret-value", "litellm-secrets-types", "litellm-tracing", "rstest", @@ -4498,6 +4524,7 @@ dependencies = [ "litellm-auth-types", "litellm-core-utils", "litellm-http", + "litellm-secret-value", "litellm-secrets-types", "percent-encoding", "reqwest", @@ -4519,6 +4546,7 @@ dependencies = [ "base64 0.22.1", "litellm-core-utils", "litellm-http", + "litellm-secret-value", "litellm-secrets-types", "litellm-tracing", "moka", @@ -4549,6 +4577,7 @@ dependencies = [ "litellm-auth-types", "litellm-core-utils", "litellm-http", + "litellm-secret-value", "litellm-secrets-types", "moka", "percent-encoding", @@ -4570,6 +4599,7 @@ dependencies = [ "litellm-core-utils", "litellm-error", "litellm-http", + "litellm-secret-value", "litellm-secrets-types", "reqwest", "rstest", @@ -4589,7 +4619,7 @@ dependencies = [ name = "litellm-secrets-types" version = "0.1.0" dependencies = [ - "litellm-auth-types", + "litellm-secret-value", "moka", "rstest", "serde", diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 49a33e08456..75cac2a5d63 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -37,6 +37,7 @@ litellm-callbacks-legacy-python = { path = "crates/callbacks-legacy-python" } litellm-framer = { path = "crates/framer" } litellm-auth = { path = "crates/auth" } litellm-auth-types = { path = "crates/auth-types" } +litellm-secret-value = { path = "crates/secret-value" } litellm-auth-aws = { path = "crates/auth-aws" } litellm-auth-azure = { path = "crates/auth-azure" } litellm-auth-gcp = { path = "crates/auth-gcp" } diff --git a/litellm-rust/crates/auth-azure/Cargo.toml b/litellm-rust/crates/auth-azure/Cargo.toml index f99972f813b..7dcb0e513a7 100644 --- a/litellm-rust/crates/auth-azure/Cargo.toml +++ b/litellm-rust/crates/auth-azure/Cargo.toml @@ -6,6 +6,7 @@ license.workspace = true repository.workspace = true [dependencies] +litellm-secret-value.workspace = true litellm-auth-types.workspace = true moka.workspace = true diff --git a/litellm-rust/crates/auth-azure/src/native.rs b/litellm-rust/crates/auth-azure/src/native.rs index 9fda67746d1..f40745ab7a2 100644 --- a/litellm-rust/crates/auth-azure/src/native.rs +++ b/litellm-rust/crates/auth-azure/src/native.rs @@ -13,7 +13,8 @@ use azure_identity::{ use sha2::{Digest, Sha256}; use litellm_auth_types::{Error, ErrorDetail}; -use litellm_auth_types::{InputSource, ResolvedCredential, SecretValue, Sourced}; +use litellm_auth_types::{InputSource, ResolvedCredential, Sourced}; +use litellm_secret_value::SecretValue; use super::credential_provider_cache::{ AzureCredentialProviderCache, AzureCredentialProviderCacheKey, @@ -498,7 +499,8 @@ mod tests { use azure_core::{Bytes, Result}; use super::{NativeAzureRequest, NativeAzureTokenAcquirer, ValidatedAzureRequest}; - use litellm_auth_types::{InputSource, SecretValue, Sourced}; + use litellm_auth_types::{InputSource, Sourced}; + use litellm_secret_value::SecretValue; fn deployment(value: T) -> Sourced { Sourced::new(value, InputSource::Deployment) diff --git a/litellm-rust/crates/auth-azure/src/resolve.rs b/litellm-rust/crates/auth-azure/src/resolve.rs index 25b71741342..a92630af26a 100644 --- a/litellm-rust/crates/auth-azure/src/resolve.rs +++ b/litellm-rust/crates/auth-azure/src/resolve.rs @@ -1,7 +1,9 @@ use litellm_auth_types::{ - CredentialFileRef, CredentialLookup, CredentialRef, InputSource, ResolvedCredential, - SecretValue, Sourced, TokenProviderHandle, + CredentialFileRef, CredentialLookup, CredentialRef, InputSource, ResolvedCredential, Sourced, + TokenProviderHandle, }; +use litellm_secret_value::SecretValue; + use litellm_auth_types::{Error, ErrorDetail}; use std::future::Future; use std::pin::Pin; @@ -500,8 +502,9 @@ mod tests { use litellm_auth_types::ResolvedCredential; use litellm_auth_types::{ CredentialFileRef, CredentialLookup, CredentialLookupFuture, CredentialRef, - CredentialResolver, CredentialResolverHandle, InputSource, SecretValue, Sourced, + CredentialResolver, CredentialResolverHandle, InputSource, Sourced, }; + use litellm_secret_value::SecretValue; #[derive(Debug)] struct FileResolver; diff --git a/litellm-rust/crates/auth-azure/src/types.rs b/litellm-rust/crates/auth-azure/src/types.rs index a042937a047..6f8290645cf 100644 --- a/litellm-rust/crates/auth-azure/src/types.rs +++ b/litellm-rust/crates/auth-azure/src/types.rs @@ -1,8 +1,10 @@ use std::collections::BTreeMap; use litellm_auth_types::{ - CredentialResolverHandle, Error, InputSource, SecretValue, Sourced, TokenProviderHandle, + CredentialResolverHandle, Error, InputSource, Sourced, TokenProviderHandle, }; +use litellm_secret_value::SecretValue; + use serde_json::{Map, Value}; use strum::EnumString; diff --git a/litellm-rust/crates/auth-gcp/Cargo.toml b/litellm-rust/crates/auth-gcp/Cargo.toml index 7844b35ecc2..ad60f0db817 100644 --- a/litellm-rust/crates/auth-gcp/Cargo.toml +++ b/litellm-rust/crates/auth-gcp/Cargo.toml @@ -9,6 +9,7 @@ repository.workspace = true google-sdk = ["dep:google-cloud-auth", "dep:http"] [dependencies] +litellm-secret-value.workspace = true litellm-auth-types.workspace = true base64.workspace = true diff --git a/litellm-rust/crates/auth-gcp/src/auth.rs b/litellm-rust/crates/auth-gcp/src/auth.rs index 8ac1e0f54ba..650d5831796 100644 --- a/litellm-rust/crates/auth-gcp/src/auth.rs +++ b/litellm-rust/crates/auth-gcp/src/auth.rs @@ -258,7 +258,7 @@ fn auth_acquisition_error(error: gcp_auth::Error) -> Error { #[cfg(test)] mod tests { - use litellm_auth_types::SecretValue; + use litellm_secret_value::SecretValue; use super::*; diff --git a/litellm-rust/crates/auth-gcp/src/config.rs b/litellm-rust/crates/auth-gcp/src/config.rs index 5444897aaaf..133e1c26903 100644 --- a/litellm-rust/crates/auth-gcp/src/config.rs +++ b/litellm-rust/crates/auth-gcp/src/config.rs @@ -4,7 +4,9 @@ use base64::{ Engine, alphabet, engine::{DecodePaddingMode, GeneralPurpose, GeneralPurposeConfig}, }; -use litellm_auth_types::{Error, ErrorDetail, InputSource, SecretValue, Sourced}; +use litellm_auth_types::{Error, ErrorDetail, InputSource, Sourced}; +use litellm_secret_value::SecretValue; + use serde_json::{Map, Value}; use litellm_auth_types::VertexParams; diff --git a/litellm-rust/crates/auth-types/Cargo.toml b/litellm-rust/crates/auth-types/Cargo.toml index 400f2640221..f607f0d1da0 100644 --- a/litellm-rust/crates/auth-types/Cargo.toml +++ b/litellm-rust/crates/auth-types/Cargo.toml @@ -7,9 +7,9 @@ repository.workspace = true [dependencies] litellm-error.workspace = true +litellm-secret-value.workspace = true serde.workspace = true serde_json.workspace = true -subtle.workspace = true thiserror.workspace = true veil.workspace = true diff --git a/litellm-rust/crates/auth-types/src/credential.rs b/litellm-rust/crates/auth-types/src/credential.rs index 4f69c6fffc8..36e21f7794a 100644 --- a/litellm-rust/crates/auth-types/src/credential.rs +++ b/litellm-rust/crates/auth-types/src/credential.rs @@ -5,7 +5,8 @@ use std::sync::Arc; use veil::Redact; -use crate::{Error, ResolvedCredential, SecretValue, TokenProviderHandle}; +use crate::{Error, ResolvedCredential, TokenProviderHandle}; +use litellm_secret_value::SecretValue; #[derive(Clone, Debug, PartialEq, Eq)] pub enum CredentialFileRef { @@ -104,8 +105,8 @@ mod tests { CredentialLookup, CredentialLookupFuture, CredentialPlan, CredentialPlanResolution, CredentialRef, CredentialResolver, CredentialResolverHandle, }; - use crate::SecretValue; use crate::{Error, ErrorDetail}; + use litellm_secret_value::SecretValue; #[derive(Debug)] struct HostResolver; diff --git a/litellm-rust/crates/auth-types/src/lib.rs b/litellm-rust/crates/auth-types/src/lib.rs index 99651977867..ab3bd3febbf 100644 --- a/litellm-rust/crates/auth-types/src/lib.rs +++ b/litellm-rust/crates/auth-types/src/lib.rs @@ -5,7 +5,6 @@ mod error; pub mod http; mod params; mod policy; -mod secret; mod token; use serde::{Deserialize, Serialize}; @@ -55,5 +54,4 @@ pub use error::{Error, ErrorDetail}; pub use http::CredentialPlacement; pub use params::{AwsParams, ParamKind, ParamSpec, VertexParams}; pub use policy::{CredentialPlanKind, CredentialRule, ExistingHeaderBehavior, ProviderAuthPolicy}; -pub use secret::SecretValue; pub use token::{ResolvedCredential, TokenFuture, TokenProvider, TokenProviderHandle}; diff --git a/litellm-rust/crates/auth-types/src/policy.rs b/litellm-rust/crates/auth-types/src/policy.rs index 8fff59e284a..2232350965a 100644 --- a/litellm-rust/crates/auth-types/src/policy.rs +++ b/litellm-rust/crates/auth-types/src/policy.rs @@ -71,7 +71,8 @@ impl ProviderAuthPolicy { #[cfg(test)] mod tests { use super::{CredentialPlanKind, CredentialRule, ExistingHeaderBehavior, ProviderAuthPolicy}; - use crate::{CredentialPlacement, ResolvedCredential, SecretValue}; + use crate::{CredentialPlacement, ResolvedCredential}; + use litellm_secret_value::SecretValue; const RULES: &[CredentialRule] = &[CredentialRule { kind: CredentialPlanKind::Static, diff --git a/litellm-rust/crates/auth-types/src/token.rs b/litellm-rust/crates/auth-types/src/token.rs index a3126052065..b948dda44fc 100644 --- a/litellm-rust/crates/auth-types/src/token.rs +++ b/litellm-rust/crates/auth-types/src/token.rs @@ -5,7 +5,8 @@ use std::time::SystemTime; use veil::Redact; -use crate::{Error, SecretValue}; +use crate::Error; +use litellm_secret_value::SecretValue; #[derive(Clone, Debug, PartialEq, Eq)] pub enum ResolvedCredential { diff --git a/litellm-rust/crates/auth-types/tests/token.rs b/litellm-rust/crates/auth-types/tests/token.rs index 7f7027942b0..39908ae7b8a 100644 --- a/litellm-rust/crates/auth-types/tests/token.rs +++ b/litellm-rust/crates/auth-types/tests/token.rs @@ -8,9 +8,9 @@ use std::{ time::{Duration, SystemTime}, }; -use litellm_auth_types::{ - Error, ErrorDetail, ResolvedCredential, SecretValue, TokenProviderHandle, -}; +use litellm_auth_types::{Error, ErrorDetail, ResolvedCredential, TokenProviderHandle}; +use litellm_secret_value::SecretValue; + use rstest::rstest; fn credential(index: usize, access_token: bool) -> ResolvedCredential { diff --git a/litellm-rust/crates/auth/Cargo.toml b/litellm-rust/crates/auth/Cargo.toml index ee4900ebcc2..674ae65a59f 100644 --- a/litellm-rust/crates/auth/Cargo.toml +++ b/litellm-rust/crates/auth/Cargo.toml @@ -16,3 +16,6 @@ litellm-auth-types.workspace = true litellm-auth-aws = { workspace = true, optional = true } litellm-auth-azure = { workspace = true, optional = true } litellm-auth-gcp = { workspace = true, optional = true } + +[dev-dependencies] +litellm-secret-value.workspace = true diff --git a/litellm-rust/crates/auth/tests/facade.rs b/litellm-rust/crates/auth/tests/facade.rs index f1092b15def..5d29bac1072 100644 --- a/litellm-rust/crates/auth/tests/facade.rs +++ b/litellm-rust/crates/auth/tests/facade.rs @@ -1,7 +1,8 @@ use litellm_auth::{ CredentialPlacement, CredentialPlanKind, CredentialRule, ExistingHeaderBehavior, - ProviderAuthPolicy, ResolvedCredential, SecretValue, + ProviderAuthPolicy, ResolvedCredential, }; +use litellm_secret_value::SecretValue; const RULES: &[CredentialRule] = &[CredentialRule { kind: CredentialPlanKind::Static, diff --git a/litellm-rust/crates/cache-gcs/Cargo.toml b/litellm-rust/crates/cache-gcs/Cargo.toml index dbd8c9ac908..878db4b8010 100644 --- a/litellm-rust/crates/cache-gcs/Cargo.toml +++ b/litellm-rust/crates/cache-gcs/Cargo.toml @@ -6,6 +6,7 @@ license.workspace = true repository.workspace = true [dependencies] +litellm-secret-value.workspace = true litellm-http.workspace = true futures-util.workspace = true litellm-auth-gcp.workspace = true diff --git a/litellm-rust/crates/cache-gcs/src/token.rs b/litellm-rust/crates/cache-gcs/src/token.rs index 079ec44dbce..852d8bbf035 100644 --- a/litellm-rust/crates/cache-gcs/src/token.rs +++ b/litellm-rust/crates/cache-gcs/src/token.rs @@ -1,7 +1,9 @@ use std::{future::Future, pin::Pin}; use litellm_auth_gcp::{VertexAuth, VertexConfig}; -use litellm_auth_types::{InputSource, SecretValue, Sourced}; +use litellm_auth_types::{InputSource, Sourced}; +use litellm_secret_value::SecretValue; + use litellm_cache::Error; pub trait TokenSource: Send + Sync + 'static { diff --git a/litellm-rust/crates/callbacks-legacy-python/Cargo.toml b/litellm-rust/crates/callbacks-legacy-python/Cargo.toml index 1cd05710366..0133af16b80 100644 --- a/litellm-rust/crates/callbacks-legacy-python/Cargo.toml +++ b/litellm-rust/crates/callbacks-legacy-python/Cargo.toml @@ -16,7 +16,7 @@ serde_json.workspace = true thiserror.workspace = true [dev-dependencies] -litellm-auth.workspace = true +litellm-secret-value.workspace = true proptest.workspace = true rstest.workspace = true serde_json.workspace = true diff --git a/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs b/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs index 493e91bb5ea..9431b9d1326 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs @@ -925,11 +925,11 @@ assert all(value is failure for name, value in logger.calls if name.endswith('_h mod payload_tests { use std::ffi::CStr; - use litellm_auth::SecretValue; use litellm_host::hooks::CallHooks; use litellm_host::interceptors::{RawResponse, RequestContext, WireRequest}; use litellm_host::lifecycle::ExecutionEvent; use litellm_host_python::{HookStep, PythonCallEvent, PythonOwned, to_py}; + use litellm_secret_value::SecretValue; use proptest::prelude::*; use pyo3::gc::{PyTraverseError, PyVisit}; use pyo3::prelude::*; diff --git a/litellm-rust/crates/config/Cargo.toml b/litellm-rust/crates/config/Cargo.toml index 5b0cd22332f..278aea6200e 100644 --- a/litellm-rust/crates/config/Cargo.toml +++ b/litellm-rust/crates/config/Cargo.toml @@ -6,7 +6,7 @@ license.workspace = true repository.workspace = true [dependencies] -litellm-auth-types.workspace = true +litellm-secret-value.workspace = true litellm-router-types.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/litellm-rust/crates/config/src/mcp.rs b/litellm-rust/crates/config/src/mcp.rs index cbe992f9956..00480fa5be1 100644 --- a/litellm-rust/crates/config/src/mcp.rs +++ b/litellm-rust/crates/config/src/mcp.rs @@ -1,6 +1,6 @@ use std::{collections::BTreeMap, fmt}; -use litellm_auth_types::SecretValue; +use litellm_secret_value::SecretValue; use serde::Deserialize; use crate::Object; diff --git a/litellm-rust/crates/config/src/redis.rs b/litellm-rust/crates/config/src/redis.rs index 5674f993025..7a91691c54b 100644 --- a/litellm-rust/crates/config/src/redis.rs +++ b/litellm-rust/crates/config/src/redis.rs @@ -1,7 +1,7 @@ use std::{collections::BTreeMap, fmt, time::Duration}; -use litellm_auth_types::SecretValue; use litellm_router_types::Spelled; +use litellm_secret_value::SecretValue; use serde::{Deserialize, Deserializer, de::Error as _}; use crate::{AdditionalFields, Error, Object, Value}; diff --git a/litellm-rust/crates/config/src/settings.rs b/litellm-rust/crates/config/src/settings.rs index 9571c4fa5de..1bfc19983f3 100644 --- a/litellm-rust/crates/config/src/settings.rs +++ b/litellm-rust/crates/config/src/settings.rs @@ -1,6 +1,6 @@ use std::fmt; -use litellm_auth_types::SecretValue; +use litellm_secret_value::SecretValue; use serde::Deserialize; use crate::{ diff --git a/litellm-rust/crates/gateway-auth/Cargo.toml b/litellm-rust/crates/gateway-auth/Cargo.toml index 27f20203843..ba9097e85d8 100644 --- a/litellm-rust/crates/gateway-auth/Cargo.toml +++ b/litellm-rust/crates/gateway-auth/Cargo.toml @@ -6,9 +6,9 @@ license.workspace = true repository.workspace = true [dependencies] +litellm-secret-value.workspace = true axum = { workspace = true, features = ["matched-path"] } axum-login.workspace = true -litellm-auth-types.workspace = true litellm-config.workspace = true litellm-error.workspace = true litellm-secrets.workspace = true diff --git a/litellm-rust/crates/gateway-auth/src/authentication.rs b/litellm-rust/crates/gateway-auth/src/authentication.rs index 4ecfaa375b4..349913619f5 100644 --- a/litellm-rust/crates/gateway-auth/src/authentication.rs +++ b/litellm-rust/crates/gateway-auth/src/authentication.rs @@ -1,8 +1,8 @@ use std::{future::Future, pin::Pin, sync::Arc, time::SystemTime}; use axum::http::{Request, request::Parts}; -use litellm_auth_types::SecretValue; use litellm_config::Config; +use litellm_secret_value::SecretValue; use litellm_secrets::source::SecretSource; use sha2::{Digest, Sha256}; use subtle::ConstantTimeEq; diff --git a/litellm-rust/crates/gateway-auth/src/http.rs b/litellm-rust/crates/gateway-auth/src/http.rs index ddbf240509a..ad1718161c0 100644 --- a/litellm-rust/crates/gateway-auth/src/http.rs +++ b/litellm-rust/crates/gateway-auth/src/http.rs @@ -4,7 +4,7 @@ use axum::{ middleware::Next, response::Response, }; -use litellm_auth_types::SecretValue; +use litellm_secret_value::SecretValue; use crate::{AccessRequest, Auth, AuthenticatedRequest, Credential, Error}; diff --git a/litellm-rust/crates/gateway-auth/src/keys.rs b/litellm-rust/crates/gateway-auth/src/keys.rs index 1fe8fa60e0e..0deaf3f7a82 100644 --- a/litellm-rust/crates/gateway-auth/src/keys.rs +++ b/litellm-rust/crates/gateway-auth/src/keys.rs @@ -1,6 +1,6 @@ use std::{future::Future, pin::Pin, time::SystemTime}; -use litellm_auth_types::SecretValue; +use litellm_secret_value::SecretValue; use crate::{Error, hash_token}; diff --git a/litellm-rust/crates/gateway-auth/src/ui.rs b/litellm-rust/crates/gateway-auth/src/ui.rs index 3d655bde187..ee4dd670aa7 100644 --- a/litellm-rust/crates/gateway-auth/src/ui.rs +++ b/litellm-rust/crates/gateway-auth/src/ui.rs @@ -3,7 +3,7 @@ use std::{convert::Infallible, sync::Arc}; use axum::extract::FromRequestParts; use axum::http::{header::AUTHORIZATION, request::Parts}; use axum_login::{AuthUser, AuthnBackend, UserId}; -use litellm_auth_types::SecretValue; +use litellm_secret_value::SecretValue; use serde::Deserialize; use sha2::{Digest, Sha256}; use subtle::ConstantTimeEq; diff --git a/litellm-rust/crates/gateway-auth/tests/auth.rs b/litellm-rust/crates/gateway-auth/tests/auth.rs index 781ba385381..54180e30504 100644 --- a/litellm-rust/crates/gateway-auth/tests/auth.rs +++ b/litellm-rust/crates/gateway-auth/tests/auth.rs @@ -8,9 +8,9 @@ use axum::{ routing::get, }; use futures_util::future::BoxFuture; -use litellm_auth_types::SecretValue; use litellm_config::Config; use litellm_gateway_auth::{Auth, RequireMasterKey, hash_token}; +use litellm_secret_value::SecretValue; use litellm_secrets::source::SecretSource; use rstest::{fixture, rstest}; use tower::ServiceExt; diff --git a/litellm-rust/crates/gateway-auth/tests/keys.rs b/litellm-rust/crates/gateway-auth/tests/keys.rs index 4d16d3573bc..1f6fc461a5b 100644 --- a/litellm-rust/crates/gateway-auth/tests/keys.rs +++ b/litellm-rust/crates/gateway-auth/tests/keys.rs @@ -1,10 +1,10 @@ use std::time::{Duration, SystemTime}; -use litellm_auth_types::SecretValue; use litellm_gateway_auth::{ Error, keys::{KeyHash, KeyLookup, KeyStatus, LookupFuture, verify_key}, }; +use litellm_secret_value::SecretValue; use rstest::{fixture, rstest}; struct Lookup { diff --git a/litellm-rust/crates/gateway-auth/tests/ui.rs b/litellm-rust/crates/gateway-auth/tests/ui.rs index 2e5f72d605d..8d465e1198d 100644 --- a/litellm-rust/crates/gateway-auth/tests/ui.rs +++ b/litellm-rust/crates/gateway-auth/tests/ui.rs @@ -1,6 +1,6 @@ use axum_login::{AuthUser, AuthnBackend}; -use litellm_auth_types::SecretValue; use litellm_gateway_auth::{UiBackend, UiCredentials}; +use litellm_secret_value::SecretValue; use rstest::{fixture, rstest}; #[fixture] diff --git a/litellm-rust/crates/gateway-inference/Cargo.toml b/litellm-rust/crates/gateway-inference/Cargo.toml index 034c3a9826e..69ca561e4b5 100644 --- a/litellm-rust/crates/gateway-inference/Cargo.toml +++ b/litellm-rust/crates/gateway-inference/Cargo.toml @@ -6,6 +6,7 @@ license.workspace = true repository.workspace = true [dependencies] +litellm-secret-value.workspace = true litellm-cache-response.workspace = true axum = { workspace = true, features = ["json", "multipart", "original-uri"] } base64.workspace = true diff --git a/litellm-rust/crates/gateway-inference/src/ocr.rs b/litellm-rust/crates/gateway-inference/src/ocr.rs index f200fd7b141..b8e6ef4ff4f 100644 --- a/litellm-rust/crates/gateway-inference/src/ocr.rs +++ b/litellm-rust/crates/gateway-inference/src/ocr.rs @@ -2,10 +2,10 @@ use litellm_gateway_auth::AuthenticatedRequest; use std::sync::Arc; use axum::{Json, extract::State, http::HeaderMap, response::IntoResponse}; -use litellm_auth::SecretValue; use litellm_inference_ocr::types::{LiteLLMOcrRequest, OcrConnectionInputs, OcrDocumentInput}; use litellm_llms::base_llm::ocr::transformation::decode_request_value; use litellm_llms_types::formats::ocr::OcrDocument; +use litellm_secret_value::SecretValue; use serde_json::Value; use crate::{ diff --git a/litellm-rust/crates/gateway-inference/tests/support/mod.rs b/litellm-rust/crates/gateway-inference/tests/support/mod.rs index 737dc0c830c..185cbc8594d 100644 --- a/litellm-rust/crates/gateway-inference/tests/support/mod.rs +++ b/litellm-rust/crates/gateway-inference/tests/support/mod.rs @@ -13,7 +13,9 @@ use futures_util::future::BoxFuture; use litellm_gateway_inference::{Deployment, Gateway, router}; use litellm_http::{HttpClientPool, HttpSettings, Resolution, media::PublicDnsResolver}; use litellm_inference::resources::CoreResources; -use litellm_secrets::{SecretValue, source::SecretSource}; +use litellm_secret_value::SecretValue; +use litellm_secrets::source::SecretSource; + use serde_json::Value; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tower::ServiceExt; diff --git a/litellm-rust/crates/gateway-management/Cargo.toml b/litellm-rust/crates/gateway-management/Cargo.toml index 502760f8fbe..1ff5dd5319d 100644 --- a/litellm-rust/crates/gateway-management/Cargo.toml +++ b/litellm-rust/crates/gateway-management/Cargo.toml @@ -6,8 +6,8 @@ license.workspace = true repository.workspace = true [dependencies] +litellm-secret-value.workspace = true base64.workspace = true -litellm-auth-types.workspace = true litellm-error.workspace = true litellm-gateway-auth.workspace = true rand.workspace = true diff --git a/litellm-rust/crates/gateway-management/src/keys.rs b/litellm-rust/crates/gateway-management/src/keys.rs index 45d915b7b3d..feeb40b2403 100644 --- a/litellm-rust/crates/gateway-management/src/keys.rs +++ b/litellm-rust/crates/gateway-management/src/keys.rs @@ -1,8 +1,8 @@ use std::{future::Future, pin::Pin, sync::Arc, time::SystemTime}; use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; -use litellm_auth_types::SecretValue; use litellm_gateway_auth::keys::{KeyHash, KeyStatus}; +use litellm_secret_value::SecretValue; use rand::{RngCore, rngs::OsRng}; use crate::Error; diff --git a/litellm-rust/crates/gateway-management/tests/keys.rs b/litellm-rust/crates/gateway-management/tests/keys.rs index bf0221cf1be..8a970bb407c 100644 --- a/litellm-rust/crates/gateway-management/tests/keys.rs +++ b/litellm-rust/crates/gateway-management/tests/keys.rs @@ -7,7 +7,6 @@ use std::{ }; use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; -use litellm_auth_types::SecretValue; use litellm_gateway_auth::{ hash_token, keys::{KeyHash, KeyLookup, KeyStatus, verify_key}, @@ -16,6 +15,7 @@ use litellm_gateway_management::{ Error, keys::{Keys, Revocation}, }; +use litellm_secret_value::SecretValue; use rstest::{fixture, rstest}; use support::{MemoryStore, Unavailable}; diff --git a/litellm-rust/crates/gateway-mcp/Cargo.toml b/litellm-rust/crates/gateway-mcp/Cargo.toml index bed48bad58d..65077eec0ea 100644 --- a/litellm-rust/crates/gateway-mcp/Cargo.toml +++ b/litellm-rust/crates/gateway-mcp/Cargo.toml @@ -7,11 +7,11 @@ license.workspace = true repository.workspace = true [dependencies] +litellm-secret-value.workspace = true axum = { workspace = true, features = ["json", "query"] } futures-util.workspace = true litellm-config.workspace = true litellm-secrets.workspace = true -litellm-auth-types.workspace = true litellm-error.workspace = true base64.workspace = true http.workspace = true diff --git a/litellm-rust/crates/gateway-mcp/src/configured.rs b/litellm-rust/crates/gateway-mcp/src/configured.rs index cc7a52fc029..33cbd243530 100644 --- a/litellm-rust/crates/gateway-mcp/src/configured.rs +++ b/litellm-rust/crates/gateway-mcp/src/configured.rs @@ -8,8 +8,8 @@ use std::{ use base64::{Engine, engine::general_purpose::STANDARD}; use futures_util::future::try_join_all; use http::{HeaderName, HeaderValue}; -use litellm_auth_types::SecretValue; use litellm_config::{McpAuth, McpServer, McpTransport}; +use litellm_secret_value::SecretValue; use litellm_secrets::source::SecretSource; use rmcp::{ RoleClient, ServiceExt, diff --git a/litellm-rust/crates/gateway-ui/Cargo.toml b/litellm-rust/crates/gateway-ui/Cargo.toml index 1c65665be72..a6104455e96 100644 --- a/litellm-rust/crates/gateway-ui/Cargo.toml +++ b/litellm-rust/crates/gateway-ui/Cargo.toml @@ -23,10 +23,10 @@ tower-http = { version = "0.6.11", features = ["fs", "set-header"] } tower-sessions.workspace = true [dev-dependencies] +litellm-secret-value.workspace = true futures-util.workspace = true serde_json.workspace = true tower = { version = "0.5", features = ["util"] } -litellm-auth-types.workspace = true rstest.workspace = true tempfile.workspace = true tokio.workspace = true diff --git a/litellm-rust/crates/gateway-ui/tests/routes.rs b/litellm-rust/crates/gateway-ui/tests/routes.rs index 95246f33a35..86b7db63ed9 100644 --- a/litellm-rust/crates/gateway-ui/tests/routes.rs +++ b/litellm-rust/crates/gateway-ui/tests/routes.rs @@ -16,8 +16,8 @@ use tower_sessions::{ session::{Id, Record}, }; -use litellm_auth_types::SecretValue; use litellm_gateway_auth::UiBackend; +use litellm_secret_value::SecretValue; struct App { router: Router, diff --git a/litellm-rust/crates/gateway/Cargo.toml b/litellm-rust/crates/gateway/Cargo.toml index 4d831aa2314..6eeaa43e013 100644 --- a/litellm-rust/crates/gateway/Cargo.toml +++ b/litellm-rust/crates/gateway/Cargo.toml @@ -6,6 +6,7 @@ license.workspace = true repository.workspace = true [dependencies] +litellm-secret-value.workspace = true axum.workspace = true envy = "0.4.2" http-body-util = "0.1" @@ -17,7 +18,6 @@ thiserror.workspace = true futures-util.workspace = true litellm-gateway-auth.workspace = true litellm-gateway-ui.workspace = true -litellm-auth-types.workspace = true litellm-config.workspace = true litellm-error.workspace = true litellm-circuit-breaker.workspace = true diff --git a/litellm-rust/crates/gateway/src/redis.rs b/litellm-rust/crates/gateway/src/redis.rs index 197f40e7988..c40b289029b 100644 --- a/litellm-rust/crates/gateway/src/redis.rs +++ b/litellm-rust/crates/gateway/src/redis.rs @@ -152,12 +152,12 @@ async fn reachable(client: RedisClient) -> Option { fn client(params: Params, breaker: Option) -> Result { tracing::info!(endpoint = ?params.endpoint, namespace = ?params.namespace, "redis client configured"); let endpoint = match params.endpoint { - RedisEndpoint::Url(url) => Endpoint::Url(url.expose().to_owned()), + RedisEndpoint::Url(url) => Endpoint::Url(url), RedisEndpoint::Host(host) => Endpoint::Host(HostEndpoint { host: host.host, port: host.port, - username: host.username.map(|value| value.expose().to_owned()), - password: host.password.map(|value| value.expose().to_owned()), + username: host.username, + password: host.password, tls: host.tls, db: host.db, }), @@ -170,8 +170,8 @@ fn client(params: Params, breaker: Option) -> Result futures_util::future::BoxFuture< 'a, - Result, litellm_secrets::Error>, + Result, litellm_secrets::Error>, > { Box::pin(async { Ok(None) }) } diff --git a/litellm-rust/crates/gateway/tests/server.rs b/litellm-rust/crates/gateway/tests/server.rs index c876c1cbcbb..7de7a97c870 100644 --- a/litellm-rust/crates/gateway/tests/server.rs +++ b/litellm-rust/crates/gateway/tests/server.rs @@ -159,7 +159,7 @@ async fn mounts_ui_without_exposing_credentials_or_authorizing_inference(inferen std::fs::write(assets.path().join("index.html"), "dashboard").unwrap(); let backend = litellm_gateway_auth::UiBackend::new( "admin".into(), - litellm_auth_types::SecretValue::new("ui-password"), + litellm_secret_value::SecretValue::new("ui-password"), ) .unwrap(); let ui = litellm_gateway_ui::router( diff --git a/litellm-rust/crates/host/Cargo.toml b/litellm-rust/crates/host/Cargo.toml index d3354fa8622..e0ccab4e0ad 100644 --- a/litellm-rust/crates/host/Cargo.toml +++ b/litellm-rust/crates/host/Cargo.toml @@ -6,8 +6,8 @@ license.workspace = true repository.workspace = true [dependencies] +litellm-secret-value.workspace = true futures-util.workspace = true -litellm-auth.workspace = true litellm-coroutine.workspace = true serde_json.workspace = true thiserror.workspace = true diff --git a/litellm-rust/crates/host/src/interceptors.rs b/litellm-rust/crates/host/src/interceptors.rs index 3cff781792f..39af7c410e0 100644 --- a/litellm-rust/crates/host/src/interceptors.rs +++ b/litellm-rust/crates/host/src/interceptors.rs @@ -22,7 +22,7 @@ pub struct RequestContext { /// Optional-param names that carry credentials and must be redacted when logged. pub secret_fields: Vec, /// The credential the route resolved for the provider call. - pub api_key: Option, + pub api_key: Option, } #[derive(Clone, Debug, PartialEq, Eq)] diff --git a/litellm-rust/crates/inference-chat/Cargo.toml b/litellm-rust/crates/inference-chat/Cargo.toml index 8a779d44976..bcde3c8d30f 100644 --- a/litellm-rust/crates/inference-chat/Cargo.toml +++ b/litellm-rust/crates/inference-chat/Cargo.toml @@ -6,6 +6,7 @@ license.workspace = true repository.workspace = true [dependencies] +litellm-secret-value.workspace = true litellm-auth = { workspace = true, features = ["aws", "azure", "gcp"] } litellm-cache-response.workspace = true litellm-core-utils.workspace = true diff --git a/litellm-rust/crates/inference-chat/src/handler.rs b/litellm-rust/crates/inference-chat/src/handler.rs index 1c724a47956..f948e8fffaf 100644 --- a/litellm-rust/crates/inference-chat/src/handler.rs +++ b/litellm-rust/crates/inference-chat/src/handler.rs @@ -58,7 +58,7 @@ pub(super) async fn execute( custom_llm_provider: provider_name.to_owned(), optional_params: Value::Object(call.optional_params), secret_fields: Vec::new(), - api_key: call.api_key.map(litellm_auth::SecretValue::new), + api_key: call.api_key.map(litellm_secret_value::SecretValue::new), }; let authenticated = resolve_auth(auth, environment, &|key| secrets.get(key)).await?; let identity = litellm_host::interceptors::ProviderIdentity { diff --git a/litellm-rust/crates/inference-chat/tests/caching.rs b/litellm-rust/crates/inference-chat/tests/caching.rs index 4b3e32f04bc..dd229d3d657 100644 --- a/litellm-rust/crates/inference-chat/tests/caching.rs +++ b/litellm-rust/crates/inference-chat/tests/caching.rs @@ -140,7 +140,7 @@ impl litellm_secrets::source::SecretSource for ChangingSecrets { name: &'a str, ) -> futures_util::future::BoxFuture< 'a, - Result, litellm_secrets::Error>, + Result, litellm_secrets::Error>, > { Box::pin(async move { let revision = self.revision.load(Ordering::SeqCst); @@ -154,7 +154,7 @@ impl litellm_secrets::source::SecretSource for ChangingSecrets { } else { None }; - Ok(value.map(litellm_secrets::SecretValue::new)) + Ok(value.map(litellm_secret_value::SecretValue::new)) }) } } diff --git a/litellm-rust/crates/inference-messages/Cargo.toml b/litellm-rust/crates/inference-messages/Cargo.toml index 694d9b4de31..ac52cb65a0d 100644 --- a/litellm-rust/crates/inference-messages/Cargo.toml +++ b/litellm-rust/crates/inference-messages/Cargo.toml @@ -6,6 +6,7 @@ license.workspace = true repository.workspace = true [dependencies] +litellm-secret-value.workspace = true bytes.workspace = true futures-util.workspace = true litellm-auth = { workspace = true, features = ["aws", "azure", "gcp"] } diff --git a/litellm-rust/crates/inference-messages/src/handler.rs b/litellm-rust/crates/inference-messages/src/handler.rs index 9d0bf09197e..2e647688a1a 100644 --- a/litellm-rust/crates/inference-messages/src/handler.rs +++ b/litellm-rust/crates/inference-messages/src/handler.rs @@ -2,7 +2,9 @@ use std::time::Duration; use bytes::Bytes; use futures_util::{FutureExt, StreamExt, TryStreamExt, stream::BoxStream}; -use litellm_auth::{AuthServices, SecretValue}; +use litellm_auth::AuthServices; +use litellm_secret_value::SecretValue; + use litellm_core_utils::{ dot_notation_indexing::delete_nested_value, get_provider_specific_headers::get_provider_specific_headers, settings::Lookup, diff --git a/litellm-rust/crates/inference-messages/tests/caching.rs b/litellm-rust/crates/inference-messages/tests/caching.rs index 480b3c4f3b7..663c87453a9 100644 --- a/litellm-rust/crates/inference-messages/tests/caching.rs +++ b/litellm-rust/crates/inference-messages/tests/caching.rs @@ -107,7 +107,7 @@ impl litellm_secrets::source::SecretSource for ChangingSecrets { name: &'a str, ) -> futures_util::future::BoxFuture< 'a, - Result, litellm_secrets::Error>, + Result, litellm_secrets::Error>, > { Box::pin(async move { let revision = self.revision.load(Ordering::SeqCst); @@ -121,7 +121,7 @@ impl litellm_secrets::source::SecretSource for ChangingSecrets { } else { None }; - Ok(value.map(litellm_secrets::SecretValue::new)) + Ok(value.map(litellm_secret_value::SecretValue::new)) }) } } diff --git a/litellm-rust/crates/inference-ocr/Cargo.toml b/litellm-rust/crates/inference-ocr/Cargo.toml index bbbdaddb050..ad53126b9fe 100644 --- a/litellm-rust/crates/inference-ocr/Cargo.toml +++ b/litellm-rust/crates/inference-ocr/Cargo.toml @@ -6,6 +6,7 @@ license.workspace = true repository.workspace = true [dependencies] +litellm-secret-value.workspace = true base64.workspace = true bytes.workspace = true futures-util.workspace = true diff --git a/litellm-rust/crates/inference-ocr/src/handler.rs b/litellm-rust/crates/inference-ocr/src/handler.rs index 426d4297f28..c126a192153 100644 --- a/litellm-rust/crates/inference-ocr/src/handler.rs +++ b/litellm-rust/crates/inference-ocr/src/handler.rs @@ -128,7 +128,7 @@ mod tests { decode_request(OcrWireRequest { model: model.into(), document, - api_key: Some(litellm_auth::SecretValue::new("test-key")), + api_key: Some(litellm_secret_value::SecretValue::new("test-key")), api_base: Some(base.into()), custom_llm_provider: None, extra_headers: None, diff --git a/litellm-rust/crates/inference-ocr/src/provider_config.rs b/litellm-rust/crates/inference-ocr/src/provider_config.rs index 19e50af0dd1..e76d291254b 100644 --- a/litellm-rust/crates/inference-ocr/src/provider_config.rs +++ b/litellm-rust/crates/inference-ocr/src/provider_config.rs @@ -1,4 +1,6 @@ -use litellm_auth::{InputSource, SecretValue, Sourced}; +use litellm_auth::{InputSource, Sourced}; +use litellm_secret_value::SecretValue; + use litellm_core_utils::get_llm_provider_logic::{ CustomLlmProvider, LlmProviders, get_custom_llm_provider, }; @@ -407,7 +409,7 @@ mod tests { fn connection_resolution_preserves_dynamic_precedence_and_input_sources() { let connection = OcrConfigKind::Mistral.resolve_connection_params(OcrCredentialInputs { api_key: Some(Sourced::new( - litellm_auth::SecretValue::new("explicit-key"), + litellm_secret_value::SecretValue::new("explicit-key"), InputSource::Deployment, )), api_base: Some(Sourced::new( @@ -415,7 +417,7 @@ mod tests { InputSource::Deployment, )), dynamic_api_key: Some(Sourced::new( - litellm_auth::SecretValue::new("dynamic-key"), + litellm_secret_value::SecretValue::new("dynamic-key"), InputSource::Environment, )), dynamic_api_base: Some(Sourced::new( @@ -455,7 +457,7 @@ mod tests { ) { let dynamic_key = dynamic_value.map(|value| { Sourced::new( - litellm_auth::SecretValue::new(value), + litellm_secret_value::SecretValue::new(value), InputSource::Environment, ) }); @@ -463,7 +465,7 @@ mod tests { dynamic_value.map(|value| Sourced::new(value.into(), InputSource::Environment)); let connection = OcrConfigKind::Mistral.resolve_connection_params(OcrCredentialInputs { api_key: Some(Sourced::new( - litellm_auth::SecretValue::new("explicit-key"), + litellm_secret_value::SecretValue::new("explicit-key"), InputSource::Deployment, )), api_base: Some(Sourced::new( @@ -502,14 +504,14 @@ mod tests { OcrCredentialInputs { api_key: explicit_key.map(|value| { Sourced::new( - litellm_auth::SecretValue::new(value), + litellm_secret_value::SecretValue::new(value), InputSource::Deployment, ) }), api_base: explicit_base .map(|value| Sourced::new(value.into(), InputSource::Deployment)), dynamic_api_key: Some(Sourced::new( - litellm_auth::SecretValue::new("dynamic-key"), + litellm_secret_value::SecretValue::new("dynamic-key"), InputSource::Environment, )), dynamic_api_base: Some(Sourced::new( diff --git a/litellm-rust/crates/inference-ocr/src/types.rs b/litellm-rust/crates/inference-ocr/src/types.rs index 98c609a158f..1b2dbb52d77 100644 --- a/litellm-rust/crates/inference-ocr/src/types.rs +++ b/litellm-rust/crates/inference-ocr/src/types.rs @@ -1,7 +1,9 @@ use std::{collections::BTreeMap, path::PathBuf, time::Duration}; use bytes::Bytes; -use litellm_auth::{InputSource, SecretValue, TokenProviderHandle}; +use litellm_auth::{InputSource, TokenProviderHandle}; +use litellm_secret_value::SecretValue; + use litellm_core_utils::call_arguments::CallArguments; use litellm_llms::base_llm::ocr::{ error::Error, diff --git a/litellm-rust/crates/inference-ocr/src/wire.rs b/litellm-rust/crates/inference-ocr/src/wire.rs index 56d22be6523..45a2865823f 100644 --- a/litellm-rust/crates/inference-ocr/src/wire.rs +++ b/litellm-rust/crates/inference-ocr/src/wire.rs @@ -1,6 +1,8 @@ use std::{collections::BTreeMap, time::Duration}; -use litellm_auth::{InputSource, SecretValue}; +use litellm_auth::InputSource; +use litellm_secret_value::SecretValue; + use litellm_llms::base_llm::ocr::{error::Error, transformation::decode_request_value}; use litellm_llms_types::formats::ocr::OcrDocument; use serde::Deserialize; diff --git a/litellm-rust/crates/inference-ocr/tests/ocr/azure_ai.rs b/litellm-rust/crates/inference-ocr/tests/ocr/azure_ai.rs index 6f591af73e0..65c7f309b82 100644 --- a/litellm-rust/crates/inference-ocr/tests/ocr/azure_ai.rs +++ b/litellm-rust/crates/inference-ocr/tests/ocr/azure_ai.rs @@ -3,9 +3,9 @@ use std::sync::{ atomic::{AtomicUsize, Ordering}, }; -use litellm_auth::{ - ResolvedCredential, SecretValue, TokenFuture, TokenProvider, TokenProviderHandle, -}; +use litellm_auth::{ResolvedCredential, TokenFuture, TokenProvider, TokenProviderHandle}; +use litellm_secret_value::SecretValue; + use rstest::rstest; use super::*; diff --git a/litellm-rust/crates/inference-ocr/tests/ocr/lifecycle.rs b/litellm-rust/crates/inference-ocr/tests/ocr/lifecycle.rs index 75a91092daf..9f59bb9a2a2 100644 --- a/litellm-rust/crates/inference-ocr/tests/ocr/lifecycle.rs +++ b/litellm-rust/crates/inference-ocr/tests/ocr/lifecycle.rs @@ -239,7 +239,7 @@ impl litellm_host_native::services::HostCallHandler for CallerTokenHost { OcrOp::AcquireAzureAdToken(reply) => { self.trace.lock().unwrap().push("token".into()); reply.send(litellm_auth::ResolvedCredential::Static( - litellm_auth::SecretValue::new("caller-token"), + litellm_secret_value::SecretValue::new("caller-token"), )); Ok(()) } diff --git a/litellm-rust/crates/inference-ocr/tests/ocr/main.rs b/litellm-rust/crates/inference-ocr/tests/ocr/main.rs index 5a48bba9e1a..6871d1ca1f6 100644 --- a/litellm-rust/crates/inference-ocr/tests/ocr/main.rs +++ b/litellm-rust/crates/inference-ocr/tests/ocr/main.rs @@ -80,7 +80,7 @@ fn wire(model: &str, base: &str, document: Value, options: Value) -> OcrWireRequ OcrWireRequest { model: model.into(), document, - api_key: Some(litellm_auth::SecretValue::new("test-key")), + api_key: Some(litellm_secret_value::SecretValue::new("test-key")), api_base: Some(base.into()), custom_llm_provider: None, extra_headers: None, diff --git a/litellm-rust/crates/inference-responses/Cargo.toml b/litellm-rust/crates/inference-responses/Cargo.toml index 083179c50c8..f90651634c5 100644 --- a/litellm-rust/crates/inference-responses/Cargo.toml +++ b/litellm-rust/crates/inference-responses/Cargo.toml @@ -25,6 +25,7 @@ tokio-tungstenite.workspace = true tracing.workspace = true [dev-dependencies] +litellm-secret-value.workspace = true litellm-http = { workspace = true, features = ["test-support"] } litellm-cache.workspace = true litellm-cache-memory.workspace = true diff --git a/litellm-rust/crates/inference-responses/tests/caching.rs b/litellm-rust/crates/inference-responses/tests/caching.rs index 462461695c3..56e25fbcb98 100644 --- a/litellm-rust/crates/inference-responses/tests/caching.rs +++ b/litellm-rust/crates/inference-responses/tests/caching.rs @@ -98,7 +98,7 @@ impl litellm_secrets::source::SecretSource for ChangingSecrets { name: &'a str, ) -> futures_util::future::BoxFuture< 'a, - Result, litellm_secrets::Error>, + Result, litellm_secrets::Error>, > { Box::pin(async move { let revision = self.revision.load(Ordering::SeqCst); @@ -112,7 +112,7 @@ impl litellm_secrets::source::SecretSource for ChangingSecrets { } else { None }; - Ok(value.map(litellm_secrets::SecretValue::new)) + Ok(value.map(litellm_secret_value::SecretValue::new)) }) } } diff --git a/litellm-rust/crates/inference-testing/Cargo.toml b/litellm-rust/crates/inference-testing/Cargo.toml index 748b1df0d3b..5f690532d9c 100644 --- a/litellm-rust/crates/inference-testing/Cargo.toml +++ b/litellm-rust/crates/inference-testing/Cargo.toml @@ -7,6 +7,7 @@ repository.workspace = true publish = false [dependencies] +litellm-secret-value.workspace = true litellm-auth.workspace = true futures-util.workspace = true litellm-http.workspace = true diff --git a/litellm-rust/crates/inference-testing/src/lib.rs b/litellm-rust/crates/inference-testing/src/lib.rs index 2e20d5a9e91..f330e02e99a 100644 --- a/litellm-rust/crates/inference-testing/src/lib.rs +++ b/litellm-rust/crates/inference-testing/src/lib.rs @@ -5,7 +5,8 @@ use litellm_http::{ ClientVariant, HttpClientConfig, HttpClientPool, HttpSettings, Resolution, media::PublicDnsResolver, }; -use litellm_secrets::{SecretValue, source::SecretSource}; +use litellm_secret_value::SecretValue; +use litellm_secrets::source::SecretSource; use litellm_inference::resources::CoreResources; diff --git a/litellm-rust/crates/llms/Cargo.toml b/litellm-rust/crates/llms/Cargo.toml index 962f61ef4ec..79ee17e8b97 100644 --- a/litellm-rust/crates/llms/Cargo.toml +++ b/litellm-rust/crates/llms/Cargo.toml @@ -9,6 +9,7 @@ repository.workspace = true test-support = ["litellm-http/test-support"] [dependencies] +litellm-secret-value.workspace = true litellm-llms-types.workspace = true litellm-core-utils.workspace = true litellm-error.workspace = true diff --git a/litellm-rust/crates/llms/src/anthropic/chat/transformation.rs b/litellm-rust/crates/llms/src/anthropic/chat/transformation.rs index eb796a900f5..164dbce1264 100644 --- a/litellm-rust/crates/llms/src/anthropic/chat/transformation.rs +++ b/litellm-rust/crates/llms/src/anthropic/chat/transformation.rs @@ -1,4 +1,3 @@ -use litellm_auth::SecretValue; use litellm_core_utils::decline::Decline; use litellm_core_utils::{ core_helpers::{finish_reason_for, unix_now, usage_from_parts}, @@ -10,6 +9,7 @@ use litellm_llms_types::{ }, providers::anthropic::DEFAULT_HEADERS, }; +use litellm_secret_value::SecretValue; use serde::Deserialize; use serde_json::{Map, Value, json}; diff --git a/litellm-rust/crates/llms/src/anthropic/common_utils.rs b/litellm-rust/crates/llms/src/anthropic/common_utils.rs index 7991a57353e..aec5d9ebc27 100644 --- a/litellm-rust/crates/llms/src/anthropic/common_utils.rs +++ b/litellm-rust/crates/llms/src/anthropic/common_utils.rs @@ -1,5 +1,7 @@ use crate::base_llm::messages::context::MessagesModelCapabilities; -use litellm_auth::{CredentialPlacement, SecretValue}; +use litellm_auth::CredentialPlacement; +use litellm_secret_value::SecretValue; + use litellm_core_utils::settings::resolve_non_empty; use litellm_http::request::{ has_header, header_value, header_values, with_header, without_headers, diff --git a/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs b/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs index d578acea68c..a2ba2cee0d3 100644 --- a/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs @@ -1,10 +1,10 @@ -use litellm_auth::SecretValue; use litellm_http::request::{has_bearer_auth, has_header}; use litellm_llms_types::{ formats::messages::{MessagesOptionalParams, MessagesRequest}, providers::anthropic::{BetaProvider, DEFAULT_HEADERS, MESSAGES_PATH}, }; use litellm_router_types::LitellmParams; +use litellm_secret_value::SecretValue; use crate::{ Error, diff --git a/litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs b/litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs index 36dfe2c890e..870306b1a83 100644 --- a/litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs +++ b/litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs @@ -1250,7 +1250,7 @@ mod tests { #[tokio::test] async fn request_endpoint_accepts_request_owned_key() { let connection = OcrConnection { - api_key: Some(litellm_auth::SecretValue::new("request-key")), + api_key: Some(litellm_secret_value::SecretValue::new("request-key")), api_key_source: InputSource::Request, api_base: Some("https://request.example".into()), api_base_source: InputSource::Request, diff --git a/litellm-rust/crates/llms/src/azure_ai/ocr/transformation.rs b/litellm-rust/crates/llms/src/azure_ai/ocr/transformation.rs index 2f7957fe56e..25f93846967 100644 --- a/litellm-rust/crates/llms/src/azure_ai/ocr/transformation.rs +++ b/litellm-rust/crates/llms/src/azure_ai/ocr/transformation.rs @@ -211,7 +211,7 @@ mod tests { #[fixture] fn connection() -> OcrConnection { OcrConnection { - api_key: Some(litellm_auth::SecretValue::new("request-key")), + api_key: Some(litellm_secret_value::SecretValue::new("request-key")), api_base: Some("https://example.com".into()), ..Default::default() } @@ -312,7 +312,7 @@ mod tests { #[tokio::test] async fn request_endpoint_accepts_request_owned_key() { let connection = OcrConnection { - api_key: Some(litellm_auth::SecretValue::new("request-key")), + api_key: Some(litellm_secret_value::SecretValue::new("request-key")), api_key_source: InputSource::Request, api_base: Some("https://request.example".into()), api_base_source: InputSource::Request, diff --git a/litellm-rust/crates/llms/src/base_llm/auth.rs b/litellm-rust/crates/llms/src/base_llm/auth.rs index de271a269d3..ab2cf00614e 100644 --- a/litellm-rust/crates/llms/src/base_llm/auth.rs +++ b/litellm-rust/crates/llms/src/base_llm/auth.rs @@ -5,7 +5,9 @@ //! that split: `validate_environment` shapes the forwarded headers and names the credential //! as an [`AuthScheme`], and [`resolve_auth`] turns the scheme into headers and a signer. -use litellm_auth::{AuthServices, CredentialPlacement, SecretValue, TokenProviderHandle}; +use litellm_auth::{AuthServices, CredentialPlacement, TokenProviderHandle}; +use litellm_secret_value::SecretValue; + use litellm_auth_aws::{AwsCredentialSource, SigV4Signer}; use litellm_auth_gcp::VertexConfig; use litellm_http::request::with_header; diff --git a/litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs b/litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs index c90fd99809e..9574e99d357 100644 --- a/litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs +++ b/litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs @@ -5,7 +5,9 @@ use serde::{ use serde_json::{Map, Value}; use std::{collections::BTreeMap, future::Future, sync::Arc, time::Duration}; -use litellm_auth::{InputSource, SecretValue, Sourced, TokenProviderHandle}; +use litellm_auth::{InputSource, Sourced, TokenProviderHandle}; +use litellm_secret_value::SecretValue; + use litellm_core_utils::{call_arguments::CallArguments, settings::ProcessEnvironment}; use litellm_http::outbound::RequestSigner; use litellm_llms_types::formats::ocr::{LiteLLMOcrResponse, OcrDocument, OcrResponseFormat}; diff --git a/litellm-rust/crates/llms/src/bedrock/chat/converse_transformation.rs b/litellm-rust/crates/llms/src/bedrock/chat/converse_transformation.rs index 976c9161dd2..6cf0167a30f 100644 --- a/litellm-rust/crates/llms/src/bedrock/chat/converse_transformation.rs +++ b/litellm-rust/crates/llms/src/bedrock/chat/converse_transformation.rs @@ -1,5 +1,7 @@ use litellm_auth::AwsParams; -use litellm_auth::{CredentialPlacement, SecretValue}; +use litellm_auth::CredentialPlacement; +use litellm_secret_value::SecretValue; + use litellm_auth_aws::{ AwsCredentialSource, bedrock_model_id_and_region, constants::{AWS_BEARER_TOKEN_BEDROCK, BEDROCK_RUNTIME_ENDPOINT_TEMPLATE, BEDROCK_SERVICE}, diff --git a/litellm-rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs b/litellm-rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs index 79c30df9b19..5b89345a1c5 100644 --- a/litellm-rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs @@ -1,6 +1,8 @@ use std::sync::LazyLock; -use litellm_auth::{AwsParams, CredentialPlacement, ParamKind, ParamSpec, SecretValue}; +use litellm_auth::{AwsParams, CredentialPlacement, ParamKind, ParamSpec}; +use litellm_secret_value::SecretValue; + use litellm_auth_aws::{AwsCredentialSource, resolve_aws_region}; use litellm_core_utils::settings::resolve_non_empty; use litellm_http::request::{header_value, with_header, without_headers}; diff --git a/litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude_transformation.rs b/litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude_transformation.rs index 3ec60f12021..1d380c917f6 100644 --- a/litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude_transformation.rs +++ b/litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude_transformation.rs @@ -7,7 +7,9 @@ use crate::{ }; use futures_util::StreamExt; use litellm_auth::AwsParams; -use litellm_auth::{CredentialPlacement, SecretValue}; +use litellm_auth::CredentialPlacement; +use litellm_secret_value::SecretValue; + use litellm_auth_aws::{ AwsCredentialSource, bedrock_model_id_and_region, constants::{ diff --git a/litellm-rust/crates/llms/src/bedrock_mantle/messages/transformation.rs b/litellm-rust/crates/llms/src/bedrock_mantle/messages/transformation.rs index ae13704f4a0..f139b93e259 100644 --- a/litellm-rust/crates/llms/src/bedrock_mantle/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/bedrock_mantle/messages/transformation.rs @@ -1,6 +1,8 @@ use std::sync::LazyLock; -use litellm_auth::{AwsParams, CredentialPlacement, SecretValue}; +use litellm_auth::{AwsParams, CredentialPlacement}; +use litellm_secret_value::SecretValue; + use litellm_auth_aws::AwsCredentialSource; use litellm_auth_aws::bedrock_model_id_and_region; use litellm_core_utils::settings::resolve_non_empty; diff --git a/litellm-rust/crates/llms/src/cohere/ocr/transformation.rs b/litellm-rust/crates/llms/src/cohere/ocr/transformation.rs index 76ff27b56fc..5bad06adda2 100644 --- a/litellm-rust/crates/llms/src/cohere/ocr/transformation.rs +++ b/litellm-rust/crates/llms/src/cohere/ocr/transformation.rs @@ -735,7 +735,7 @@ mod tests { assert!(matches!( CohereParseConfig.resolve_headers( &OcrConnection { - api_key: Some(litellm_auth::SecretValue::new(" ")), + api_key: Some(litellm_secret_value::SecretValue::new(" ")), ..Default::default() }, &|_| None, diff --git a/litellm-rust/crates/llms/src/deepseek/messages/transformation.rs b/litellm-rust/crates/llms/src/deepseek/messages/transformation.rs index 168c3e63920..0dbcea01439 100644 --- a/litellm-rust/crates/llms/src/deepseek/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/deepseek/messages/transformation.rs @@ -1,4 +1,6 @@ -use litellm_auth::{CredentialPlacement, SecretValue}; +use litellm_auth::CredentialPlacement; +use litellm_secret_value::SecretValue; + use litellm_core_utils::settings::resolve_non_empty; use litellm_llms_types::{ formats::messages::{MessagesOptionalParams, MessagesRequest, MessagesTool}, diff --git a/litellm-rust/crates/llms/src/edenai/messages/transformation.rs b/litellm-rust/crates/llms/src/edenai/messages/transformation.rs index 2c3a7eaef1d..d5944484522 100644 --- a/litellm-rust/crates/llms/src/edenai/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/edenai/messages/transformation.rs @@ -1,4 +1,6 @@ -use litellm_auth::{CredentialPlacement, SecretValue}; +use litellm_auth::CredentialPlacement; +use litellm_secret_value::SecretValue; + use litellm_core_utils::settings::resolve_non_empty; use litellm_llms_types::formats::messages::{ MessagesOptionalParams, MessagesRequest, MessagesResponse, diff --git a/litellm-rust/crates/llms/src/minimax/messages/transformation.rs b/litellm-rust/crates/llms/src/minimax/messages/transformation.rs index 012a657fceb..dafe5fbd16b 100644 --- a/litellm-rust/crates/llms/src/minimax/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/minimax/messages/transformation.rs @@ -1,4 +1,6 @@ -use litellm_auth::{CredentialPlacement, SecretValue}; +use litellm_auth::CredentialPlacement; +use litellm_secret_value::SecretValue; + use litellm_core_utils::settings::resolve_non_empty; use litellm_llms_types::{ formats::messages::{MessagesOptionalParams, MessagesRequest}, diff --git a/litellm-rust/crates/llms/src/mistral/ocr/transformation.rs b/litellm-rust/crates/llms/src/mistral/ocr/transformation.rs index fa4e2254e86..679aa65f7fd 100644 --- a/litellm-rust/crates/llms/src/mistral/ocr/transformation.rs +++ b/litellm-rust/crates/llms/src/mistral/ocr/transformation.rs @@ -219,7 +219,7 @@ mod tests { #[default(vec![])] extra_headers: Vec<(String, String)>, ) -> OcrConnection { OcrConnection { - api_key: api_key.map(litellm_auth::SecretValue::new), + api_key: api_key.map(litellm_secret_value::SecretValue::new), extra_headers, ..OcrConnection::default() } diff --git a/litellm-rust/crates/llms/src/openai/responses/transformation.rs b/litellm-rust/crates/llms/src/openai/responses/transformation.rs index cf74871894b..deb163daa07 100644 --- a/litellm-rust/crates/llms/src/openai/responses/transformation.rs +++ b/litellm-rust/crates/llms/src/openai/responses/transformation.rs @@ -3,7 +3,8 @@ use litellm_llms_types::formats::responses::{ }; use serde_json::{Map, Value}; -use litellm_auth::{CredentialPlacement, SecretValue}; +use litellm_auth::CredentialPlacement; +use litellm_secret_value::SecretValue; use crate::{ Error, ErrorDetail, diff --git a/litellm-rust/crates/llms/src/openai_like/chat/transformation.rs b/litellm-rust/crates/llms/src/openai_like/chat/transformation.rs index e998f8785cb..83fb8ac3fa2 100644 --- a/litellm-rust/crates/llms/src/openai_like/chat/transformation.rs +++ b/litellm-rust/crates/llms/src/openai_like/chat/transformation.rs @@ -4,7 +4,9 @@ //! the `max_completion_tokens` -> `max_tokens` rename and the usage //! `*_tokens` null-to-zero sanitize. -use litellm_auth::{CredentialPlacement, SecretValue}; +use litellm_auth::CredentialPlacement; +use litellm_secret_value::SecretValue; + use litellm_core_utils::core_helpers::unix_now; use litellm_llms_types::formats::chat_completions::{ ChatCompletionsChoice, ChatCompletionsChoiceMessage, ChatCompletionsResponse, diff --git a/litellm-rust/crates/llms/src/openrouter/messages/transformation.rs b/litellm-rust/crates/llms/src/openrouter/messages/transformation.rs index 5b4e4f77f2a..eeb3d1cb7f8 100644 --- a/litellm-rust/crates/llms/src/openrouter/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/openrouter/messages/transformation.rs @@ -1,4 +1,6 @@ -use litellm_auth::{CredentialPlacement, SecretValue}; +use litellm_auth::CredentialPlacement; +use litellm_secret_value::SecretValue; + use litellm_core_utils::settings::resolve_non_empty; use litellm_http::request::{has_bearer_auth, without_headers}; use litellm_llms_types::{ diff --git a/litellm-rust/crates/llms/src/openrouter/responses/transformation.rs b/litellm-rust/crates/llms/src/openrouter/responses/transformation.rs index 900fba9e916..76155ba55e5 100644 --- a/litellm-rust/crates/llms/src/openrouter/responses/transformation.rs +++ b/litellm-rust/crates/llms/src/openrouter/responses/transformation.rs @@ -1,4 +1,6 @@ -use litellm_auth::{CredentialPlacement, SecretValue}; +use litellm_auth::CredentialPlacement; +use litellm_secret_value::SecretValue; + use litellm_core_utils::settings::resolve_non_empty; use litellm_llms_types::formats::responses::ResponsesApiResponse; use serde_json::{Map, Value}; diff --git a/litellm-rust/crates/llms/src/reducto/ocr/transformation.rs b/litellm-rust/crates/llms/src/reducto/ocr/transformation.rs index bf0ddc1c7a8..beda6899d04 100644 --- a/litellm-rust/crates/llms/src/reducto/ocr/transformation.rs +++ b/litellm-rust/crates/llms/src/reducto/ocr/transformation.rs @@ -690,7 +690,7 @@ mod tests { #[test] fn explicit_key_precedes_environment_key() { let connection = OcrConnection { - api_key: Some(litellm_auth::SecretValue::new("passed-key")), + api_key: Some(litellm_secret_value::SecretValue::new("passed-key")), ..Default::default() }; let headers = resolve_headers(&connection, &|_| Some("env-key".into())).unwrap(); @@ -700,7 +700,7 @@ mod tests { #[test] fn blank_explicit_key_uses_environment_key() { let connection = OcrConnection { - api_key: Some(litellm_auth::SecretValue::new(" ")), + api_key: Some(litellm_secret_value::SecretValue::new(" ")), ..Default::default() }; let headers = resolve_headers(&connection, &|_| Some(" env-key ".into())).unwrap(); diff --git a/litellm-rust/crates/llms/src/tencent/messages/transformation.rs b/litellm-rust/crates/llms/src/tencent/messages/transformation.rs index a70336748ad..621a09f4fb0 100644 --- a/litellm-rust/crates/llms/src/tencent/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/tencent/messages/transformation.rs @@ -1,4 +1,6 @@ -use litellm_auth::{CredentialPlacement, SecretValue}; +use litellm_auth::CredentialPlacement; +use litellm_secret_value::SecretValue; + use litellm_core_utils::settings::resolve_non_empty; use litellm_llms_types::{ formats::messages::{MessagesOptionalParams, MessagesRequest}, diff --git a/litellm-rust/crates/llms/src/vertex_ai/ocr/transformation.rs b/litellm-rust/crates/llms/src/vertex_ai/ocr/transformation.rs index 77881cd2e78..b902967d9dc 100644 --- a/litellm-rust/crates/llms/src/vertex_ai/ocr/transformation.rs +++ b/litellm-rust/crates/llms/src/vertex_ai/ocr/transformation.rs @@ -137,7 +137,7 @@ impl VertexAiOcrConfig { connection .api_key .as_ref() - .map(litellm_auth::SecretValue::expose), + .map(litellm_secret_value::SecretValue::expose), config, &|name: &str| connection.secret(name), ) diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index a4e1c57ab1e..e3ef45f7876 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -19,6 +19,7 @@ huggingface = ["litellm-token-counter/huggingface"] tiktoken = ["litellm-token-counter/tiktoken"] [dependencies] +litellm-secret-value.workspace = true fancy-regex.workspace = true litellm-tracing.workspace = true litellm-spend-clickhouse.workspace = true diff --git a/litellm-rust/crates/python-bridge/src/credentials.rs b/litellm-rust/crates/python-bridge/src/credentials.rs index a4bb064aae0..adc52a7e77a 100644 --- a/litellm-rust/crates/python-bridge/src/credentials.rs +++ b/litellm-rust/crates/python-bridge/src/credentials.rs @@ -2,7 +2,9 @@ //! keyword arguments and acquired on the host's own thread when the call asks for one. use crate::callable::wrap_failure; -use litellm_auth::{ResolvedCredential, SecretValue}; +use litellm_auth::ResolvedCredential; +use litellm_secret_value::SecretValue; + use pyo3::{ exceptions::PyTypeError, gc::{PyTraverseError, PyVisit}, diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs index c2c3fe51600..73a2922804a 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs @@ -1,4 +1,3 @@ -use litellm_auth::SecretValue; use litellm_auth::VertexParams; use litellm_host_python::{from_py, present}; use litellm_inference_ocr::{ @@ -6,6 +5,7 @@ use litellm_inference_ocr::{ wire::{OcrWireRequest, consumed_optional_params, decode_document, decode_request_input}, }; use litellm_llms::base_llm::ocr::{error::Error, transformation::DROP_PARAMS_PARAM}; +use litellm_secret_value::SecretValue; use pyo3::{prelude::*, types::PyDict}; use serde_json::{Map, Value}; diff --git a/litellm-rust/crates/python-bridge/src/secrets/callback.rs b/litellm-rust/crates/python-bridge/src/secrets/callback.rs index 669c10e602a..af6485a56e1 100644 --- a/litellm-rust/crates/python-bridge/src/secrets/callback.rs +++ b/litellm-rust/crates/python-bridge/src/secrets/callback.rs @@ -2,9 +2,11 @@ use std::{future::Future, pin::Pin, sync::Arc}; use litellm_core_utils::settings::Lookup; use litellm_host_python::{PythonContext, attach_blocking}; +use litellm_secret_value::SecretValue; use litellm_secrets::{ - Error, ExternalSecretManager, KeyManagementSettings, KeyManagementSystem, Secret, SecretValue, + Error, ExternalSecretManager, KeyManagementSettings, KeyManagementSystem, Secret, }; + use pyo3::{ exceptions::PyException, prelude::*, diff --git a/litellm-rust/crates/python-bridge/src/secrets/config.rs b/litellm-rust/crates/python-bridge/src/secrets/config.rs index 7afd06e2373..c1203dcbd89 100644 --- a/litellm-rust/crates/python-bridge/src/secrets/config.rs +++ b/litellm-rust/crates/python-bridge/src/secrets/config.rs @@ -1,7 +1,9 @@ use std::sync::Arc; +use litellm_secret_value::SecretValue; use litellm_secrets::{SecretManager, SecretManagerState}; -use litellm_secrets_types::{AccessMode, KeyManagementSettings, KeyManagementSystem, SecretValue}; +use litellm_secrets_types::{AccessMode, KeyManagementSettings, KeyManagementSystem}; + use pyo3::prelude::*; use serde_json::Value; diff --git a/litellm-rust/crates/python-bridge/src/secrets/operations.rs b/litellm-rust/crates/python-bridge/src/secrets/operations.rs index b458e3f34e8..027da1e4718 100644 --- a/litellm-rust/crates/python-bridge/src/secrets/operations.rs +++ b/litellm-rust/crates/python-bridge/src/secrets/operations.rs @@ -81,7 +81,7 @@ impl From for PythonMutationError { pub(super) async fn write_python_provider( manager: &SecretManager, name: &str, - value: &litellm_secrets::SecretValue, + value: &litellm_secret_value::SecretValue, ) -> Result { match manager { SecretManager::Cyberark(client) => { @@ -128,7 +128,7 @@ pub(super) async fn rotate_python_provider( manager: &SecretManager, current_name: &str, new_name: &str, - value: &litellm_secrets::SecretValue, + value: &litellm_secret_value::SecretValue, ) -> Result { match manager { SecretManager::Cyberark(client) => { @@ -186,7 +186,7 @@ pub(super) enum PythonMutationResponse { pub(super) async fn write_python_provider_with_context( manager: &SecretManager, name: &str, - value: &litellm_secrets::SecretValue, + value: &litellm_secret_value::SecretValue, context: &litellm_secrets_types::SecretWriteContext, ) -> Result { if let (SecretManager::HashicorpVault(client), SecretOperationContext::Hashicorp(operation)) = @@ -235,7 +235,7 @@ pub(super) async fn rotate_python_provider_with_context( manager: &SecretManager, current_name: &str, new_name: &str, - value: &litellm_secrets::SecretValue, + value: &litellm_secret_value::SecretValue, context: &SecretOperationContext, ) -> Result { if let (SecretManager::HashicorpVault(client), SecretOperationContext::Hashicorp(context)) = diff --git a/litellm-rust/crates/python-bridge/src/secrets/provider.rs b/litellm-rust/crates/python-bridge/src/secrets/provider.rs index ff10f596311..55343fb63bb 100644 --- a/litellm-rust/crates/python-bridge/src/secrets/provider.rs +++ b/litellm-rust/crates/python-bridge/src/secrets/provider.rs @@ -1,7 +1,9 @@ use std::time::Duration; use super::operations::PythonReadRequest; -use litellm_secrets::{KeyManagementSystem, SecretValue}; +use litellm_secret_value::SecretValue; +use litellm_secrets::KeyManagementSystem; + use litellm_secrets_types::{ AwsOperationContext, CyberarkOperationContext, GoogleOperationContext, HashicorpOperationContext, SecretOperationContext, diff --git a/litellm-rust/crates/python-bridge/src/secrets/python.rs b/litellm-rust/crates/python-bridge/src/secrets/python.rs index 2f040bff773..c5d9e3565b3 100644 --- a/litellm-rust/crates/python-bridge/src/secrets/python.rs +++ b/litellm-rust/crates/python-bridge/src/secrets/python.rs @@ -2,7 +2,9 @@ use std::sync::Arc; use futures_util::future::BoxFuture; use litellm_host_python::{PythonContext, attach_blocking}; -use litellm_secrets::{Error, SecretValue, source::SecretSource}; +use litellm_secret_value::SecretValue; +use litellm_secrets::{Error, source::SecretSource}; + use pyo3::prelude::*; use super::error::external_error; diff --git a/litellm-rust/crates/python-bridge/src/secrets/resolved.rs b/litellm-rust/crates/python-bridge/src/secrets/resolved.rs index deb07175e7a..752b533886e 100644 --- a/litellm-rust/crates/python-bridge/src/secrets/resolved.rs +++ b/litellm-rust/crates/python-bridge/src/secrets/resolved.rs @@ -4,10 +4,9 @@ use futures_util::future::BoxFuture; use litellm_core_utils::settings::ProcessEnvironment; use litellm_host_python::PythonContext; use litellm_http::Client; +use litellm_secret_value::SecretValue; use litellm_secrets::source::SecretSource; -use litellm_secrets::{ - Error, FailurePolicy, OidcResolver, SecretManagerState, SecretResolver, SecretValue, -}; +use litellm_secrets::{Error, FailurePolicy, OidcResolver, SecretManagerState, SecretResolver}; use super::config::SecretManagerSnapshot; diff --git a/litellm-rust/crates/python-bridge/src/secrets/runtime.rs b/litellm-rust/crates/python-bridge/src/secrets/runtime.rs index b956ec297b6..0d52b0aa629 100644 --- a/litellm-rust/crates/python-bridge/src/secrets/runtime.rs +++ b/litellm-rust/crates/python-bridge/src/secrets/runtime.rs @@ -307,7 +307,7 @@ impl NativeSecretManager { super::operations::write_python_provider_with_context( &backend, &secret_name, - &litellm_secrets::SecretValue::new(secret_value), + &litellm_secret_value::SecretValue::new(secret_value), &context, ) .await, @@ -365,7 +365,7 @@ impl NativeSecretManager { &backend, ¤t_secret_name, &new_secret_name, - &litellm_secrets::SecretValue::new(new_secret_value), + &litellm_secret_value::SecretValue::new(new_secret_value), &context, ) .await, diff --git a/litellm-rust/crates/python-bridge/src/secrets/vault/operation.rs b/litellm-rust/crates/python-bridge/src/secrets/vault/operation.rs index 41658f0baf8..b99dff7aeac 100644 --- a/litellm-rust/crates/python-bridge/src/secrets/vault/operation.rs +++ b/litellm-rust/crates/python-bridge/src/secrets/vault/operation.rs @@ -1,9 +1,8 @@ use std::collections::HashMap; -use litellm_secrets::{ - SecretValue, - hashicorp::{HashicorpVault, RawOperationError}, -}; +use litellm_secret_value::SecretValue; +use litellm_secrets::hashicorp::{HashicorpVault, RawOperationError}; + use litellm_secrets_types::{HashicorpOperationContext, SecretWriteContext}; use serde_json::value::RawValue; diff --git a/litellm-rust/crates/redis-client/Cargo.toml b/litellm-rust/crates/redis-client/Cargo.toml index eab57b36423..3e9447a6bfa 100644 --- a/litellm-rust/crates/redis-client/Cargo.toml +++ b/litellm-rust/crates/redis-client/Cargo.toml @@ -7,6 +7,7 @@ repository.workspace = true [dependencies] litellm-circuit-breaker.workspace = true +litellm-secret-value.workspace = true futures-util.workspace = true redis = { workspace = true, features = ["cluster-async", "connection-manager", "tokio-rustls-comp"] } thiserror.workspace = true diff --git a/litellm-rust/crates/redis-client/src/client.rs b/litellm-rust/crates/redis-client/src/client.rs index a02b0847663..48dbc2b2dfb 100644 --- a/litellm-rust/crates/redis-client/src/client.rs +++ b/litellm-rust/crates/redis-client/src/client.rs @@ -29,7 +29,7 @@ impl RedisClient { pub fn new(config: RedisConfig) -> Result { let connection = match &config.endpoint { Endpoint::Url(url) => { - Connection::standalone(url.as_str().into_connection_info()?, config.timeout)? + Connection::standalone(url.expose().into_connection_info()?, config.timeout)? } Endpoint::Host(host) => { Connection::standalone(host.connection_info()?, config.timeout)? @@ -204,6 +204,7 @@ mod tests { use std::time::{Duration, Instant}; use litellm_redis_testing::RedisServer; + use litellm_secret_value::SecretValue; use rstest::rstest; use litellm_circuit_breaker::{Breaker, Config, State}; @@ -630,8 +631,12 @@ mod tests { db: 3, })) .unwrap(); - let in_db_0 = connect(Endpoint::Url(server.url().to_owned())).unwrap(); - let via_url = connect(Endpoint::Url(format!("{}/3", server.url()))).unwrap(); + let in_db_0 = connect(Endpoint::Url(SecretValue::new(server.url().to_owned()))).unwrap(); + let via_url = connect(Endpoint::Url(SecretValue::new(format!( + "{}/3", + server.url() + )))) + .unwrap(); in_db_3 .set("k", "3", Duration::from_secs(60)) diff --git a/litellm-rust/crates/redis-client/src/config.rs b/litellm-rust/crates/redis-client/src/config.rs index 64934c68891..c40b9eb6b36 100644 --- a/litellm-rust/crates/redis-client/src/config.rs +++ b/litellm-rust/crates/redis-client/src/config.rs @@ -1,8 +1,9 @@ -use std::{fmt, time::Duration}; +use std::time::Duration; use redis::{ConnectionAddr, ConnectionInfo, IntoConnectionInfo, RedisConnectionInfo, RedisResult}; use litellm_circuit_breaker::Breaker; +use litellm_secret_value::SecretValue; use crate::Namespace; @@ -15,28 +16,28 @@ pub struct RedisConfig { pub breaker: Option, } -#[derive(Clone, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub enum Endpoint { - Url(String), + Url(SecretValue), Host(HostEndpoint), Cluster(ClusterEndpoint), } -#[derive(Clone, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub struct HostEndpoint { pub host: String, pub port: u16, - pub username: Option, - pub password: Option, + pub username: Option, + pub password: Option, pub tls: bool, pub db: i64, } -#[derive(Clone, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub struct ClusterEndpoint { pub nodes: Vec, - pub username: Option, - pub password: Option, + pub username: Option, + pub password: Option, pub tls: bool, } @@ -58,11 +59,11 @@ impl HostEndpoint { pub(crate) fn connection_info(&self) -> RedisResult { let settings = RedisConnectionInfo::default().set_db(self.db); let settings = match &self.username { - Some(username) => settings.set_username(username), + Some(username) => settings.set_username(username.expose()), None => settings, }; let settings = match &self.password { - Some(password) => settings.set_password(password), + Some(password) => settings.set_password(password.expose()), None => settings, }; Ok(addr(&self.host, self.port, self.tls) @@ -83,70 +84,35 @@ fn addr(host: &str, port: u16, tls: bool) -> ConnectionAddr { } } -impl fmt::Debug for Endpoint { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Url(url) => formatter - .debug_tuple("Url") - .field(&redacted_url(url)) - .finish(), - Self::Host(host) => formatter - .debug_struct("Host") - .field("host", &host.host) - .field("port", &host.port) - .field("username", &host.username) - .field("password", &host.password.as_ref().map(|_| "")) - .field("tls", &host.tls) - .field("db", &host.db) - .finish(), - Self::Cluster(cluster) => formatter - .debug_struct("Cluster") - .field("nodes", &cluster.nodes) - .field("username", &cluster.username) - .field("password", &cluster.password.as_ref().map(|_| "")) - .field("tls", &cluster.tls) - .finish(), - } - } -} - -fn redacted_url(url: &str) -> String { - match url - .split_once("://") - .and_then(|(scheme, rest)| Some((scheme, rest.rsplit_once('@')?.1))) - { - Some((scheme, host)) => format!("{scheme}://@{host}"), - None => url.to_owned(), - } -} - #[cfg(test)] mod tests { + use litellm_secret_value::SecretValue; use rstest::rstest; - use super::{Endpoint, HostEndpoint}; + use super::{ClusterEndpoint, Endpoint, HostEndpoint}; #[rstest] - #[case::url_with_password(Endpoint::Url("redis://user:secret@cache:6379/0".to_owned()), "redis://@cache:6379/0")] - #[case::url_with_at_in_password(Endpoint::Url("rediss://:p@ss@cache:6380".to_owned()), "rediss://@cache:6380")] - #[case::host_password( + #[case::url(Endpoint::Url(SecretValue::new("redis://credential-value@cache:6379/0")))] + #[case::host( Endpoint::Host(HostEndpoint { host: "cache".to_owned(), port: 6379, - username: None, - password: Some("secret".to_owned()), + username: Some(SecretValue::new("credential-value")), + password: Some(SecretValue::new("credential-value")), tls: false, db: 0, - }), - "" + }) )] - fn debug_output_never_shows_the_password(#[case] endpoint: Endpoint, #[case] shown: &str) { + #[case::cluster(Endpoint::Cluster(ClusterEndpoint { + nodes: Vec::new(), + username: Some(SecretValue::new("credential-value")), + password: Some(SecretValue::new("credential-value")), + tls: false, + }))] + fn debug_output_never_shows_the_secret(#[case] endpoint: Endpoint) { let debug = format!("{endpoint:?}"); - assert!( - !debug.contains("secret") && !debug.contains("p@ss"), - "{debug}" - ); - assert!(debug.contains(shown), "{debug}"); + assert!(!debug.contains("credential-value"), "{debug}"); + assert!(debug.contains("[REDACTED]"), "{debug}"); } } diff --git a/litellm-rust/crates/redis-client/src/connection.rs b/litellm-rust/crates/redis-client/src/connection.rs index aaf26f105da..41729bfd107 100644 --- a/litellm-rust/crates/redis-client/src/connection.rs +++ b/litellm-rust/crates/redis-client/src/connection.rs @@ -33,11 +33,11 @@ impl Connection { .response_timeout(timeout) .retries(RETRIES); let builder = match &endpoint.username { - Some(username) => builder.username(username), + Some(username) => builder.username(username.expose()), None => builder, }; let builder = match &endpoint.password { - Some(password) => builder.password(password), + Some(password) => builder.password(password.expose()), None => builder, }; let config = ClusterConfig::new() diff --git a/litellm-rust/crates/redis-client/src/test_support.rs b/litellm-rust/crates/redis-client/src/test_support.rs index 64a174b1543..e503739a942 100644 --- a/litellm-rust/crates/redis-client/src/test_support.rs +++ b/litellm-rust/crates/redis-client/src/test_support.rs @@ -2,6 +2,7 @@ use std::time::Duration; use litellm_circuit_breaker::Breaker; use litellm_redis_testing::{RedisCluster, RedisServer}; +use litellm_secret_value::SecretValue; use redis::cluster_routing::{Route, RoutingInfo, SingleNodeRoutingInfo, SlotAddr}; use crate::{ @@ -30,7 +31,7 @@ impl Server { pub(crate) fn endpoint(&self) -> Endpoint { match self { - Self::Standalone(server) => Endpoint::Url(server.url().to_owned()), + Self::Standalone(server) => Endpoint::Url(SecretValue::new(server.url().to_owned())), Self::Cluster(cluster) => Endpoint::Cluster(ClusterEndpoint { nodes: cluster .nodes() diff --git a/litellm-rust/crates/router-types/Cargo.toml b/litellm-rust/crates/router-types/Cargo.toml index 2e9fec28685..f0470ce8a8e 100644 --- a/litellm-rust/crates/router-types/Cargo.toml +++ b/litellm-rust/crates/router-types/Cargo.toml @@ -7,6 +7,7 @@ repository.workspace = true description = "The types of litellm/types/router.py: a deployment's litellm_params as a config or a caller spells them" [dependencies] +litellm-secret-value.workspace = true litellm-auth-types.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/litellm-rust/crates/router-types/src/litellm_params.rs b/litellm-rust/crates/router-types/src/litellm_params.rs index 186a1d98036..aa2c47f6e3c 100644 --- a/litellm-rust/crates/router-types/src/litellm_params.rs +++ b/litellm-rust/crates/router-types/src/litellm_params.rs @@ -1,6 +1,8 @@ use std::{collections::BTreeMap, fmt, ops::Deref}; -use litellm_auth_types::{AwsParams, ParamSpec, SecretValue, VertexParams}; +use litellm_auth_types::{AwsParams, ParamSpec, VertexParams}; +use litellm_secret_value::SecretValue; + use serde::Deserialize; use serde_json::Value; use serde_with::{OneOrMany, formats::PreferMany, serde_as}; diff --git a/litellm-rust/crates/router-types/tests/litellm_params.rs b/litellm-rust/crates/router-types/tests/litellm_params.rs index 7d021fd857a..c5c0d16ee8f 100644 --- a/litellm-rust/crates/router-types/tests/litellm_params.rs +++ b/litellm-rust/crates/router-types/tests/litellm_params.rs @@ -1,4 +1,6 @@ -use litellm_auth_types::{AwsParams, SecretValue, VertexParams}; +use litellm_auth_types::{AwsParams, VertexParams}; +use litellm_secret_value::SecretValue; + use litellm_router_types::{LitellmParams, Spelled}; use rstest::rstest; use serde_json::{Map, Value, json}; diff --git a/litellm-rust/crates/secret-value/Cargo.toml b/litellm-rust/crates/secret-value/Cargo.toml new file mode 100644 index 00000000000..467f852a278 --- /dev/null +++ b/litellm-rust/crates/secret-value/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "litellm-secret-value" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +serde.workspace = true +subtle.workspace = true +veil.workspace = true + +[dev-dependencies] +rstest.workspace = true diff --git a/litellm-rust/crates/secret-value/src/lib.rs b/litellm-rust/crates/secret-value/src/lib.rs new file mode 100644 index 00000000000..f11d22d3701 --- /dev/null +++ b/litellm-rust/crates/secret-value/src/lib.rs @@ -0,0 +1,5 @@ +#![forbid(unsafe_code)] + +mod secret_value; + +pub use secret_value::SecretValue; diff --git a/litellm-rust/crates/auth-types/src/secret.rs b/litellm-rust/crates/secret-value/src/secret_value.rs similarity index 95% rename from litellm-rust/crates/auth-types/src/secret.rs rename to litellm-rust/crates/secret-value/src/secret_value.rs index 7e6789deef7..27e4227080d 100644 --- a/litellm-rust/crates/auth-types/src/secret.rs +++ b/litellm-rust/crates/secret-value/src/secret_value.rs @@ -32,9 +32,11 @@ impl Hash for SecretValue { #[cfg(test)] mod tests { + use rstest::rstest; + use super::SecretValue; - #[test] + #[rstest] fn debug_redacts_plaintext() { let debug = format!("{:?}", SecretValue::new("credential-value")); @@ -42,7 +44,7 @@ mod tests { assert!(debug.contains("REDACTED")); } - #[test] + #[rstest] fn equality_compares_plaintext_values() { assert_eq!(SecretValue::new("same"), SecretValue::new("same")); assert_ne!(SecretValue::new("same"), SecretValue::new("different")); diff --git a/litellm-rust/crates/secrets-aws/Cargo.toml b/litellm-rust/crates/secrets-aws/Cargo.toml index 5d3bd413484..d17dfcfeec2 100644 --- a/litellm-rust/crates/secrets-aws/Cargo.toml +++ b/litellm-rust/crates/secrets-aws/Cargo.toml @@ -6,6 +6,7 @@ license.workspace = true repository.workspace = true [dependencies] +litellm-secret-value.workspace = true litellm-auth-aws.workspace = true litellm-secrets-types.workspace = true litellm-core-utils.workspace = true diff --git a/litellm-rust/crates/secrets-aws/src/secret_manager.rs b/litellm-rust/crates/secrets-aws/src/secret_manager.rs index 2220e387838..b25872ab3f6 100644 --- a/litellm-rust/crates/secrets-aws/src/secret_manager.rs +++ b/litellm-rust/crates/secrets-aws/src/secret_manager.rs @@ -21,11 +21,12 @@ use litellm_auth_aws::constants::{ AWS_ACCESS_KEY_ID, AWS_REGION, AWS_REGION_NAME, AWS_SECRET_ACCESS_KEY, }; use litellm_core_utils::settings::Lookup; +use litellm_secret_value::SecretValue; use litellm_secrets_types::{ AwsOperationContext, BaseSecretManager, KeyManagementSettings, RotationError, Secret, - SecretDeleter, SecretRotator, SecretValue, SecretWriteContext, SecretWriter, - async_rotate_secret, + SecretDeleter, SecretRotator, SecretWriteContext, SecretWriter, async_rotate_secret, }; + use serde_json::Value; use crate::{Error, auth}; diff --git a/litellm-rust/crates/secrets-aws/tests/secret_manager.rs b/litellm-rust/crates/secrets-aws/tests/secret_manager.rs index c482a168090..56999633a78 100644 --- a/litellm-rust/crates/secrets-aws/tests/secret_manager.rs +++ b/litellm-rust/crates/secrets-aws/tests/secret_manager.rs @@ -10,11 +10,13 @@ use aws_sdk_secretsmanager::{ Client, config::{BehaviorVersion, Credentials, Region, retry::RetryConfig}, }; +use litellm_secret_value::SecretValue; use litellm_secrets_aws::{AwsSecretsManagerV2, Error, RotationResponse}; use litellm_secrets_types::{ AwsOperationContext, BaseSecretManager, KeyManagementSettings, Secret, SecretDeleter, - SecretValue, SecretWriteContext, SecretWriter, + SecretWriteContext, SecretWriter, }; + use rstest::{fixture, rstest}; use serde_json::json; use wiremock::{ diff --git a/litellm-rust/crates/secrets-azure/Cargo.toml b/litellm-rust/crates/secrets-azure/Cargo.toml index 2aff6129c37..c4bfb24898c 100644 --- a/litellm-rust/crates/secrets-azure/Cargo.toml +++ b/litellm-rust/crates/secrets-azure/Cargo.toml @@ -6,6 +6,7 @@ license.workspace = true repository.workspace = true [dependencies] +litellm-secret-value.workspace = true litellm-http.workspace = true tokio.workspace = true litellm-auth-azure.workspace = true diff --git a/litellm-rust/crates/secrets-azure/src/key_vault.rs b/litellm-rust/crates/secrets-azure/src/key_vault.rs index d97b7a22cfc..40028828298 100644 --- a/litellm-rust/crates/secrets-azure/src/key_vault.rs +++ b/litellm-rust/crates/secrets-azure/src/key_vault.rs @@ -3,7 +3,9 @@ use std::sync::Arc; use litellm_auth_azure::{AzureAuthInputs, AzureAuthService, ConfigValue}; use litellm_auth_types::{InputSource, Sourced}; use litellm_core_utils::settings::Lookup; -use litellm_secrets_types::{AzureOperationContext, BaseSecretManager, Secret, SecretValue}; +use litellm_secret_value::SecretValue; +use litellm_secrets_types::{AzureOperationContext, BaseSecretManager, Secret}; + use percent_encoding::{AsciiSet, NON_ALPHANUMERIC}; use serde::Deserialize; diff --git a/litellm-rust/crates/secrets-azure/tests/key_vault.rs b/litellm-rust/crates/secrets-azure/tests/key_vault.rs index a0ba9e6ddf8..c3ba01672a8 100644 --- a/litellm-rust/crates/secrets-azure/tests/key_vault.rs +++ b/litellm-rust/crates/secrets-azure/tests/key_vault.rs @@ -1,7 +1,9 @@ use std::sync::Arc; +use litellm_secret_value::SecretValue; use litellm_secrets_azure::{AzureKeyVault, Error}; -use litellm_secrets_types::{Secret, SecretValue}; +use litellm_secrets_types::Secret; + use rstest::{fixture, rstest}; use serde::Deserialize; use wiremock::{ diff --git a/litellm-rust/crates/secrets-cyberark/Cargo.toml b/litellm-rust/crates/secrets-cyberark/Cargo.toml index 2ae2c1ea5c3..643f322f7fa 100644 --- a/litellm-rust/crates/secrets-cyberark/Cargo.toml +++ b/litellm-rust/crates/secrets-cyberark/Cargo.toml @@ -6,6 +6,7 @@ license.workspace = true repository.workspace = true [dependencies] +litellm-secret-value.workspace = true litellm-secrets-types.workspace = true litellm-core-utils.workspace = true litellm-http.workspace = true diff --git a/litellm-rust/crates/secrets-cyberark/src/secret_manager.rs b/litellm-rust/crates/secrets-cyberark/src/secret_manager.rs index c5cb0308b13..cf077a2bbc8 100644 --- a/litellm-rust/crates/secrets-cyberark/src/secret_manager.rs +++ b/litellm-rust/crates/secrets-cyberark/src/secret_manager.rs @@ -9,11 +9,12 @@ use litellm_core_utils::settings::Lookup; use litellm_http::{ Client, ClientIdentity, ClientVariant, HttpClientConfig, HttpClientPool, TlsSource, Verify, }; +use litellm_secret_value::SecretValue; use litellm_secrets_types::{ BaseSecretManager, CyberarkOperationContext, RotationError, SecretCache, SecretDeleter, - SecretRotator, SecretValue, SecretWriteContext, SecretWriter, async_rotate_secret, - validate_secret_name, + SecretRotator, SecretWriteContext, SecretWriter, async_rotate_secret, validate_secret_name, }; + use moka::future::Cache; use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, utf8_percent_encode}; diff --git a/litellm-rust/crates/secrets-cyberark/tests/secret_manager.rs b/litellm-rust/crates/secrets-cyberark/tests/secret_manager.rs index 783ba6ff67a..e9b480e026b 100644 --- a/litellm-rust/crates/secrets-cyberark/tests/secret_manager.rs +++ b/litellm-rust/crates/secrets-cyberark/tests/secret_manager.rs @@ -9,8 +9,10 @@ use std::{ use base64::{Engine, engine::general_purpose::STANDARD}; use litellm_core_utils::settings::Lookup; use litellm_http::{HttpClientPool, HttpSettings, Resolution, media::PublicDnsResolver}; +use litellm_secret_value::SecretValue; use litellm_secrets_cyberark::{CyberArkSecretManager, DeleteOutcome, Error}; -use litellm_secrets_types::{BaseSecretManager, CyberarkOperationContext, SecretValue}; +use litellm_secrets_types::{BaseSecretManager, CyberarkOperationContext}; + use rstest::{fixture, rstest}; use serde::Deserialize; use wiremock::{ diff --git a/litellm-rust/crates/secrets-google/Cargo.toml b/litellm-rust/crates/secrets-google/Cargo.toml index 06b3747def5..3c5c2130788 100644 --- a/litellm-rust/crates/secrets-google/Cargo.toml +++ b/litellm-rust/crates/secrets-google/Cargo.toml @@ -6,6 +6,7 @@ license.workspace = true repository.workspace = true [dependencies] +litellm-secret-value.workspace = true litellm-http.workspace = true moka.workspace = true tokio.workspace = true diff --git a/litellm-rust/crates/secrets-google/src/auth.rs b/litellm-rust/crates/secrets-google/src/auth.rs index 45fc99d8d5f..7022ecc5ae4 100644 --- a/litellm-rust/crates/secrets-google/src/auth.rs +++ b/litellm-rust/crates/secrets-google/src/auth.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use litellm_auth_gcp::{GoogleCredentials, VertexConfig}; use litellm_auth_types::{InputSource, Sourced}; use litellm_core_utils::settings::Lookup; -use litellm_secrets_types::SecretValue; +use litellm_secret_value::SecretValue; pub(crate) fn credentials( project: Option, diff --git a/litellm-rust/crates/secrets-google/src/kms.rs b/litellm-rust/crates/secrets-google/src/kms.rs index 78ebae008f7..5622a3a4d56 100644 --- a/litellm-rust/crates/secrets-google/src/kms.rs +++ b/litellm-rust/crates/secrets-google/src/kms.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use google_cloud_kms_v1::client::KeyManagementService; use litellm_core_utils::settings::Lookup; -use litellm_secrets_types::SecretValue; +use litellm_secret_value::SecretValue; use crate::{Error, auth}; diff --git a/litellm-rust/crates/secrets-google/src/secret_manager.rs b/litellm-rust/crates/secrets-google/src/secret_manager.rs index 463d1f6fd73..577a2ccbd4d 100644 --- a/litellm-rust/crates/secrets-google/src/secret_manager.rs +++ b/litellm-rust/crates/secrets-google/src/secret_manager.rs @@ -2,9 +2,9 @@ use std::{sync::Arc, time::Duration}; use base64::{Engine, engine::general_purpose::STANDARD}; use litellm_core_utils::settings::Lookup; -use litellm_secrets_types::{ - BaseSecretManager, GoogleOperationContext, Secret, SecretCache, SecretValue, -}; +use litellm_secret_value::SecretValue; +use litellm_secrets_types::{BaseSecretManager, GoogleOperationContext, Secret, SecretCache}; + use serde::Deserialize; use litellm_auth_gcp::GoogleCredentials; diff --git a/litellm-rust/crates/secrets-hashicorp/Cargo.toml b/litellm-rust/crates/secrets-hashicorp/Cargo.toml index 8731a8deb7a..4fcd496931d 100644 --- a/litellm-rust/crates/secrets-hashicorp/Cargo.toml +++ b/litellm-rust/crates/secrets-hashicorp/Cargo.toml @@ -6,6 +6,7 @@ license.workspace = true repository.workspace = true [dependencies] +litellm-secret-value.workspace = true litellm-error.workspace = true litellm-core-utils.workspace = true litellm-http.workspace = true diff --git a/litellm-rust/crates/secrets-hashicorp/src/config.rs b/litellm-rust/crates/secrets-hashicorp/src/config.rs index 4232667ae16..a63161b17b2 100644 --- a/litellm-rust/crates/secrets-hashicorp/src/config.rs +++ b/litellm-rust/crates/secrets-hashicorp/src/config.rs @@ -1,7 +1,7 @@ use std::{path::PathBuf, time::Duration}; use litellm_core_utils::settings::Lookup; -use litellm_secrets_types::SecretValue; +use litellm_secret_value::SecretValue; use crate::Error; diff --git a/litellm-rust/crates/secrets-hashicorp/src/secret_manager.rs b/litellm-rust/crates/secrets-hashicorp/src/secret_manager.rs index 447beacda85..dae14439630 100644 --- a/litellm-rust/crates/secrets-hashicorp/src/secret_manager.rs +++ b/litellm-rust/crates/secrets-hashicorp/src/secret_manager.rs @@ -12,11 +12,12 @@ use std::{ }; use litellm_core_utils::settings::Lookup; +use litellm_secret_value::SecretValue; use litellm_secrets_types::{ BaseSecretManager, HashicorpOperationContext, RotationError, SecretCache, SecretDeleter, - SecretRotator, SecretValue, SecretWriteContext, SecretWriter, async_rotate_secret, - validate_secret_name, + SecretRotator, SecretWriteContext, SecretWriter, async_rotate_secret, validate_secret_name, }; + use rustify::errors::ClientError as RustifyClientError; use serde_json::Value; use tokio::sync::Mutex; diff --git a/litellm-rust/crates/secrets-hashicorp/tests/secret_manager.rs b/litellm-rust/crates/secrets-hashicorp/tests/secret_manager.rs index 668871d3fc7..d431e472400 100644 --- a/litellm-rust/crates/secrets-hashicorp/tests/secret_manager.rs +++ b/litellm-rust/crates/secrets-hashicorp/tests/secret_manager.rs @@ -1,11 +1,13 @@ use std::{collections::HashMap, sync::Arc, time::Duration}; use litellm_core_utils::settings::Lookup; +use litellm_secret_value::SecretValue; use litellm_secrets_hashicorp::{Error, HashicorpVault, HashicorpVaultConfig}; use litellm_secrets_types::{ - BaseSecretManager, HashicorpOperationContext, RotationError, SecretDeleter, SecretValue, - SecretWriteContext, SecretWriter, + BaseSecretManager, HashicorpOperationContext, RotationError, SecretDeleter, SecretWriteContext, + SecretWriter, }; + use rstest::{fixture, rstest}; use serde::Deserialize; use serde_json::json; diff --git a/litellm-rust/crates/secrets-types/Cargo.toml b/litellm-rust/crates/secrets-types/Cargo.toml index 6dd847ec989..b45c41bcf1c 100644 --- a/litellm-rust/crates/secrets-types/Cargo.toml +++ b/litellm-rust/crates/secrets-types/Cargo.toml @@ -6,7 +6,7 @@ license.workspace = true repository.workspace = true [dependencies] -litellm-auth-types.workspace = true +litellm-secret-value.workspace = true moka.workspace = true tokio = { workspace = true, features = ["sync"] } serde.workspace = true diff --git a/litellm-rust/crates/secrets-types/src/base_secret_manager.rs b/litellm-rust/crates/secrets-types/src/base_secret_manager.rs index 737fad28662..41dadd1baa6 100644 --- a/litellm-rust/crates/secrets-types/src/base_secret_manager.rs +++ b/litellm-rust/crates/secrets-types/src/base_secret_manager.rs @@ -1,4 +1,5 @@ -use crate::{Error, SecretValue, SecretWriteContext}; +use crate::{Error, SecretWriteContext}; +use litellm_secret_value::SecretValue; pub fn validate_secret_name(name: &str) -> Result<(), Error> { if name.split('/').any(|segment| segment == "..") diff --git a/litellm-rust/crates/secrets-types/src/config.rs b/litellm-rust/crates/secrets-types/src/config.rs index 9b6e82c3a6b..a45acb4df11 100644 --- a/litellm-rust/crates/secrets-types/src/config.rs +++ b/litellm-rust/crates/secrets-types/src/config.rs @@ -3,7 +3,7 @@ use std::collections::BTreeMap; use serde::{Deserialize, Serialize}; use strum::IntoStaticStr; -use crate::SecretValue; +use litellm_secret_value::SecretValue; #[derive( Clone, diff --git a/litellm-rust/crates/secrets-types/src/context.rs b/litellm-rust/crates/secrets-types/src/context.rs index 1ec493edf42..b71195e2d21 100644 --- a/litellm-rust/crates/secrets-types/src/context.rs +++ b/litellm-rust/crates/secrets-types/src/context.rs @@ -1,6 +1,7 @@ use std::{collections::BTreeMap, time::Duration}; -use crate::{Error, KeyManagementSystem, SecretValue}; +use crate::{Error, KeyManagementSystem}; +use litellm_secret_value::SecretValue; #[derive(Clone, Debug, Default, Eq, PartialEq)] pub enum SecretOperationContext { diff --git a/litellm-rust/crates/secrets-types/src/lib.rs b/litellm-rust/crates/secrets-types/src/lib.rs index 4d7eaf8313e..2563ccbe39d 100644 --- a/litellm-rust/crates/secrets-types/src/lib.rs +++ b/litellm-rust/crates/secrets-types/src/lib.rs @@ -18,5 +18,4 @@ pub use context::{ HashicorpOperationContext, SecretOperationContext, SecretWriteContext, }; pub use error::Error; -pub use litellm_auth_types::SecretValue; pub use value::{PythonSecretRead, Secret}; diff --git a/litellm-rust/crates/secrets-types/src/value.rs b/litellm-rust/crates/secrets-types/src/value.rs index 11f57ef4448..8c92902edad 100644 --- a/litellm-rust/crates/secrets-types/src/value.rs +++ b/litellm-rust/crates/secrets-types/src/value.rs @@ -1,4 +1,4 @@ -use crate::SecretValue; +use litellm_secret_value::SecretValue; #[derive(Clone, PartialEq, Eq, veil::Redact)] pub enum Secret { diff --git a/litellm-rust/crates/secrets-types/tests/config.rs b/litellm-rust/crates/secrets-types/tests/config.rs index 45fd9d0ee03..e7ba63343da 100644 --- a/litellm-rust/crates/secrets-types/tests/config.rs +++ b/litellm-rust/crates/secrets-types/tests/config.rs @@ -1,6 +1,6 @@ -use litellm_secrets_types::{ - AccessMode, KeyManagementSettings, KeyManagementSystem, Secret, SecretValue, -}; +use litellm_secret_value::SecretValue; +use litellm_secrets_types::{AccessMode, KeyManagementSettings, KeyManagementSystem, Secret}; + use rstest::{fixture, rstest}; use serde_json::json; diff --git a/litellm-rust/crates/secrets-types/tests/context.rs b/litellm-rust/crates/secrets-types/tests/context.rs index 8f8eeabdb62..f8d754564dc 100644 --- a/litellm-rust/crates/secrets-types/tests/context.rs +++ b/litellm-rust/crates/secrets-types/tests/context.rs @@ -1,10 +1,11 @@ use std::{collections::BTreeMap, time::Duration}; +use litellm_secret_value::SecretValue; use litellm_secrets_types::{ AwsOperationContext, AzureOperationContext, CyberarkOperationContext, GoogleOperationContext, - HashicorpOperationContext, KeyManagementSystem, SecretOperationContext, SecretValue, - SecretWriteContext, + HashicorpOperationContext, KeyManagementSystem, SecretOperationContext, SecretWriteContext, }; + use rstest::{fixture, rstest}; #[fixture] diff --git a/litellm-rust/crates/secrets-types/tests/rotation.rs b/litellm-rust/crates/secrets-types/tests/rotation.rs index 36c1a065ffa..089890c60bd 100644 --- a/litellm-rust/crates/secrets-types/tests/rotation.rs +++ b/litellm-rust/crates/secrets-types/tests/rotation.rs @@ -1,10 +1,12 @@ use std::sync::atomic::{AtomicUsize, Ordering}; +use litellm_secret_value::SecretValue; use litellm_secrets_types::{ BaseSecretManager, Error, HashicorpOperationContext, RotationError, SecretDeleter, - SecretOperationContext, SecretRotator, SecretValue, SecretWriteContext, SecretWriter, - async_rotate_secret, validate_secret_name, + SecretOperationContext, SecretRotator, SecretWriteContext, SecretWriter, async_rotate_secret, + validate_secret_name, }; + use rstest::{fixture, rstest}; struct Manager { diff --git a/litellm-rust/crates/secrets/Cargo.toml b/litellm-rust/crates/secrets/Cargo.toml index 023d61136e4..21ee1bdc760 100644 --- a/litellm-rust/crates/secrets/Cargo.toml +++ b/litellm-rust/crates/secrets/Cargo.toml @@ -14,6 +14,7 @@ azure = ["dep:litellm-secrets-azure"] cyberark = ["dep:litellm-secrets-cyberark"] [dependencies] +litellm-secret-value.workspace = true futures-util.workspace = true litellm-python-compat.workspace = true litellm-secrets-types.workspace = true diff --git a/litellm-rust/crates/secrets/src/compatibility.rs b/litellm-rust/crates/secrets/src/compatibility.rs index 368c9931b51..3d6ba854325 100644 --- a/litellm-rust/crates/secrets/src/compatibility.rs +++ b/litellm-rust/crates/secrets/src/compatibility.rs @@ -3,9 +3,10 @@ use litellm_python_compat::{Value, literal::literal_eval}; use litellm_secrets_types::PythonSecretRead; use crate::{ - Error, KeyManagementSettings, KeyManagementSystem, Secret, SecretManager, SecretValue, + Error, KeyManagementSettings, KeyManagementSystem, Secret, SecretManager, get_secret_from_manager, }; +use litellm_secret_value::SecretValue; pub async fn get_secret_from_python_manager( manager: &SecretManager, diff --git a/litellm-rust/crates/secrets/src/handler.rs b/litellm-rust/crates/secrets/src/handler.rs index f42e09c6bdb..2d2b81d197f 100644 --- a/litellm-rust/crates/secrets/src/handler.rs +++ b/litellm-rust/crates/secrets/src/handler.rs @@ -5,7 +5,7 @@ use litellm_core_utils::settings::Lookup; use crate::{Error, KeyManagementSettings, KeyManagementSystem, Secret}; #[cfg(any(feature = "aws", feature = "google"))] -use crate::SecretValue; +use litellm_secret_value::SecretValue; #[cfg(any( feature = "google", diff --git a/litellm-rust/crates/secrets/src/lib.rs b/litellm-rust/crates/secrets/src/lib.rs index 8d0f0561608..e4459576e94 100644 --- a/litellm-rust/crates/secrets/src/lib.rs +++ b/litellm-rust/crates/secrets/src/lib.rs @@ -12,9 +12,7 @@ mod state; pub use compatibility::{get_secret_from_python_manager, read_secret_from_python_manager}; pub use error::Error; pub use handler::{ExternalSecretManager, SecretManager, get_secret_from_manager}; -pub use litellm_secrets_types::{ - AccessMode, KeyManagementSettings, KeyManagementSystem, Secret, SecretValue, -}; +pub use litellm_secrets_types::{AccessMode, KeyManagementSettings, KeyManagementSystem, Secret}; pub use native::load_native_manager; pub use oidc::{OidcProvider, OidcReference, OidcResolver}; pub use resolver::{FailurePolicy, SecretResolver, normalize_nonempty_secret_str}; diff --git a/litellm-rust/crates/secrets/src/oidc.rs b/litellm-rust/crates/secrets/src/oidc.rs index 24e6912c797..18fc9479ef1 100644 --- a/litellm-rust/crates/secrets/src/oidc.rs +++ b/litellm-rust/crates/secrets/src/oidc.rs @@ -9,7 +9,8 @@ use litellm_http::Client; use moka::future::Cache; use serde::Deserialize; -use crate::{Error, SecretValue}; +use crate::Error; +use litellm_secret_value::SecretValue; const GOOGLE_TOKEN_MAX_TTL: Duration = Duration::from_secs(3540); const GITHUB_TOKEN_TTL: Duration = Duration::from_secs(295); diff --git a/litellm-rust/crates/secrets/src/resolver.rs b/litellm-rust/crates/secrets/src/resolver.rs index 830e7c22cd5..631b60cba10 100644 --- a/litellm-rust/crates/secrets/src/resolver.rs +++ b/litellm-rust/crates/secrets/src/resolver.rs @@ -4,7 +4,8 @@ use crate::compatibility::python_manager_string; use litellm_core_utils::{serde_compat::parse_str_bool, settings::Lookup}; use crate::state::{LookupTarget, normalize_secret_name}; -use crate::{Error, OidcResolver, Secret, SecretManagerState, SecretValue}; +use crate::{Error, OidcResolver, Secret, SecretManagerState}; +use litellm_secret_value::SecretValue; #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub enum FailurePolicy { diff --git a/litellm-rust/crates/secrets/src/source.rs b/litellm-rust/crates/secrets/src/source.rs index 3c86fd25e9a..1ee224b763a 100644 --- a/litellm-rust/crates/secrets/src/source.rs +++ b/litellm-rust/crates/secrets/src/source.rs @@ -3,7 +3,8 @@ use std::{collections::HashMap, sync::Arc}; use futures_util::future::{BoxFuture, try_join_all}; use litellm_core_utils::settings::Lookup; -use crate::{Error, SecretResolver, SecretValue}; +use crate::{Error, SecretResolver}; +use litellm_secret_value::SecretValue; pub type Secrets = Arc; diff --git a/litellm-rust/crates/secrets/tests/aws.rs b/litellm-rust/crates/secrets/tests/aws.rs index abad04f31c5..6f30d1b5161 100644 --- a/litellm-rust/crates/secrets/tests/aws.rs +++ b/litellm-rust/crates/secrets/tests/aws.rs @@ -2,11 +2,13 @@ use std::sync::Arc; +use litellm_secret_value::SecretValue; use litellm_secrets::{ AccessMode, Error, FailurePolicy, KeyManagementSettings, OidcResolver, Secret, SecretManager, - SecretManagerState, SecretResolver, SecretValue, aws::AwsSecretsManagerV2, + SecretManagerState, SecretResolver, aws::AwsSecretsManagerV2, secret_manager_would_be_consulted, }; + use wiremock::{Mock, MockServer, ResponseTemplate, matchers::method}; fn state(server: &MockServer, settings: KeyManagementSettings) -> SecretManagerState { diff --git a/litellm-rust/crates/secrets/tests/azure.rs b/litellm-rust/crates/secrets/tests/azure.rs index 6e81017b0f6..201e31f44b8 100644 --- a/litellm-rust/crates/secrets/tests/azure.rs +++ b/litellm-rust/crates/secrets/tests/azure.rs @@ -68,10 +68,11 @@ async fn azure_handler_reads_missing_and_failed_secrets() { async fn successful_azure_responses_do_not_fall_back_when_the_value_is_empty_or_null( #[case] body: serde_json::Value, ) { + use litellm_secret_value::SecretValue; use litellm_secrets::{ - OidcResolver, SecretManager, SecretManagerState, SecretResolver, SecretValue, - azure::AzureKeyVault, + OidcResolver, SecretManager, SecretManagerState, SecretResolver, azure::AzureKeyVault, }; + use std::sync::Arc; use wiremock::{Mock, MockServer, ResponseTemplate, matchers::any}; let server = MockServer::start().await; diff --git a/litellm-rust/crates/secrets/tests/common_read_contract.rs b/litellm-rust/crates/secrets/tests/common_read_contract.rs index e6928038799..4cf3513fadf 100644 --- a/litellm-rust/crates/secrets/tests/common_read_contract.rs +++ b/litellm-rust/crates/secrets/tests/common_read_contract.rs @@ -9,8 +9,9 @@ use std::sync::Arc; use base64::{Engine, engine::general_purpose::STANDARD}; use litellm_core_utils::settings::Lookup; +use litellm_secret_value::SecretValue; use litellm_secrets::{ - KeyManagementSettings, SecretManager, SecretValue, + KeyManagementSettings, SecretManager, aws::AwsSecretsManagerV2, azure::AzureKeyVault, cyberark::CyberArkSecretManager, @@ -18,6 +19,7 @@ use litellm_secrets::{ google::GoogleSecretManager, hashicorp::{HashicorpVault, HashicorpVaultConfig}, }; + use rstest::rstest; use serde_json::json; use wiremock::{ diff --git a/litellm-rust/crates/secrets/tests/cyberark.rs b/litellm-rust/crates/secrets/tests/cyberark.rs index 0a6c394937e..09342234118 100644 --- a/litellm-rust/crates/secrets/tests/cyberark.rs +++ b/litellm-rust/crates/secrets/tests/cyberark.rs @@ -4,10 +4,12 @@ async fn cyberark_handler_reads_values_and_surfaces_errors() { use std::time::Duration; + use litellm_secret_value::SecretValue; use litellm_secrets::{ - Error, KeyManagementSettings, SecretManager, SecretValue, cyberark::CyberArkSecretManager, + Error, KeyManagementSettings, SecretManager, cyberark::CyberArkSecretManager, get_secret_from_manager, }; + use wiremock::{ Mock, MockServer, ResponseTemplate, matchers::{body_string, path}, diff --git a/litellm-rust/crates/secrets/tests/google.rs b/litellm-rust/crates/secrets/tests/google.rs index 5542c864dfd..a75e34335de 100644 --- a/litellm-rust/crates/secrets/tests/google.rs +++ b/litellm-rust/crates/secrets/tests/google.rs @@ -7,10 +7,12 @@ use std::sync::Arc; #[case::failure(503)] #[tokio::test] async fn google_resolver_distinguishes_absence_from_failure(#[case] status: u16) { + use litellm_secret_value::SecretValue; use litellm_secrets::{ Error, FailurePolicy, KeyManagementSettings, OidcResolver, SecretManager, - SecretManagerState, SecretResolver, SecretValue, google::GoogleSecretManager, + SecretManagerState, SecretResolver, google::GoogleSecretManager, }; + use wiremock::{Mock, MockServer, ResponseTemplate, matchers::method}; let server = MockServer::start().await; Mock::given(method("GET")) diff --git a/litellm-rust/crates/secrets/tests/oidc.rs b/litellm-rust/crates/secrets/tests/oidc.rs index 867a662cbbb..2d250e1de9d 100644 --- a/litellm-rust/crates/secrets/tests/oidc.rs +++ b/litellm-rust/crates/secrets/tests/oidc.rs @@ -315,7 +315,7 @@ async fn azure_oidc_acquires_the_requested_scope_and_preserves_failures(#[case] Box< dyn std::future::Future< Output = Result< - litellm_secrets::SecretValue, + litellm_secret_value::SecretValue, litellm_secrets::azure::Error, >, > + Send @@ -331,7 +331,7 @@ async fn azure_oidc_acquires_the_requested_scope_and_preserves_failures(#[case] if self.0 { Err(litellm_secrets::azure::Error::MissingCredentials) } else { - Ok(litellm_secrets::SecretValue::new("azure-token")) + Ok(litellm_secret_value::SecretValue::new("azure-token")) } }) } @@ -346,7 +346,7 @@ async fn azure_oidc_acquires_the_requested_scope_and_preserves_failures(#[case] let result = resolver .get_secret_str( "oidc/azure/api://audience/path", - Some(litellm_secrets::SecretValue::new("fallback")), + Some(litellm_secret_value::SecretValue::new("fallback")), ) .await; if failed { diff --git a/litellm-rust/crates/secrets/tests/resolution.rs b/litellm-rust/crates/secrets/tests/resolution.rs index 8435e051315..185f2256780 100644 --- a/litellm-rust/crates/secrets/tests/resolution.rs +++ b/litellm-rust/crates/secrets/tests/resolution.rs @@ -1,9 +1,10 @@ use std::{future::Future, pin::Pin, sync::Arc}; use litellm_core_utils::settings::Lookup; +use litellm_secret_value::SecretValue; use litellm_secrets::{ Error, ExternalSecretManager, FailurePolicy, KeyManagementSettings, KeyManagementSystem, - OidcResolver, Secret, SecretManager, SecretManagerState, SecretResolver, SecretValue, + OidcResolver, Secret, SecretManager, SecretManagerState, SecretResolver, secret_manager_would_be_consulted, }; From 24b8889965a6f86bee6f743d61c77da0896eaa15 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 11:35:18 -0700 Subject: [PATCH 22/74] chore(cost-map): add openai gpt-6-rosalind from the pricing page and changelog (#49268) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 56 +++++++++++++++++++ model_prices_and_context_window.json | 56 +++++++++++++++++++ 2 files changed, 112 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index ea9cacbdabe..dac7af6ab7a 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -81156,5 +81156,61 @@ "output_cost_per_token": 0.0, "source": "https://inference.baseten.co/v1/models", "supports_vision": true + }, + "gpt-6-rosalind": { + "cache_creation_input_token_cost": 1.875e-05, + "cache_creation_input_token_cost_above_12hr": 3.75e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 7.5e-05, + "cache_creation_input_token_cost_above_272k_tokens": 3.75e-05, + "cache_read_input_token_cost": 1.5e-06, + "cache_read_input_token_cost_above_272k_tokens": 3e-06, + "input_cost_per_token": 1.5e-05, + "input_cost_per_token_above_272k_tokens": 3e-05, + "litellm_provider": "openai", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 7.5e-05, + "output_cost_per_token_above_272k_tokens": 0.0001125, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, + "source": "https://developers.openai.com/api/docs/pricing", + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, + "supports_native_streaming": true, + "supports_none_reasoning_effort": false, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_cache_breakpoint": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_tool_search": true, + "supports_vision": true, + "supports_web_search": true, + "supports_xhigh_reasoning_effort": true + }, + "chatgpt/gpt-6-rosalind": { + "litellm_provider": "chatgpt", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "mode": "responses", + "source": "https://developers.openai.com/api/docs/pricing", + "supported_endpoints": [ + "/v1/responses" + ], + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, + "supports_none_reasoning_effort": false, + "supports_parallel_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index ea9cacbdabe..dac7af6ab7a 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -81156,5 +81156,61 @@ "output_cost_per_token": 0.0, "source": "https://inference.baseten.co/v1/models", "supports_vision": true + }, + "gpt-6-rosalind": { + "cache_creation_input_token_cost": 1.875e-05, + "cache_creation_input_token_cost_above_12hr": 3.75e-05, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": 7.5e-05, + "cache_creation_input_token_cost_above_272k_tokens": 3.75e-05, + "cache_read_input_token_cost": 1.5e-06, + "cache_read_input_token_cost_above_272k_tokens": 3e-06, + "input_cost_per_token": 1.5e-05, + "input_cost_per_token_above_272k_tokens": 3e-05, + "litellm_provider": "openai", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 7.5e-05, + "output_cost_per_token_above_272k_tokens": 0.0001125, + "regional_processing_uplift_multiplier_eu": 1.1, + "regional_processing_uplift_multiplier_us": 1.1, + "source": "https://developers.openai.com/api/docs/pricing", + "supports_computer_use": true, + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, + "supports_native_streaming": true, + "supports_none_reasoning_effort": false, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_cache_breakpoint": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_tool_search": true, + "supports_vision": true, + "supports_web_search": true, + "supports_xhigh_reasoning_effort": true + }, + "chatgpt/gpt-6-rosalind": { + "litellm_provider": "chatgpt", + "max_input_tokens": 922000, + "max_output_tokens": 128000, + "mode": "responses", + "source": "https://developers.openai.com/api/docs/pricing", + "supported_endpoints": [ + "/v1/responses" + ], + "supports_function_calling": true, + "supports_max_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, + "supports_none_reasoning_effort": false, + "supports_parallel_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true } } From 1ccb970d520aa24a078d5b2a40ccc7f5249a770c Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Fri, 30 Oct 2026 11:43:13 -0700 Subject: [PATCH 23/74] refactor(ui): remove the deprecated Old Usage page (#49284) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/ui/fixtures/menuMappings.ts | 1 - tests/e2e/ui/fixtures/migratedPages.ts | 6 - tests/e2e/ui/fixtures/pages.ts | 1 - .../e2e/ui/tests/logs/logsPagination.spec.ts | 2 +- tests/e2e/ui/tests/usage/usagePage.spec.ts | 2 +- ui/litellm-dashboard/eslint-suppressions.json | 28 - .../app/(dashboard)/legacyPageRoutes.test.ts | 2 +- .../src/app/(dashboard)/legacyPageRoutes.ts | 2 +- .../old-usage/_components/usage.test.tsx | 257 ----- .../old-usage/_components/usage.tsx | 950 ------------------ .../src/app/(dashboard)/old-usage/page.tsx | 22 - .../components/UsagePageView.test.tsx | 4 - .../src/components/leftnav.test.tsx | 26 - .../src/components/leftnav.tsx | 8 - .../src/components/page_metadata.ts | 1 - .../src/components/view_user_spend.tsx | 144 --- .../tests/CreateKeyPage.expiredToken.test.tsx | 1 - 17 files changed, 4 insertions(+), 1453 deletions(-) delete mode 100644 ui/litellm-dashboard/src/app/(dashboard)/old-usage/_components/usage.test.tsx delete mode 100644 ui/litellm-dashboard/src/app/(dashboard)/old-usage/_components/usage.tsx delete mode 100644 ui/litellm-dashboard/src/app/(dashboard)/old-usage/page.tsx delete mode 100644 ui/litellm-dashboard/src/components/view_user_spend.tsx diff --git a/tests/e2e/ui/fixtures/menuMappings.ts b/tests/e2e/ui/fixtures/menuMappings.ts index d6e7ea86982..4cf90c00cd0 100644 --- a/tests/e2e/ui/fixtures/menuMappings.ts +++ b/tests/e2e/ui/fixtures/menuMappings.ts @@ -32,7 +32,6 @@ export const menuLabelToPage: Record = { Budgets: Page.Budgets, "API Playground": Page.TransformRequest, "Tag Management": Page.TagManagement, - "Old Usage": Page.Usage, // Tools submenu items "MCP Servers": Page.McpServers, "Vector Stores": Page.VectorStores, diff --git a/tests/e2e/ui/fixtures/migratedPages.ts b/tests/e2e/ui/fixtures/migratedPages.ts index 5fa1f9d2444..6fa8d0c2587 100644 --- a/tests/e2e/ui/fixtures/migratedPages.ts +++ b/tests/e2e/ui/fixtures/migratedPages.ts @@ -118,12 +118,6 @@ export const MIGRATED_E2E_PAGES: Readonly> = { content: { role: "heading", name: "AI Hub" }, }, new_usage: { segment: "usage", linkName: "Usage", content: { role: "heading", name: "Usage" } }, - usage: { - segment: "old-usage", - linkName: "Old Usage", - group: "Experimental", - content: { role: "tab", name: "All Up" }, - }, agents: { segment: "agents", linkName: "Agents", group: "Agentic", content: { role: "heading", name: "Agents" } }, "router-settings": { segment: "router-settings", diff --git a/tests/e2e/ui/fixtures/pages.ts b/tests/e2e/ui/fixtures/pages.ts index 8210334c166..a850dcb4755 100644 --- a/tests/e2e/ui/fixtures/pages.ts +++ b/tests/e2e/ui/fixtures/pages.ts @@ -30,5 +30,4 @@ export enum Page { TagManagement = "tag-management", VectorStores = "vector-stores", NewUsage = "new_usage", - Usage = "usage", } diff --git a/tests/e2e/ui/tests/logs/logsPagination.spec.ts b/tests/e2e/ui/tests/logs/logsPagination.spec.ts index 585eb31c37c..a611f760e8b 100644 --- a/tests/e2e/ui/tests/logs/logsPagination.spec.ts +++ b/tests/e2e/ui/tests/logs/logsPagination.spec.ts @@ -84,7 +84,7 @@ test.describe("Logs pagination with Aria in the header", () => { await expect(aria).toBeFocused(); await visibleTestId(page, "pagination-first").click(); await expect(visibleTestId(page, "pagination-range")).toHaveText("Showing 1-25 of 26"); - for (const destination of [Page.Usage, Page.LlmPlayground, Page.ApiKeys]) { + for (const destination of [Page.NewUsage, Page.LlmPlayground, Page.ApiKeys]) { await navigateToPage(page, destination); await aria.click(); await expect(page.getByRole("dialog", { name: "Aria Agent", exact: true })).toBeVisible(); diff --git a/tests/e2e/ui/tests/usage/usagePage.spec.ts b/tests/e2e/ui/tests/usage/usagePage.spec.ts index d532042c7af..2a58a1ef59d 100644 --- a/tests/e2e/ui/tests/usage/usagePage.spec.ts +++ b/tests/e2e/ui/tests/usage/usagePage.spec.ts @@ -11,7 +11,7 @@ import { waitForSpendLog, } from "../../helpers/traffic"; -/** Covers /ui/usage. The legacy /ui/old-usage view is deprecated and deliberately not covered. */ +/** Covers /ui/usage. */ const topKeysCard = (page: PlaywrightPage): Locator => page.locator("section").filter({ has: page.getByRole("heading", { name: "Top Virtual Keys", exact: true }) }); diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 9fce93782b2..c14f3e6f036 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -625,23 +625,6 @@ "count": 1 } }, - "src/app/(dashboard)/old-usage/_components/usage.tsx": { - "local/filename-pascal-case": { - "count": 1 - }, - "max-lines": { - "count": 1 - }, - "prefer-const": { - "count": 6 - }, - "react-hooks/immutability": { - "count": 1 - }, - "react-hooks/purity": { - "count": 1 - } - }, "src/app/(dashboard)/playground/components/chat_ui/AdditionalModelSettings.tsx": { "react-hooks/set-state-in-effect": { "count": 2 @@ -2090,17 +2073,6 @@ "count": 1 } }, - "src/components/view_user_spend.tsx": { - "local/filename-pascal-case": { - "count": 1 - }, - "prefer-const": { - "count": 3 - }, - "react-hooks/set-state-in-effect": { - "count": 2 - } - }, "src/contexts/AuthContext.tsx": { "react-hooks/set-state-in-effect": { "count": 1 diff --git a/ui/litellm-dashboard/src/app/(dashboard)/legacyPageRoutes.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/legacyPageRoutes.test.ts index 6b02b79b04c..315d988db54 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/legacyPageRoutes.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/legacyPageRoutes.test.ts @@ -10,7 +10,7 @@ describe("legacyPageRedirectHref", () => { expect(redirect("page=models")).toBe("/ui/models-and-endpoints"); expect(redirect("page=llm-playground")).toBe("/ui/playground"); expect(redirect("page=new_usage")).toBe("/ui/usage"); - expect(redirect("page=usage")).toBe("/ui/old-usage"); + expect(redirect("page=usage")).toBe("/ui/usage"); }); it("keeps the older aliases for renamed pages", () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/legacyPageRoutes.ts b/ui/litellm-dashboard/src/app/(dashboard)/legacyPageRoutes.ts index e3025266ddb..fe054c7b39a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/legacyPageRoutes.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/legacyPageRoutes.ts @@ -36,7 +36,7 @@ const LEGACY_PAGE_ROUTES: ReadonlyMap = new Map( "logging-and-alerts": "logging-and-alerts", "model-hub-table": "model-hub-table", new_usage: "usage", - usage: "old-usage", + usage: "usage", "cost-optimization": "cost-optimization", "model-insights": "model-insights", agents: "agents", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/old-usage/_components/usage.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/old-usage/_components/usage.test.tsx deleted file mode 100644 index 0e4455ee912..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/old-usage/_components/usage.test.tsx +++ /dev/null @@ -1,257 +0,0 @@ -import React from "react"; -import { describe, it, expect, vi, beforeEach } from "vitest"; -import { act, screen, waitFor, within } from "@testing-library/react"; -import userEvent from "@testing-library/user-event"; -import { renderWithProviders } from "../../../../../tests/test-utils"; -import UsagePage from "./usage"; - -const networking = vi.hoisted(() => ({ - adminSpendLogsCall: vi.fn(), - adminTopKeysCall: vi.fn(), - adminTopModelsCall: vi.fn(), - adminTopEndUsersCall: vi.fn(), - teamSpendLogsCall: vi.fn(), - tagsSpendLogsCall: vi.fn(), - allTagNamesCall: vi.fn(), - adminspendByProvider: vi.fn(), - adminGlobalActivity: vi.fn(), - adminGlobalActivityPerModel: vi.fn(), - getProxyUISettings: vi.fn(), - modelAvailableCall: vi.fn(), - keyInfoV1Call: vi.fn(), -})); - -vi.mock("@/components/networking", () => networking); -vi.mock("../../../../components/networking", () => networking); - -vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ - default: () => ({ - accessToken: "sk-test", - token: "tok", - userRole: "Admin", - userId: "u1", - premiumUser: true, - }), -})); - -const UNLIMITED_SETTINGS = { DISABLE_EXPENSIVE_DB_QUERIES: false, NUM_SPEND_LOGS_ROWS: 10 }; - -const renderUsage = (overrides: Partial> = {}) => - renderWithProviders( - , - ); - -// Width of this window is guarded by "proves the flush window is wide enough". -const flushPendingRequests = async () => { - await act(async () => { - await new Promise((resolve) => setTimeout(resolve, 0)); - await new Promise((resolve) => setTimeout(resolve, 0)); - }); -}; - -beforeEach(() => { - vi.clearAllMocks(); - networking.getProxyUISettings.mockResolvedValue(UNLIMITED_SETTINGS); - networking.adminSpendLogsCall.mockResolvedValue([{ date: "2026-07-01", spend: 12.5 }]); - networking.adminTopKeysCall.mockResolvedValue([ - { api_key: "sk-abcdefghijk", key_alias: "prod-key", total_spend: 9.5 }, - ]); - networking.adminTopModelsCall.mockResolvedValue([{ model: "gpt-5.1", total_spend: 7.25 }]); - networking.adminTopEndUsersCall.mockResolvedValue([ - { end_user: "customer-alpha", total_spend: 3.5, total_count: 42 }, - ]); - networking.teamSpendLogsCall.mockResolvedValue({ - daily_spend: [{ date: "2026-07-01", "team-a": 5 }], - teams: ["team-a"], - total_spend_per_team: [{ team_id: "team-a", total_spend: 5 }], - }); - networking.tagsSpendLogsCall.mockResolvedValue({ spend_per_tag: [{ name: "prod", spend: 4 }] }); - networking.allTagNamesCall.mockResolvedValue({ tag_names: ["prod", "staging"] }); - networking.adminspendByProvider.mockResolvedValue([{ provider: "openai", spend: 6.75 }]); - networking.adminGlobalActivity.mockResolvedValue({ - sum_api_requests: 120, - sum_total_tokens: 4500, - daily_data: [{ date: "2026-07-01", api_requests: 120, total_tokens: 4500 }], - }); - networking.adminGlobalActivityPerModel.mockResolvedValue([]); - networking.modelAvailableCall.mockResolvedValue({ data: [] }); - networking.keyInfoV1Call.mockResolvedValue({ info: {} }); -}); - -describe("old usage page", () => { - describe("when the proxy has disabled expensive DB queries", () => { - beforeEach(() => { - networking.getProxyUISettings.mockResolvedValue({ - DISABLE_EXPENSIVE_DB_QUERIES: true, - NUM_SPEND_LOGS_ROWS: 2500000, - }); - }); - - it("shows the database query limit warning instead of the usage dashboard", async () => { - renderUsage(); - - expect(await screen.findByText("Database Query Limit Reached")).toBeInTheDocument(); - expect(screen.getByText(/SpendLogs in DB has/)).toHaveTextContent("2500000"); - expect(screen.getByText(/Please follow our guide to view usage when SpendLogs has more than 1M rows/i)); - expect(screen.queryByRole("tab", { name: "All Up" })).not.toBeInTheDocument(); - }); - - it("links to the cost tracking guide in a new tab", async () => { - renderUsage(); - - const link = await screen.findByRole("link", { name: "View Usage Guide" }); - expect(link).toHaveAttribute("href", "https://docs.litellm.ai/docs/proxy/cost_tracking"); - expect(link).toHaveAttribute("target", "_blank"); - }); - - it("skips every expensive usage query", async () => { - renderUsage(); - - await screen.findByText("Database Query Limit Reached"); - await waitFor(() => expect(networking.getProxyUISettings).toHaveBeenCalled()); - - expect(networking.adminSpendLogsCall).not.toHaveBeenCalled(); - expect(networking.adminspendByProvider).not.toHaveBeenCalled(); - expect(networking.adminTopKeysCall).not.toHaveBeenCalled(); - expect(networking.adminTopModelsCall).not.toHaveBeenCalled(); - expect(networking.adminGlobalActivity).not.toHaveBeenCalled(); - expect(networking.adminGlobalActivityPerModel).not.toHaveBeenCalled(); - expect(networking.teamSpendLogsCall).not.toHaveBeenCalled(); - expect(networking.adminTopEndUsersCall).not.toHaveBeenCalled(); - expect(networking.tagsSpendLogsCall).not.toHaveBeenCalled(); - }); - }); - - describe("as an admin", () => { - it("renders the admin tabs", async () => { - renderUsage(); - - expect(await screen.findByRole("tab", { name: "All Up" })).toBeInTheDocument(); - expect(screen.getByRole("tab", { name: "Team Based Usage" })).toBeInTheDocument(); - expect(screen.getByRole("tab", { name: "Customer Usage" })).toBeInTheDocument(); - expect(screen.getByRole("tab", { name: "Tag Based Usage" })).toBeInTheDocument(); - }); - - it("renders the cost panel cards", async () => { - renderUsage(); - - expect(await screen.findByText("Monthly Spend")).toBeInTheDocument(); - expect(screen.getByText("Top Virtual Keys")).toBeInTheDocument(); - expect(screen.getByText("Top Models")).toBeInTheDocument(); - expect(screen.getByText("Spend by Provider")).toBeInTheDocument(); - }); - - it("lists spend by provider in a table", async () => { - renderUsage(); - - const providerCell = await screen.findByText("openai"); - const row = providerCell.closest("tr"); - expect(row).not.toBeNull(); - expect(within(row as HTMLElement).getByText("$6.75")).toBeInTheDocument(); - expect(screen.getByRole("columnheader", { name: "Provider" })).toBeInTheDocument(); - }); - - it("shows the customer usage table when its tab is selected", async () => { - const user = userEvent.setup(); - renderUsage(); - - await user.click(await screen.findByRole("tab", { name: "Customer Usage" })); - - const customerCell = await screen.findByText("customer-alpha"); - const row = customerCell.closest("tr"); - expect(row).not.toBeNull(); - expect(within(row as HTMLElement).getByText("$3.50")).toBeInTheDocument(); - expect(within(row as HTMLElement).getByText("42")).toBeInTheDocument(); - expect(screen.getByRole("columnheader", { name: "Total Events" })).toBeInTheDocument(); - }); - - it("shows the tag spend panel when its tab is selected", async () => { - const user = userEvent.setup(); - renderUsage(); - - await user.click(await screen.findByRole("tab", { name: "Tag Based Usage" })); - - expect(await screen.findByText("Spend Per Tag")).toBeInTheDocument(); - }); - - it("shows the team spend panel when its tab is selected", async () => { - const user = userEvent.setup(); - renderUsage(); - - await user.click(await screen.findByRole("tab", { name: "Team Based Usage" })); - - expect(await screen.findByText("Total Spend Per Team")).toBeInTheDocument(); - expect(screen.getByText("Daily Spend Per Team")).toBeInTheDocument(); - }); - }); - - // org_admin is an organization membership role; those users reach the UI as "Internal User". - describe.each(["Internal User", "Internal Viewer", "internal_user", "internal_user_viewer", "Org Admin"])( - "as %s", - (userRole) => { - it("shows the admin-only notice instead of the usage dashboard", async () => { - renderUsage({ userRole }); - - expect(await screen.findByText(/Proxy-wide usage is only available to admin users/i)).toBeInTheDocument(); - expect(screen.queryByRole("tab", { name: "All Up" })).not.toBeInTheDocument(); - }); - - it("fires no /global/spend or /global/activity request", async () => { - renderUsage({ userRole }); - - await screen.findByText(/Proxy-wide usage is only available to admin users/i); - await flushPendingRequests(); - - expect(networking.getProxyUISettings).not.toHaveBeenCalled(); - expect(networking.adminSpendLogsCall).not.toHaveBeenCalled(); - expect(networking.adminTopKeysCall).not.toHaveBeenCalled(); - expect(networking.adminTopModelsCall).not.toHaveBeenCalled(); - expect(networking.adminTopEndUsersCall).not.toHaveBeenCalled(); - expect(networking.teamSpendLogsCall).not.toHaveBeenCalled(); - expect(networking.tagsSpendLogsCall).not.toHaveBeenCalled(); - expect(networking.allTagNamesCall).not.toHaveBeenCalled(); - expect(networking.adminspendByProvider).not.toHaveBeenCalled(); - expect(networking.adminGlobalActivity).not.toHaveBeenCalled(); - expect(networking.adminGlobalActivityPerModel).not.toHaveBeenCalled(); - }); - }, - ); - - describe("the admin-only gate", () => { - it("proves the flush window is wide enough to catch a leaked request", async () => { - renderUsage({ userRole: "Admin" }); - - await flushPendingRequests(); - - expect(networking.getProxyUISettings).toHaveBeenCalled(); - expect(networking.adminSpendLogsCall).toHaveBeenCalled(); - expect(networking.tagsSpendLogsCall).toHaveBeenCalled(); - expect(networking.adminGlobalActivity).toHaveBeenCalled(); - }); - - it("still lets an admin through, so the notice is a real gate and not a dead branch", async () => { - renderUsage({ userRole: "Admin" }); - - expect(await screen.findByRole("tab", { name: "All Up" })).toBeInTheDocument(); - expect(screen.queryByText(/Proxy-wide usage is only available to admin users/i)).not.toBeInTheDocument(); - await waitFor(() => expect(networking.adminSpendLogsCall).toHaveBeenCalled()); - }); - - it("does not put the session token in the provider spend query", async () => { - renderUsage({ userRole: "Admin", token: "session-jwt-value" }); - - await waitFor(() => expect(networking.adminspendByProvider).toHaveBeenCalled()); - const callArgs = networking.adminspendByProvider.mock.calls[0]; - expect(callArgs).not.toContain("session-jwt-value"); - expect(callArgs[0]).toBe("sk-test"); - }); - }); -}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/old-usage/_components/usage.tsx b/ui/litellm-dashboard/src/app/(dashboard)/old-usage/_components/usage.tsx deleted file mode 100644 index 15b2a30b50c..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/old-usage/_components/usage.tsx +++ /dev/null @@ -1,950 +0,0 @@ -import React, { useState, useEffect } from "react"; - -import ViewUserSpend from "@/components/view_user_spend"; -import { ProxySettings } from "@/app/(dashboard)/hooks/proxySettings/useProxySettings"; -import AdvancedDatePicker from "@/components/shared/advanced_date_picker"; -import { Button } from "@/components/ui/button"; -import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; -import { - Combobox, - ComboboxChip, - ComboboxChips, - ComboboxChipsInput, - ComboboxContent, - ComboboxEmpty, - ComboboxItem, - ComboboxList, - ComboboxValue, - useComboboxAnchor, -} from "@/components/ui/combobox"; -import { Meter, MeterIndicator, MeterTrack } from "@/components/shared/Meter"; -import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; -import { - NUMERIC_CELL_CLASS, - Table, - TableBody, - TableCell, - TableHead, - TableHeader, - TableRow, -} from "@/components/ui/table"; -import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; -import { AreaChart, BarChart, DonutChart } from "@/components/shared/charts"; - -import { - adminSpendLogsCall, - adminTopKeysCall, - adminTopModelsCall, - adminTopEndUsersCall, - teamSpendLogsCall, - tagsSpendLogsCall, - allTagNamesCall, - adminspendByProvider, - adminGlobalActivity, - adminGlobalActivityPerModel, - getProxyUISettings, -} from "@/components/networking"; -import TopKeyView from "@/components/UsagePage/components/EntityUsage/TopKeyView"; -import { MoneyCell } from "@/components/shared/table_cells"; -import { hasCapability } from "@/utils/capabilities"; -import { formatNumberWithCommas } from "@/utils/dataUtils"; - -interface UsagePageProps { - accessToken: string | null; - token: string | null; - userRole: string | null; - userID: string | null; - keys: any[] | null; - premiumUser: boolean; -} - -interface GlobalActivityData { - sum_api_requests: number; - sum_total_tokens: number; - daily_data: { date: string; api_requests: number; total_tokens: number }[]; -} - -const EMPTY_GLOBAL_ACTIVITY: GlobalActivityData = { sum_api_requests: 0, sum_total_tokens: 0, daily_data: [] }; - -type UsageDateRange = { from?: Date; to?: Date }; - -type TeamSpendTotal = { name: string; value: number }; - -type TagOption = { value: string; label: string; disabled: boolean }; - -const ALL_TAGS = "all-tags"; - -const isAdminOrAdminViewer = (role: string | null): boolean => { - if (role === null) return false; - return role === "Admin" || role === "Admin Viewer"; -}; - -const TeamSpendBarList: React.FC<{ data: TeamSpendTotal[] }> = ({ data }) => { - const max = Math.max(0, ...data.map((team) => team.value)); - - return ( -

- {data.map((team) => ( -
-

{team.name}

- - - - - -

- {formatNumberWithCommas(team.value, 2)} -

-
- ))} -
- ); -}; - -const UsagePage: React.FC = ({ accessToken, token, userRole, userID, keys, premiumUser }) => { - const anchor = useComboboxAnchor(); - const canViewGlobalSpend = hasCapability(userRole, "viewGlobalSpend"); - const currentDate = new Date(); - const [keySpendData, setKeySpendData] = useState([]); - const [topKeys, setTopKeys] = useState([]); - const [topModels, setTopModels] = useState([]); - const [topUsers, setTopUsers] = useState([]); - const [teamSpendData, setTeamSpendData] = useState([]); - const [topTagsData, setTopTagsData] = useState([]); - const [allTagNames, setAllTagNames] = useState([]); - const [uniqueTeamIds, setUniqueTeamIds] = useState([]); - const [totalSpendPerTeam, setTotalSpendPerTeam] = useState([]); - const [spendByProvider, setSpendByProvider] = useState([]); - const [globalActivity, setGlobalActivity] = useState(EMPTY_GLOBAL_ACTIVITY); - const [globalActivityPerModel, setGlobalActivityPerModel] = useState([]); - const [selectedKeyToken, setSelectedKeyToken] = useState(null); - const [selectedTags, setSelectedTags] = useState([ALL_TAGS]); - const [dateValue, setDateValue] = useState({ - from: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000), - to: new Date(), - }); - const [proxySettings, setProxySettings] = useState(null); - const [totalMonthlySpend, setTotalMonthlySpend] = useState(0); - - const firstDay = new Date(currentDate.getFullYear(), currentDate.getMonth(), 1); - const lastDay = new Date(currentDate.getFullYear(), currentDate.getMonth() + 1, 0); - - let startTime = formatDate(firstDay); - let endTime = formatDate(lastDay); - - const selectableKeys: { token: string; alias: string }[] = (keys ?? []) - .filter((key: any) => key && typeof key["key_alias"] === "string" && key["key_alias"].length > 0) - .map((key: any) => ({ token: String(key["token"]), alias: String(key["key_alias"]) })); - - const tagOptions: TagOption[] = [ - { value: ALL_TAGS, label: "All Tags", disabled: false }, - ...allTagNames - .filter((tag) => tag !== ALL_TAGS) - .map((tag) => ({ - value: tag, - label: premiumUser ? tag : `✨ ${tag} (Enterprise only Feature)`, - disabled: !premiumUser, - })), - ]; - - function valueFormatterNumbers(number: number) { - const formatter = new Intl.NumberFormat("en-US", { - maximumFractionDigits: 0, - notation: "compact", - compactDisplay: "short", - }); - - return formatter.format(number); - } - - const fetchProxySettings = async () => { - if (accessToken) { - try { - const proxy_settings: ProxySettings = await getProxyUISettings(accessToken); - return proxy_settings; - } catch (error) { - console.error("Error fetching proxy settings:", error); - } - } - }; - - useEffect(() => { - if (!canViewGlobalSpend) { - return; - } - updateTagSpendData(dateValue.from, dateValue.to); - }, [canViewGlobalSpend, dateValue, selectedTags]); - - const updateEndUserData = async ( - startTime: Date | undefined, - endTime: Date | undefined, - uiSelectedKey: string | null, - ) => { - if (!startTime || !endTime || !accessToken) { - return; - } - - let newTopUserData = await adminTopEndUsersCall( - accessToken, - uiSelectedKey, - startTime.toISOString(), - endTime.toISOString(), - ); - setTopUsers(newTopUserData); - }; - - const updateTagSpendData = async (startTime: Date | undefined, endTime: Date | undefined) => { - if (!startTime || !endTime || !accessToken) { - return; - } - - // we refetch because the state variable can be None when the user refreshes the page - const proxy_settings: ProxySettings | undefined = await fetchProxySettings(); - - if (proxy_settings?.DISABLE_EXPENSIVE_DB_QUERIES) { - return; // Don't run expensive DB queries - return out when SpendLogs has more than 1M rows - } - - let top_tags = await tagsSpendLogsCall( - accessToken, - startTime.toISOString(), - endTime.toISOString(), - selectedTags.length === 0 ? undefined : selectedTags, - ); - setTopTagsData(top_tags.spend_per_tag); - }; - - function formatDate(date: Date) { - const year = date.getFullYear(); - let month = date.getMonth() + 1; // JS month index starts from 0 - let day = date.getDate(); - - // Pad with 0 if month or day is less than 10 - const monthStr = month < 10 ? "0" + month : month; - const dayStr = day < 10 ? "0" + day : day; - - return `${year}-${monthStr}-${dayStr}`; - } - - const valueFormatter = (number: number) => `$ ${formatNumberWithCommas(number, 2)}`; - - const fetchAndSetData = async ( - fetchFunction: () => Promise, - setStateFunction: React.Dispatch>, - errorMessage: string, - ) => { - try { - const data = await fetchFunction(); - setStateFunction(data); - } catch (error) { - console.error(errorMessage, error); - // Optionally, update UI to reflect error state for this specific data - } - }; - - // Update the fillMissingDates function to handle different date formats - const fillMissingDates = (data: any[], startDate: Date, endDate: Date, categories: string[]) => { - const filledData = []; - const currentDate = new Date(startDate); - - // Helper function to standardize date format - const standardizeDate = (dateStr: string) => { - if (dateStr.includes("-")) { - // Already in YYYY-MM-DD format - return dateStr; - } else { - // Convert "Jan 06" format - const [month, day] = dateStr.split(" "); - const year = new Date().getFullYear(); - const monthIndex = new Date(`${month} 01 2024`).getMonth(); - const fullDate = new Date(year, monthIndex, parseInt(day)); - return fullDate.toISOString().split("T")[0]; - } - }; - - // Create a map of existing dates for quick lookup - const existingDates = new Map( - data.map((item) => { - const standardizedDate = standardizeDate(item.date); - return [ - standardizedDate, - { - ...item, - date: standardizedDate, // Store standardized date format - }, - ]; - }), - ); - - // Iterate through each date in the range - while (currentDate <= endDate) { - const dateStr = currentDate.toISOString().split("T")[0]; - - if (existingDates.has(dateStr)) { - // Use existing data if we have it - filledData.push(existingDates.get(dateStr)); - } else { - // Create an entry with zero values - const emptyEntry: any = { - date: dateStr, - api_requests: 0, - total_tokens: 0, - }; - - // Add zero values for each model/team if needed - categories.forEach((category) => { - if (!emptyEntry[category]) { - emptyEntry[category] = 0; - } - }); - - filledData.push(emptyEntry); - } - - // Move to next day - currentDate.setDate(currentDate.getDate() + 1); - } - - return filledData; - }; - - // Update the fetchOverallSpend function - const fetchOverallSpend = async () => { - if (!accessToken) { - return; - } - try { - const data = await adminSpendLogsCall(accessToken); - - // Get the first and last day of the current month - const now = new Date(); - const firstDay = new Date(now.getFullYear(), now.getMonth(), 1); - const lastDay = new Date(now.getFullYear(), now.getMonth() + 1, 0); - - // Fill in missing dates - const filledData = fillMissingDates(data, firstDay, lastDay, []); - - // Calculate total spend for the month and round to 2 decimal places - const monthlyTotal = Number(filledData.reduce((sum, day) => sum + (day.spend || 0), 0).toFixed(2)); - setTotalMonthlySpend(monthlyTotal); - - setKeySpendData(filledData); - } catch (error) { - console.error("Error fetching overall spend:", error); - } - }; - - const fetchProviderSpend = () => - fetchAndSetData( - () => (accessToken ? adminspendByProvider(accessToken, startTime, endTime) : Promise.reject("No access token")), - setSpendByProvider, - "Error fetching provider spend", - ); - - const fetchTopKeys = async () => { - if (!accessToken) return; - await fetchAndSetData( - async () => { - const top_keys = await adminTopKeysCall(accessToken); - return top_keys.map((k: any) => ({ - key: k["api_key"].substring(0, 10), - api_key: k["api_key"], - key_alias: k["key_alias"], - spend: Number(k["total_spend"].toFixed(2)), - })); - }, - setTopKeys, - "Error fetching top keys", - ); - }; - - const fetchTopModels = async () => { - if (!accessToken) return; - await fetchAndSetData( - async () => { - const top_models = await adminTopModelsCall(accessToken); - return top_models.map((k: any) => ({ - key: k["model"], - spend: formatNumberWithCommas(k["total_spend"], 2), - })); - }, - setTopModels, - "Error fetching top models", - ); - }; - - // Update the fetchTeamSpend function - const fetchTeamSpend = async () => { - if (!accessToken) return; - await fetchAndSetData( - async () => { - const teamSpend = await teamSpendLogsCall(accessToken); - - // Get the first and last day of the current month - const now = new Date(); - const firstDay = new Date(now.getFullYear(), now.getMonth(), 1); - const lastDay = new Date(now.getFullYear(), now.getMonth() + 1, 0); - - // Fill in missing dates with zero values for all teams - const filledData = fillMissingDates(teamSpend.daily_spend, firstDay, lastDay, teamSpend.teams); - - setTeamSpendData(filledData); - setUniqueTeamIds(teamSpend.teams); - return teamSpend.total_spend_per_team.map((tspt: any) => ({ - name: tspt["team_id"] || "", - value: Number(tspt["total_spend"] || 0), - })); - }, - setTotalSpendPerTeam, - "Error fetching team spend", - ); - }; - - const fetchTagNames = () => { - if (!accessToken) return; - fetchAndSetData( - async () => { - const all_tag_names = await allTagNamesCall(accessToken); - return all_tag_names.tag_names; - }, - setAllTagNames, - "Error fetching tag names", - ); - }; - - const fetchTopTags = () => { - if (!accessToken) return; - fetchAndSetData( - () => tagsSpendLogsCall(accessToken, dateValue.from?.toISOString(), dateValue.to?.toISOString(), undefined), - (data) => setTopTagsData(data.spend_per_tag), - "Error fetching top tags", - ); - }; - - const fetchTopEndUsers = () => { - if (!accessToken) return; - fetchAndSetData( - () => adminTopEndUsersCall(accessToken, null, undefined, undefined), - setTopUsers, - "Error fetching top end users", - ); - }; - - // Update the fetchGlobalActivity function - const fetchGlobalActivity = async () => { - if (!accessToken) return; - try { - const data = await adminGlobalActivity(accessToken, startTime, endTime); - - // Get the date range from the current month - const now = new Date(); - const firstDay = new Date(now.getFullYear(), now.getMonth(), 1); - const lastDay = new Date(now.getFullYear(), now.getMonth() + 1, 0); - - // Fill in missing dates for daily_data - const filledDailyData = fillMissingDates(data.daily_data || [], firstDay, lastDay, [ - "api_requests", - "total_tokens", - ]); - - setGlobalActivity({ - ...data, - daily_data: filledDailyData, - }); - } catch (error) { - console.error("Error fetching global activity:", error); - } - }; - - // Update the fetchGlobalActivityPerModel function - const fetchGlobalActivityPerModel = async () => { - if (!accessToken) return; - try { - const data = await adminGlobalActivityPerModel(accessToken, startTime, endTime); - - // Get the date range from the current month - const now = new Date(); - const firstDay = new Date(now.getFullYear(), now.getMonth(), 1); - const lastDay = new Date(now.getFullYear(), now.getMonth() + 1, 0); - - // Fill in missing dates for each model's daily data - const filledModelData = data.map((modelData: any) => ({ - ...modelData, - daily_data: fillMissingDates(modelData.daily_data || [], firstDay, lastDay, ["api_requests", "total_tokens"]), - })); - - setGlobalActivityPerModel(filledModelData); - } catch (error) { - console.error("Error fetching global activity per model:", error); - } - }; - - useEffect(() => { - const initlizeUsageData = async () => { - if (!canViewGlobalSpend) { - return; - } - if (accessToken && token && userRole && userID) { - const proxy_settings: ProxySettings | undefined = await fetchProxySettings(); - if (proxy_settings) { - setProxySettings(proxy_settings); // saved in state so it can be used when rendering UI - if (proxy_settings?.DISABLE_EXPENSIVE_DB_QUERIES) { - return; // Don't run expensive UI queries - return out of initlizeUsageData at this point - } - } - - fetchOverallSpend(); - fetchProviderSpend(); - fetchTopKeys(); - fetchTopModels(); - fetchGlobalActivity(); - fetchGlobalActivityPerModel(); - - if (isAdminOrAdminViewer(userRole)) { - fetchTeamSpend(); - fetchTagNames(); - fetchTopTags(); - fetchTopEndUsers(); - } - } - }; - - initlizeUsageData(); - }, [canViewGlobalSpend, accessToken, token, userRole, userID, startTime, endTime]); - - if (!canViewGlobalSpend) { - return ( -
- - - Usage - - -

- Proxy-wide usage is only available to admin users. Your own usage is on the Usage page. -

- - -
- ); - } - - if (proxySettings?.DISABLE_EXPENSIVE_DB_QUERIES) { - return ( -
- - - Database Query Limit Reached - - -

- SpendLogs in DB has {proxySettings.NUM_SPEND_LOGS_ROWS} rows. -

- Please follow our guide to view usage when SpendLogs has more than 1M rows. -

-
- ); - } - - return ( -
- - - All Up - - {isAdminOrAdminViewer(userRole) && ( - <> - Team Based Usage - Customer Usage - Tag Based Usage - - )} - - - - - - Cost - Activity - - - -
-
-

- Project Spend {new Date().toLocaleString("default", { month: "long" })} 1 -{" "} - {new Date(new Date().getFullYear(), new Date().getMonth() + 1, 0).getDate()} -

- -
-
- - - Monthly Spend - - - - - -
-
- - - Top Virtual Keys - - - {}} /> - - -
-
- - - Top Models - - - `$${formatNumberWithCommas(value, 2)}`} - /> - - -
-
-
- - - Spend by Provider - - -
-
- `$${formatNumberWithCommas(value, 2)}`} - /> -
-
- - - - Provider - Spend - - - - {spendByProvider.map((provider) => ( - - {provider.provider} - - - - - ))} - -
-
-
- - -
-
- - - -
- - - All Up - - -
-
-

- API Requests {valueFormatterNumbers(globalActivity.sum_api_requests)} -

- -
-
-

- Tokens {valueFormatterNumbers(globalActivity.sum_total_tokens)} -

- -
-
- - - - {globalActivityPerModel.map((globalActivity, index) => ( - - - {globalActivity.model} - - -
-
-

- API Requests {valueFormatterNumbers(globalActivity.sum_api_requests)} -

- -
-
-

- Tokens {valueFormatterNumbers(globalActivity.sum_total_tokens)} -

- -
-
- - - ))} -
- - - - - -
-
- - - Total Spend Per Team - - - - - - - - Daily Spend Per Team - - - - - -
-
- - - -

- Customers of your LLM API calls. Tracked when a `user` param is passed in your LLM calls{" "} - - docs here - -

-
-
- { - setDateValue(value); - updateEndUserData(value.from, value.to, null); - }} - /> -
-
-

Select Key

- -
-
- - - -
- - - - Customer - Spend - Total Events - - - - - {topUsers?.map((user: any, index: number) => ( - - {user.end_user} - - - - {user.total_count} - - ))} - -
-
- - - - - -
-
- { - setDateValue(value); - updateTagSpendData(value.from, value.to); - }} - /> -
- -
- selectedTags.includes(option.value))} - onValueChange={(options: TagOption[]) => setSelectedTags(options.map((option) => option.value))} - isItemEqualToValue={(a: TagOption, b: TagOption) => a.value === b.value} - itemToStringLabel={(option: TagOption) => option.label} - > - }> - - {(options: TagOption[]) => - options.map((option) => ( - - {option.label} - - )) - } - - - - - No tags found - - {(option: TagOption) => ( - - {option.label} - - )} - - - -
-
-
-
- - - Spend Per Tag - - -

- Get Started by Tracking cost per tag{" "} - - here - -

- - - -
-
- - -
- ); -}; - -export default UsagePage; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/old-usage/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/old-usage/page.tsx deleted file mode 100644 index 138dd97e5e8..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/old-usage/page.tsx +++ /dev/null @@ -1,22 +0,0 @@ -"use client"; - -import Usage from "./_components/usage"; -import { DeprecationBanner } from "@/components/DeprecationBanner"; -import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; - -export default function OldUsagePage() { - const { accessToken, token, userRole, userId: userID, premiumUser } = useAuthorized(); - return ( - <> - - - - ); -} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx index 9d39e9c080c..f7ffd6d52cf 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx @@ -102,10 +102,6 @@ vi.mock("@/components/activity_metrics", () => ({ processActivityData: vi.fn(), })); -vi.mock("@/components/view_user_spend", () => ({ - default: () =>
View User Spend
, -})); - vi.mock("@/components/UsagePage/components/EntityUsage/TopKeyView", () => ({ default: () =>
Top Keys
, })); diff --git a/ui/litellm-dashboard/src/components/leftnav.test.tsx b/ui/litellm-dashboard/src/components/leftnav.test.tsx index 6cf05af1ed1..f3bff85e2d5 100644 --- a/ui/litellm-dashboard/src/components/leftnav.test.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.test.tsx @@ -444,30 +444,6 @@ describe("Sidebar (leftnav)", () => { }); expect(screen.queryByText("Prompts")).not.toBeInTheDocument(); }); - - it("should hide Old Usage from internal users while keeping other Experimental children", async () => { - mockUseAuthorized.mockReturnValue(internalAuth); - renderWithProviders(); - - act(() => { - fireEvent.click(screen.getByText("Experimental")); - }); - await waitFor(() => { - expect(screen.getByText("API Playground")).toBeInTheDocument(); - }); - expect(screen.queryByText("Old Usage")).not.toBeInTheDocument(); - }); - - it("should show Old Usage to admins", async () => { - renderWithProviders(); - - act(() => { - fireEvent.click(screen.getByText("Experimental")); - }); - await waitFor(() => { - expect(screen.getByText("Old Usage")).toBeInTheDocument(); - }); - }); }); // Workflow Runs, Memory and Guardrails Monitor render a shell and then 401 @@ -704,7 +680,6 @@ describe("Sidebar (leftnav)", () => { expectHref("Models + Endpoints", "/ui/models-and-endpoints"); expectHref("Usage", "/ui/usage"); expectHref("API Reference", "/ui/api-reference"); - expectHref("Old Usage", "/ui/old-usage"); }); it("never links a leaf to the legacy ?page= switch", () => { @@ -760,7 +735,6 @@ describe("getBreadcrumb", () => { it("resolves routes whose segment differs from the sidebar page id", () => { expect(getBreadcrumb("/ui/models-and-endpoints")).toEqual({ section: "AI Gateway", title: "Models + Endpoints" }); expect(getBreadcrumb("/ui/usage")).toEqual({ section: "Observability", title: "Usage" }); - expect(getBreadcrumb("/ui/old-usage")).toEqual({ section: "Developer Tools", title: "Old Usage" }); }); it("titles the dashboard root as Home", () => { diff --git a/ui/litellm-dashboard/src/components/leftnav.tsx b/ui/litellm-dashboard/src/components/leftnav.tsx index 6795cf1c7d1..2cf6f8219db 100644 --- a/ui/litellm-dashboard/src/components/leftnav.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.tsx @@ -414,14 +414,6 @@ const menuGroups: MenuGroup[] = [ icon: , roles: all_admin_roles, }, - { - key: "4", - page: "usage", - route: "old-usage", - label: "Old Usage", - icon: , - roles: rolesWithCapability("viewGlobalSpend"), - }, ], }, ], diff --git a/ui/litellm-dashboard/src/components/page_metadata.ts b/ui/litellm-dashboard/src/components/page_metadata.ts index 18f136c3b28..97220a0ef76 100644 --- a/ui/litellm-dashboard/src/components/page_metadata.ts +++ b/ui/litellm-dashboard/src/components/page_metadata.ts @@ -41,7 +41,6 @@ export const pageDescriptions: Record = { "tag-management": "Organize resources with tags", prompts: "Manage and version prompt templates", skills: "Browse and manage Claude Code skills", - usage: "View legacy usage dashboard", "router-settings": "Configure routing and load balancing settings", telemetry: "Choose which anonymous usage telemetry this proxy collects and see the last report", "logging-and-alerts": "Set up logging and alert configurations", diff --git a/ui/litellm-dashboard/src/components/view_user_spend.tsx b/ui/litellm-dashboard/src/components/view_user_spend.tsx deleted file mode 100644 index 655fc092673..00000000000 --- a/ui/litellm-dashboard/src/components/view_user_spend.tsx +++ /dev/null @@ -1,144 +0,0 @@ -"use client"; -import React, { useEffect, useState } from "react"; -import { modelAvailableCall } from "./networking"; -import { formatNumberWithCommas } from "@/utils/dataUtils"; -import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; - -// Define the props type -interface ViewUserSpendProps { - userSpend: number | null; - userMaxBudget: number | null; - selectedTeam: any | null; -} -const ViewUserSpend: React.FC = ({ userSpend, userMaxBudget, selectedTeam }) => { - const { accessToken, userRole, userId: userID } = useAuthorized(); - let [spend, setSpend] = useState(userSpend !== null ? userSpend : 0.0); - const [maxBudget, setMaxBudget] = useState( - selectedTeam ? Number(formatNumberWithCommas(selectedTeam.max_budget, 4)) : null, - ); - useEffect(() => { - if (selectedTeam) { - if (selectedTeam.team_alias === "Default Team") { - setMaxBudget(userMaxBudget); - } else { - let setMaxBudgetFlag = false; - if (selectedTeam.team_memberships) { - /** - * What 'team_memberships' looks like: - * "team_memberships": [ - * { - * "user_id": "2c315de3-e7ce-4269-b73e-b039a06187b1", - * "team_id": "test-team_515e6f42-ded2-4f0d-8919-0a1f43c5a45f", - * "budget_id": "0880769f-716a-4149-ab19-7f7651ad4db5", - * "litellm_budget_table": { - "soft_budget": null, - "max_budget": 20.0, - "max_parallel_requests": null, - "tpm_limit": null, - "rpm_limit": null, - "model_max_budget": null, - "budget_duration": null - } - */ - for (const member of selectedTeam.team_memberships) { - if ( - member.user_id === userID && - "max_budget" in member.litellm_budget_table && - member.litellm_budget_table.max_budget !== null - ) { - setMaxBudget(member.litellm_budget_table.max_budget); - setMaxBudgetFlag = true; - } - } - } - if (!setMaxBudgetFlag) { - setMaxBudget(selectedTeam.max_budget); - } - } - } else { - setMaxBudget(userMaxBudget); - } - }, [selectedTeam, userMaxBudget]); - const [userModels, setUserModels] = useState([]); - useEffect(() => { - const fetchData = async () => { - if (!accessToken || !userID || !userRole) { - return; - } - }; - const fetchUserModels = async () => { - try { - if (userID === null || userRole === null) { - return; - } - - if (accessToken !== null) { - const model_available = await modelAvailableCall(accessToken, userID, userRole); - let available_model_names = model_available["data"].map((element: { id: string }) => element.id); - setUserModels(available_model_names); - } - } catch (error) { - console.error("Error fetching user models:", error); - } - }; - - fetchUserModels(); - fetchData(); - }, [userRole, accessToken, userID]); - - useEffect(() => { - if (userSpend !== null) { - setSpend(userSpend); - } - }, [userSpend]); - - // logic to decide what models to display - let modelsToDisplay = []; - if (selectedTeam && selectedTeam.models) { - modelsToDisplay = selectedTeam.models; - } - - // check if "all-proxy-models" is in modelsToDisplay - if (modelsToDisplay && modelsToDisplay.includes("all-proxy-models")) { - modelsToDisplay = userModels; - } else if (modelsToDisplay && modelsToDisplay.includes("all-team-models")) { - modelsToDisplay = selectedTeam.models; - } else if (modelsToDisplay && modelsToDisplay.length === 0) { - modelsToDisplay = userModels; - } - - const displayMaxBudget = maxBudget !== null ? `$${formatNumberWithCommas(Number(maxBudget), 4)} limit` : "No limit"; - - const roundedSpend = spend !== undefined ? formatNumberWithCommas(spend, 4) : null; - - return ( -
-
-
-

Total Spend

-

${roundedSpend}

-
-
-

Max Budget

-

{displayMaxBudget}

-
-
- {/*
- - Team Models - - - {modelsToDisplay.map((model: string) => ( - - {model} - - ))} - - - -
*/} -
- ); -}; - -export default ViewUserSpend; diff --git a/ui/litellm-dashboard/tests/CreateKeyPage.expiredToken.test.tsx b/ui/litellm-dashboard/tests/CreateKeyPage.expiredToken.test.tsx index 58d7309c078..e5efd4c8e9f 100644 --- a/ui/litellm-dashboard/tests/CreateKeyPage.expiredToken.test.tsx +++ b/ui/litellm-dashboard/tests/CreateKeyPage.expiredToken.test.tsx @@ -127,7 +127,6 @@ vi.mock("@/components/new_usage", () => ({ default: stub("new-usage") })); vi.mock("@/components/api_ref", () => ({ default: stub("api-ref") })); vi.mock("@/components/chat_ui/ChatUI", () => ({ default: stub("chat-ui") })); vi.mock("@/components/leftnav", () => ({ default: stub("sidebar") })); -vi.mock("@/app/(dashboard)/old-usage/_components/usage", () => ({ default: stub("usage") })); vi.mock("@/components/cache_dashboard", () => ({ default: stub("cache-dashboard") })); vi.mock("@/app/(dashboard)/guardrails/_components", () => ({ default: stub("guardrails") })); vi.mock("@/components/prompts", () => ({ default: stub("prompts") })); From e752528ac8f9e1399e0e5bff98b7f404bffda1b4 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 11:53:06 -0700 Subject: [PATCH 24/74] feat(keys): allow assigning a project to an unassigned key on /key/update (#41700) * feat(keys): allow assigning a project to an unassigned key on /key/update Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore(keys): suppress LIT002 on project assignment error payloads Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore(keys): format project assignment error detail Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(keys): assert accepted project assignment flows into update payload Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore(keys): drop stale LIT002 suppressions removed by main Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(keys): mock writer_db team lookup used by check_db_only path Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(keys): write project assignment only while key row is still unassigned Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore(keys): satisfy type gate on guarded project assignment write Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(keys): default soft-budget update path to unguarded write Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(keys): run guarded project assignment in one writer transaction Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(keys): include object permission link in assignment guard Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(keys): return permission-linked payload without mutating the update dict Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore(keys): fix type errors and stale noqa after main merge Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore(keys): use Sequence in assignment guard model annotation Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore(keys): type update fields as a mapping in the key update path Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore(keys): place cast reason on the cast call Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(keys): honor explicit null max_budget when assigning a project * test(keys): assert explicit null max_budget flows into update payload * fix(keys): match NULL models column in guarded project assignment * fix(keys): authorize caller before project lookup on /key/update * fix(keys): drop explicit null object_permission before project assignment write --------- Co-authored-by: jesus Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../key_management_endpoints.py | 249 +++++++- .../management_endpoints/key_requests.py | 2 +- .../test_key_management_endpoints.py | 530 +++++++++++++++++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 +- 4 files changed, 759 insertions(+), 26 deletions(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 3021b21de51..61da5599886 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -162,7 +162,7 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( # noqa: F401 # _is_master_key, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export is_master_key, ) -from litellm.proxy.utils import ( # noqa: F401 # legacy module exports +from litellm.proxy.utils import ( PrismaClient, ProxyLogging, _hash_token_if_needed, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export @@ -280,6 +280,10 @@ class _BudgetRowSoftBudgetCreate(TypedDict): updated_by: ReadOnly[str] +class _KeyRowDumpable(Protocol): + def model_dump(self) -> Mapping[str, object]: ... + + class _KeyUpdateTx(Protocol): @property def litellm_verificationtoken(self) -> "TableActions[prisma_models.LiteLLM_VerificationToken]": ... @@ -287,6 +291,9 @@ class _KeyUpdateTx(Protocol): @property def litellm_budgettable(self) -> "TableActions[prisma_models.LiteLLM_BudgetTable]": ... + @property + def litellm_objectpermissiontable(self) -> "TableActions[prisma_models.LiteLLM_ObjectPermissionTable]": ... + class _ConfigTableActions(Protocol): """Config table surface this module needs; the shared repository seam exposes no ``update``.""" @@ -2072,6 +2079,56 @@ async def _check_project_key_limits( ) +async def _validate_project_assignment( + data: UpdateKeyRequest, + existing_key_row: LiteLLM_VerificationToken, + prisma_client: PrismaClient, + user_api_key_cache: UserApiKeyCache, +) -> None: + """Validate assigning a project to a key that does not have one yet.""" + if data.project_id is None or data.project_id == existing_key_row.project_id: + return + if existing_key_row.project_id is not None: + raise HTTPException( + status_code=400, detail="Project reassignment is not supported. Use null to detach the key." + ) + project_obj: Final = await get_project_object( + project_id=data.project_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + ) + if project_obj is None: + raise HTTPException( + status_code=404, + detail={"error": f"Project not found, project_id={data.project_id}"}, + ) + team: Final = data.team_id if "team_id" in data.model_fields_set else existing_key_row.team_id + if team is None or team != project_obj.team_id: + raise HTTPException( + status_code=400, + detail={ + "error": f"Project {data.project_id} belongs to team {project_obj.team_id}. Keys can only be assigned to a project owned by their own team (key team: {team})" + }, + ) + await _check_project_key_limits( + project_id=data.project_id, + data=data.model_copy( + update={ + "models": ( + data.models + if "models" in data.model_fields_set + else cast( # cast-ok: LiteLLM_VerificationToken.models is a bare list + "list[str] | None", existing_key_row.models + ) + ), + "max_budget": data.max_budget if "max_budget" in data.model_fields_set else existing_key_row.max_budget, + } + ), + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + ) + + async def _check_key_project_team( project_id: str, key_team_id: str | None, @@ -2771,6 +2828,115 @@ async def _apply_soft_budget_update( return remaining +async def _apply_object_permission_update( + data_json: Mapping[str, object], + existing_object_permission_id: str | None, + prisma_client: PrismaClient, + table: "TableActions[prisma_models.LiteLLM_ObjectPermissionTable]", +) -> Mapping[str, object]: + """Upsert the requested object permission row inside the caller's writer transaction.""" + new_object_permission: Final = data_json.get("object_permission") + if new_object_permission is None: + return MappingProxyType({k: v for k, v in data_json.items() if k != "object_permission"}) + loaded: Final[object] = ( + json.loads(new_object_permission) if isinstance(new_object_permission, str) else new_object_permission + ) + grants: Final[Mapping[str, object]] = ( + cast( # cast-ok: object_permission payloads are grant-name to grant-list mappings + "Mapping[str, object]", loaded + ) + if isinstance(loaded, dict) + else MappingProxyType({}) + ) + upsert: Final = await prepare_object_permission_upsert( + new_object_permission=grants, + existing_object_permission_id=existing_object_permission_id, + prisma_client=prisma_client, + ) + row: Final = await table.upsert( + where={"object_permission_id": upsert.object_permission_id}, + data={"create": upsert.record, "update": upsert.record}, + ) + return MappingProxyType( + { + **{k: v for k, v in data_json.items() if k != "object_permission"}, + "object_permission_id": row.object_permission_id, + } + ) + + +async def _write_guarded_project_assignment( + table: "TableActions[_KeyRowDumpable]", + hashed_token: str, + existing_key_row: LiteLLM_VerificationToken, + data: Mapping[str, object], +) -> "_KeyRowDumpable | None": + validated_models: Final[Sequence[str]] = ( + cast( # cast-ok: LiteLLM_VerificationToken.models is a bare list + "Sequence[str] | None", existing_key_row.models + ) + or [] + ) + models_guard: Final[Mapping[str, object]] = ( + {"OR": [{"models": {"equals": list[str]()}}, {"models": {"equals": None}}]} + if not validated_models + else {"models": {"equals": validated_models}} + ) + updated_count: Final = await table.update_many( + where={ + "token": hashed_token, + "project_id": None, + "team_id": existing_key_row.team_id, + **models_guard, + "max_budget": existing_key_row.max_budget, + "object_permission_id": existing_key_row.object_permission_id, + }, + data=data, + ) + if updated_count == 0: + raise HTTPException( + status_code=400, + detail="Project assignment failed because the key was updated concurrently. Retry the request.", + ) + return await table.find_unique( + where={"token": hashed_token}, + include={"object_permission": True}, + ) + + +async def _update_key_row_assigning_project( + prisma_client: PrismaClient, + key: str, + non_default_values: Mapping[str, object], + existing_key_row: LiteLLM_VerificationToken, +) -> _KeyUpdateResult: + """Assign a project in one writer transaction, guarded on the key's validated state.""" + hashed_token: Final = _hash_token_if_needed(key) + async with prisma_client.tx() as tx_ctx: + tx: Final[_KeyUpdateTx] = cast( # cast-ok: the transaction object exposes the same table actions + "_KeyUpdateTx", tx_ctx + ) + update_values: Final[Mapping[str, object]] = await _apply_object_permission_update( + data_json=non_default_values, + existing_object_permission_id=existing_key_row.object_permission_id, + prisma_client=prisma_client, + table=tx.litellm_objectpermissiontable, + ) + updated_row: Final = await _write_guarded_project_assignment( + table=tx.litellm_verificationtoken, + hashed_token=hashed_token, + existing_key_row=existing_key_row, + data=with_settings_updated_at( + prisma_client.jsonify_object(MappingProxyType({**update_values, "token": hashed_token})) + ), + ) + updated_data: Final[Mapping[str, object]] = ( + updated_row.model_dump() if updated_row is not None else MappingProxyType({}) + ) + result: Final[_KeyUpdateResult] = {"token": hashed_token, "data": updated_data} + return result + + async def _update_key_row_with_soft_budget( prisma_client: PrismaClient, key: str, @@ -2778,25 +2944,48 @@ async def _update_key_row_with_soft_budget( non_default_values: Mapping[str, object], existing_key_row: LiteLLM_VerificationToken, changed_by: str, + expect_unassigned_project: bool = False, ) -> _KeyUpdateResult: hashed_token: Final = hash_token_if_needed(key) key_where: Final[_KeyRowWhere] = {"token": hashed_token} - tx: _KeyUpdateTx - async with prisma_client.tx() as tx: + async with prisma_client.tx() as tx_ctx: + tx: Final[_KeyUpdateTx] = cast( # cast-ok: the transaction object exposes the same table actions + "_KeyUpdateTx", tx_ctx + ) + update_input: Final[Mapping[str, object]] = ( + await _apply_object_permission_update( + data_json=non_default_values, + existing_object_permission_id=existing_key_row.object_permission_id, + prisma_client=prisma_client, + table=tx.litellm_objectpermissiontable, + ) + if expect_unassigned_project + else non_default_values + ) update_values: Final = await _apply_soft_budget_update( data=data, - non_default_values=non_default_values, + non_default_values=update_input, db=tx, existing_key_row=existing_key_row, changed_by=changed_by, ) include_object_permission: Final[prisma.types.LiteLLM_VerificationTokenInclude] = {"object_permission": True} - updated_row: Final = await tx.litellm_verificationtoken.update( - where=key_where, - data=with_settings_updated_at( - prisma_client.jsonify_object(MappingProxyType({**update_values, "token": hashed_token})) - ), - include=include_object_permission, + update_payload: Final = with_settings_updated_at( + prisma_client.jsonify_object(MappingProxyType({**update_values, "token": hashed_token})) + ) + updated_row: Final = ( + await _write_guarded_project_assignment( + table=tx.litellm_verificationtoken, + hashed_token=hashed_token, + existing_key_row=existing_key_row, + data=update_payload, + ) + if expect_unassigned_project + else await tx.litellm_verificationtoken.update( + where=key_where, + data=update_payload, + include=include_object_permission, + ) ) updated_data: Final[Mapping[str, object]] = ( updated_row.model_dump() if updated_row is not None else MappingProxyType({}) @@ -3563,10 +3752,6 @@ async def _validate_update_key_data( user_api_key_dict=user_api_key_dict, ) - if data.project_id is not None and data.project_id != existing_key_row.project_id: - raise HTTPException( - status_code=400, detail="Project reassignment is not supported. Use null to detach the key." - ) is_project_change: Final = "project_id" in data.model_fields_set and data.project_id != existing_key_row.project_id acting_as_team_admin: Final = await _acting_as_team_admin_for_key_update( @@ -3698,6 +3883,12 @@ async def _validate_update_key_data( user_api_key_cache=user_api_key_cache, route=("/key/update (max_budget/spend)" if _is_budget_change else "/key/update"), ) + await _validate_project_assignment( + data=data, + existing_key_row=existing_key_row, + prisma_client=checked_prisma_client, + user_api_key_cache=user_api_key_cache, + ) check_denied_passthrough_routes_caller_permission( data, user_api_key_dict, @@ -3847,7 +4038,7 @@ async def update_key_fn( - user_id: Optional[str] - User ID associated with key - team_id: Optional[str] - Team ID associated with key - agent_id: Optional[str] - The agent id associated with the key. - - project_id: Optional[str] - Omit to retain the project, or send null to detach. A different project ID is rejected. + - project_id: Optional[str] - Omit to retain the project, send null to detach, or send a project id to assign an unassigned key to a project on the key's team. Moving a key between projects is rejected. - organization_id: Optional[str] - The organization id of the key. - budget_id: Optional[str] - The budget id associated with the key. Created by calling `/budget/new`. - end_user_budget_id: Optional[str] - Proxy admin only. Budget id applied to end users first seen through this key that carry no budget of their own. Omit to keep the current value, pass an empty string to clear it. @@ -3995,23 +4186,31 @@ async def update_key_fn( if prisma_client is None: raise Exception("Not connected to DB!") + is_project_assignment: Final = data.project_id is not None and existing_key_row.project_id is None + update_fields: Final[Mapping[str, object]] = cast( # cast-ok: prepare_key_update_data returns a bare dict + "Mapping[str, object]", non_default_values + ) object_permission_upsert: Final = await _prepare_key_update_object_permission( - object_permission_data=non_default_values.get("object_permission"), + object_permission_data=update_fields.get("object_permission"), existing_key_row=existing_key_row, prisma_client=prisma_client, ) key_update_values: Final = MappingProxyType( - {field: value for field, value in non_default_values.items() if field != "object_permission"} + {field: value for field, value in update_fields.items() if field != "object_permission"} ) await _check_key_project_team_on_mutation( data=data, existing_key_row=existing_key_row, prisma_client=prisma_client, ) - update_values: Final = await _write_prepared_key_update_object_permission( - data_json=key_update_values, - upsert=object_permission_upsert, - prisma_client=prisma_client, + update_values: Final = ( + update_fields + if is_project_assignment + else await _write_prepared_key_update_object_permission( + data_json=key_update_values, + upsert=object_permission_upsert, + prisma_client=prisma_client, + ) ) changed_by: Final = user_api_key_dict.user_id or litellm_proxy_admin_name response: Final = ( @@ -4022,8 +4221,16 @@ async def update_key_fn( non_default_values=update_values, existing_key_row=existing_key_row, changed_by=changed_by, + expect_unassigned_project=is_project_assignment, ) if "soft_budget" in data.model_fields_set + else await _update_key_row_assigning_project( + prisma_client=prisma_client, + key=key, + non_default_values=update_fields, + existing_key_row=existing_key_row, + ) + if is_project_assignment else await _update_key_row( prisma_client=prisma_client, key=key, diff --git a/litellm/types/proxy/management_endpoints/key_requests.py b/litellm/types/proxy/management_endpoints/key_requests.py index b49cb63c8e0..f42a412cf2d 100644 --- a/litellm/types/proxy/management_endpoints/key_requests.py +++ b/litellm/types/proxy/management_endpoints/key_requests.py @@ -118,7 +118,7 @@ class UpdateKeyRequest(KeyRequestBase): project_id: str | None = Field( default=None, - description="Omit to retain the project, or send null to detach. Assigning a different project is not supported.", + description="Omit to retain the project, send null to detach, or send a project id to assign an unassigned key to a project on the key's team. Moving a key between projects is not supported.", ) @model_validator(mode="before") diff --git a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py index c26a4bf19b8..0198f582be7 100644 --- a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py @@ -67,6 +67,7 @@ from litellm.proxy.management_endpoints.key_management_endpoints import ( _enforce_upperbound_key_params, _execute_virtual_key_regeneration, _get_and_validate_existing_key, + _hash_token_if_needed, _is_team_key, _key_generation_required_param_check, _member_key_request, @@ -77,6 +78,7 @@ from litellm.proxy.management_endpoints.key_management_endpoints import ( _team_admin_requests_member_key, _team_key_generation_check, _transform_verification_tokens_to_deleted_records, + _update_key_row_assigning_project, _validate_end_user_budget_id_change, _validate_max_budget, _validate_reset_spend_value, @@ -20975,12 +20977,20 @@ async def test_regenerate_key_repoints_live_membership_not_the_key_row_it_read( async def _cache_with_project( - project_id: str, project_models: list[str], team_id: str | None = "team-lit-5823" + project_id: str, project_models: list[str], team_id: str | None = "team-lit-5823", + project_max_budget: float | None = None, ) -> UserApiKeyCache: user_api_key_cache = UserApiKeyCache() await user_api_key_cache.async_set_cache( key=project_cache_key(project_id), - value=LiteLLM_ProjectTableCachedObj(project_id=project_id, team_id=team_id, models=project_models), + value=LiteLLM_ProjectTableCachedObj( + project_id=project_id, + team_id=team_id, + models=project_models, + litellm_budget_table=( + LiteLLM_BudgetTable(max_budget=project_max_budget) if project_max_budget is not None else None + ), + ), model_type=LiteLLM_ProjectTableCachedObj, ) return user_api_key_cache @@ -21413,6 +21423,487 @@ async def test_project_detachment_uses_effective_project_for_validation_on_unown assert expected in str(exc.value.detail) +def _project_assignment_prisma(stored_row: Mapping[str, object] | None = None) -> MagicMock: + database = MagicMock() + team_row: Final = LiteLLM_TeamTable(team_id="team-lit-5823", members=[]) + database.db.litellm_teamtable.find_unique = AsyncMock(return_value=team_row) + database.writer_db.litellm_teamtable.find_unique = AsyncMock(return_value=team_row) + if stored_row is None: + return database + + database.jsonify_object = lambda data: dict(data) + tx: Final = database.tx.return_value.__aenter__.return_value + + def _where_matches(where: Mapping[str, object], row: Mapping[str, object]) -> bool: + for key, clause in where.items(): + if key == "OR": + if not any(_where_matches(part, row) for part in clause): + return False + elif key == "models": + equals = clause["equals"] + if equals is None: + if row[key] is not None: + return False + elif row[key] is None or list(row[key]) != list(equals): + return False + elif row.get(key) != clause: + return False + return True + + async def _update_many(*, where, data): + if "object_permission" in data: + raise AssertionError("object_permission must not reach update_many") + return 1 if _where_matches(where, stored_row) else 0 + + tx.litellm_verificationtoken.update_many = AsyncMock(side_effect=_update_many) + written = MagicMock() + written.model_dump = MagicMock(return_value={**stored_row, "project_id": "project-orbit"}) + tx.litellm_verificationtoken.find_unique = AsyncMock(return_value=written) + return database + + +@pytest.mark.asyncio +async def test_project_assignment_to_unassigned_key_on_same_team(): + existing: Final = LiteLLM_VerificationToken( + token="project-assign-token", project_id=None, team_id="team-lit-5823", models=["model-orbit"] + ) + cache: Final = await _cache_with_project("project-orbit", ["model-orbit"]) + data: Final = UpdateKeyRequest(key=existing.token, project_id="project-orbit") + + await _validate_update_key_data( + data, existing, + UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), + None, False, _project_assignment_prisma(), cache, + ) + + update: Final = await prepare_key_update_data(data=data, existing_key_row=existing) + assert update["project_id"] == "project-orbit" + + +@pytest.mark.asyncio +async def test_project_assignment_rejects_project_on_another_team(): + existing: Final = LiteLLM_VerificationToken( + token="project-assign-token", project_id=None, team_id="team-other", models=["model-orbit"] + ) + cache: Final = await _cache_with_project("project-orbit", ["model-orbit"]) + + with pytest.raises(HTTPException) as exc: + await _validate_update_key_data( + UpdateKeyRequest(key=existing.token, project_id="project-orbit"), existing, + UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), + None, False, _project_assignment_prisma(), cache, + ) + assert exc.value.status_code == 400 + assert "team" in str(exc.value.detail) + + +@pytest.mark.asyncio +async def test_project_assignment_rejects_key_with_no_team(): + existing: Final = LiteLLM_VerificationToken( + token="project-assign-token", project_id=None, team_id=None, models=["model-orbit"] + ) + cache: Final = await _cache_with_project("project-orbit", ["model-orbit"]) + + with pytest.raises(HTTPException) as exc: + await _validate_update_key_data( + UpdateKeyRequest(key=existing.token, project_id="project-orbit"), existing, + UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), + None, False, _project_assignment_prisma(), cache, + ) + assert exc.value.status_code == 400 + assert "team" in str(exc.value.detail) + + +@pytest.mark.asyncio +async def test_project_assignment_rejects_nonexistent_project(): + existing: Final = LiteLLM_VerificationToken( + token="project-assign-token", project_id=None, team_id="team-lit-5823", models=["model-orbit"] + ) + database: Final = _project_assignment_prisma() + database.db.litellm_projecttable.find_unique = AsyncMock(return_value=None) + + with pytest.raises(HTTPException) as exc: + await _validate_update_key_data( + UpdateKeyRequest(key=existing.token, project_id="project-ghost"), existing, + UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), + None, False, database, UserApiKeyCache(), + ) + assert exc.value.status_code == 404 + assert "Project not found" in str(exc.value.detail) + + +@pytest.mark.asyncio +async def test_project_assignment_validates_existing_models_against_project(): + existing: Final = LiteLLM_VerificationToken( + token="project-assign-token", project_id=None, team_id="team-lit-5823", models=["model-elsewhere"] + ) + cache: Final = await _cache_with_project("project-orbit", ["model-orbit"]) + + with pytest.raises(HTTPException) as exc: + await _validate_update_key_data( + UpdateKeyRequest(key=existing.token, project_id="project-orbit"), existing, + UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), + None, False, _project_assignment_prisma(), cache, + ) + assert exc.value.status_code == 400 + assert "not in project's allowed models" in str(exc.value.detail) + + +@pytest.mark.asyncio +async def test_project_assignment_with_explicit_null_max_budget_ignores_stored_budget(): + existing: Final = LiteLLM_VerificationToken( + token="project-assign-token", project_id=None, team_id="team-lit-5823", models=[], max_budget=100.0 + ) + cache: Final = await _cache_with_project("project-orbit", [], project_max_budget=10.0) + + request: Final = UpdateKeyRequest(key=existing.token, project_id="project-orbit", max_budget=None) + await _validate_update_key_data( + request, existing, + UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), + None, False, _project_assignment_prisma(), cache, + ) + + result: Final = await prepare_key_update_data(data=request, existing_key_row=existing) + assert result["max_budget"] is None + assert result["project_id"] == "project-orbit" + + +@pytest.mark.asyncio +async def test_project_assignment_without_max_budget_uses_stored_budget(): + existing: Final = LiteLLM_VerificationToken( + token="project-assign-token", project_id=None, team_id="team-lit-5823", models=[], max_budget=100.0 + ) + cache: Final = await _cache_with_project("project-orbit", [], project_max_budget=10.0) + + with pytest.raises(HTTPException) as exc: + await _validate_update_key_data( + UpdateKeyRequest(key=existing.token, project_id="project-orbit"), existing, + UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), + None, False, _project_assignment_prisma(), cache, + ) + assert exc.value.status_code == 400 + assert "exceeds project's max_budget" in str(exc.value.detail) + + +@pytest.mark.asyncio +async def test_prepare_key_update_data_includes_new_project_assignment(): + existing: Final = LiteLLM_VerificationToken(token="project-assign-token", project_id=None) + + result: Final = await prepare_key_update_data( + data=UpdateKeyRequest(key=existing.token, project_id="project-orbit"), existing_key_row=existing, + ) + + assert result["project_id"] == "project-orbit" + + +@pytest.mark.asyncio +async def test_project_assignment_write_requires_row_still_unassigned(): + database = MagicMock() + database.jsonify_object = lambda data: dict(data) + tx: Final = database.tx.return_value.__aenter__.return_value + tx.litellm_verificationtoken.update_many = AsyncMock(return_value=1) + row = MagicMock() + row.model_dump = MagicMock(return_value={"project_id": "project-orbit"}) + tx.litellm_verificationtoken.find_unique = AsyncMock(return_value=row) + existing: Final = LiteLLM_VerificationToken( + token="project-assign-token", project_id=None, team_id="team-lit-5823", models=["model-orbit"] + ) + + result: Final = await _update_key_row_assigning_project( + prisma_client=database, + key="sk-assign", + non_default_values={"project_id": "project-orbit"}, + existing_key_row=existing, + ) + + update_where: Final = tx.litellm_verificationtoken.update_many.await_args.kwargs["where"] + assert update_where["token"] == result["token"] + assert result["data"] == {"project_id": "project-orbit"} + + +@pytest.mark.asyncio +async def test_project_assignment_write_rejects_when_row_gained_project(): + database = MagicMock() + database.jsonify_object = lambda data: dict(data) + tx: Final = database.tx.return_value.__aenter__.return_value + tx.litellm_verificationtoken.update_many = AsyncMock(return_value=0) + existing: Final = LiteLLM_VerificationToken( + token="project-assign-token", project_id=None, team_id="team-lit-5823", models=["model-orbit"] + ) + + with pytest.raises(HTTPException) as exc: + await _update_key_row_assigning_project( + prisma_client=database, + key="sk-assign", + non_default_values={"project_id": "project-orbit"}, + existing_key_row=existing, + ) + + assert exc.value.status_code == 400 + assert "concurrently" in str(exc.value.detail) + tx.litellm_verificationtoken.find_unique.assert_not_called() + + +@pytest.mark.asyncio +async def test_project_assignment_write_upserts_object_permission_in_same_tx(): + database = MagicMock() + database.jsonify_object = lambda data: dict(data) + database.db.litellm_objectpermissiontable.find_unique = AsyncMock(return_value=None) + tx: Final = database.tx.return_value.__aenter__.return_value + tx.litellm_verificationtoken.update_many = AsyncMock(return_value=1) + permission_row = MagicMock() + permission_row.object_permission_id = "perm-lit-01" + tx.litellm_objectpermissiontable.upsert = AsyncMock(return_value=permission_row) + row = MagicMock() + row.model_dump = MagicMock(return_value={"project_id": "project-orbit"}) + tx.litellm_verificationtoken.find_unique = AsyncMock(return_value=row) + existing: Final = LiteLLM_VerificationToken( + token="project-assign-token", project_id=None, team_id="team-lit-5823" + ) + + result: Final = await _update_key_row_assigning_project( + prisma_client=database, + key="sk-assign", + non_default_values={"project_id": "project-orbit", "object_permission": {"agents": ["agent-a"]}}, + existing_key_row=existing, + ) + + tx.litellm_objectpermissiontable.upsert.assert_awaited_once() + update_data: Final = tx.litellm_verificationtoken.update_many.await_args.kwargs["data"] + assert update_data["object_permission_id"] == "perm-lit-01" + assert "object_permission" not in update_data + assert result["data"] == {"project_id": "project-orbit"} + + +@pytest.mark.asyncio +async def test_project_assignment_write_matches_null_models_column(): + stored: Final = { + "token": _hash_token_if_needed("sk-assign"), + "project_id": None, + "team_id": "team-lit-5823", + "models": None, + "max_budget": None, + "object_permission_id": None, + } + database: Final = _project_assignment_prisma(stored_row=stored) + existing: Final = LiteLLM_VerificationToken( + token=stored["token"], project_id=None, team_id="team-lit-5823", models=[] + ) + + result: Final = await _update_key_row_assigning_project( + prisma_client=database, + key="sk-assign", + non_default_values={"project_id": "project-orbit"}, + existing_key_row=existing, + ) + + assert result["data"]["project_id"] == "project-orbit" + + +@pytest.mark.asyncio +async def test_project_assignment_write_strips_explicit_null_object_permission(): + stored: Final = { + "token": _hash_token_if_needed("sk-assign"), + "project_id": None, + "team_id": "team-lit-5823", + "models": None, + "max_budget": None, + "object_permission_id": None, + } + database: Final = _project_assignment_prisma(stored_row=stored) + existing: Final = LiteLLM_VerificationToken( + token=stored["token"], project_id=None, team_id="team-lit-5823", models=[] + ) + update_values: Final = await prepare_key_update_data( + data=UpdateKeyRequest(key="sk-assign", project_id="project-orbit", object_permission=None), + existing_key_row=existing, + ) + assert "object_permission" in update_values + + result: Final = await _update_key_row_assigning_project( + prisma_client=database, + key="sk-assign", + non_default_values=update_values, + existing_key_row=existing, + ) + + assert result["data"]["project_id"] == "project-orbit" + + +@pytest.mark.asyncio +async def test_project_assignment_write_rejects_when_max_budget_changed(): + stored: Final = { + "token": _hash_token_if_needed("sk-assign"), + "project_id": None, + "team_id": "team-lit-5823", + "models": ["model-orbit"], + "max_budget": 50.0, + "object_permission_id": None, + } + database: Final = _project_assignment_prisma(stored_row=stored) + existing: Final = LiteLLM_VerificationToken( + token=stored["token"], project_id=None, team_id="team-lit-5823", + models=["model-orbit"], max_budget=100.0, + ) + + with pytest.raises(HTTPException) as exc: + await _update_key_row_assigning_project( + prisma_client=database, + key="sk-assign", + non_default_values={"project_id": "project-orbit"}, + existing_key_row=existing, + ) + + assert exc.value.status_code == 400 + assert "concurrently" in str(exc.value.detail) + + +@pytest.mark.asyncio +async def test_project_assignment_write_rejects_when_object_permission_changed(): + stored: Final = { + "token": _hash_token_if_needed("sk-assign"), + "project_id": None, + "team_id": "team-lit-5823", + "models": ["model-orbit"], + "max_budget": None, + "object_permission_id": "perm-new", + } + database: Final = _project_assignment_prisma(stored_row=stored) + existing: Final = LiteLLM_VerificationToken( + token=stored["token"], project_id=None, team_id="team-lit-5823", models=["model-orbit"] + ) + + with pytest.raises(HTTPException) as exc: + await _update_key_row_assigning_project( + prisma_client=database, + key="sk-assign", + non_default_values={"project_id": "project-orbit"}, + existing_key_row=existing, + ) + + assert exc.value.status_code == 400 + assert "concurrently" in str(exc.value.detail) + + +@pytest.mark.asyncio +async def test_project_assignment_write_rejects_when_team_changed(): + stored: Final = { + "token": _hash_token_if_needed("sk-assign"), + "project_id": None, + "team_id": "team-moved", + "models": ["model-orbit"], + "max_budget": None, + "object_permission_id": None, + } + database: Final = _project_assignment_prisma(stored_row=stored) + existing: Final = LiteLLM_VerificationToken( + token=stored["token"], project_id=None, team_id="team-lit-5823", models=["model-orbit"] + ) + + with pytest.raises(HTTPException) as exc: + await _update_key_row_assigning_project( + prisma_client=database, + key="sk-assign", + non_default_values={"project_id": "project-orbit"}, + existing_key_row=existing, + ) + + assert exc.value.status_code == 400 + assert "concurrently" in str(exc.value.detail) + + +def _assignment_project_update_prisma(stored_row: Mapping[str, object]) -> MagicMock: + database: Final = _project_assignment_prisma(stored_row=stored_row) + project_row: Final = LiteLLM_ProjectTable(project_id="project-orbit", team_id="team-lit-5823") + database.db.litellm_projecttable.find_unique = AsyncMock(return_value=project_row) + database.writer_db.litellm_projecttable.find_unique = AsyncMock(return_value=project_row) + return database + + +def _setup_assignment_update_mocks(monkeypatch: pytest.MonkeyPatch, database: MagicMock, cache: UserApiKeyCache) -> None: + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", database) + monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", cache) + proxy_logging_obj: Final = MagicMock() + proxy_logging_obj.service_logging_obj.async_service_success_hook = AsyncMock() + proxy_logging_obj.service_logging_obj.async_service_failure_hook = AsyncMock() + monkeypatch.setattr("litellm.proxy.proxy_server.proxy_logging_obj", proxy_logging_obj) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None) + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) + monkeypatch.setattr("litellm.store_audit_logs", False) + + +@pytest.mark.asyncio +async def test_update_key_fn_assigns_project_through_guarded_write(monkeypatch: pytest.MonkeyPatch): + stored: Final = { + "token": _hash_token_if_needed("sk-assign"), + "project_id": None, + "team_id": "team-lit-5823", + "models": ["model-orbit"], + "max_budget": None, + "object_permission_id": None, + } + existing: Final = LiteLLM_VerificationToken( + token=stored["token"], project_id=None, team_id="team-lit-5823", models=["model-orbit"] + ) + database: Final = _assignment_project_update_prisma(stored) + database.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=existing) + cache: Final = await _cache_with_project("project-orbit", ["model-orbit"]) + _setup_assignment_update_mocks(monkeypatch, database, cache) + + result: Final = await update_key_fn( + request=MagicMock(), + data=UpdateKeyRequest(key="sk-assign", project_id="project-orbit"), + user_api_key_dict=UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-admin", user_id="admin-user" + ), + litellm_changed_by=None, + ) + + tx: Final = database.tx.return_value.__aenter__.return_value + tx.litellm_verificationtoken.update_many.assert_awaited_once() + update_where: Final = tx.litellm_verificationtoken.update_many.await_args.kwargs["where"] + assert update_where["project_id"] is None + tx.litellm_verificationtoken.update.assert_not_called() + assert result["project_id"] == "project-orbit" + + +@pytest.mark.asyncio +async def test_update_key_fn_soft_budget_runs_guarded_assignment(monkeypatch: pytest.MonkeyPatch): + stored: Final = { + "token": _hash_token_if_needed("sk-assign"), + "project_id": None, + "team_id": "team-lit-5823", + "models": ["model-orbit"], + "max_budget": None, + "object_permission_id": None, + } + existing: Final = LiteLLM_VerificationToken( + token=stored["token"], project_id=None, team_id="team-lit-5823", models=["model-orbit"] + ) + database: Final = _assignment_project_update_prisma(stored) + database.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=existing) + tx: Final = database.tx.return_value.__aenter__.return_value + tx.litellm_budgettable.create = AsyncMock(return_value=SimpleNamespace(budget_id="budget-new")) + cache: Final = await _cache_with_project("project-orbit", ["model-orbit"]) + _setup_assignment_update_mocks(monkeypatch, database, cache) + + result: Final = await update_key_fn( + request=MagicMock(), + data=UpdateKeyRequest(key="sk-assign", project_id="project-orbit", soft_budget=5.0), + user_api_key_dict=UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-admin", user_id="admin-user" + ), + litellm_changed_by=None, + ) + + tx.litellm_budgettable.create.assert_awaited_once() + tx.litellm_verificationtoken.update_many.assert_awaited_once() + update_where: Final = tx.litellm_verificationtoken.update_many.await_args.kwargs["where"] + assert update_where["project_id"] is None + tx.litellm_verificationtoken.update.assert_not_called() + assert result["project_id"] == "project-orbit" + + @pytest.mark.asyncio async def test_key_creator_cannot_detach_project_without_admin_access(): existing: Final = LiteLLM_VerificationToken( @@ -21430,6 +21921,41 @@ async def test_key_creator_cannot_detach_project_without_admin_access(): assert "Only proxy admins, team admins, or org admins" in str(exc.value.detail) +@pytest.mark.asyncio +async def test_unauthorized_caller_gets_403_not_project_lookup_result(): + existing: Final = LiteLLM_VerificationToken( + token="project-assign-token", project_id=None, team_id="team-lit-5823", + user_id="user-orbit", created_by="user-orbit", + ) + database: Final = _project_assignment_prisma() + with pytest.raises(HTTPException) as exc: + await _validate_update_key_data( + UpdateKeyRequest(key=existing.token, project_id="project-missing"), existing, + UserAPIKeyAuth(user_id="user-other", user_role=LitellmUserRoles.INTERNAL_USER), + None, False, database, UserApiKeyCache(), + ) + assert exc.value.status_code == 403 + assert "Project not found" not in str(exc.value.detail) + + +@pytest.mark.asyncio +async def test_unauthorized_caller_gets_403_not_project_team_leak(): + existing: Final = LiteLLM_VerificationToken( + token="project-assign-token", project_id=None, team_id="team-lit-5823", + user_id="user-orbit", created_by="user-orbit", + ) + cache: Final = await _cache_with_project("project-orbit", [], team_id="team-other") + database: Final = _project_assignment_prisma() + with pytest.raises(HTTPException) as exc: + await _validate_update_key_data( + UpdateKeyRequest(key=existing.token, project_id="project-orbit"), existing, + UserAPIKeyAuth(user_id="user-other", user_role=LitellmUserRoles.INTERNAL_USER), + None, False, database, cache, + ) + assert exc.value.status_code == 403 + assert "belongs to team" not in str(exc.value.detail) + + _OWNED_PROJECT: Final = "proj-owned-1" _OWNERSHIP_PROJECT_TEAM: Final = "ownership-project-team" _OWNERSHIP_KEY_TEAM: Final = "ownership-key-team" diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index aafa732276b..96c7c34621b 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -8953,7 +8953,7 @@ export interface paths { * - user_id: Optional[str] - User ID associated with key * - team_id: Optional[str] - Team ID associated with key * - agent_id: Optional[str] - The agent id associated with the key. - * - project_id: Optional[str] - Omit to retain the project, or send null to detach. A different project ID is rejected. + * - project_id: Optional[str] - Omit to retain the project, send null to detach, or send a project id to assign an unassigned key to a project on the key's team. Moving a key between projects is rejected. * - organization_id: Optional[str] - The organization id of the key. * - budget_id: Optional[str] - The budget id associated with the key. Created by calling `/budget/new`. * - end_user_budget_id: Optional[str] - Proxy admin only. Budget id applied to end users first seen through this key that carry no budget of their own. Omit to keep the current value, pass an empty string to clear it. @@ -48704,7 +48704,7 @@ export interface components { policies?: string[] | null; /** * Project Id - * @description Omit to retain the project, or send null to detach. Assigning a different project is not supported. + * @description Omit to retain the project, send null to detach, or send a project id to assign an unassigned key to a project on the key's team. Moving a key between projects is not supported. */ project_id?: string | null; /** Prompts */ From b20acb48699e91fbe4d64a8878eaa0c92078a895 Mon Sep 17 00:00:00 2001 From: Maxime Boudier <113669264+MaximeBOUDIER@users.noreply.github.com> Date: Fri, 30 Oct 2026 20:10:13 +0100 Subject: [PATCH 25/74] fix(proxy)!: persist sent model_info fields on POST /model/update (#46902) * fix(proxy): persist sent model_info fields on POST /model/update POST /model/update rebuilt litellm_params from the request but dropped the request's model_info, so changing base_model (or mode, tier, etc.) on an existing DB model was silently ignored. Clients that only use this endpoint, like the Terraform provider, could never fix a deployment's base_model without deleting and recreating it. Persist the model_info fields the caller actually sent, on top of the stored ones. ModelInfo defaults never overwrite stored values, server-owned, team, PTU and pricing fields stay on their dedicated write paths, and values echoed from a /model/info response are not pinned. * fix(proxy): keep echoed and blank model_info values out of POST /model/update writes Blank strings from typed clients no longer overwrite stored values. Values echoed from /model/info (the stored base model's cost map entry and discovered limits) are not pinned, and an echoed catalog value resets a differing stored override like PATCH does. direct_access and access_via_team_ids stay display-only. The tests stop patching the auth check class and give the team lookup a real row instead. * fix(proxy): lock the model row and skip unchanged model_info on POST /model/update * fix(proxy): treat only keyed /model/update bodies as echoes and write the stored model_info as saved A typed model_info value that matched the cost map, or the previous base model's entry, was dropped and could erase a stored override. Echo detection now follows PATCH, only a body carrying the /model/info key counts, compared against the base model the row has after the write. The locked write patches the stored JSON instead of dumping the hydrated Deployment, so litellm_params pricing is no longer copied into model_info. Tests run the real Router with discovery over a mock upstream * docs(terraform): note that model_info updates now take effect and overwrite values set outside Terraform * test: expect POST /model/update to save the cache point flag, and wrap long test lines test_x06 pinned the old contract, where POST /model/update answered 200 and ignored model_info. This PR makes POST save model_info the way PATCH does, so x06 now asserts what its PATCH sibling x01 asserts, the stored flag flips to true and cache points come back on. * fix(proxy): answer 400 when POST /model/update gets an access_groups that is not a list of strings * fix(proxy): read discovered model info through the router registry entry point --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../model_management_endpoints.py | 167 ++++++++- terraform/provider/CHANGELOG.md | 3 +- .../provider/litellm/resource_model_crud.go | 2 +- .../test_bedrock_kimi_k3_cache_point_wire.py | 18 +- .../test_model_management_endpoints.py | 345 +++++++++++++++++- 5 files changed, 506 insertions(+), 29 deletions(-) diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 74fa8e3b3c1..dbd77e45d57 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -486,6 +486,7 @@ async def _raise_on_invalid_credential_name( AUTO_ROUTER_CAPABILITY_SLOT_LOCK_KEY: Final = 5_872_301 _CAPABILITY_LOCK_SQL: Final = "SELECT 1 AS locked FROM pg_advisory_xact_lock($1)" +_MODEL_ROW_LOCK_SQL: Final = 'SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_id = $1 FOR UPDATE' _STORED_LITELLM_PARAMS_SQL: Final = ( "(CASE jsonb_typeof(litellm_params) WHEN 'string' THEN (litellm_params #>> '{}')::jsonb ELSE litellm_params END)" ) @@ -633,6 +634,7 @@ async def _auto_router_capability_slot( effective_params: Mapping[str, object], model_id: str | None, member_write: MemberAutoRouterWrite | None = None, + lock_model_row: bool = False, ) -> AsyncGenerator[_ProxyModelTable, None]: """Hand out the model table to write through while the row's claim on a licensed capability is settled. @@ -649,6 +651,9 @@ async def _auto_router_capability_slot( A heuristic-v1 router whose tuning has moved off its recorded baseline is judged the same way under the same lock, against the DB rows plus this proxy's config.yaml routers. + + ``lock_model_row`` runs the write in a transaction holding the row's ``FOR UPDATE`` lock, + so a caller that re-reads the row through the yielded table writes onto its committed state. """ from litellm.proxy.proxy_server import ( _license_check, # pyright: ignore[reportPrivateUsage] # existing capability slot reads the proxy license singleton @@ -662,7 +667,8 @@ async def _auto_router_capability_slot( baselines: Final = heuristic_v1_tuning_baselines tuning_candidate: Final = _tuning_candidate(effective_params, model_id=model_id) judges_tuning: Final = baselines is not None and is_mutable_tuned_candidate(tuning_candidate, baselines) - if member_write is None and (limit is None or (not capabilities and not judges_tuning)): + checks_license: Final = member_write is not None or (limit is not None and (bool(capabilities) or judges_tuning)) + if not checks_license and not lock_model_row: yield _proxy_model_table(prisma_client) return transaction_client: Final = _ModelTransactionClient.model_validate(prisma_client.db) @@ -673,14 +679,15 @@ async def _auto_router_capability_slot( ) async with transaction as tx_ctx: tables: Final[_TxModelTables] = tx_ctx - await tx_ctx.query_raw(_CAPABILITY_LOCK_SQL, AUTO_ROUTER_CAPABILITY_SLOT_LOCK_KEY) + if checks_license: + await tx_ctx.query_raw(_CAPABILITY_LOCK_SQL, AUTO_ROUTER_CAPABILITY_SLOT_LOCK_KEY) + locked_model_id: Final = ( + member_write.model_id if member_write is not None else (model_id if lock_model_row else None) + ) + if locked_model_id is not None: + await tx_ctx.query_raw(_MODEL_ROW_LOCK_SQL, locked_model_id) config_rows: Final = () if llm_router is None else tuple(llm_router.registry().deployments(source="config")) if member_write is not None: - if member_write.model_id is not None: - await tx_ctx.query_raw( - 'SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_id = $1 FOR UPDATE', - member_write.model_id, - ) pinned_client: Final = _TransactionClient(tx_ctx) team_where: Final[prisma_types.LiteLLM_TeamTableWhereUniqueInput] = {"team_id": member_write.team_id} team_include: Final[prisma_types.LiteLLM_TeamTableInclude] = {"litellm_model_table": True} @@ -752,7 +759,8 @@ async def _auto_router_capability_slot( prisma_client=pinned_client, llm_router=llm_router, ) - for capability in capabilities: + judged_capabilities: Final = capabilities if checks_license else () + for capability in judged_capabilities: if ( violation := await _claimed_capability_violation( tables, capability=capability, config_rows=config_rows, model_id=model_id, limit=limit @@ -761,7 +769,7 @@ async def _auto_router_capability_slot( raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail=f"{violation} {AUTO_ROUTER_LICENSE_REMEDY}" ) - if judges_tuning and baselines is not None: + if checks_license and judges_tuning and baselines is not None: model_rows: Final = await ModelRepository(_TransactionClient(tx_ctx)).find_all_except(model_id or "") _raise_on_tuning_quota_violation( candidate=tuning_candidate, @@ -1049,6 +1057,16 @@ def _raise_if_ptu_cost_attribution_disabled(incoming_model_info: Mapping[str, ob ) +_ACCESS_GROUPS: Final[TypeAdapter[tuple[str, ...] | None]] = TypeAdapter(tuple[str, ...] | None) + + +def _raise_if_access_groups_malformed(model_info_writes: Mapping[str, object]) -> None: + try: + _ACCESS_GROUPS.validate_python(model_info_writes.get("access_groups")) + except ValidationError: + raise HTTPException(status_code=400, detail="model_info.access_groups must be a list of strings") from None + + def _validate_ptu_model_info(model_info: Mapping[str, object]) -> None: """Enforce the PTU cross-field invariant on the effective model_info. @@ -1291,6 +1309,109 @@ def _loaded_catalog_entry( return loaded_catalog().get(catalog_key, MappingProxyType({})) +_LEGACY_UPDATE_IGNORED_MODEL_INFO_FIELDS: Final = frozenset( + { + "id", + "db_model", + "created_at", + "created_by", + "updated_at", + "updated_by", + "blocked", + "member_auto_router", + "team_id", + "team_public_model_name", + "direct_access", + "access_via_team_ids", + COST_MAP_LOOKUP_KEY, + *PTU_MODEL_INFO_FIELDS, + } +) + + +def legacy_update_model_info_changes( + db_model: Deployment, + incoming_model_info: ModelInfo, + discovered_model_info: Mapping[str, object] = MappingProxyType({}), + loaded_catalog: LoadedCatalog = GetModelCostMap.loaded_model_cost_map, +) -> Mapping[str, object]: + """The ``model_info`` fields ``POST /model/update`` persists from its request. + + Only fields the caller sent with a value count, so ``ModelInfo`` defaults and the blank + strings typed clients send for unset fields never overwrite stored values. Server-owned, + display-only, team, PTU and pricing fields keep their dedicated write paths. As on + ``update_db_model``, only a body carrying the ``key`` that ``/model/info`` emits is an echo: + its values equal to the cost map entry of the base model the row has after the write, the + entry ``key`` names, or a limit discovered from the upstream stay display-only, and an echoed + value that differs from the stored override resets that override (``None``). + """ + stored: Final = db_model.model_info.model_dump(exclude_none=True) + requested: Final = incoming_model_info.model_dump(exclude_unset=True, exclude_none=True) + sent: Final = {k: v for k, v in requested.items() if k not in _LEGACY_UPDATE_IGNORED_MODEL_INFO_FIELDS and v != ""} + echoed: Final = frozenset( + echoed_cost_map_fields( + requested, + _cost_map_entry(db_model, {**stored, **sent}), + _loaded_catalog_entry(requested, loaded_catalog), + discovered_model_info, + ) + ) + kept: Final = without_server_derived_pricing(sent) + return MappingProxyType( + { + **{k: getattr(incoming_model_info, k) for k in kept if k not in echoed}, + **{k: None for k, v in kept.items() if k in echoed and stored.get(k) not in (None, v)}, + } + ) + + +def _legacy_update_model_info_writes( + current: Deployment, + incoming_model_info: ModelInfo, + discovered_model_info: Mapping[str, object], + member_marker: bool | None, +) -> Mapping[str, object]: + changes: Final = { + **legacy_update_model_info_changes(current, incoming_model_info, discovered_model_info), + **({"member_auto_router": member_marker} if member_marker is not None else {}), + } + return MappingProxyType({k: v for k, v in changes.items() if getattr(current.model_info, k, None) != v}) + + +async def _locked_legacy_update_model_info_json( + table: _ProxyModelTable, + model_id: str, + incoming_model_info: ModelInfo, + discovered_model_info: Mapping[str, object], + member_marker: bool | None, +) -> str | None: + row: Final = await table.find_unique(where={"model_id": model_id}) + if row is None: + return None + payload: Final = row.model_dump() + writes: Final = _legacy_update_model_info_writes( + Deployment(**payload), incoming_model_info, discovered_model_info, member_marker + ) + if not writes: + return None + stored_model_info: Final[object] = payload.get("model_info") + stored: Final = ( + TypeAdapter(dict[str, object]).validate_json(stored_model_info) + if isinstance(stored_model_info, str) + else TypeAdapter(dict[str, object]).validate_python(stored_model_info or {}) + ) + return ( + TypeAdapter(dict[str, object]) + .dump_json( + { + **{k: v for k, v in stored.items() if k not in writes}, + **{k: v for k, v in writes.items() if v is not None}, + } + ) + .decode() + ) + + def _param_keys_to_remove(patch: updateDeployment) -> tuple[str, ...]: return patch.litellm_params_to_remove if isinstance(patch, PatchModelRequest) else () @@ -3279,22 +3400,32 @@ async def update_model( renamed_update: Final[PrismaCompatibleUpdateDBModel] = ( {**base_update, "model_name": renamed_to} if renamed_to is not None else base_update ) - _data: Final[PrismaCompatibleUpdateDBModel] = ( - { - **renamed_update, - "model_info": deployment.model_info.model_copy( - update=MappingProxyType({"member_auto_router": member_marker}) - ).model_dump_json(exclude_none=True), - } - if member_marker is not None - else renamed_update + discovered_model_info: Final[Mapping[str, int]] = ( + llm_router.registry().discovered_model_info(_model_id) + if llm_router is not None + else MappingProxyType({}) ) + requested_model_info: Final = _legacy_update_model_info_writes( + deployment, _model_info, discovered_model_info, member_marker + ) + _raise_if_access_groups_malformed(requested_model_info) async with _auto_router_capability_slot( prisma_client, effective_params=effective_params, model_id=_model_id, member_write=member_write, + lock_model_row=bool(requested_model_info), ) as table: + model_info_json: Final = ( + await _locked_legacy_update_model_info_json( + table, _model_id, _model_info, discovered_model_info, member_marker + ) + if requested_model_info + else None + ) + _data: Final[PrismaCompatibleUpdateDBModel] = ( + {**renamed_update, "model_info": model_info_json} if model_info_json is not None else renamed_update + ) model_response: Final = await table.update( where={"model_id": _model_id}, data=_data, diff --git a/terraform/provider/CHANGELOG.md b/terraform/provider/CHANGELOG.md index 597e0c6d4d8..4a1838a7b60 100644 --- a/terraform/provider/CHANGELOG.md +++ b/terraform/provider/CHANGELOG.md @@ -18,7 +18,7 @@ longer signal it. - **credential, model, mcp_server**: Write-only secret attributes so secrets never land in Terraform state (Terraform 1.11 or later). `litellm_credential` gains `credential_values_wo` (a JSON object, `jsonencode({...})`, since the SDK cannot mark a map write-only) with `credential_values_wo_version`, and `credential_values` becomes optional with exactly one of the two required. `litellm_model` gains `model_api_key_wo`, `aws_secret_access_key_wo` and `vertex_credentials_wo`, each with a `_wo_version` companion and each conflicting with its in-state twin. `litellm_mcp_server` gains `client_id`, `client_secret_wo` and `auth_value_wo` (with `_wo_version` companions), sent to the proxy as `credentials`, and `auth_type` now accepts every value the proxy does (`oauth2`, `api_key`, `bearer_token`, ...). Every `_wo` value is sent on each apply from the configuration and produces no diff, so bump its `_wo_version` to push a rotated secret. `litellm_vector_store.litellm_params` warns at plan time when it carries a secret-looking key such as `api_key` - **team**: Optional `team_member_model_max_budget` argument on `litellm_team`, a JSON string of per-model budgets applied to every team member and sent as `team_member_model_max_budget` on `/team/new` and `/team/update`; read back from `team_member_budget_table.model_max_budget` on `/team/info` so out-of-band changes show as drift, with `budget_limit`/`time_period` treated as equal to the `max_budget`/`budget_duration` spelling the proxy stores; removing it sends `{}`, which clears every per-model member budget; `tpm_limit`/`rpm_limit` are rejected at plan time because the proxy rejects them on team budgets. Needs a proxy that includes [#43887](https://github.com/BerriAI/litellm/pull/43887): v1.105.0 and v1.106.0-rc.1 accept the field but silently drop it, so every plan shows a diff -- **model**: Optional `display_name` argument on `litellm_model`, sent as `model_info.display_name` and returned as `display_name` by `/v1/models`, so client model pickers show a readable name; changes are persisted through `/model/{id}/update` since `/model/update` ignores `model_info`; also exported by the `litellm_model` and `litellm_models` data sources +- **model**: Optional `display_name` argument on `litellm_model`, sent as `model_info.display_name` and returned as `display_name` by `/v1/models`, so client model pickers show a readable name; changes are also persisted through `/model/{id}/update`, since proxies older than this provider drop the `model_info` sent to `/model/update`; also exported by the `litellm_model` and `litellm_models` data sources - **key**: Computed `server_metadata` attribute on `litellm_key` exposing every metadata entry the proxy stores, so metadata created outside Terraform is visible in state and drift on it shows on refresh, while `metadata` keeps tracking only the declared entries and updates keep preserving undeclared ones - **team_member_add**: `tpm_limit`, `rpm_limit`, `budget_duration`, and `allowed_models` attributes on `litellm_team_member_add`, applied to every member of the resource; `budget_duration` and `allowed_models` ride on `/team/member_add`, while the limits are sent through `/team/member_update`, which is where the proxy accepts them - **team**: Optional `team_id` argument on `litellm_team`, so teams can be created with a stable, human-readable ID instead of a provider-generated UUID; changing it forces replacement @@ -70,6 +70,7 @@ longer signal it. ### Changed +- **model** (breaking): on a proxy that includes [#46902](https://github.com/BerriAI/litellm/pull/46902), `litellm_model` updates now save the `model_info` the provider sends to `/model/update`, so `pricing_base_model`, `tier` and `mode` changes take effect. The first apply after upgrading the proxy overwrites values set outside Terraform with the configuration's, including `tier`, which defaults to `free`, and `base_model`, which falls back to the deployment name when `pricing_base_model` is unset. Run `terraform plan` before that apply and copy any value to keep into the configuration - **model** (breaking): a key named in `additional_litellm_params.additional_drop_params` is no longer deleted from the deployment's `litellm_params`. A configuration that also sets that key, such as `vertex_location`, stores it on the deployment at its next apply, and the proxy applies the drop list to requests instead. For `vertex_location` this changes routing. `/v1/messages` calls now go to the configured region, while chat completions drop a listed `vertex_location` and use the default region. To keep such a key off a deployment, remove its attribute and replace the model with `terraform apply -replace`, which gives it a new model id, since an update keeps a stored value the provider leaves out. `model`, `custom_llm_provider` and `merge_reasoning_content_in_choices` are always sent. A value that is not a JSON list of names, such as a bare `"store"` or an empty string, now fails the plan, and with it `terraform destroy`, instead of being stored as a string the proxy ignores. Rewrite it as `jsonencode([...])`, or remove the model without fixing it through `terraform destroy -refresh=false` - **key** (breaking): `model_max_budget` on `litellm_key` is now a JSON string of per-model budget objects (`jsonencode({"gpt-4o-mini" = {budget_limit = 50, time_period = "30d"}})`), matching `litellm_user`, `litellm_budget` and `litellm_tag`. The old `map(number)` form sent bare numbers to `/key/generate`, which the proxy rejects with a 500 (`'int' object is not iterable`), so every key with a non-empty `model_max_budget` failed to apply. Existing state upgrades automatically (schema version 1) and the attribute is refilled from the proxy on the next read; configurations still using the map form must be rewritten - **Versioning**: the provider is now published at the LiteLLM version, from the same commit as the proxy, on every LiteLLM release (dev, rc, stable). The `0.x` line ends at `0.4.0`; a `~> 0.4` constraint will not receive further releases, so re-pin to the LiteLLM version your proxy runs (for example `~> 1.99.0`). Existing `0.x` versions remain in the registry and keep verifying diff --git a/terraform/provider/litellm/resource_model_crud.go b/terraform/provider/litellm/resource_model_crud.go index 3db6f68760d..1f5aa62560f 100644 --- a/terraform/provider/litellm/resource_model_crud.go +++ b/terraform/provider/litellm/resource_model_crud.go @@ -357,7 +357,7 @@ func createOrUpdateModel(d *schema.ResourceData, m interface{}, isUpdate bool) e return retryModelRead(d, m, 5) } -// /model/update only merges litellm_params, so model_info changes go through the PATCH endpoint. +// Older proxies drop the model_info sent to /model/update, so display_name also goes through the PATCH endpoint. func patchModelDisplayName(client *Client, modelID, displayName string) error { resp, err := MakeRequest(client, "PATCH", fmt.Sprintf(endpointModelPatch, url.PathEscape(modelID)), ModelInfoPatch{ ModelInfo: ModelInfoPatchFields{ID: modelID, DisplayName: displayName}, diff --git a/tests/integration/providers/test_bedrock_kimi_k3_cache_point_wire.py b/tests/integration/providers/test_bedrock_kimi_k3_cache_point_wire.py index 0772e928b35..4441582fb6b 100644 --- a/tests/integration/providers/test_bedrock_kimi_k3_cache_point_wire.py +++ b/tests/integration/providers/test_bedrock_kimi_k3_cache_point_wire.py @@ -1325,7 +1325,9 @@ def test_x01_flipping_the_flag_to_true_through_a_patch_update_turns_cache_points @pytest.mark.timeout(600) -def test_x06_the_legacy_post_update_answers_200_and_leaves_the_stored_flag_alone(gateway: Gateway) -> None: +def test_x06_flipping_the_flag_to_true_through_the_legacy_post_update_turns_cache_points_back_on( + gateway: Gateway, +) -> None: arn: Final = _arn("legacy") with wire_server(_peer()) as wire, gateway.scenario() as scenario: name: Final = _deployment( @@ -1333,6 +1335,7 @@ def test_x06_the_legacy_post_update_answers_200_and_leaves_the_stored_flag_alone ) before, before_received = _observe_at(gateway, wire, "chat", _chat_body(name, _prompt(), _SYSTEM_AND_USER)) _assert_answered(before) + assert before_received.cache_points == 0, before_received.body identity: Final = _model_id(name) updated: Final = gateway.request( "POST", @@ -1344,13 +1347,14 @@ def test_x06_the_legacy_post_update_answers_200_and_leaves_the_stored_flag_alone }, ) assert updated.status_code == 200, updated.text - assert _stored_flag(identity) is False - _settled(gateway, name, wire) - after, after_received = _observe_at(gateway, wire, "chat", _chat_body(name, _prompt(), _SYSTEM_AND_USER)) - _assert_answered(after) - assert after_received.cache_points == before_received.cache_points, (before_received.body, after_received.body) + assert _stored_flag(identity) is True + points: Final = eventually( + lambda: _six_cache_point_counts(gateway, wire, name), + lambda counts: len(counts) == 6 and all(count == 2 for count in counts), + seconds=60, + ) + assert points == (2,) * 6, points _success_row(before.response_id) - _success_row(after.response_id) @dataclass(frozen=True, slots=True) diff --git a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py index 872d1607351..abaa802de2c 100644 --- a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py @@ -3,17 +3,19 @@ import contextlib import inspect import json import socket -from collections.abc import Iterator, Mapping -from types import SimpleNamespace +from collections.abc import AsyncIterator, Iterator, Mapping +from types import MappingProxyType, SimpleNamespace from typing import Dict, Final, Optional from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest from fastapi import HTTPException from fastapi.encoders import jsonable_encoder from fastapi.testclient import TestClient from litellm._uuid import uuid +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.models.credentials import CredentialItem from litellm.models.project import LiteLLM_ProjectTable @@ -1639,11 +1641,350 @@ class TestDeletedAutoRouterAvailability: } +class _ContendedModelDb: + def __init__(self, row: LiteLLM_ProxyModelTable, other_writers: tuple[Mapping[str, object], ...]) -> None: + self.row = row + self.writes: tuple[Mapping[str, object], ...] = () + self._other_writers = other_writers + self._row_locked = False + self.db = self + self.litellm_proxymodeltable = self + self.litellm_teamtable = SimpleNamespace(find_unique=self._find_team) + + async def _find_team(self, *, where: Mapping[str, str]) -> LiteLLM_TeamTable: + return LiteLLM_TeamTable(team_id=where["team_id"]) + + async def find_unique(self, *, where: Mapping[str, object]) -> LiteLLM_ProxyModelTable: + read: Final = self.row + if not self._row_locked and self._other_writers: + self._commit_next_other_writer() + return read + + async def update(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> LiteLLM_ProxyModelTable: + self.writes = (*self.writes, data) + self.row = self.row.model_copy( + update={k: json.loads(v) if k in ("litellm_params", "model_info") else v for k, v in data.items()} + ) + return self.row + + async def query_raw(self, query: str, *args: object) -> tuple[()]: + self._row_locked = self._row_locked or "FOR UPDATE" in query + return () + + @contextlib.asynccontextmanager + async def tx(self, timeout: object = None) -> AsyncIterator["_ContendedModelDb"]: + yield self + self._row_locked = False + while self._other_writers: + self._commit_next_other_writer() + + def _commit_next_other_writer(self) -> None: + self.row = self.row.model_copy( + update={"model_info": {**(self.row.model_info or {}), **self._other_writers[0]}} + ) + self._other_writers = self._other_writers[1:] + + +_AZURE_DEPLOYMENT_PARAMS: Final = MappingProxyType({"model": "azure/my-deployment", "api_key": "sk-existing"}) +_LOCAL_DEPLOYMENT_PARAMS: Final = MappingProxyType( + {"model": "hosted_vllm/local-model", "api_base": "https://local.test/v1", "api_key": "local-key"} +) + + +async def _discover_from_upstream(router: Router, model_card: Mapping[str, object]) -> None: + handler: Final = AsyncHTTPHandler() + await handler.client.aclose() + async with httpx.AsyncClient( + transport=httpx.MockTransport(lambda request: httpx.Response(200, json={"data": [dict(model_card)]})) + ) as client: + handler.client = client + await router.arefresh_model_info(client=handler) + + class TestUpdateModel: """ Tests for the update_model (POST /model/update) handler. """ + @staticmethod + async def _update_model( + stored_model_info: Mapping[str, object], + model_info: ModelInfo, + litellm_params: Mapping[str, object] = _AZURE_DEPLOYMENT_PARAMS, + upstream_model_card: Mapping[str, object] | None = None, + other_writers: tuple[Mapping[str, object], ...] = (), + ) -> "_ContendedModelDb": + db: Final = _ContendedModelDb( + LiteLLM_ProxyModelTable( + model_id="m1", + model_name="my-model", + litellm_params=dict(litellm_params), + model_info=dict(stored_model_info), + ), + other_writers, + ) + router: Final = Router( + model_list=[ + {"model_name": "my-model", "litellm_params": dict(litellm_params), "model_info": {"id": "m1"}} + ] + ) + if upstream_model_card is not None: + await _discover_from_upstream(router, upstream_model_card) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", db), + patch("litellm.proxy.proxy_server.llm_router", router), + patch("litellm.proxy.proxy_server.store_model_in_db", True), + patch("litellm.proxy.proxy_server.premium_user", True), + patch( # test-quality-ok: [TQ008] the route reloads the proxy-global router with no seam to inject + "litellm.proxy.management_endpoints.model_management_endpoints.clear_cache", + new=AsyncMock(return_value=ReconcileOutcome(still_desired=None, live_after=None)), + ), + ): + await update_model( + model_params=updateDeployment(litellm_params=updateLiteLLMParams(), model_info=model_info), + user_api_key_dict=UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + + return db + + @pytest.mark.asyncio + async def test_update_model_persists_sent_model_info_over_stored_fields(self): + stored: Final = { + "id": "m1", + "db_model": True, + "base_model": "my-deployment", + "tier": "paid", + "team_id": "team-a", + } + + db: Final = await self._update_model( + stored, + ModelInfo(id="m1", base_model="test-provider/test-model", mode="chat", db_model=False, team_id="team-b"), + ) + + persisted: Final = db.row.model_info + assert persisted is not None + assert persisted["base_model"] == "test-provider/test-model" + assert persisted["mode"] == "chat" + assert {k: persisted[k] for k in ("db_model", "tier", "team_id")} == { + "db_model": True, + "tier": "paid", + "team_id": "team-a", + } + + @pytest.mark.asyncio + async def test_update_model_with_only_an_id_leaves_model_info_untouched(self): + db: Final = await self._update_model({"id": "m1", "base_model": "my-deployment"}, ModelInfo(id="m1")) + + assert tuple("model_info" in write for write in db.writes) == (False,) + + @pytest.mark.asyncio + async def test_update_model_resending_the_stored_model_info_does_not_rewrite_it(self): + db: Final = await self._update_model( + {"id": "m1", "db_model": True, "tier": "paid", "mode": "chat"}, + ModelInfo(id="m1", tier="paid", mode="chat"), + ) + + assert tuple("model_info" in write for write in db.writes) == (False,) + + @pytest.mark.asyncio + async def test_update_model_keeps_model_info_fields_a_concurrent_writer_stored(self): + db: Final = await self._update_model( + {"id": "m1", "db_model": True, "mode": "embedding"}, + ModelInfo(id="m1", mode="chat"), + other_writers=(MappingProxyType({"access_groups": ["group-a"]}), MappingProxyType({"tier": "paid"})), + ) + + persisted: Final = db.row.model_info + assert persisted is not None + assert {k: persisted.get(k) for k in ("mode", "access_groups", "tier")} == { + "mode": "chat", + "access_groups": ["group-a"], + "tier": "paid", + } + + @pytest.mark.asyncio + async def test_update_model_does_not_pin_values_echoed_from_model_info(self): + import litellm + + catalog_entry: Final = litellm.get_model_info(model="azure/gpt-4o") + echoed: Final = ModelInfo( + id="m1", + base_model="azure/gpt-4o", + key=catalog_entry["key"], + max_tokens=catalog_entry["max_tokens"], + input_cost_per_token=catalog_entry["input_cost_per_token"], + ) + + db: Final = await self._update_model({"id": "m1"}, echoed) + + persisted: Final = db.row.model_info + assert persisted is not None + assert persisted["base_model"] == "azure/gpt-4o" + assert not {"key", "max_tokens", "input_cost_per_token"} & persisted.keys() + + @pytest.mark.asyncio + async def test_update_model_saves_typed_values_that_equal_the_catalog(self): + import litellm + + catalog_entry: Final = litellm.get_model_info(model="azure/gpt-4o") + typed: Final = { + "max_tokens": catalog_entry["max_tokens"], + "max_input_tokens": catalog_entry["max_input_tokens"], + "supports_vision": catalog_entry["supports_vision"], + } + + db: Final = await self._update_model( + {"id": "m1", "max_input_tokens": catalog_entry["max_input_tokens"] // 2}, + ModelInfo(id="m1", base_model="azure/gpt-4o", **typed), + ) + + persisted: Final = db.row.model_info + assert persisted is not None + assert {k: persisted.get(k) for k in typed} == typed + + @pytest.mark.asyncio + async def test_update_model_keeps_stored_value_when_a_field_is_sent_blank(self): + db: Final = await self._update_model( + {"id": "m1", "base_model": "my-deployment", "mode": "chat"}, + ModelInfo(id="m1", base_model="test-provider/test-model", mode=""), + ) + + persisted: Final = db.row.model_info + assert persisted is not None + assert persisted["base_model"] == "test-provider/test-model" + assert persisted["mode"] == "chat" + + @pytest.mark.asyncio + async def test_update_model_ignores_ptu_blocked_and_pricing_fields(self): + db: Final = await self._update_model( + {"id": "m1"}, + ModelInfo( + id="m1", base_model="test-provider/test-model", ptu_count=5, blocked=True, input_cost_per_token=0.5 + ), + ) + + persisted: Final = db.row.model_info + assert persisted is not None + assert persisted["base_model"] == "test-provider/test-model" + assert not {"ptu_count", "blocked", "input_cost_per_token"} & persisted.keys() + + @pytest.mark.asyncio + @pytest.mark.parametrize("access_groups", [5, "group-a", ["group-a", 5], {"group": "group-a"}]) + async def test_update_model_rejects_access_groups_that_are_not_a_list_of_strings(self, access_groups: object): + with pytest.raises(ProxyException) as error: + await self._update_model({"id": "m1"}, ModelInfo.model_validate({"id": "m1", "access_groups": access_groups})) + + assert error.value.code == "400" + + @pytest.mark.asyncio + async def test_update_model_saves_access_groups_sent_as_a_list_of_strings(self): + db: Final = await self._update_model( + {"id": "m1"}, ModelInfo.model_validate({"id": "m1", "access_groups": ["group-a", "group-b"]}) + ) + + persisted: Final = db.row.model_info + assert persisted is not None + assert persisted["access_groups"] == ["group-a", "group-b"] + + def test_update_model_does_not_pin_the_previous_base_models_echoed_values(self): + from litellm.proxy.management_endpoints.model_management_endpoints import legacy_update_model_info_changes + + catalog: Final = MappingProxyType( + {"test-provider/previous": MappingProxyType({"supported_openai_params": ["previous-only-param"]})} + ) + stored: Final = Deployment( + model_name="my-model", + litellm_params=LiteLLM_Params(model="test-provider/my-deployment"), + model_info=ModelInfo(id="m1", base_model="test-provider/previous"), + ) + + changes: Final = legacy_update_model_info_changes( + stored, + ModelInfo( + id="m1", + base_model="test-provider/next", + key="test-provider/previous", + supported_openai_params=["previous-only-param"], + ), + loaded_catalog=lambda: catalog, + ) + + assert dict(changes) == {"base_model": "test-provider/next"} + + @pytest.mark.asyncio + async def test_update_model_saves_a_value_the_previous_base_model_has_in_the_catalog(self): + import litellm + + previous: Final = litellm.get_model_info(model="azure/gpt-4o") + + db: Final = await self._update_model( + {"id": "m1", "base_model": "azure/gpt-4o", "max_input_tokens": previous["max_input_tokens"] // 2}, + ModelInfo(id="m1", base_model="azure/gpt-4.1", max_input_tokens=previous["max_input_tokens"]), + ) + + persisted: Final = db.row.model_info + assert persisted is not None + assert {k: persisted.get(k) for k in ("base_model", "max_input_tokens")} == { + "base_model": "azure/gpt-4.1", + "max_input_tokens": previous["max_input_tokens"], + } + + @pytest.mark.asyncio + async def test_update_model_does_not_pin_discovered_limits_or_team_access_echoed_from_model_info(self): + db: Final = await self._update_model( + {"id": "m1"}, + ModelInfo( + id="m1", + base_model="test-provider/test-model", + key="test-provider/test-model", + max_input_tokens=128000, + max_output_tokens=64000, + direct_access=True, + access_via_team_ids=["team-a"], + ), + litellm_params=_LOCAL_DEPLOYMENT_PARAMS, + upstream_model_card=MappingProxyType( + {"id": "local-model", "max_model_len": 128000, "max_output_tokens": 32000} + ), + ) + + persisted: Final = db.row.model_info + assert persisted is not None + assert persisted["base_model"] == "test-provider/test-model" + assert persisted["max_output_tokens"] == 64000 + assert not {"max_input_tokens", "direct_access", "access_via_team_ids"} & persisted.keys() + + @pytest.mark.asyncio + async def test_update_model_does_not_copy_litellm_params_pricing_into_model_info(self): + db: Final = await self._update_model( + {"id": "m1"}, + ModelInfo(id="m1", tier="paid"), + litellm_params={**_AZURE_DEPLOYMENT_PARAMS, "input_cost_per_token": 0.000003}, + ) + + persisted: Final = db.row.model_info + assert persisted == {"id": "m1", "tier": "paid"} + + @pytest.mark.asyncio + async def test_update_model_resets_a_stored_override_echoed_back_at_its_catalog_value(self): + import litellm + + catalog_entry: Final = litellm.get_model_info(model="azure/gpt-4o") + stored_override: Final = catalog_entry["max_tokens"] // 2 + + db: Final = await self._update_model( + {"id": "m1", "base_model": "azure/gpt-4o", "max_tokens": stored_override}, + ModelInfo( + id="m1", base_model="azure/gpt-4o", key=catalog_entry["key"], max_tokens=catalog_entry["max_tokens"] + ), + ) + + persisted: Final = db.row.model_info + assert persisted is not None + assert "max_tokens" not in persisted + @pytest.mark.asyncio async def test_update_model_clears_cache_after_db_write(self): """ From 6ad4f9cfdf3f2c4a8db3e39acb02d3d7730d2dce Mon Sep 17 00:00:00 2001 From: Nate Armstrong Date: Fri, 30 Oct 2026 12:14:17 -0700 Subject: [PATCH 26/74] ci: cache Rust compilation with sccache (#49117) This should make compilation much faster --- .github/workflows/test-rust-db.yml | 10 ++++++++++ .github/workflows/test-rust.yml | 20 ++++++++++++++++++++ .github/workflows/test-unit.yml | 12 ++++++++++++ 3 files changed, 42 insertions(+) diff --git a/.github/workflows/test-rust-db.yml b/.github/workflows/test-rust-db.yml index bb88b593d5b..80ec256fa6f 100644 --- a/.github/workflows/test-rust-db.yml +++ b/.github/workflows/test-rust-db.yml @@ -42,6 +42,11 @@ concurrency: env: CARGO_TERM_COLOR: always + RUSTC_WRAPPER: sccache + SCCACHE_GHA_ENABLED: "true" + SCCACHE_GHA_VERSION: litellm-ci-v1 + SCCACHE_GHA_RW_MODE: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'READ_WRITE' || 'READ_ONLY' }} + CARGO_INCREMENTAL: "0" jobs: db-queries: @@ -87,6 +92,11 @@ jobs: workspaces: litellm-rust cache-on-failure: true + - name: Set up sccache + uses: mozilla-actions/sccache-action@d651010b8da762cde178750d8eda7b5febfe147a + with: + version: "v0.18.0" + - name: Install Python dependencies working-directory: . run: uv sync --frozen --extra extra-proxy diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index 04af770b24f..cf9fed192f6 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -78,6 +78,11 @@ concurrency: env: CARGO_TERM_COLOR: always + RUSTC_WRAPPER: sccache + SCCACHE_GHA_ENABLED: "true" + SCCACHE_GHA_VERSION: litellm-ci-v1 + SCCACHE_GHA_RW_MODE: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'READ_WRITE' || 'READ_ONLY' }} + CARGO_INCREMENTAL: "0" jobs: rust-lint: @@ -101,6 +106,11 @@ jobs: cache-on-failure: true save-if: ${{ github.ref == 'refs/heads/main' }} + - name: Set up sccache + uses: mozilla-actions/sccache-action@d651010b8da762cde178750d8eda7b5febfe147a + with: + version: "v0.18.0" + - run: cargo clippy --workspace --all-targets --locked -- -D warnings rust-test: @@ -143,6 +153,11 @@ jobs: cache-on-failure: true save-if: ${{ github.ref == 'refs/heads/main' }} + - name: Set up sccache + uses: mozilla-actions/sccache-action@d651010b8da762cde178750d8eda7b5febfe147a + with: + version: "v0.18.0" + - name: Check generated trace contracts working-directory: . run: uv run scripts/generate_trace_types.py --check @@ -189,6 +204,11 @@ jobs: cache-on-failure: true save-if: ${{ github.ref == 'refs/heads/main' }} + - name: Set up sccache + uses: mozilla-actions/sccache-action@d651010b8da762cde178750d8eda7b5febfe147a + with: + version: "v0.18.0" + - run: uv build --wheel --out-dir dist - run: python .github/scripts/verify_linux_native_wheel.py dist/*.whl diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 68823d99519..5ec5975d83b 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -73,9 +73,21 @@ jobs: timeout-minutes: 5 uses: ./.github/actions/cache-cargo-build + - name: Set up sccache + if: steps.changes.outputs.decision != 'skip' && steps.rust-bridge-cache.outputs.cache-hit != 'true' + uses: mozilla-actions/sccache-action@d651010b8da762cde178750d8eda7b5febfe147a + with: + version: "v0.18.0" + - name: Build editable Rust bridge if: steps.changes.outputs.decision != 'skip' && steps.rust-bridge-cache.outputs.cache-hit != 'true' timeout-minutes: 15 + env: + RUSTC_WRAPPER: sccache + SCCACHE_GHA_ENABLED: "true" + SCCACHE_GHA_VERSION: litellm-ci-v1 + SCCACHE_GHA_RW_MODE: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'READ_WRITE' || 'READ_ONLY' }} + CARGO_INCREMENTAL: "0" run: | mkdir -p rust-bridge-dist uv run --no-project --with maturin==1.15.0 python -c 'import maturin; maturin.build_editable("rust-bridge-dist")' From 5898a42f5aac37fc9fa2392f3c7fe16644e3e958 Mon Sep 17 00:00:00 2001 From: norkelbyte <348963847+norkelbyte@users.noreply.github.com> Date: Fri, 30 Oct 2026 12:19:51 -0700 Subject: [PATCH 27/74] fix(openrouter)!: list OpenRouter's live catalog for openrouter/* when check_provider_endpoint is on (#48820) * fix(openrouter): list OpenRouter's live catalog for openrouter/* when check_provider_endpoint is on * fix(openrouter): list models off the event loop and keep only chat-callable catalog ids The model-list builders are sync, and with check_provider_endpoint on they call OpenRouter's catalog with a 10 second timeout and no caching of failures. They ran on the event loop inside /v1/models, /model/info, /model_group/info and the per-user listing, so one slow catalog froze every request on the worker. Those calls now run in a worker thread with asyncio.to_thread File, batch and RAG credential lookups only need each name's deployment, and a wildcard that is itself a deployment sits in the list ahead of its expansions and resolves its own credentials. get_complete_model_list takes expand_deployed_wildcard_routes, which that lookup sets to False, so a deployed wildcard never fetches its provider catalog there. A key wildcard wider than any deployment, like anthropic/* over a deployed anthropic/claude-*, still expands so it finds the matching deployment's credentials. Every other caller keeps the default The pattern router's model lookup tolerates a pattern removed between its two reads of the pattern table, which a deployment delete on the event loop can now do while a listing runs in a thread Listing now reads only OpenRouter's models/user catalog. OpenRouter's own /models list carries no embedding models, the OpenAI-format /v1/models has no mode field so chat pickers cannot tell embeddings apart, /model_group/info labels them mode chat, and one failing catalog emptied the whole list. Ids ending in :batch are dropped because chat completions answers 404 for them, while :free and every other variant stay. An api_base that already ends in /chat/completions, which chat accepts as is, has that suffix stripped before the catalog route is joined * fix(proxy): keep model listing off the shared thread pool and out of credential lookups Listings run on their own thread pool so many slow catalogs cannot starve the default executor that chat and Responses calls share. File and batch credential lookups match deployed names against a key's wildcard grants instead of listing any provider catalog. The token counter keeps OpenRouter deployments on their previous path, and a catch-all "*" over provider/* lists bare ids that route back through it * test(proxy): give the off-loop listing tests a generous deadlock guard and typed ids * fix(proxy): keep routed models in wildcard model info, skip key-only OpenRouter catalogs, cache failed listings briefly * fix(utils): cache failed model listings under their own key so a late failure never hides a good listing * fix(proxy): remember only timed-out provider listings so a provider that failed fast is listed again * fix(proxy): ask each provider catalog once per worker, keep deployed names out of catch-all ids, and match file credentials by name Listings that start together for one deployment now wait for the first catalog request. A catch-all wildcard no longer lists a name a deployment serves by name. A key wildcard only finds file and batch credentials on deployments named under it, so a bare name keeps the default credentials as before. * fix(proxy): keep the listing failure log line the same for timed-out listings * refactor(proxy): fold the cached listing lookup back into the provider listing function * fix(proxy): list each model id once and keep a named deployment's row over a wildcard's * fix(proxy): drop a wildcard's model info row when named deployments serve every name it lists --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/llms/openrouter/common_utils.py | 87 +++++++ litellm/proxy/auth/model_checks.py | 74 +++++- .../openai_files_endpoints/common_utils.py | 53 +++-- litellm/proxy/proxy_server.py | 59 +++-- litellm/proxy/utils.py | 55 +++-- .../router_utils/pattern_match_deployments.py | 4 +- litellm/utils.py | 81 +++++-- .../test_openrouter_common_utils.py | 159 +++++++++++++ tests/unit/proxy/auth/test_model_checks.py | 219 ++++++++++++++++++ .../test_files_common_utils.py | 112 +++++++++ .../proxy_server/test_routes_model_info.py | 12 +- .../proxy/proxy_server/test_routes_models.py | 73 ++++++ tests/unit/proxy/test_proxy_token_counter.py | 33 +++ .../proxy/utils/helpers/test_model_access.py | 67 +++++- tests/unit/test_utils.py | 71 ++++++ 15 files changed, 1055 insertions(+), 104 deletions(-) create mode 100644 tests/unit/llms/openrouter/test_openrouter_common_utils.py diff --git a/litellm/llms/openrouter/common_utils.py b/litellm/llms/openrouter/common_utils.py index 96e53a5aae7..64caf83d745 100644 --- a/litellm/llms/openrouter/common_utils.py +++ b/litellm/llms/openrouter/common_utils.py @@ -1,5 +1,92 @@ +from collections.abc import Mapping, Sequence +from typing import Final + +import httpx +from pydantic import BaseModel, ConfigDict + +import litellm +from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.base_llm.base_utils import BaseLLMModelInfo from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import AllMessageValues + +_DEFAULT_API_BASE: Final = "https://openrouter.ai/api/v1" +_CHAT_CATALOG_ROUTE: Final = "models/user" +_CHAT_COMPLETIONS_PATH: Final = "/chat/completions" +_BATCH_ONLY_SUFFIX: Final = ":batch" +_CATALOG_TIMEOUT: Final = httpx.Timeout(10.0) class OpenRouterException(BaseLLMException): pass + + +class _CatalogModel(BaseModel): + model_config = ConfigDict(frozen=True) + + id: str + + +class _Catalog(BaseModel): + model_config = ConfigDict(frozen=True) + + data: tuple[_CatalogModel, ...] + + +def _chat_catalog_url(api_base: str) -> str: + return join_api_base(api_base.rstrip("/").removesuffix(_CHAT_COMPLETIONS_PATH), _CHAT_CATALOG_ROUTE) + + +class OpenRouterModelInfo(BaseLLMModelInfo): + @staticmethod + def get_api_key(api_key: str | None = None) -> str | None: + return ( + api_key + or litellm.api_key + or litellm.openrouter_key + or get_secret_str("OPENROUTER_API_KEY") + or get_secret_str("OR_API_KEY") + ) + + @staticmethod + def get_api_base(api_base: str | None = None) -> str: + return api_base or litellm.api_base or get_secret_str("OPENROUTER_API_BASE") or _DEFAULT_API_BASE + + @staticmethod + def get_base_model(model: str) -> str: + return model.removeprefix("openrouter/") + + def validate_environment( + self, + headers: Mapping[str, str], + model: str, + messages: Sequence[AllMessageValues], + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + api_key: str | None = None, + api_base: str | None = None, + ) -> dict[str, str]: # mutable-ok: BaseLLMModelInfo.validate_environment returns a dict + resolved_key: Final = self.get_api_key(api_key) + if resolved_key is None: + return {**headers} + return {**headers, "Authorization": f"Bearer {resolved_key}"} + + def get_models( + self, api_key: str | None = None, api_base: str | None = None + ) -> list[str]: # mutable-ok: matches get_models' list[str] contract shared by every provider override + resolved_key: Final = self.get_api_key(api_key) + if resolved_key is None: + raise ValueError("Set OPENROUTER_API_KEY or the deployment's api_key to list OpenRouter models") + headers: Final = self.validate_environment( + headers={}, model="", messages=(), optional_params={}, litellm_params={}, api_key=resolved_key + ) + response: Final = litellm.module_level_client.get( # pyright: ignore[reportUnknownMemberType] # legacy HTTP wrapper has untyped query parameters + url=_chat_catalog_url(self.get_api_base(api_base)), headers=headers, timeout=_CATALOG_TIMEOUT + ) + response.raise_for_status() + return [ + f"openrouter/{model.id}" + for model in _Catalog.model_validate_json(response.content).data + if not model.id.endswith(_BATCH_ONLY_SUFFIX) + ] diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index b21315529eb..cfc0d2750bd 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -1,12 +1,15 @@ # What is this? ## Common checks for /v1/models and `/model/info` +import asyncio +import contextvars import copy import functools import itertools import re from collections.abc import Callable, Mapping, Sequence +from concurrent.futures import ThreadPoolExecutor from types import MappingProxyType -from typing import Final +from typing import Final, TypeVar import litellm from litellm._logging import verbose_proxy_logger @@ -30,6 +33,17 @@ _CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields) _CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields) +_MODEL_LISTING_POOL: Final = ThreadPoolExecutor(thread_name_prefix="litellm-model-listing") + +_PROVIDERS_LISTED_ONLY_THROUGH_A_DEPLOYMENT: Final = frozenset({LlmProviders.OPENROUTER.value}) + +_Listed: Final = TypeVar("_Listed") + + +async def run_model_listing(listing: Callable[[], _Listed]) -> _Listed: + context: Final = contextvars.copy_context() + return await asyncio.get_running_loop().run_in_executor(_MODEL_LISTING_POOL, context.run, listing) + def _check_wildcard_routing(model: str) -> bool: """ @@ -209,6 +223,7 @@ def get_complete_model_list( include_model_access_groups: bool | None = False, only_model_access_groups: bool | None = False, team_id: str | None = None, + expand_wildcard_routes: bool = True, ) -> list[str]: """Logic for returning complete model list for a given key + team pair""" @@ -254,11 +269,10 @@ def get_complete_model_list( return_wildcard_routes=return_wildcard_routes, llm_router=llm_router, team_id=team_id, + expand_wildcard_routes=expand_wildcard_routes, ) - complete_model_list: Final = unique_models + all_wildcard_models - - return complete_model_list + return list(dict.fromkeys(unique_models + all_wildcard_models)) def _hydrate_litellm_credential_name( @@ -425,6 +439,15 @@ def _cached_wildcard_routing(llm_router: Router | None) -> Callable[[str, str | return functools.cache(wildcard_routed_models) +def _deployed_by_name(llm_router: Router | None) -> Callable[[str, str | None], bool]: + if llm_router is None: + return lambda _model, _team_id: False + registry: Final = llm_router.registry() + deployed_names: Final = frozenset(registry.names(kind="group_set")) + team_deployed_names: Final = frozenset(registry.index("team_model")) + return lambda model, team_id: model in deployed_names or (team_id, model) in team_deployed_names + + def _routes_to_model( routed_models: Callable[[str, str | None], frozenset[str]] | None, team_id: str | None ) -> Callable[[str, str], bool]: @@ -455,7 +478,11 @@ def get_known_models_from_wildcard( wildcard_provider_prefix, wildcard_suffix = wildcard_model_to_expand.split("/", 1) except ValueError: # safely fail return [] - deployment_pattern_naming_listed_ids: Final = _deployment_pattern_naming_listed_ids(wildcard_suffix, litellm_params) + deployment_pattern_naming_listed_ids: Final = ( + wildcard_model_to_expand + if wildcard_model_to_expand != wildcard_model + else _deployment_pattern_naming_listed_ids(wildcard_suffix, litellm_params) + ) # Use provider from litellm_params when available, otherwise from wildcard prefix # (e.g., "openai" from "openai/*" - needed for BYOK where wildcard isn't in router) @@ -514,6 +541,27 @@ def _under_wildcard_prefix(model: str, wildcard_provider_prefix: str, listed_pro return f"{wildcard_provider_prefix}/{model_suffix if leading_is_listed_providers_own_prefix else model}" +@functools.lru_cache(maxsize=256) +def _wildcard_name_regex(name_pattern: str) -> re.Pattern[str]: + return re.compile(PatternMatchRouter().pattern_to_regex(name_pattern)) + + +def _routed_deployment_model(name_pattern: str, listed_name: str, deployment_model: str) -> str: + name_match: Final = _wildcard_name_regex(name_pattern).match(listed_name) + if name_match is None: + return listed_name + return PatternMatchRouter.set_deployment_model_name( + matched_pattern=name_match, litellm_deployment_litellm_model=deployment_model + ) + + +def _names_no_deployment_serves_by_name( + deployment: Mapping[str, object], names: Sequence[str], deployed_by_name: Callable[[str, str | None], bool] +) -> tuple[str, ...]: + team_id: Final = _deployment_team_id(deployment) + return tuple(name for name in names if not deployed_by_name(name, team_id)) + + def expand_wildcard_deployments_for_model_info( deployments: list[dict[str, object]], llm_router: Router | None = None, @@ -527,6 +575,7 @@ def expand_wildcard_deployments_for_model_info( """ expanded: Final[list[dict[str, object]]] = [] routed_models: Final = _cached_wildcard_routing(llm_router) + deployed_by_name: Final = _deployed_by_name(llm_router) for deployment in deployments: model_name = str(deployment.get("model_name") or "") raw_params = deployment.get("litellm_params") @@ -560,22 +609,29 @@ def expand_wildcard_deployments_for_model_info( expanded.append(deployment) continue - for name in expanded_names: + for name in _names_no_deployment_serves_by_name(deployment, expanded_names, deployed_by_name): row = copy.deepcopy(deployment) row["model_name"] = name params = row.get("litellm_params") if isinstance(params, dict): - params["model"] = name + params["model"] = _routed_deployment_model(model_name, name, litellm_model) expanded.append(row) return expanded +def _known_models_without_a_deployment(wildcard_model: str) -> Sequence[str]: + if wildcard_model.partition("/")[0] in _PROVIDERS_LISTED_ONLY_THROUGH_A_DEPLOYMENT: + return () + return get_known_models_from_wildcard(wildcard_model=wildcard_model, litellm_params=None) + + def _get_wildcard_models( unique_models: list[str], return_wildcard_routes: bool | None = False, llm_router: Router | None = None, team_id: str | None = None, + expand_wildcard_routes: bool = True, ) -> list[str]: models_to_remove: Final = set() routes_to_model: Final = _routes_to_model(_cached_wildcard_routing(llm_router), team_id) @@ -586,6 +642,8 @@ def _get_wildcard_models( all_wildcard_models.append(model) models_to_remove.add(model) + if not expand_wildcard_routes: + continue model_list = llm_router.registry().deployments(model, team_id=team_id) if llm_router else None if model_list: @@ -598,7 +656,7 @@ def _get_wildcard_models( ) ) else: - all_wildcard_models.extend(get_known_models_from_wildcard(wildcard_model=model, litellm_params=None)) + all_wildcard_models.extend(_known_models_without_a_deployment(model)) for model in models_to_remove: unique_models.remove(model) diff --git a/litellm/proxy/openai_files_endpoints/common_utils.py b/litellm/proxy/openai_files_endpoints/common_utils.py index 907c2ca303e..c0d81abeba7 100644 --- a/litellm/proxy/openai_files_endpoints/common_utils.py +++ b/litellm/proxy/openai_files_endpoints/common_utils.py @@ -1,4 +1,5 @@ import base64 +import itertools import logging import mimetypes import re @@ -541,7 +542,8 @@ def get_team_provider_credentials( listings on the team's own provider account/key instead of a shared global one. 2. Fallback: any deployment the caller is granted access to for this - provider, expanding wildcard routes and the all-proxy-models sentinel. + provider, matching wildcard grants against deployed model names without + listing any provider's catalog, and expanding the all-proxy-models sentinel. Skipped when ``include_accessible_deployments`` is False. Credential lookup is scoped to both the team's allowlist and the key's own @@ -559,6 +561,7 @@ def get_team_provider_credentials( return None from litellm.proxy._types import SpecialModelNames + from litellm.proxy.auth.auth_checks import is_model_allowed_by_pattern from litellm.proxy.auth.model_checks import get_complete_model_list, get_key_models team_id: Final = user_api_key_dict.team_id @@ -645,28 +648,36 @@ def get_team_provider_credentials( grants_all_models: Final = SpecialModelNames.all_proxy_models.value in team_models effective_team_models: Final = [] if grants_all_models else team_models - models_to_try: Final = list( - dict.fromkeys( - get_complete_model_list( - key_models=list(key_model_allowlist), - team_models=effective_team_models, - proxy_model_list=proxy_model_list, - user_model=None, - infer_model_from_keys=False, - return_wildcard_routes=True, - llm_router=llm_router, - model_access_groups=model_access_groups, - include_model_access_groups=True, - team_id=team_id, - ) + granted_models: Final = get_complete_model_list( + key_models=list(key_model_allowlist), + team_models=effective_team_models, + proxy_model_list=proxy_model_list, + user_model=None, + infer_model_from_keys=False, + return_wildcard_routes=True, + llm_router=llm_router, + model_access_groups=model_access_groups, + include_model_access_groups=True, + team_id=team_id, + expand_wildcard_routes=False, + ) + deployments_granted_by_wildcard: Final = ( + model_name + for model_name in proxy_model_list + if any( + is_model_allowed_by_pattern(model=model_name, allowed_model_pattern=granted) for granted in granted_models ) ) - for model_name in models_to_try: - credentials = _provider_credentials(model_name) - if credentials is not None: - return credentials - - return None + return next( + ( + credentials + for credentials in map( + _provider_credentials, dict.fromkeys(itertools.chain(granted_models, deployments_granted_by_wildcard)) + ) + if credentials is not None + ), + None, + ) HTTP_HEADER_VALUE: Final = re.compile(r"[\x21-\x7e]+(?:[ \t]+[\x21-\x7e]+)*") diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index f078267c6b2..7ea1c246da5 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -199,6 +199,7 @@ from litellm.types.caching import RedisPipelineIncrementOperation from litellm.types.utils import ( PRICING_OVERRIDES_KEY, CountContextManagement, + LlmProviders, ModelResponse, ModelResponseStream, TextCompletionResponse, @@ -443,6 +444,7 @@ from litellm.proxy.auth.model_checks import ( get_key_models, get_mcp_server_ids, get_team_models, + run_model_listing, ) from litellm.proxy.auth.password_policy import validate_password_not_breached, validate_password_policy from litellm.proxy.auth.user_api_key_auth import ( # noqa: F401, RUF100 # legacy module exports @@ -12487,20 +12489,23 @@ async def model_list( # Get complete model list including wildcard routes if requested from litellm.proxy.auth.model_checks import get_complete_model_list - all_models = get_complete_model_list( - key_models=[], - team_models=[], - proxy_model_list=[ - *proxy_model_list, - *(typesafe_allowlist_model_names() if asks_for_allowlist_names else ()), - ], - user_model=None, - infer_model_from_keys=False, - return_wildcard_routes=include_wildcard_routes, - llm_router=llm_router, - model_access_groups=model_access_groups, - include_model_access_groups=include_model_access_groups or False, - only_model_access_groups=only_model_access_groups or False, + all_models = await run_model_listing( + partial( + get_complete_model_list, + key_models=[], + team_models=[], + proxy_model_list=[ + *proxy_model_list, + *(typesafe_allowlist_model_names() if asks_for_allowlist_names else ()), + ], + user_model=None, + infer_model_from_keys=False, + return_wildcard_routes=include_wildcard_routes, + llm_router=llm_router, + model_access_groups=model_access_groups, + include_model_access_groups=include_model_access_groups or False, + only_model_access_groups=only_model_access_groups or False, + ) ) expanded_undiscoverable_names: Final = undiscoverable_model_names( @@ -14747,6 +14752,8 @@ from litellm.responses.litellm_completion_transformation.count_request import ( with_provider_file_ids, ) +_PROVIDERS_WITH_LISTING_ONLY_MODEL_INFO: Final = frozenset({LlmProviders.OPENROUTER}) + def _with_client_oauth_token( counter: BaseTokenCounter | None, client_oauth_token: str | None @@ -14782,14 +14789,13 @@ def _get_provider_token_counter( ) # Switch case pattern using existing get_provider_model_info - from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager # Convert string provider to LlmProviders enum llm_provider_enum: Final = LlmProviders(custom_llm_provider) # Add more provider mappings as needed - if llm_provider_enum: + if llm_provider_enum not in _PROVIDERS_WITH_LISTING_ONLY_MODEL_INFO: provider_model_info: Final = ProviderConfigManager.get_provider_model_info( model=full_model, provider=llm_provider_enum ) @@ -17941,7 +17947,7 @@ async def model_info_v1( alias_models: Final = copy.deepcopy(registry.deployments(source="alias")) all_models.extend(alias_models) - all_models = expand_wildcard_deployments_for_model_info(all_models, llm_router) + all_models = await run_model_listing(partial(expand_wildcard_deployments_for_model_info, all_models, llm_router)) allowed_model_names: Final = await _get_v1_model_info_allowed_model_names_with_access_groups( user_api_key_dict=user_api_key_dict, @@ -18279,14 +18285,17 @@ async def model_group_info( LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, ) all_models_str: Final = ( - get_complete_model_list( - key_models=(), - team_models=(), - proxy_model_list=llm_router.registry().names(), - user_model=user_model, - infer_model_from_keys=general_settings.get("infer_model_from_keys", False), - return_wildcard_routes=False, - llm_router=llm_router, + await run_model_listing( + partial( + get_complete_model_list, + key_models=(), + team_models=(), + proxy_model_list=llm_router.registry().names(), + user_model=user_model, + infer_model_from_keys=general_settings.get("infer_model_from_keys", False), + return_wildcard_routes=False, + llm_router=llm_router, + ) ) if is_proxy_admin else await get_available_models_for_user( diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index ddaa8d2e69d..00dbcd0f9a8 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -9500,7 +9500,7 @@ async def _agent_access_group_visible_models( ) -> frozenset[str] | None: """Models an agent key may still list once its attached access groups cap it, ``None`` when nothing caps it, so ``/v1/models`` never advertises a model the same key would be denied on.""" - from litellm.proxy.auth.model_checks import get_complete_model_list, get_team_models + from litellm.proxy.auth.model_checks import get_complete_model_list, get_team_models, run_model_listing if not user_api_key_dict.agent_id: return None @@ -9521,17 +9521,20 @@ async def _agent_access_group_visible_models( if not granted: return frozenset() return frozenset( - get_complete_model_list( - key_models=granted, - team_models=(), - proxy_model_list=proxy_model_list, - user_model=None, - infer_model_from_keys=False, - return_wildcard_routes=return_wildcard_routes, - llm_router=llm_router, - model_access_groups=model_access_groups, - include_model_access_groups=include_model_access_groups, - team_id=team_id, + await run_model_listing( + partial( + get_complete_model_list, + key_models=granted, + team_models=(), + proxy_model_list=proxy_model_list, + user_model=None, + infer_model_from_keys=False, + return_wildcard_routes=return_wildcard_routes, + llm_router=llm_router, + model_access_groups=model_access_groups, + include_model_access_groups=include_model_access_groups, + team_id=team_id, + ) ) ) @@ -9576,6 +9579,7 @@ async def get_available_models_for_user( get_complete_model_list, get_key_models, get_team_models, + run_model_listing, ) # Get proxy model list and access groups @@ -9656,18 +9660,21 @@ async def get_available_models_for_user( ) # Get complete model list - complete_models: Final = get_complete_model_list( - key_models=granted_key_models, - team_models=granted_team_models, - proxy_model_list=proxy_model_list, - user_model=user_model, - infer_model_from_keys=general_settings.get("infer_model_from_keys", False), - return_wildcard_routes=return_wildcard_routes, - llm_router=llm_router, - model_access_groups=model_access_groups, - include_model_access_groups=include_model_access_groups, - only_model_access_groups=only_model_access_groups, - team_id=effective_team_id, + complete_models: Final = await run_model_listing( + partial( + get_complete_model_list, + key_models=granted_key_models, + team_models=granted_team_models, + proxy_model_list=proxy_model_list, + user_model=user_model, + infer_model_from_keys=general_settings.get("infer_model_from_keys", False), + return_wildcard_routes=return_wildcard_routes, + llm_router=llm_router, + model_access_groups=model_access_groups, + include_model_access_groups=include_model_access_groups, + only_model_access_groups=only_model_access_groups, + team_id=effective_team_id, + ) ) all_models: Final = ( list( diff --git a/litellm/router_utils/pattern_match_deployments.py b/litellm/router_utils/pattern_match_deployments.py index 4dd217cf59b..2b8d1e9c11b 100644 --- a/litellm/router_utils/pattern_match_deployments.py +++ b/litellm/router_utils/pattern_match_deployments.py @@ -279,7 +279,9 @@ class PatternMatchRouter: PatternMatchRouter.set_deployment_model_name( matched_pattern=pattern_match, litellm_deployment_litellm_model=deployment["litellm_params"]["model"] ) - for deployment in self.patterns[pattern] + for deployment in self.patterns.get( # pyright: ignore[reportUnknownMemberType] # untyped deployment dicts + pattern, () + ) ) diff --git a/litellm/utils.py b/litellm/utils.py index 2ca8a49f968..560d0c31e4a 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8724,11 +8724,13 @@ class AvailableModelsCache(InMemoryCache): cache_key: Final = self._get_cache_key(custom_llm_provider, litellm_params) - result: Final = cast(list[str] | None, self.get_cache(cache_key)) + result: Final = self._cached(cache_key) - if result is not None: - return copy.deepcopy(result) - return result + if isinstance(result, Sequence): + return list(result) + if self._cached(_failed_listing_key(cache_key)) is not None: + return [] + return None def set_cached_model_info( self, @@ -8738,12 +8740,46 @@ class AvailableModelsCache(InMemoryCache): ): """Set cached model info""" cache_key: Final = self._get_cache_key(custom_llm_provider, litellm_params) - self.set_cache(cache_key, copy.deepcopy(available_models)) + self._store(cache_key, copy.deepcopy(available_models)) + + def set_failed_model_listing( + self, custom_llm_provider: str, litellm_params: LiteLLM_Params | None, ttl_seconds: float + ) -> None: + cache_key: Final = self._get_cache_key(custom_llm_provider, litellm_params) + self._store(_failed_listing_key(cache_key), True, ttl=ttl_seconds) + + def listing_lock(self, custom_llm_provider: str, litellm_params: LiteLLM_Params | None) -> threading.RLock: + return _provider_listing_lock(self._get_cache_key(custom_llm_provider, litellm_params)) + + def _cached(self, key: str) -> Sequence[str] | bool | None: + return cast(Sequence[str] | bool | None, self.get_cache(key)) + + def _store(self, key: str, value: object, ttl: float | None = None) -> None: + self.set_cache(key, value, ttl=ttl) + + +def _failed_listing_key(cache_key: str) -> str: + return f"{cache_key}:failed" + + +_PROVIDER_LISTING_LOCK_GUARD: Final = threading.Lock() + + +@lru_cache(maxsize=1024) +def _cached_provider_listing_lock(cache_key: str) -> threading.RLock: + return threading.RLock() + + +def _provider_listing_lock(cache_key: str) -> threading.RLock: + with _PROVIDER_LISTING_LOCK_GUARD: + return _cached_provider_listing_lock(cache_key) # Global cache instance _model_cache: Final = AvailableModelsCache() +_FAILED_PROVIDER_LISTING_TTL_SECONDS: Final = 30 + def _infer_valid_provider_from_env_vars( custom_llm_provider: str | None = None, @@ -8791,20 +8827,27 @@ def _get_valid_models_from_provider_api( custom_llm_provider: str, litellm_params: LiteLLM_Params | None = None, ) -> list[str]: - try: - cached_result: Final = _model_cache.get_cached_model_info(custom_llm_provider, litellm_params) + with _model_cache.listing_lock(custom_llm_provider, litellm_params): + try: + cached_result: Final = _model_cache.get_cached_model_info(custom_llm_provider, litellm_params) - if cached_result is not None: - return cached_result - models: Final = provider_config.discover_models( - litellm_params=_provider_listing_params(custom_llm_provider, litellm_params) - ) + if cached_result is not None: + return cached_result + models: Final = provider_config.discover_models( + litellm_params=_provider_listing_params(custom_llm_provider, litellm_params) + ) - _model_cache.set_cached_model_info(custom_llm_provider, litellm_params, models) - return models - except Exception as e: - verbose_logger.warning("Error getting valid models: %s", e) - return [] + _model_cache.set_cached_model_info(custom_llm_provider, litellm_params, models) + return models + except httpx.TimeoutException as e: + verbose_logger.warning("Error getting valid models: %s", e) + _model_cache.set_failed_model_listing( + custom_llm_provider, litellm_params, ttl_seconds=_FAILED_PROVIDER_LISTING_TTL_SECONDS + ) + return [] + except Exception as e: + verbose_logger.warning("Error getting valid models: %s", e) + return [] def bundled_provider_models(provider: str) -> tuple[str, ...]: @@ -10470,6 +10513,10 @@ class ProviderConfigManager: from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo return AzureFoundryModelInfo(model=model) + elif LlmProviders.OPENROUTER == provider: + from litellm.llms.openrouter.common_utils import OpenRouterModelInfo + + return OpenRouterModelInfo() return None @staticmethod diff --git a/tests/unit/llms/openrouter/test_openrouter_common_utils.py b/tests/unit/llms/openrouter/test_openrouter_common_utils.py new file mode 100644 index 00000000000..087c05561b5 --- /dev/null +++ b/tests/unit/llms/openrouter/test_openrouter_common_utils.py @@ -0,0 +1,159 @@ +from typing import Final + +import httpx +import pytest +import respx + +import litellm +from litellm.llms.openrouter.common_utils import OpenRouterModelInfo + +DEFAULT_BASE: Final = "https://openrouter.ai/api/v1" +KEY_SOURCES: Final = ("api_key", "litellm.api_key", "litellm.openrouter_key", "OPENROUTER_API_KEY", "OR_API_KEY") +BASE_SOURCES: Final = ("api_base", "litellm.api_base", "OPENROUTER_API_BASE") + + +@pytest.fixture(autouse=True) +def _no_ambient_openrouter_settings(monkeypatch: pytest.MonkeyPatch): + for env_var in ("OPENROUTER_API_BASE", "OPENROUTER_API_KEY", "OR_API_KEY"): + monkeypatch.delenv(env_var, raising=False) + monkeypatch.setattr(litellm, "api_key", None) + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.setattr(litellm, "openrouter_key", None) + + +def _catalog(*model_ids: str) -> httpx.Response: + return httpx.Response(200, json={"data": [{"id": model_id, "name": model_id} for model_id in model_ids]}) + + +def _configure(monkeypatch: pytest.MonkeyPatch, source: str, value: str) -> None: + if source.startswith("litellm."): + monkeypatch.setattr(litellm, source.removeprefix("litellm."), value) + else: + monkeypatch.setenv(source, value) + + +def test_get_models_lists_only_the_chat_catalog_under_openrouter_keeping_the_org_namespace(): + with respx.mock() as respx_mock: + respx_mock.get(f"{DEFAULT_BASE}/models/user").mock( + return_value=_catalog("openai/gpt-5.4-mini", "anthropic/claude-haiku-5-5") + ) + + assert OpenRouterModelInfo().get_models(api_key="sk-or-test") == [ + "openrouter/openai/gpt-5.4-mini", + "openrouter/anthropic/claude-haiku-5-5", + ] + assert [call.request.url.path for call in respx_mock.calls] == ["/api/v1/models/user"] + + +@respx.mock +def test_get_models_drops_batch_only_ids_and_keeps_every_other_variant(): + respx.get(f"{DEFAULT_BASE}/models/user").mock( + return_value=_catalog("openai/gpt-5.4-mini:batch", "openai/gpt-5.4-mini", "qwen/qwen3-coder:free") + ) + + assert OpenRouterModelInfo().get_models(api_key="sk-or-test") == [ + "openrouter/openai/gpt-5.4-mini", + "openrouter/qwen/qwen3-coder:free", + ] + + +@pytest.mark.parametrize( + "api_base", + [ + "https://gateway.example/openrouter/api/v1/", + "https://gateway.example/openrouter/api/v1/chat/completions", + "https://gateway.example/openrouter/api/v1/chat/completions/", + ], +) +@respx.mock +def test_get_models_keeps_the_api_base_path_and_sends_the_deployment_key(api_base: str): + chat_route: Final = respx.get("https://gateway.example/openrouter/api/v1/models/user").mock( + return_value=_catalog("openai/gpt-5.4-mini") + ) + + assert OpenRouterModelInfo().get_models(api_key="sk-or-deployment", api_base=api_base) == [ + "openrouter/openai/gpt-5.4-mini" + ] + assert chat_route.calls.last.request.headers["authorization"] == "Bearer sk-or-deployment" + + +@pytest.mark.parametrize("winner", range(len(KEY_SOURCES)), ids=KEY_SOURCES) +@respx.mock +def test_get_models_sends_the_first_configured_key_in_the_chat_paths_order( + monkeypatch: pytest.MonkeyPatch, winner: int +): + for source in KEY_SOURCES[winner + 1 :]: + _configure(monkeypatch, source, f"sk-or-from-{source}") + if winner > 0: + _configure(monkeypatch, KEY_SOURCES[winner], "sk-or-winner") + chat_route: Final = respx.get(f"{DEFAULT_BASE}/models/user").mock(return_value=_catalog("qwen/qwen3-coder")) + + assert OpenRouterModelInfo().get_models(api_key="sk-or-winner" if winner == 0 else None) == [ + "openrouter/qwen/qwen3-coder" + ] + assert chat_route.calls.last.request.headers["authorization"] == "Bearer sk-or-winner" + + +@pytest.mark.parametrize("winner", range(len(BASE_SOURCES) + 1), ids=(*BASE_SOURCES, "default")) +@respx.mock +def test_get_models_calls_the_first_configured_base_in_the_chat_paths_order( + monkeypatch: pytest.MonkeyPatch, winner: int +): + for source in BASE_SOURCES[winner + 1 :]: + _configure(monkeypatch, source, f"https://{source.lower().replace('.', '-')}.example/api/v1") + if 0 < winner < len(BASE_SOURCES): + _configure(monkeypatch, BASE_SOURCES[winner], "https://winner.example/api/v1") + expected_base: Final = "https://winner.example/api/v1" if winner < len(BASE_SOURCES) else DEFAULT_BASE + respx.get(f"{expected_base}/models/user").mock(return_value=_catalog("qwen/qwen3-coder")) + + assert OpenRouterModelInfo().get_models( + api_key="sk-or-test", api_base="https://winner.example/api/v1" if winner == 0 else None + ) == ["openrouter/qwen/qwen3-coder"] + + +def test_get_models_without_any_key_raises_before_calling_openrouter(): + with respx.mock(assert_all_called=False) as respx_mock: + chat_route: Final = respx_mock.get(f"{DEFAULT_BASE}/models/user").mock( + return_value=_catalog("openai/gpt-5.4-mini") + ) + + with pytest.raises(ValueError, match="OPENROUTER_API_KEY"): + OpenRouterModelInfo().get_models() + + assert not chat_route.called + + +@respx.mock +def test_get_models_raises_when_the_catalog_answers_an_error(): + respx.get(f"{DEFAULT_BASE}/models/user").mock(return_value=httpx.Response(401, json={"error": {"code": 401}})) + + with pytest.raises(httpx.HTTPStatusError): + OpenRouterModelInfo().get_models(api_key="sk-or-revoked") + + +def test_validate_environment_adds_the_resolved_key_and_keeps_the_callers_headers(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("OPENROUTER_API_KEY", "sk-or-env") + headers: Final = {"X-Title": "my-app"} + + assert OpenRouterModelInfo().validate_environment( + headers=headers, model="openrouter/openai/gpt-5.4-mini", messages=(), optional_params={}, litellm_params={} + ) == {"X-Title": "my-app", "Authorization": "Bearer sk-or-env"} + assert headers == {"X-Title": "my-app"} + + +def test_validate_environment_without_any_key_sends_no_authorization(): + assert OpenRouterModelInfo().validate_environment( + headers={"X-Title": "my-app"}, + model="openrouter/openai/gpt-5.4-mini", + messages=(), + optional_params={}, + litellm_params={}, + ) == {"X-Title": "my-app"} + + +@pytest.mark.parametrize( + ("model", "base_model"), + [("openrouter/openai/gpt-5.4-mini", "openai/gpt-5.4-mini"), ("openai/gpt-5.4-mini", "openai/gpt-5.4-mini")], +) +def test_get_base_model_drops_only_the_openrouter_prefix(model: str, base_model: str): + assert OpenRouterModelInfo.get_base_model(model) == base_model diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index c67610c24a1..d0156ff0195 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -7,6 +7,8 @@ import httpx import pytest import respx +from litellm import Router + def test_get_team_models_for_all_models_and_team_only_models(): from litellm.proxy.auth.model_checks import get_team_models @@ -1261,6 +1263,223 @@ def test_partial_bedrock_wildcard_lists_nothing_when_no_discovered_id_carries_it assert get_known_models_from_wildcard("bedrock/anthropic.*", deployment) == [] +@pytest.mark.parametrize( + ("wildcard", "deployment_model", "listed_ids"), + [ + ( + "openrouter/*", + "openrouter/*", + ["openrouter/openai/gpt-5.4-mini", "openrouter/anthropic/claude-haiku-5-5"], + ), + ("or-team/*", "openrouter/*", ["or-team/openai/gpt-5.4-mini", "or-team/anthropic/claude-haiku-5-5"]), + ("openrouter/openai/*", "openrouter/openai/*", ["openrouter/openai/gpt-5.4-mini"]), + ("*", "openrouter/*", ["openai/gpt-5.4-mini", "anthropic/claude-haiku-5-5"]), + ], +) +@respx.mock +def test_openrouter_wildcard_lists_the_live_catalog_with_each_vendor_namespace( + monkeypatch: pytest.MonkeyPatch, wildcard: str, deployment_model: str, listed_ids: list[str] +) -> None: + import litellm + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + monkeypatch.delenv("OPENROUTER_API_BASE", raising=False) + respx.get("https://openrouter.ai/api/v1/models/user").mock( + return_value=httpx.Response( + 200, json={"data": [{"id": "openai/gpt-5.4-mini"}, {"id": "anthropic/claude-haiku-5-5"}]} + ) + ) + + assert ( + get_known_models_from_wildcard(wildcard, LiteLLM_Params(model=deployment_model, api_key=f"sk-or-{wildcard}")) + == listed_ids + ) + + +@respx.mock +def test_key_granted_openrouter_wildcard_without_an_openrouter_deployment_lists_no_environment_catalog( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm + from litellm import Router + from litellm.proxy.auth.model_checks import get_complete_model_list + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + monkeypatch.setenv("OPENROUTER_API_KEY", "sk-or-environment-only") + catalog: Final = respx.get("https://openrouter.ai/api/v1/models/user").mock( + return_value=httpx.Response(200, json={"data": [{"id": "anthropic/claude-haiku-5-5"}]}) + ) + router: Final = Router( + model_list=[{"model_name": "or-team/*", "litellm_params": {"model": "openrouter/*", "api_key": "sk-or-team"}}] + ) + + listed: Final = get_complete_model_list( + key_models=["openrouter/*"], + team_models=[], + proxy_model_list=router.model_names, + user_model=None, + infer_model_from_keys=False, + llm_router=router, + ) + + assert (listed, catalog.call_count) == ([], 0) + + +@contextmanager +def _openai_catalog_behind_a_catch_all( + monkeypatch: pytest.MonkeyPatch, concrete_name: str, concrete_model_info: Mapping[str, str] +) -> Generator[Router, None, None]: + import litellm + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + with respx.mock(assert_all_called=False) as respx_mock: + respx_mock.get("https://api.openai.com/v1/models").mock( + return_value=httpx.Response(200, json={"data": [{"id": "gpt-4o"}, {"id": "gpt-4o-mini"}]}) + ) + yield Router( + model_list=[ + {"model_name": "*", "litellm_params": {"model": "openai/*", "api_key": "sk-openai-catch-all"}}, + { + "model_name": concrete_name, + "litellm_params": {"model": "openrouter/openai/gpt-4o", "api_key": "sk-or-gpt-4o"}, + "model_info": dict(concrete_model_info), + }, + ] + ) + + +@pytest.mark.parametrize( + ("concrete_name", "concrete_model_info", "team_id"), + [ + ("gpt-4o", {}, None), + ("model_name_team-x_gpt-4o", {"team_id": "team-x", "team_public_model_name": "gpt-4o"}, "team-x"), + ], +) +def test_catch_all_wildcard_lists_a_name_a_concrete_deployment_serves_once( + monkeypatch: pytest.MonkeyPatch, concrete_name: str, concrete_model_info: Mapping[str, str], team_id: str | None +) -> None: + from litellm.proxy.auth.model_checks import get_complete_model_list + + with _openai_catalog_behind_a_catch_all(monkeypatch, concrete_name, concrete_model_info) as router: + listed: Final = get_complete_model_list( + key_models=[], + team_models=[], + proxy_model_list=router.get_model_names(team_id=team_id), + user_model=None, + infer_model_from_keys=False, + llm_router=router, + team_id=team_id, + ) + + assert sorted(listed) == ["gpt-4o", "gpt-4o-mini"] + + +def test_catch_all_wildcard_model_info_keeps_only_the_concrete_row_for_a_deployed_name( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info + + with _openai_catalog_behind_a_catch_all(monkeypatch, "gpt-4o", {}) as router: + rows: Final = expand_wildcard_deployments_for_model_info( + [dict(deployment) for deployment in router.get_model_list() or ()], llm_router=router + ) + + assert sorted((row["model_name"], row["litellm_params"]["model"]) for row in rows) == [ + ("gpt-4o", "openrouter/openai/gpt-4o"), + ("gpt-4o-mini", "openai/gpt-4o-mini"), + ] + + +@contextmanager +def _openrouter_catalog_beside_a_named_deployment( + monkeypatch: pytest.MonkeyPatch, + wildcard_name: str, + named: str, + catalog: tuple[str, ...] = ("openai/gpt-5.4-mini", "anthropic/claude-haiku-5-5"), + wildcard_key: str | None = None, +) -> Generator[Router, None, None]: + import litellm + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + monkeypatch.delenv("OPENROUTER_API_BASE", raising=False) + with respx.mock(assert_all_called=False) as respx_mock: + respx_mock.get("https://openrouter.ai/api/v1/models/user").mock( + return_value=httpx.Response(200, json={"data": [{"id": catalog_id} for catalog_id in catalog]}) + ) + yield Router( + model_list=[ + { + "model_name": wildcard_name, + "litellm_params": { + "model": "openrouter/*", + "api_key": wildcard_key or f"sk-or-wildcard-{wildcard_name}", + }, + }, + { + "model_name": named, + "litellm_params": {"model": "openrouter/openai/gpt-5.4-mini", "api_key": "sk-or-named"}, + }, + ] + ) + + +@pytest.mark.parametrize( + ("wildcard_name", "named"), + [("openrouter/*", "openrouter/openai/gpt-5.4-mini"), ("or-team/*", "or-team/openai/gpt-5.4-mini")], +) +def test_provider_wildcard_lists_a_name_a_named_deployment_serves_once( + monkeypatch: pytest.MonkeyPatch, wildcard_name: str, named: str +) -> None: + from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info, get_complete_model_list + + with _openrouter_catalog_beside_a_named_deployment(monkeypatch, wildcard_name, named) as router: + listed: Final = get_complete_model_list( + key_models=[], + team_models=[], + proxy_model_list=list(router.model_names), + user_model=None, + infer_model_from_keys=False, + llm_router=router, + ) + rows: Final = expand_wildcard_deployments_for_model_info( + [dict(deployment) for deployment in router.get_model_list() or ()], llm_router=router + ) + + vendor_prefix: Final = wildcard_name.removesuffix("*") + assert sorted(listed) == [f"{vendor_prefix}anthropic/claude-haiku-5-5", named] + assert sorted((row["model_name"], row["litellm_params"]["api_key"]) for row in rows) == [ + (f"{vendor_prefix}anthropic/claude-haiku-5-5", f"sk-or-wildcard-{wildcard_name}"), + (named, "sk-or-named"), + ] + + +@pytest.mark.parametrize( + ("wildcard_name", "named"), + [("openrouter/*", "openrouter/openai/gpt-5.4-mini"), ("or-team/*", "or-team/openai/gpt-5.4-mini")], +) +def test_provider_wildcard_whose_whole_catalog_is_deployed_by_name_shows_only_the_named_row( + monkeypatch: pytest.MonkeyPatch, wildcard_name: str, named: str +) -> None: + from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info + + with _openrouter_catalog_beside_a_named_deployment( + monkeypatch, + wildcard_name, + named, + catalog=("openai/gpt-5.4-mini",), + wildcard_key=f"sk-or-one-model-{wildcard_name}", + ) as router: + rows: Final = expand_wildcard_deployments_for_model_info( + [dict(deployment) for deployment in router.get_model_list() or ()], llm_router=router + ) + + assert [(row["model_name"], row["litellm_params"]["api_key"]) for row in rows] == [(named, "sk-or-named")] + + @respx.mock def test_custom_prefix_that_starts_a_discovered_id_still_prefixes_every_listed_model( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py b/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py index 82523572b28..238f1fa6495 100644 --- a/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py +++ b/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py @@ -3,10 +3,13 @@ from types import MappingProxyType from typing import Final from unittest.mock import AsyncMock, MagicMock +import httpx import pytest +import respx from fastapi import HTTPException from litellm_enterprise.proxy.hooks.managed_files import _PROXY_LiteLLMManagedFiles +import litellm from litellm import Router from litellm.caching.caching import DualCache from litellm.proxy._types import UserAPIKeyAuth @@ -852,3 +855,112 @@ def test_a_team_keeps_its_first_config_file_openai_deployment_over_older_db_ones assert credentials is not None assert credentials["api_key"] == "deployment-z-key" + + +@pytest.mark.parametrize( + ("custom_llm_provider", "openrouter_key", "api_key"), + [("openai", "sk-or-files-openai", "openai-key"), ("openrouter", "sk-or-files-openrouter", "sk-or-files-openrouter")], +) +def test_a_provider_credential_lookup_never_lists_an_openrouter_wildcards_live_catalog( + monkeypatch: pytest.MonkeyPatch, custom_llm_provider: str, openrouter_key: str, api_key: str +): + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + router: Final = Router( + model_list=[ + { + "model_name": "openrouter/*", + "litellm_params": { + "model": "openrouter/*", + "api_key": openrouter_key, + "api_base": "https://openrouter.ai/api/v1", + }, + }, + {"model_name": "gpt-5.4-mini", "litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": "openai-key"}}, + ] + ) + + with respx.mock(assert_all_called=False) as respx_mock: + catalog_route: Final = respx_mock.get("https://openrouter.ai/api/v1/models/user").mock( + return_value=httpx.Response(200, json={"data": [{"id": "openai/gpt-5.4-mini"}]}) + ) + credentials: Final = get_team_provider_credentials( + llm_router=router, + user_api_key_dict=UserAPIKeyAuth(api_key="test-key", user_id="test-user"), + custom_llm_provider=custom_llm_provider, + include_accessible_deployments=True, + ) + + assert credentials is not None + assert (credentials["custom_llm_provider"], credentials["api_key"]) == (custom_llm_provider, api_key) + assert catalog_route.call_count == 0 + + +@pytest.mark.parametrize( + ("key_models", "deployments", "custom_llm_provider", "api_key"), + [ + ( + ["anthropic/*"], + (("anthropic/claude-*", "anthropic/claude-*", "anthropic-narrow-key"),), + "anthropic", + "anthropic-narrow-key", + ), + (["anthropic/claude-3*"], (("anthropic/*", "anthropic/*", "anthropic-wide-key"),), "anthropic", "anthropic-wide-key"), + (["openrouter/*"], (("openrouter/openai/*", "openrouter/openai/*", "sk-or-narrow"),), "openrouter", "sk-or-narrow"), + ( + ["team/*"], + (("team/*", "openrouter/*", "sk-or-team"), ("team/openai/*", "openai/*", "openai-team-key")), + "openai", + "openai-team-key", + ), + ], +) +def test_a_key_wildcard_finds_the_deployment_it_grants_without_listing_any_provider_catalog( + monkeypatch: pytest.MonkeyPatch, + key_models: list[str], + deployments: tuple[tuple[str, str, str], ...], + custom_llm_provider: str, + api_key: str, +): + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + router: Final = Router( + model_list=[ + {"model_name": model_name, "litellm_params": {"model": model, "api_key": deployment_key}} + for model_name, model, deployment_key in deployments + ] + ) + + with respx.mock(assert_all_called=False) as respx_mock: + catalog_requests: Final = respx_mock.route().mock(return_value=httpx.Response(200, json={"data": []})) + credentials: Final = get_team_provider_credentials( + llm_router=router, + user_api_key_dict=UserAPIKeyAuth(api_key="test-key", user_id="test-user", models=key_models), + custom_llm_provider=custom_llm_provider, + include_accessible_deployments=True, + ) + + assert credentials is not None + assert (credentials["custom_llm_provider"], credentials["api_key"]) == (custom_llm_provider, api_key) + assert catalog_requests.call_count == 0 + + +@pytest.mark.parametrize( + ("deployed_name", "api_key"), [("gpt-4o-mini", None), ("openai/gpt-4o-mini", "openai-deployment-key")] +) +def test_a_provider_wildcard_grant_finds_only_deployments_named_under_it(deployed_name: str, api_key: str | None): + router: Final = Router( + model_list=[ + { + "model_name": deployed_name, + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "openai-deployment-key"}, + } + ] + ) + + credentials: Final = get_team_provider_credentials( + llm_router=router, + user_api_key_dict=UserAPIKeyAuth(api_key="test-key", user_id="test-user", models=["openai/*"]), + custom_llm_provider="openai", + include_accessible_deployments=True, + ) + + assert (None if credentials is None else credentials["api_key"]) == api_key diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py index 86364d93ed0..bffa4612964 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_info.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py @@ -550,15 +550,12 @@ async def test_v1_model_info_star_wildcard_filter_keeps_provider_expansion(monke deployment = { "model_name": "*", - "litellm_params": {"model": "openai/*"}, + "litellm_params": {"model": "openai/*", "api_key": "sk-star"}, } - router = route_entry_points_to_stubs(MagicMock()) - router.get_model_access_groups = MagicMock(return_value={}) - router.get_model_names = MagicMock(return_value=["*"]) - router.get_model_list = MagicMock(return_value=[deployment]) + router = litellm.Router(model_list=[deployment]) monkeypatch.setattr(model_checks, "get_provider_models", fake_get_provider_models) - expanded_deployments = proxy_server.expand_wildcard_deployments_for_model_info([deployment]) + expanded_deployments = proxy_server.expand_wildcard_deployments_for_model_info(router.get_model_list(), router) allowed_model_names = await proxy_server._get_v1_model_info_allowed_model_names_with_access_groups( user_api_key_dict=UserAPIKeyAuth( api_key="sk-test", @@ -575,7 +572,8 @@ async def test_v1_model_info_star_wildcard_filter_keeps_provider_expansion(monke allowed_model_names=allowed_model_names, ) - assert [model["model_name"] for model in result] == ["openai/gpt-4o"] + assert [(model["model_name"], model["litellm_params"]["model"]) for model in result] == [("gpt-4o", "openai/gpt-4o")] + assert router.registry().names("gpt-4o", kind="wildcard") == frozenset({"openai/gpt-4o"}) # --------------------------------------------------------------------------- diff --git a/tests/unit/proxy/proxy_server/test_routes_models.py b/tests/unit/proxy/proxy_server/test_routes_models.py index ffd23fa3a6f..874f471f3e7 100644 --- a/tests/unit/proxy/proxy_server/test_routes_models.py +++ b/tests/unit/proxy/proxy_server/test_routes_models.py @@ -9,10 +9,16 @@ Pins (PR2): from __future__ import annotations +import asyncio +import threading +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass, field from typing import Final from unittest.mock import MagicMock +import httpx import pytest +import respx import litellm from litellm import Router @@ -541,6 +547,73 @@ def test_codex_service_tiers_resolved_via_internal_team_key(client, auth_as, pat assert (entry["slug"], [tier["id"] for tier in entry["service_tiers"]]) == ("gpt-4-team", ["ultrafast"]) +OPENROUTER_BASE: Final = "https://openrouter.ai/api/v1" + + +@dataclass(frozen=True, slots=True) +class _CatalogHeldUntilTheLoopRuns: + requested: threading.Event = field(default_factory=threading.Event) + released: threading.Event = field(default_factory=threading.Event) + + def answer(self, _request: httpx.Request) -> httpx.Response: + self.requested.set() + if not self.released.wait(timeout=30): + return httpx.Response(504) + return httpx.Response(200, json={"data": [{"id": "openai/gpt-5.4-mini"}]}) + + async def release_from_the_loop(self) -> None: + assert await asyncio.to_thread(self.requested.wait, 30) + self.released.set() + + +@pytest.mark.parametrize("scope", [None, "expand"]) +def test_v1_models_fetches_a_live_openrouter_catalog_without_holding_the_event_loop_or_its_default_thread_pool( + monkeypatch: pytest.MonkeyPatch, scope: str | None +): + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setattr(proxy_server, "user_model", None) + monkeypatch.setattr(proxy_server, "prisma_client", None) + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "openrouter/*", + "litellm_params": { + "model": "openrouter/*", + "api_key": f"sk-or-v1-models-off-loop-{scope}", + "api_base": OPENROUTER_BASE, + }, + } + ] + ), + ) + catalog: Final = _CatalogHeldUntilTheLoopRuns() + + async def listed_ids_on_a_one_thread_default_pool() -> list[str]: + asyncio.get_running_loop().set_default_executor(ThreadPoolExecutor(max_workers=1)) + listing: Final = asyncio.ensure_future( + proxy_server.model_list( + user_api_key_dict=UserAPIKeyAuth( + api_key="sk-test-key", user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN + ), + scope=scope, + ) + ) + await catalog.release_from_the_loop() + response: Final = await listing + return [str(model["id"]) for model in response["data"]] + + with respx.mock() as respx_mock: + catalog_route: Final = respx_mock.get(f"{OPENROUTER_BASE}/models/user").mock(side_effect=catalog.answer) + listed_ids: Final = asyncio.run(listed_ids_on_a_one_thread_default_pool()) + + assert catalog_route.calls.last.response.status_code == 200 + assert listed_ids == ["openrouter/openai/gpt-5.4-mini"] + + def _priced_jev_models() -> frozenset[str]: return frozenset(name.removeprefix("typesafe/") for name in litellm.model_cost if name.startswith("typesafe/")) diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index c182cd1ec75..68eb05e60dd 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -179,6 +179,39 @@ async def test_token_counting_model_not_in_model_list(): assert response.model_used == "special-alias" +@pytest.mark.asyncio +async def test_asking_the_provider_endpoint_leaves_an_openrouter_deployments_count_unchanged( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr( + litellm.proxy.proxy_server, + "llm_router", + Router( + model_list=[ + { + "model_name": "or-gpt", + "litellm_params": { + "model": "openai/gpt-4o", + "custom_llm_provider": "openrouter", + "api_key": "sk-or-token-count", + }, + } + ] + ), + ) + request: Final = TokenCountRequest( + model="or-gpt", messages=[{"role": "user", "content": "Count these tokens, ünïcödé 你好 🙂"}] + ) + + via_provider_endpoint: Final = await token_counter(request=request, call_endpoint=True) + counted_locally: Final = await token_counter(request=request, call_endpoint=False) + + assert (via_provider_endpoint.total_tokens, via_provider_endpoint.model_used) == ( + counted_locally.total_tokens, + counted_locally.model_used, + ) + + @pytest.mark.asyncio async def test_gpt_token_counting(): """ diff --git a/tests/unit/proxy/utils/helpers/test_model_access.py b/tests/unit/proxy/utils/helpers/test_model_access.py index 324360cc2ee..30d24e9eddc 100644 --- a/tests/unit/proxy/utils/helpers/test_model_access.py +++ b/tests/unit/proxy/utils/helpers/test_model_access.py @@ -1,10 +1,17 @@ +import asyncio +import threading +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass, field +from typing import Final from unittest.mock import MagicMock +import httpx import pytest +import respx from fastapi import HTTPException import litellm -from litellm import ModelResponse +from litellm import ModelResponse, Router from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.utils import ( create_model_info_response, @@ -650,3 +657,61 @@ async def test_listing_is_unchanged_without_an_agent_or_without_attached_groups( resolve_agent_ceiling=_agent_ceiling(None), ) assert (plain_key, agent_without_groups) == (["model-a", "model-b"], ["model-a", "model-b"]) + + +OPENROUTER_BASE: Final = "https://openrouter.ai/api/v1" + + +@dataclass(frozen=True, slots=True) +class _CatalogHeldUntilTheLoopRuns: + requested: threading.Event = field(default_factory=threading.Event) + released: threading.Event = field(default_factory=threading.Event) + + def answer(self, _request: httpx.Request) -> httpx.Response: + self.requested.set() + if not self.released.wait(timeout=30): + return httpx.Response(504) + return httpx.Response(200, json={"data": [{"id": "openai/gpt-5.4-mini"}]}) + + async def release_from_the_loop(self) -> None: + assert await asyncio.to_thread(self.requested.wait, 30) + self.released.set() + + +def test_get_available_models_for_user_fetches_a_live_catalog_without_holding_the_event_loop_or_its_default_pool( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + catalog: Final = _CatalogHeldUntilTheLoopRuns() + router: Final = Router( + model_list=[ + { + "model_name": "openrouter/*", + "litellm_params": { + "model": "openrouter/*", + "api_key": "sk-or-available-models-off-loop", + "api_base": OPENROUTER_BASE, + }, + } + ] + ) + + async def list_models_on_a_one_thread_default_pool() -> list[str]: + asyncio.get_running_loop().set_default_executor(ThreadPoolExecutor(max_workers=1)) + listing: Final = asyncio.ensure_future( + get_available_models_for_user( + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test-key", user_id="user-1"), + llm_router=router, + general_settings={}, + user_model=None, + ) + ) + await catalog.release_from_the_loop() + return await listing + + with respx.mock() as respx_mock: + catalog_route: Final = respx_mock.get(f"{OPENROUTER_BASE}/models/user").mock(side_effect=catalog.answer) + models: Final = asyncio.run(list_models_on_a_one_thread_default_pool()) + + assert catalog_route.calls.last.response.status_code == 200 + assert models == ["openrouter/openai/gpt-5.4-mini"] diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 0f471b1de71..5cf872f4ea7 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -9269,6 +9269,77 @@ def reset_mock_cache() -> None: _model_cache.flush_cache() +@respx.mock +def test_get_valid_models_does_not_ask_a_timed_out_provider_again_on_the_next_listing(reset_mock_cache: None) -> None: + catalog: Final = respx.get("https://openrouter.ai/api/v1/models/user").mock( + side_effect=httpx.ReadTimeout("catalog hung") + ) + deployment: Final = LiteLLM_Params(model="openrouter/*", api_key="sk-or-during-an-outage") + + listings: Final = tuple( + get_valid_models(check_provider_endpoint=True, custom_llm_provider="openrouter", litellm_params=deployment) + for _ in range(2) + ) + + assert (listings, catalog.call_count) == (([], []), 1) + + +@respx.mock +def test_get_valid_models_lists_a_provider_again_right_after_a_fast_failure(reset_mock_cache: None) -> None: + respx.get("https://openrouter.ai/api/v1/models/user").mock( + side_effect=(httpx.Response(503), httpx.Response(200, json={"data": [{"id": "anthropic/claude-haiku-5-5"}]})) + ) + deployment: Final = LiteLLM_Params(model="openrouter/*", api_key="sk-or-recovers") + + listings: Final = tuple( + get_valid_models(check_provider_endpoint=True, custom_llm_provider="openrouter", litellm_params=deployment) + for _ in range(2) + ) + + assert listings == ([], ["openrouter/anthropic/claude-haiku-5-5"]) + + +@respx.mock +def test_get_valid_models_keeps_a_good_listing_when_an_overlapping_listing_fails_after_it( + reset_mock_cache: None, +) -> None: + deployment: Final = LiteLLM_Params(model="openrouter/*", api_key="sk-or-overlapping-listings") + first_request: Final = iter((True, False)) + + def answer_after_an_overlapping_listing(request: httpx.Request) -> httpx.Response: + if next(first_request): + get_valid_models(check_provider_endpoint=True, custom_llm_provider="openrouter", litellm_params=deployment) + raise httpx.ReadTimeout("catalog hung", request=request) + return httpx.Response(200, json={"data": [{"id": "anthropic/claude-haiku-5-5"}]}) + + respx.get("https://openrouter.ai/api/v1/models/user").mock(side_effect=answer_after_an_overlapping_listing) + get_valid_models(check_provider_endpoint=True, custom_llm_provider="openrouter", litellm_params=deployment) + + assert get_valid_models( + check_provider_endpoint=True, custom_llm_provider="openrouter", litellm_params=deployment + ) == ["openrouter/anthropic/claude-haiku-5-5"] + + +@respx.mock +def test_get_valid_models_asks_a_provider_once_when_many_listings_start_together(reset_mock_cache: None) -> None: + def slow_catalog(request: httpx.Request) -> httpx.Response: + threading.Event().wait(0.2) + return httpx.Response(200, json={"data": [{"id": "anthropic/claude-haiku-5-5"}]}) + + catalog: Final = respx.get("https://openrouter.ai/api/v1/models/user").mock(side_effect=slow_catalog) + deployment: Final = LiteLLM_Params(model="openrouter/*", api_key="sk-or-many-listings") + + def list_models(_: int) -> tuple[str, ...]: + return tuple( + get_valid_models(check_provider_endpoint=True, custom_llm_provider="openrouter", litellm_params=deployment) + ) + + with ThreadPoolExecutor(max_workers=8) as pool: + listings: Final = frozenset(pool.map(list_models, range(8))) + + assert (listings, catalog.call_count) == (frozenset({("openrouter/anthropic/claude-haiku-5-5",)}), 1) + + @pytest.fixture def isolated_openai_model_sets(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr( From 4f7150f832b3180a6dfcce2be5a052038de45eac Mon Sep 17 00:00:00 2001 From: moe-berri Date: Fri, 30 Oct 2026 12:21:30 -0700 Subject: [PATCH 28/74] fix(ollama): forward assistant tool call ids in ollama_chat requests (#49278) * test(ollama): cover assistant tool call ids in ollama_chat requests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ollama): forward assistant tool call ids in ollama_chat requests Co-authored-by: Kenneth Bingham Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(ollama): format tool call id regression assertion Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ollama): keep tool call ID handling type safe Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(ollama): format tool-call id helper Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ollama): preserve typed tool call arguments Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Kenneth Bingham --- litellm/llms/ollama/chat/transformation.py | 24 +++-- litellm/types/llms/ollama.py | 3 + .../ollama/test_ollama_chat_transformation.py | 93 +++++++++++++++++++ 3 files changed, 110 insertions(+), 10 deletions(-) diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index 755d647da1d..168b562b358 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -235,6 +235,19 @@ class OllamaChatConfig(BaseConfig): litellm_params["function_name"] = function_name tools: Final = optional_params.pop("tools", None) + def transform_tool_call(typed_tool: ChatCompletionAssistantToolCall) -> OllamaToolCall: + function: Final = OllamaToolCallFunction( + name=typed_tool["function"].get("name") or "", + arguments=json.loads(typed_tool["function"]["arguments"]) + if "arguments" in typed_tool["function"] + else {}, + ) + tool_id: Final = typed_tool.get("id") + ollama_tool_call: Final = ( + OllamaToolCall(function=function) if tool_id is None else OllamaToolCall(id=tool_id, function=function) + ) + return ollama_tool_call + new_messages: Final = [] for m in messages: if isinstance( @@ -248,16 +261,7 @@ class OllamaChatConfig(BaseConfig): for tool in tool_calls: typed_tool = ChatCompletionAssistantToolCall(**tool) if typed_tool["type"] == "function": - arguments = {} - if "arguments" in typed_tool["function"]: - arguments = json.loads(typed_tool["function"]["arguments"]) - ollama_tool_call = OllamaToolCall( - function=OllamaToolCallFunction( - name=typed_tool["function"].get("name") or "", - arguments=arguments, - ) - ) - new_tools.append(ollama_tool_call) + new_tools.append(transform_tool_call(typed_tool)) reasoning_content, parsed_content = extract_reasoning_content(cast(dict, m)) content_str = convert_content_list_to_str(cast(AllMessageValues, m)) images = extract_images_from_message(cast(AllMessageValues, m)) diff --git a/litellm/types/llms/ollama.py b/litellm/types/llms/ollama.py index 4783eac9a8d..fb9d9e8c363 100644 --- a/litellm/types/llms/ollama.py +++ b/litellm/types/llms/ollama.py @@ -1,4 +1,6 @@ from typing_extensions import ( + NotRequired, + ReadOnly, Required, TypedDict, ) @@ -12,6 +14,7 @@ class OllamaToolCallFunction( class OllamaToolCall(TypedDict): + id: NotRequired[ReadOnly[str]] function: OllamaToolCallFunction diff --git a/tests/unit/llms/ollama/test_ollama_chat_transformation.py b/tests/unit/llms/ollama/test_ollama_chat_transformation.py index af1b85cf325..6a80f29ac43 100644 --- a/tests/unit/llms/ollama/test_ollama_chat_transformation.py +++ b/tests/unit/llms/ollama/test_ollama_chat_transformation.py @@ -856,6 +856,99 @@ class TestOllamaReasoningContentStreaming: class TestOllamaToolCallTransformation: + @pytest.mark.parametrize("stream", [False, True]) + def test_transform_request_keeps_parallel_call_ids_paired_with_results(self, stream: bool): + config: Final = OllamaChatConfig() + messages: Final = cast( + list[AllMessageValues], + [ + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_first", + "type": "function", + "function": { + "name": "read_fixture", + "arguments": '{"path": "first.txt"}', + }, + }, + { + "id": "call_second", + "type": "function", + "function": { + "name": "read_fixture", + "arguments": '{"path": "second.txt"}', + }, + }, + ], + }, + { + "role": "tool", + "tool_call_id": "call_second", + "content": "second result", + }, + { + "role": "tool", + "tool_call_id": "call_first", + "content": "first result", + }, + ], + ) + + result: Final = config.transform_request( + model="ollama_chat/test-model", + messages=messages, + optional_params={"stream": stream}, + litellm_params={}, + headers={}, + ) + + assert result["messages"][0]["tool_calls"] == [ + { + "id": "call_first", + "function": {"name": "read_fixture", "arguments": {"path": "first.txt"}}, + }, + { + "id": "call_second", + "function": {"name": "read_fixture", "arguments": {"path": "second.txt"}}, + }, + ] + assert [message.get("tool_call_id") for message in result["messages"][1:]] == [ + "call_second", + "call_first", + ] + + def test_transform_request_does_not_invent_an_id_for_an_unidentified_call(self): + config: Final = OllamaChatConfig() + messages: Final = cast( + list[AllMessageValues], + [ + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": None, + "type": "function", + "function": {"name": "read_fixture", "arguments": "{}"}, + } + ], + } + ], + ) + + result: Final = config.transform_request( + model="ollama_chat/test-model", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert result["messages"][0]["tool_calls"] == [{"function": {"name": "read_fixture", "arguments": {}}}] + def test_transform_request_preserves_tool_calls(self): """ tool_calls on assistant messages must survive transform_request. From b2466f8d7b2c783c04d424009b4b5b1066e06d23 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 30 Oct 2026 12:31:16 -0700 Subject: [PATCH 29/74] chore(cost-map): flag Vertex Gemini computer use and add Cohere Rerank 5 (#49297) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 33 +++++++++++++++++++ model_prices_and_context_window.json | 33 +++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index dac7af6ab7a..1d18903bb07 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -29561,6 +29561,7 @@ ], "supports_audio_input": true, "supports_audio_output": false, + "supports_computer_use": true, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -30167,6 +30168,7 @@ "text" ], "supports_audio_input": true, + "supports_computer_use": true, "supports_function_calling": true, "supports_native_streaming": true, "supports_pdf_input": true, @@ -30210,6 +30212,7 @@ "text" ], "supports_audio_input": true, + "supports_computer_use": true, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -30274,6 +30277,7 @@ "text" ], "supports_audio_input": true, + "supports_computer_use": true, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -30334,6 +30338,7 @@ "text" ], "supports_audio_input": true, + "supports_computer_use": true, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -30401,6 +30406,7 @@ "text" ], "supports_audio_input": true, + "supports_computer_use": true, "supports_function_calling": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, @@ -32405,6 +32411,7 @@ "text" ], "supports_audio_output": false, + "supports_computer_use": true, "supports_function_calling": true, "supports_native_streaming": true, "supports_parallel_function_calling": true, @@ -32479,6 +32486,7 @@ ], "supports_audio_output": false, "supports_audio_input": true, + "supports_computer_use": true, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -32543,6 +32551,7 @@ ], "supports_audio_output": false, "supports_audio_input": true, + "supports_computer_use": true, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -32603,6 +32612,7 @@ ], "supports_audio_output": false, "supports_audio_input": true, + "supports_computer_use": true, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -32670,6 +32680,7 @@ ], "supports_audio_input": true, "supports_audio_output": false, + "supports_computer_use": true, "supports_function_calling": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, @@ -47624,6 +47635,24 @@ "output_cost_per_token": 0.0, "source": "https://cohere.com/pricing" }, + "rerank-v5.0-fast": { + "input_cost_per_token": 2e-08, + "litellm_provider": "cohere", + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "mode": "rerank", + "output_cost_per_token": 0.0, + "source": "https://cohere.com/blog/rerank-5" + }, + "rerank-v5.0-pro": { + "input_cost_per_token": 5e-08, + "litellm_provider": "cohere", + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "mode": "rerank", + "output_cost_per_token": 0.0, + "source": "https://cohere.com/blog/rerank-5" + }, "nvidia_nim/nvidia/nv-rerankqa-mistral-4b-v3": { "input_cost_per_query": 0.0, "input_cost_per_token": 0.0, @@ -52713,6 +52742,7 @@ ], "supports_audio_input": true, "supports_audio_output": false, + "supports_computer_use": true, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -76498,6 +76528,7 @@ ], "supports_audio_input": true, "supports_audio_output": false, + "supports_computer_use": true, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -76543,6 +76574,7 @@ ], "supports_audio_input": true, "supports_audio_output": false, + "supports_computer_use": true, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -76654,6 +76686,7 @@ "text" ], "supports_audio_input": true, + "supports_computer_use": true, "supports_function_calling": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index dac7af6ab7a..1d18903bb07 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -29561,6 +29561,7 @@ ], "supports_audio_input": true, "supports_audio_output": false, + "supports_computer_use": true, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -30167,6 +30168,7 @@ "text" ], "supports_audio_input": true, + "supports_computer_use": true, "supports_function_calling": true, "supports_native_streaming": true, "supports_pdf_input": true, @@ -30210,6 +30212,7 @@ "text" ], "supports_audio_input": true, + "supports_computer_use": true, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -30274,6 +30277,7 @@ "text" ], "supports_audio_input": true, + "supports_computer_use": true, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -30334,6 +30338,7 @@ "text" ], "supports_audio_input": true, + "supports_computer_use": true, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -30401,6 +30406,7 @@ "text" ], "supports_audio_input": true, + "supports_computer_use": true, "supports_function_calling": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, @@ -32405,6 +32411,7 @@ "text" ], "supports_audio_output": false, + "supports_computer_use": true, "supports_function_calling": true, "supports_native_streaming": true, "supports_parallel_function_calling": true, @@ -32479,6 +32486,7 @@ ], "supports_audio_output": false, "supports_audio_input": true, + "supports_computer_use": true, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -32543,6 +32551,7 @@ ], "supports_audio_output": false, "supports_audio_input": true, + "supports_computer_use": true, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -32603,6 +32612,7 @@ ], "supports_audio_output": false, "supports_audio_input": true, + "supports_computer_use": true, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -32670,6 +32680,7 @@ ], "supports_audio_input": true, "supports_audio_output": false, + "supports_computer_use": true, "supports_function_calling": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, @@ -47624,6 +47635,24 @@ "output_cost_per_token": 0.0, "source": "https://cohere.com/pricing" }, + "rerank-v5.0-fast": { + "input_cost_per_token": 2e-08, + "litellm_provider": "cohere", + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "mode": "rerank", + "output_cost_per_token": 0.0, + "source": "https://cohere.com/blog/rerank-5" + }, + "rerank-v5.0-pro": { + "input_cost_per_token": 5e-08, + "litellm_provider": "cohere", + "max_input_tokens": 32768, + "max_output_tokens": 32768, + "mode": "rerank", + "output_cost_per_token": 0.0, + "source": "https://cohere.com/blog/rerank-5" + }, "nvidia_nim/nvidia/nv-rerankqa-mistral-4b-v3": { "input_cost_per_query": 0.0, "input_cost_per_token": 0.0, @@ -52713,6 +52742,7 @@ ], "supports_audio_input": true, "supports_audio_output": false, + "supports_computer_use": true, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -76498,6 +76528,7 @@ ], "supports_audio_input": true, "supports_audio_output": false, + "supports_computer_use": true, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -76543,6 +76574,7 @@ ], "supports_audio_input": true, "supports_audio_output": false, + "supports_computer_use": true, "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, @@ -76654,6 +76686,7 @@ "text" ], "supports_audio_input": true, + "supports_computer_use": true, "supports_function_calling": true, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, From 0d5cf9b2f7d8cba315cd6c550bd98aa9a2fa88e1 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Fri, 30 Oct 2026 12:32:03 -0700 Subject: [PATCH 30/74] fix(headroom): forward tenant-scoped session affinity (#49255) --- .../litellm_core_utils/provider_affinity.py | 42 ++++- .../guardrail_hooks/headroom/headroom.py | 91 ++++----- litellm/proxy/litellm_pre_call_utils.py | 1 + litellm/types/litellm_params.py | 2 + .../guardrails/guardrail_hooks/headroom.py | 12 +- .../test_provider_affinity.py | 16 +- .../guardrail_hooks/test_headroom.py | 174 +++++++++--------- tests/unit/types/test_litellm_params.py | 3 + 8 files changed, 209 insertions(+), 132 deletions(-) diff --git a/litellm/litellm_core_utils/provider_affinity.py b/litellm/litellm_core_utils/provider_affinity.py index c831e92c09d..767eab962c8 100644 --- a/litellm/litellm_core_utils/provider_affinity.py +++ b/litellm/litellm_core_utils/provider_affinity.py @@ -2,7 +2,12 @@ import re from collections.abc import Mapping from typing import Final -from litellm.constants import SESSION_ID_FROM_TRACE_ID_PARAM_KEY, SESSION_ID_GENERATED_METADATA_KEY +from litellm.constants import ( + CALLER_SESSION_ID_METADATA_KEY, + SESSION_ID_FROM_TRACE_ID_PARAM_KEY, + SESSION_ID_GENERATED_METADATA_KEY, +) +from litellm.litellm_core_utils.core_helpers import as_str_mapping _HTTP_HEADER_NAME_PATTERN: Final = re.compile(r"^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$") _FORBIDDEN_AFFINITY_HEADERS: Final = frozenset( @@ -98,6 +103,41 @@ def get_stable_session_id(litellm_params: object | None) -> str | None: return _non_generated_litellm_session_id(litellm_params) or _metadata_session_id(litellm_params) +def get_caller_session_id(litellm_params: object | None) -> str | None: + """Resolve a caller conversation without accepting a generated session or a promoted trace id.""" + if litellm_params is None: + return None + metadata_values: Final = _metadata_values(litellm_params) + preserved_session: Final = next( + ( + value + for metadata in metadata_values + if isinstance(value := _get_value(metadata, CALLER_SESSION_ID_METADATA_KEY), str) and value + ), + None, + ) + if preserved_session: + return preserved_session + direct_session: Final = _get_value(litellm_params, "session_id") + if isinstance(direct_session, str) and direct_session: + return direct_session + + proxy_request: Final = _get_value(litellm_params, "proxy_server_request") + headers: Final = as_str_mapping(_get_value(proxy_request, "headers")) + trace_header: Final = next( + (value for key, value in (headers or {}).items() if key.lower() == "x-litellm-trace-id"), None + ) + candidates: Final = ( + get_caller_litellm_session_id(litellm_params), + *( + _get_value(metadata, "session_id") + for metadata in metadata_values + if not _get_value(metadata, SESSION_ID_GENERATED_METADATA_KEY) + ), + ) + return next((value for value in candidates if isinstance(value, str) and value and value != trace_header), None) + + def add_provider_affinity_header( headers: Mapping[str, object], litellm_params: object | None, default_header: str | None = None ) -> dict[str, object]: # mutable-ok: downstream handlers add auth and signing headers diff --git a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py index dac802a1ddc..7f48766bf26 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py +++ b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py @@ -1,10 +1,10 @@ from __future__ import annotations +import hashlib import json import math import re import time -import uuid from collections.abc import Mapping, Sequence from dataclasses import dataclass from typing import TYPE_CHECKING, ClassVar, Final, Literal, Protocol, TypeGuard @@ -28,6 +28,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( group_tool_exchanges, has_tool_with_name, ) +from litellm.litellm_core_utils.provider_affinity import get_caller_session_id from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] httpxSpecialProvider, @@ -39,6 +40,7 @@ from litellm.proxy.guardrails.guardrail_hooks.content_text import ( merge_rewritten_text_parts, ) from litellm.proxy.spend_tracking.compression_savings import HEADROOM_GUARDRAIL_PROVIDER +from litellm.router_utils.common_utils import request_user_api_key_auth from litellm.secret_managers.main import get_secret_str from litellm.types.guardrails import GuardrailEventHooks, Mode from litellm.types.integrations.custom_logger import ( @@ -47,6 +49,11 @@ from litellm.types.integrations.custom_logger import ( AgenticLoopRequestPatch, as_converted_stream, ) +from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth +from litellm.types.proxy.guardrails.guardrail_hooks.headroom import ( + HEADROOM_REQUEST_CONTEXT_KEY, + HeadroomRequestContext, +) from litellm.types.utils import CallTypes, GenericGuardrailAPIInputs if TYPE_CHECKING: @@ -357,15 +364,18 @@ def _build_headroom_retrieve_tool() -> dict[str, object]: } -def _resolve_call_id(logging_obj: object, request_state: dict[str, object]) -> str | None: - """Resolve the litellm_call_id shared by a request's pre-call hook and its - agentic-loop hooks, so CCR hash validation can be scoped per call instead - of trusting any hash-shaped string that shows up in message text.""" - logging_call_id: Final = getattr(logging_obj, "litellm_call_id", None) - if isinstance(logging_call_id, str) and logging_call_id: - return logging_call_id - kwargs_call_id: Final = request_state.get("litellm_call_id") - return kwargs_call_id if isinstance(kwargs_call_id, str) else None +def _session_affinity(request_data: Mapping[str, object]) -> str | None: + auth: Final = request_user_api_key_auth(request_data) + session_id: Final = get_caller_session_id(request_data) + if not isinstance(auth, UserAPIKeyAuth) or not session_id: + return None + tenant: Final = ( + ("key", auth.api_key) if auth.api_key else ("principal", auth.org_id, auth.team_id, auth.user_id, auth.agent_id) + ) + if not any(tenant[1:]): + return None + identity: Final = json.dumps(["headroom-affinity-v1", tenant, session_id], separators=(",", ":")) + return hashlib.sha256(identity.encode("utf-8")).hexdigest() def has_headroom_retrieve_tool(tools: object) -> bool: @@ -521,7 +531,6 @@ class HeadroomGuardrail(CustomGuardrail): self.async_handler = get_async_httpx_client( llm_provider=httpxSpecialProvider.GuardrailCallback, ) - self._issued_hashes_by_call_id: dict[str, tuple[frozenset[str], float]] = {} super().__init__( # pyright: ignore[reportUnknownMemberType] guardrail_name=guardrail_name, event_hook=event_hook, @@ -540,10 +549,12 @@ class HeadroomGuardrail(CustomGuardrail): value: Final = headers.get(BYPASS_HEADER) return str(value).lower() == "true" - def _request_headers(self) -> dict[str, str]: + def _request_headers(self, affinity: str | None) -> dict[str, str]: headers: Final[dict[str, str]] = {"Content-Type": "application/json"} if self.headroom_api_key: headers["Authorization"] = f"Bearer {self.headroom_api_key}" + if affinity is not None: + headers["x-headroom-affinity"] = affinity return headers @staticmethod @@ -569,14 +580,6 @@ class HeadroomGuardrail(CustomGuardrail): super().update_in_memory_litellm_params(litellm_params) self.timeout = self._resolve_timeout(litellm_params.timeout) - def _prune_expired_hashes(self) -> None: - now: Final = time.monotonic() - self._issued_hashes_by_call_id = { - call_id: (hashes, expiry) - for call_id, (hashes, expiry) in self._issued_hashes_by_call_id.items() - if expiry > now - } - def _handle_compress_failure( self, messages: list[dict[str, object]], @@ -596,16 +599,19 @@ class HeadroomGuardrail(CustomGuardrail): self, messages: list[dict[str, object]], model: str | None, + affinity: str | None, ) -> _CompressResult: payload: Final[dict[str, object]] = {"messages": messages} if model: payload["model"] = model + if self.ccr_retrieval: + payload["config"] = {"mode": "ccr"} try: raw_response: HttpxResponse = await self.async_handler.post( # pyright: ignore[reportUnknownMemberType] # AsyncHTTPHandler.post is untyped url=f"{self.headroom_api_base}/v1/compress", json=payload, - headers=self._request_headers(), + headers=self._request_headers(affinity), timeout=self.timeout, ) except httpx.HTTPStatusError as e: @@ -734,7 +740,7 @@ class HeadroomGuardrail(CustomGuardrail): stats["tokens_saved"] = tokens_before - tokens_after return _CompressResult(filtered, True, stats, _read_ccr_hashes(body)) - async def _call_retrieve(self, hash_value: str, query: str | None = None) -> str: + async def _call_retrieve(self, hash_value: str, affinity: str | None, query: str | None = None) -> str: params: Final[dict[str, str]] = {} if query: params["query"] = query @@ -743,7 +749,7 @@ class HeadroomGuardrail(CustomGuardrail): raw_response: HttpxResponse = await self.async_handler.get( # pyright: ignore[reportUnknownMemberType] # AsyncHTTPHandler.get is untyped url=f"{self.headroom_api_base}/v1/retrieve/{hash_value}", params=params, - headers=self._request_headers(), + headers=self._request_headers(affinity), timeout=self.timeout, ) except (httpx.ConnectError, httpx.TimeoutException, httpx.TransportError, litellm.Timeout) as e: @@ -777,13 +783,14 @@ class HeadroomGuardrail(CustomGuardrail): async def apply_guardrail( self, inputs: GenericGuardrailAPIInputs, - request_data: dict, + request_data: dict[str, object], input_type: Literal["request", "response"], logging_obj: LiteLLMLoggingObj | None = None, ) -> GenericGuardrailAPIInputs: if input_type != "request": return inputs + request_data.pop(HEADROOM_REQUEST_CONTEXT_KEY, None) if self._should_bypass(request_data): verbose_proxy_logger.debug("Headroom: %s header set; skipping compression", BYPASS_HEADER) return inputs @@ -809,9 +816,7 @@ class HeadroomGuardrail(CustomGuardrail): # /v1/compress grows a field for sending the live turn as the retrieval # query without compressing it: query-aware compression reads the newest # user message, so it is withheld here at some cost to history ranking. - # request_data is a bare dict on the base signature; narrow it before - # reading the untranslated messages so long tool names can be recovered. - raw_messages: Final = _REQUEST_DATA_ADAPTER.validate_python(request_data).get("messages") + raw_messages: Final = request_data.get("messages") raw_retrieve_call_ids: Final = _raw_retrieve_call_ids(raw_messages) protected_indices: Final = _protected_indices(messages, raw_retrieve_call_ids) compressible: Final = [m for i, m in enumerate(messages) if i not in protected_indices] @@ -820,9 +825,11 @@ class HeadroomGuardrail(CustomGuardrail): model: Final = self.headroom_model or request_data.get("model") start_time: Final = time.time() + affinity: Final = _session_affinity(request_data) result: Final = await self._call_compress( messages=_flatten_messages_for_compression(compressible), model=model if isinstance(model, str) else None, + affinity=affinity, ) end_time: Final = time.time() @@ -868,12 +875,9 @@ class HeadroomGuardrail(CustomGuardrail): if not hashes: return {**inputs, "structured_messages": compressed} # pyright: ignore[reportReturnType] - self._prune_expired_hashes() - call_id = _resolve_call_id(logging_obj, request_data) - if not call_id: - call_id = str(uuid.uuid4()) - request_data["litellm_call_id"] = call_id - self._issued_hashes_by_call_id[call_id] = (frozenset(hashes), time.monotonic() + _HASH_CACHE_TTL_SECONDS) + request_data[HEADROOM_REQUEST_CONTEXT_KEY] = HeadroomRequestContext( + frozenset(hashes), affinity, time.monotonic() + _HASH_CACHE_TTL_SECONDS + ) existing_tools: Final = inputs.get("tools") retrieve_tool: Final = _build_headroom_retrieve_tool() @@ -934,9 +938,11 @@ class HeadroomGuardrail(CustomGuardrail): ) -> AgenticLoopPlan: tool_calls: Final[list[dict[str, object]]] = tools.get("tool_calls", []) - self._prune_expired_hashes() - call_id: Final = _resolve_call_id(logging_obj, kwargs) - valid_hashes = self._issued_hashes_by_call_id.get(call_id, (frozenset(), 0.0))[0] if call_id else frozenset() + context: Final = _REQUEST_DATA_ADAPTER.validate_python(kwargs).get(HEADROOM_REQUEST_CONTEXT_KEY) + valid_context: Final = ( + context if isinstance(context, HeadroomRequestContext) and context.expires_at > time.monotonic() else None + ) + valid_hashes: Final = valid_context.issued_hashes if valid_context is not None else frozenset() retrieved: Final[list[tuple[dict[str, object], str]]] = [] for tc in tool_calls: @@ -944,11 +950,6 @@ class HeadroomGuardrail(CustomGuardrail): raw_hash = arguments.get("hash", "") if isinstance(arguments, dict) else "" hash_value = str(raw_hash).lower() query = arguments.get("query") if isinstance(arguments, dict) else None - # A hash is only honored if it was issued by *this request's own* - # Headroom /v1/compress call, scoped by litellm_call_id. Scoping by - # message text alone is forgeable -- an attacker can plant a - # hash-shaped string in their own prompt, and a hash issued for one - # request would validate for any other request that echoes it back. if hash_value not in valid_hashes: verbose_proxy_logger.warning( "Headroom CCR: rejecting hash=%s not produced by current request compression", @@ -958,6 +959,7 @@ class HeadroomGuardrail(CustomGuardrail): else: content = await self._call_retrieve( hash_value=hash_value, + affinity=valid_context.affinity if valid_context is not None else None, query=str(query) if query else None, ) verbose_proxy_logger.debug("Headroom CCR: retrieved hash=%s (%d chars)", hash_value, len(content)) @@ -996,7 +998,12 @@ class HeadroomGuardrail(CustomGuardrail): max_tokens=max_tokens, optional_params=optional_params_without_max_tokens, kwargs={ - k: v for k, v in kwargs.items() if not k.startswith("_headroom") and k != "litellm_logging_obj" + **{ + k: v + for k, v in _REQUEST_DATA_ADAPTER.validate_python(kwargs).items() + if not k.startswith("_headroom") and k != "litellm_logging_obj" + }, + **({HEADROOM_REQUEST_CONTEXT_KEY: valid_context} if valid_context is not None else {}), }, ), metadata={"tool_type": "headroom_ccr"}, diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 9b52f2f8a8d..ae9a6d4b959 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -2566,6 +2566,7 @@ def refresh_proxy_server_request_body_snapshot( VECTOR_STORE_CALLER_METADATA_KWARG, STREAM_USAGE_INJECTED_FIELD, "_litellm_strip_stream_usage", + "_headroom_request_context", } ) | _TRANSPORT_ONLY_CREDENTIAL_KEYS diff --git a/litellm/types/litellm_params.py b/litellm/types/litellm_params.py index 2065093f295..3c2c636b5ed 100644 --- a/litellm/types/litellm_params.py +++ b/litellm/types/litellm_params.py @@ -21,6 +21,7 @@ if TYPE_CHECKING: from litellm.router_utils.fallback_event_handlers import AttemptedFallbackTargets from litellm.types.caching import DynamicCacheControl from litellm.types.llms.openai import ChatCompletionAssistantMessage, ChatCompletionUserMessage + from litellm.types.proxy.guardrails.guardrail_hooks.headroom import HeadroomRequestContext from litellm.types.proxy.litellm_pre_call_utils import SecretFields from litellm.types.router import ConfigurableClientsideParamsCustomAuth, DeploymentTypedDict, RetryPolicy from litellm.types.router_weights import RouterWeights @@ -352,6 +353,7 @@ class CallState: @dataclass(frozen=True, slots=True, kw_only=True) class AgenticLoopState: + headroom_context: "HeadroomRequestContext | None" = field(default=None, metadata=wire("_headroom_request_context")) depth: int | None = field(default=None, metadata=wire("_agentic_loop_depth")) fingerprints: Sequence[str] | None = field(default=None, metadata=wire("_agentic_loop_fingerprints")) api_surface: Literal["chat_completions", "responses"] | None = field( diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/headroom.py b/litellm/types/proxy/guardrails/guardrail_hooks/headroom.py index 0df0f0c80ef..76e806e3899 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/headroom.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/headroom.py @@ -1,9 +1,19 @@ -from typing import Literal +from dataclasses import dataclass +from typing import Final, Literal from pydantic import BaseModel, Field from .base import GuardrailConfigModel +HEADROOM_REQUEST_CONTEXT_KEY: Final = "_headroom_request_context" + + +@dataclass(frozen=True, slots=True) +class HeadroomRequestContext: + issued_hashes: frozenset[str] + affinity: str | None + expires_at: float + class HeadroomGuardrailConfigModel(GuardrailConfigModel[BaseModel]): api_base: str | None = Field( diff --git a/tests/unit/litellm_core_utils/test_provider_affinity.py b/tests/unit/litellm_core_utils/test_provider_affinity.py index edf4f5169b5..6ccd6b17c01 100644 --- a/tests/unit/litellm_core_utils/test_provider_affinity.py +++ b/tests/unit/litellm_core_utils/test_provider_affinity.py @@ -1,12 +1,16 @@ +from collections.abc import Callable + import pytest from litellm.constants import SESSION_ID_GENERATED_METADATA_KEY from litellm.litellm_core_utils.provider_affinity import ( add_provider_affinity_header, + get_caller_session_id, get_stable_session_id, ) +@pytest.mark.parametrize("resolve", [get_stable_session_id, get_caller_session_id]) @pytest.mark.parametrize( ("litellm_params", "expected"), [ @@ -16,18 +20,22 @@ from litellm.litellm_core_utils.provider_affinity import ( ({"litellm_metadata": {"session_id": "litellm-metadata-session"}}, "litellm-metadata-session"), ], ) -def test_get_stable_session_id_uses_explicit_session_sources(litellm_params: dict, expected: str): - assert get_stable_session_id(litellm_params) == expected +def test_session_ids(litellm_params: dict[str, object], expected: str, resolve: Callable[[object], str | None]) -> None: + assert resolve(litellm_params) == expected def test_get_stable_session_id_does_not_use_trace_id(): assert get_stable_session_id({"litellm_trace_id": "per-request-trace"}) is None + assert get_caller_session_id( + {"litellm_session_id": "trace", "proxy_server_request": {"headers": {"x-litellm-trace-id": "trace"}}} + ) is None +@pytest.mark.parametrize("resolve", [get_stable_session_id, get_caller_session_id]) @pytest.mark.parametrize("metadata_key", ["metadata", "litellm_metadata"]) -def test_get_stable_session_id_ignores_proxy_generated_session(metadata_key: str): +def test_generated_sessions_are_ignored(metadata_key: str, resolve: Callable[[object], str | None]) -> None: assert ( - get_stable_session_id( + resolve( { "litellm_session_id": "generated-session", metadata_key: { diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_headroom.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_headroom.py index 741fea5f2b9..0cdcb2e2d07 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_headroom.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_headroom.py @@ -23,7 +23,6 @@ Tests cover: import datetime import json -import time from typing import Final, Optional from unittest.mock import AsyncMock, MagicMock, patch @@ -33,17 +32,22 @@ import respx from fastapi import HTTPException import litellm - from litellm.litellm_core_utils.litellm_logging import Logging from litellm.proxy.guardrails.guardrail_hooks.headroom.headroom import ( - HeadroomGuardrail, - has_headroom_retrieve_tool, HEADROOM_RETRIEVE_TOOL_NAME, + HeadroomGuardrail, + _session_affinity, + has_headroom_retrieve_tool, ) from litellm.proxy.spend_tracking.compression_savings import ( extract_compression_saved_tokens, ) from litellm.types.integrations.custom_logger import HEADROOM_CONVERTED_STREAM_KEY, HEADROOM_STREAM_OPTIONS_KEY +from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth +from litellm.types.proxy.guardrails.guardrail_hooks.headroom import ( + HEADROOM_REQUEST_CONTEXT_KEY, + HeadroomRequestContext, +) from litellm.types.utils import ( CallTypes, GenericGuardrailAPIInputs, @@ -148,6 +152,10 @@ def guardrail() -> HeadroomGuardrail: return _make_guardrail() +def _issued_context(*hashes: str, affinity: str | None = None) -> HeadroomRequestContext: + return HeadroomRequestContext(frozenset(hashes), affinity, float("inf")) + + def _recorded_guardrail_entries(request_data: dict) -> list: for container_key in ("metadata", "litellm_metadata"): container = request_data.get(container_key) @@ -175,20 +183,22 @@ async def test_apply_guardrail_compresses_and_returns_structured_messages( structured_messages=ORIGINAL_MESSAGES, ) mock_response = _make_compress_response(COMPRESSED_MESSAGES) - request_data = {"model": "gpt-4o"} + auth: Final = UserAPIKeyAuth(api_key="tenant") + request_data: Final = {"session_id": "conversation", "metadata": {"user_api_key_auth": auth}} with patch.object( guardrail.async_handler, "post", new_callable=AsyncMock, return_value=mock_response, - ): + ) as mock_post: result = await guardrail.apply_guardrail( inputs=inputs, request_data=request_data, input_type="request", ) + assert mock_post.call_args.kwargs["headers"]["x-headroom-affinity"] == _session_affinity(request_data) assert result.get("structured_messages") == EXPECTED_MESSAGES entries = _recorded_guardrail_entries(request_data) @@ -510,10 +520,6 @@ async def test_async_build_agentic_loop_plan_calls_retrieve_and_builds_messages( tool_id="call_abc123", ) messages = [{"role": "user", "content": "What does it say? hash=b573993006976af767214fac"}] - guardrail._issued_hashes_by_call_id["call-1"] = ( - frozenset({"b573993006976af767214fac"}), - time.monotonic() + 999, - ) with patch.object( guardrail.async_handler, @@ -530,11 +536,12 @@ async def test_async_build_agentic_loop_plan_calls_retrieve_and_builds_messages( anthropic_messages_optional_request_params={}, logging_obj=None, stream=False, - kwargs={"litellm_call_id": "call-1"}, + kwargs={HEADROOM_REQUEST_CONTEXT_KEY: _issued_context("b573993006976af767214fac", affinity="original")}, ) assert plan.run_agentic_loop is True assert plan.request_patch is not None + assert HEADROOM_REQUEST_CONTEXT_KEY in plan.request_patch.kwargs follow_up = plan.request_patch.messages assert follow_up is not None @@ -545,6 +552,7 @@ async def test_async_build_agentic_loop_plan_calls_retrieve_and_builds_messages( assert tool_result_message["tool_call_id"] == "call_abc123" mock_get.assert_called_once() + assert mock_get.call_args.kwargs["headers"]["x-headroom-affinity"] == "original" call_url = mock_get.call_args.kwargs.get("url") or mock_get.call_args.args[0] assert "b573993006976af767214fac" in call_url @@ -576,10 +584,6 @@ async def test_async_build_agentic_loop_plan_handles_retrieve_404( "content": "Retrieve more: hash=deadbeef000000000000dead", } ] - guardrail._issued_hashes_by_call_id["call-1"] = ( - frozenset({"deadbeef000000000000dead"}), - time.monotonic() + 999, - ) with patch.object( guardrail.async_handler, @@ -596,7 +600,7 @@ async def test_async_build_agentic_loop_plan_handles_retrieve_404( anthropic_messages_optional_request_params={}, logging_obj=None, stream=False, - kwargs={"litellm_call_id": "call-1"}, + kwargs={HEADROOM_REQUEST_CONTEXT_KEY: _issued_context("deadbeef000000000000dead")}, ) follow_up = plan.request_patch.messages # type: ignore[union-attr] @@ -606,9 +610,13 @@ async def test_async_build_agentic_loop_plan_handles_retrieve_404( @pytest.mark.asyncio +@pytest.mark.parametrize("context", [ + None, {"issued_hashes": ["deadbeef000000000000dead"]}, + HeadroomRequestContext(frozenset({"deadbeef000000000000dead"}), None, 0.0), +]) async def test_async_build_agentic_loop_plan_rejects_hash_with_no_known_call( - guardrail: HeadroomGuardrail, -): + guardrail: HeadroomGuardrail, context: object, +) -> None: """A hash-shaped string planted in message text must not be honored when this guardrail has no record of ever issuing it, even if it's echoed back in the current request's own messages (e.g. via prompt injection).""" @@ -625,8 +633,6 @@ async def test_async_build_agentic_loop_plan_rejects_hash_with_no_known_call( arguments={"hash": "deadbeef000000000000dead"}, tool_id="call_xyz", ) - assert not guardrail._issued_hashes_by_call_id - with patch.object( guardrail.async_handler, "get", @@ -641,7 +647,7 @@ async def test_async_build_agentic_loop_plan_rejects_hash_with_no_known_call( anthropic_messages_optional_request_params={}, logging_obj=None, stream=False, - kwargs={"litellm_call_id": "call-unknown"}, + kwargs={HEADROOM_REQUEST_CONTEXT_KEY: context}, ) mock_get.assert_not_called() @@ -653,17 +659,11 @@ async def test_async_build_agentic_loop_plan_rejects_hash_with_no_known_call( @pytest.mark.asyncio -async def test_async_build_agentic_loop_plan_rejects_hash_issued_for_different_call( +async def test_async_build_agentic_loop_plan_rejects_hash_issued_for_different_request( guardrail: HeadroomGuardrail, ): - """A hash issued for one request must not be retrievable by a different - request just because the second request echoes that hash-shaped string - back in its own messages -- retrieval must be scoped per litellm_call_id, - not derived by re-scanning attacker-controlled message text.""" - guardrail._issued_hashes_by_call_id["call-A"] = ( - frozenset({"b573993006976af767214fac"}), - time.monotonic() + 999, - ) + issuer_data: Final[dict[str, object]] = {"model": "gpt-4o", "litellm_call_id": "call-A"} + await _apply(guardrail, ORIGINAL_MESSAGES, ["b573993006976af767214fac"], request_data=issuer_data) tool_calls = [ { @@ -693,7 +693,7 @@ async def test_async_build_agentic_loop_plan_rejects_hash_issued_for_different_c anthropic_messages_optional_request_params={}, logging_obj=None, stream=False, - kwargs={"litellm_call_id": "call-B"}, + kwargs={"litellm_call_id": "call-A"}, ) mock_get.assert_not_called() @@ -736,10 +736,6 @@ async def test_async_build_agentic_loop_plan_builds_responses_api_function_call_ } ] messages = [{"role": "user", "content": "What does it say? hash=b573993006976af767214fac"}] - guardrail._issued_hashes_by_call_id["call-1"] = ( - frozenset({"b573993006976af767214fac"}), - time.monotonic() + 999, - ) with patch.object( guardrail.async_handler, @@ -756,7 +752,7 @@ async def test_async_build_agentic_loop_plan_builds_responses_api_function_call_ anthropic_messages_optional_request_params={}, logging_obj=None, stream=False, - kwargs={"litellm_call_id": "call-1"}, + kwargs={HEADROOM_REQUEST_CONTEXT_KEY: _issued_context("b573993006976af767214fac")}, ) follow_up = plan.request_patch.messages # type: ignore[union-attr] @@ -806,7 +802,6 @@ async def test_responses_retrieval_followup_reuses_the_model_the_caller_requeste "arguments": json.dumps({"hash": retrieve_hash}), } ] - guardrail._issued_hashes_by_call_id["call-1"] = (frozenset({retrieve_hash}), time.monotonic() + 999) with patch.object( guardrail.async_handler, @@ -832,7 +827,7 @@ async def test_responses_retrieval_followup_reuses_the_model_the_caller_requeste anthropic_messages_optional_request_params={}, logging_obj=logging_obj, stream=False, - kwargs={"litellm_call_id": "call-1"}, + kwargs={HEADROOM_REQUEST_CONTEXT_KEY: _issued_context(retrieve_hash)}, ) assert plan.request_patch is not None @@ -876,10 +871,6 @@ async def test_async_build_agentic_loop_plan_builds_anthropic_tool_result_messag } ] messages = [{"role": "user", "content": "What does it say? hash=b573993006976af767214fac"}] - guardrail._issued_hashes_by_call_id["call-1"] = ( - frozenset({"b573993006976af767214fac"}), - time.monotonic() + 999, - ) with patch.object( guardrail.async_handler, @@ -896,7 +887,7 @@ async def test_async_build_agentic_loop_plan_builds_anthropic_tool_result_messag anthropic_messages_optional_request_params={}, logging_obj=None, stream=False, - kwargs={"litellm_call_id": "call-1"}, + kwargs={HEADROOM_REQUEST_CONTEXT_KEY: _issued_context("b573993006976af767214fac")}, ) follow_up = plan.request_patch.messages # type: ignore[union-attr] @@ -929,16 +920,20 @@ HASH_SHAPED_HISTORY = [ ] -async def _apply(guardrail: HeadroomGuardrail, messages: list, ccr_hashes: list | None = None) -> dict: - request_data = {"model": "claude-sonnet-5"} +async def _apply( + guardrail: HeadroomGuardrail, messages: list, ccr_hashes: list | None = None, + request_data: dict[str, object] | None = None, +) -> dict: + data: Final = request_data if request_data is not None else {"model": "claude-sonnet-5"} def _echo(**kwargs): + assert kwargs["json"].get("config") == ({"mode": "ccr"} if guardrail.ccr_retrieval else None) return _make_compress_response(json.loads(json.dumps(kwargs["json"]["messages"])), ccr_hashes=ccr_hashes) with patch.object(guardrail.async_handler, "post", new_callable=AsyncMock, side_effect=_echo): return await guardrail.apply_guardrail( inputs=GenericGuardrailAPIInputs(texts=["x"], structured_messages=json.loads(json.dumps(messages))), - request_data=request_data, + request_data=data, input_type="request", ) @@ -950,10 +945,11 @@ async def test_hash_shaped_text_in_history_never_injects_retrieve_tool(guardrail tool must not appear and no hash may be registered as issued. Covers a service that omits ccr_hashes, returns it empty, or returns a non-list.""" for ccr_hashes in (None, [], "b573993006976af767214fac"): - result = await _apply(guardrail, HASH_SHAPED_HISTORY, ccr_hashes=ccr_hashes) + request_data: Final[dict[str, object]] = {} + result = await _apply(guardrail, HASH_SHAPED_HISTORY, ccr_hashes=ccr_hashes, request_data=request_data) assert not has_headroom_retrieve_tool(result.get("tools") or []) - assert not guardrail._issued_hashes_by_call_id + assert HEADROOM_REQUEST_CONTEXT_KEY not in request_data @pytest.mark.asyncio @@ -961,15 +957,18 @@ async def test_service_declared_ccr_hashes_drive_injection_and_validation(guardr """Only the hashes the service reports in ccr_hashes are honored, in the service's own 12 to 24 hex grammar; anything else is dropped because each entry is interpolated into the /v1/retrieve URL.""" + request_data: Final[dict[str, object]] = {} result = await _apply( guardrail, HASH_SHAPED_HISTORY, ccr_hashes=["98CA69107318", "b573993006976af767214fac", "../../etc/passwd", "tooshort", 42], + request_data=request_data, ) assert has_headroom_retrieve_tool(result.get("tools") or []) - ((issued, _expiry),) = guardrail._issued_hashes_by_call_id.values() - assert issued == frozenset({"98ca69107318", "b573993006976af767214fac"}) + context: Final = request_data[HEADROOM_REQUEST_CONTEXT_KEY] + assert isinstance(context, HeadroomRequestContext) + assert context.issued_hashes == frozenset({"98ca69107318", "b573993006976af767214fac"}) @pytest.mark.asyncio @@ -987,7 +986,6 @@ async def test_ccr_retrieval_disabled_ignores_service_declared_hashes(monkeypatc assert result["structured_messages"][-1] == HASH_SHAPED_HISTORY[-1] assert not has_headroom_retrieve_tool(result.get("tools") or []) - assert not guardrail._issued_hashes_by_call_id @pytest.mark.asyncio @@ -1498,6 +1496,7 @@ async def test_apply_guardrail_fail_open_does_not_register_hashes_from_original_ ] guardrail = _make_guardrail(unreachable_fallback="fail_open") + request_data: Final = {HEADROOM_REQUEST_CONTEXT_KEY: _issued_context("deadbeef000000000000dead")} inputs = GenericGuardrailAPIInputs( texts=["hello"], structured_messages=messages_with_fake_hash, @@ -1511,13 +1510,13 @@ async def test_apply_guardrail_fail_open_does_not_register_hashes_from_original_ ): result = await guardrail.apply_guardrail( inputs=inputs, - request_data={}, + request_data=request_data, input_type="request", ) assert result["structured_messages"] == messages_with_fake_hash assert not has_headroom_retrieve_tool(result.get("tools") or []) - assert not guardrail._issued_hashes_by_call_id + assert HEADROOM_REQUEST_CONTEXT_KEY not in request_data @pytest.mark.asyncio @@ -2164,18 +2163,18 @@ def _openai_completion_payload(message: dict, finish_reason: str) -> dict: } -def _openai_tool_call_payload() -> dict: +def _openai_tool_call_payload(hash_value: str = CCR_HASH) -> dict: return _openai_completion_payload( { "role": "assistant", "content": None, "tool_calls": [ { - "id": "call_ccr", + "id": f"call_{hash_value}", "type": "function", "function": { "name": HEADROOM_RETRIEVE_TOOL_NAME, - "arguments": json.dumps({"hash": CCR_HASH}), + "arguments": json.dumps({"hash": hash_value}), }, } ], @@ -2215,14 +2214,14 @@ def _openai_responses_payload(output_item: dict) -> dict: } -def _openai_responses_retrieve_call_payload() -> dict: +def _openai_responses_retrieve_call_payload(hash_value: str = CCR_HASH) -> dict: return _openai_responses_payload( { "type": "function_call", "id": "fc_ccr", - "call_id": "call_ccr", + "call_id": f"call_{hash_value}", "name": HEADROOM_RETRIEVE_TOOL_NAME, - "arguments": json.dumps({"hash": CCR_HASH}), + "arguments": json.dumps({"hash": hash_value}), "status": "completed", } ) @@ -2381,10 +2380,6 @@ async def test_streaming_chat_completion_resolves_ccr_retrieval_end_to_end( to a client that never declared the tool.""" original_content = "the full uncompressed document" final_answer = "the document says hello" - guardrail._issued_hashes_by_call_id["ccr-call-id"] = ( - frozenset({CCR_HASH}), - time.monotonic() + 999, - ) monkeypatch.setenv("OPENAI_API_KEY", "sk-test") monkeypatch.setattr(litellm, "callbacks", [guardrail]) @@ -2392,6 +2387,7 @@ async def test_streaming_chat_completion_resolves_ccr_retrieval_end_to_end( upstream = respx_mock.post("https://api.openai.com/v1/chat/completions").mock( side_effect=[ httpx.Response(200, json=_openai_tool_call_payload()), + httpx.Response(200, json=_openai_tool_call_payload("a" * 24)), httpx.Response(200, json=_openai_text_payload(final_answer)), ] ) @@ -2408,20 +2404,22 @@ async def test_streaming_chat_completion_resolves_ccr_retrieval_end_to_end( tools=[_retrieve_tool_definition()], stream=True, litellm_call_id="ccr-call-id", + **{HEADROOM_REQUEST_CONTEXT_KEY: _issued_context(CCR_HASH, "a" * 24, affinity="original-affinity")}, ) chunks = [chunk async for chunk in response] streamed_text = "".join(chunk.choices[0].delta.content or "" for chunk in chunks if chunk.choices) assert streamed_text == final_answer assert not any(chunk.choices and chunk.choices[0].delta.tool_calls for chunk in chunks) - mock_get.assert_called_once() - assert CCR_HASH in (mock_get.call_args.kwargs.get("url") or mock_get.call_args.args[0]) + assert mock_get.call_count == 2 + assert mock_get.call_args.kwargs["url"].endswith("a" * 24) + assert all(call.kwargs["headers"]["x-headroom-affinity"] == "original-affinity" for call in mock_get.call_args_list) - assert len(upstream.calls) == 2 - followup_body = json.loads(upstream.calls[1].request.content) + assert len(upstream.calls) == 3 + followup_body = json.loads(upstream.calls[2].request.content) assert not followup_body.get("stream") assert original_content in json.dumps(followup_body["messages"]) - assert not any(key.startswith("_headroom_interception") for key in followup_body) + assert not any(key.startswith("_headroom") for key in followup_body) @pytest.mark.asyncio @@ -2435,10 +2433,6 @@ async def test_streaming_responses_resolves_ccr_retrieval_end_to_end( instead of streaming a headroom_retrieve function_call to the client.""" original_content = "the full uncompressed document" final_answer = "the document says hello" - guardrail._issued_hashes_by_call_id["ccr-call-id"] = ( - frozenset({CCR_HASH}), - time.monotonic() + 999, - ) monkeypatch.setenv("OPENAI_API_KEY", "sk-test") monkeypatch.setattr(litellm, "callbacks", [guardrail]) @@ -2446,6 +2440,7 @@ async def test_streaming_responses_resolves_ccr_retrieval_end_to_end( upstream = respx_mock.post("https://api.openai.com/v1/responses").mock( side_effect=[ httpx.Response(200, json=_openai_responses_retrieve_call_payload()), + httpx.Response(200, json=_openai_responses_retrieve_call_payload("a" * 24)), httpx.Response(200, json=_openai_responses_text_payload(final_answer)), ] ) @@ -2462,6 +2457,7 @@ async def test_streaming_responses_resolves_ccr_retrieval_end_to_end( tools=[_responses_retrieve_tool_definition()], stream=True, litellm_call_id="ccr-call-id", + **{HEADROOM_REQUEST_CONTEXT_KEY: _issued_context(CCR_HASH, "a" * 24, affinity="original-affinity")}, ) events = [event async for event in response] @@ -2471,14 +2467,15 @@ async def test_streaming_responses_resolves_ccr_retrieval_end_to_end( assert streamed_text == final_answer assert not any("function_call" in str(getattr(event, "type", "")) for event in events) assert not any(getattr(getattr(event, "item", None), "type", None) == "function_call" for event in events) - mock_get.assert_called_once() - assert CCR_HASH in (mock_get.call_args.kwargs.get("url") or mock_get.call_args.args[0]) + assert mock_get.call_count == 2 + assert mock_get.call_args.kwargs["url"].endswith("a" * 24) + assert all(call.kwargs["headers"]["x-headroom-affinity"] == "original-affinity" for call in mock_get.call_args_list) - assert len(upstream.calls) == 2 - followup_body = json.loads(upstream.calls[1].request.content) + assert len(upstream.calls) == 3 + followup_body = json.loads(upstream.calls[2].request.content) assert not followup_body.get("stream") assert original_content in json.dumps(followup_body["input"]) - assert not any(key.startswith("_headroom_interception") for key in followup_body) + assert not any(key.startswith("_headroom") for key in followup_body) def test_sync_streaming_responses_resolves_ccr_retrieval_end_to_end( @@ -2491,10 +2488,6 @@ def test_sync_streaming_responses_resolves_ccr_retrieval_end_to_end( completed response object.""" original_content = "the full uncompressed document" final_answer = "the document says hello" - guardrail._issued_hashes_by_call_id["ccr-call-id"] = ( - frozenset({CCR_HASH}), - time.monotonic() + 999, - ) monkeypatch.setenv("OPENAI_API_KEY", "sk-test") monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) @@ -2518,6 +2511,7 @@ def test_sync_streaming_responses_resolves_ccr_retrieval_end_to_end( tools=[_responses_retrieve_tool_definition()], stream=True, litellm_call_id="ccr-call-id", + **{HEADROOM_REQUEST_CONTEXT_KEY: _issued_context(CCR_HASH)}, ) events = list(response) @@ -2917,7 +2911,6 @@ def _anthropic_response_with_text_and_tool_call() -> dict: async def _plan_for(guardrail: HeadroomGuardrail, response, messages: list): - guardrail._issued_hashes_by_call_id["call-1"] = (frozenset({"h" * 24}), time.monotonic() + 60) logging_obj = MagicMock() logging_obj.litellm_call_id = "call-1" logging_obj.model_call_details = {} @@ -2938,7 +2931,7 @@ async def _plan_for(guardrail: HeadroomGuardrail, response, messages: list): anthropic_messages_optional_request_params={}, logging_obj=logging_obj, stream=False, - kwargs={}, + kwargs={HEADROOM_REQUEST_CONTEXT_KEY: _issued_context("h" * 24)}, ) @@ -3033,7 +3026,7 @@ async def test_retrieve_call_passes_bounded_timeout(guardrail: HeadroomGuardrail new_callable=AsyncMock, return_value=_make_retrieve_response("original"), ) as mock_get: - result = await guardrail._call_retrieve("a" * 24) + result = await guardrail._call_retrieve("a" * 24, affinity=None) assert result == "original" timeout = _timeout_of(mock_get.call_args) @@ -3143,3 +3136,16 @@ def test_unusable_timeout_falls_back_to_the_default(configured: float): assert guardrail.timeout.read == 60.0 assert guardrail.timeout.connect == 5.0 + + +@pytest.mark.parametrize("session", [None, "conversation", "other"]) +@pytest.mark.parametrize("scope", [None, {}, {"api_key": "key"}, {"api_key": "other"}, {"user_id": "user"}]) +def test_affinity_requires_trusted_identity_and_caller_session( + scope: dict[str, str] | None, session: str | None +) -> None: + baseline: Final = {"session_id": "conversation", "metadata": {"user_api_key_auth": UserAPIKeyAuth(api_key="key")}} + auth: Final = UserAPIKeyAuth(**scope) if scope else scope + candidate: Final = {"session_id": session, "metadata": {"user_api_key_auth": auth}} + affinity: Final = _session_affinity(candidate) + assert (affinity is None) is (session is None or not isinstance(auth, UserAPIKeyAuth)) + assert (affinity == _session_affinity(baseline)) is (candidate == baseline) diff --git a/tests/unit/types/test_litellm_params.py b/tests/unit/types/test_litellm_params.py index e369b7e93ab..22eb9929e7f 100644 --- a/tests/unit/types/test_litellm_params.py +++ b/tests/unit/types/test_litellm_params.py @@ -33,6 +33,7 @@ from litellm.types.litellm_params import ( wire_names, ) from litellm.types.llms.openai import ChatCompletionAssistantMessage, ChatCompletionUserMessage +from litellm.types.proxy.guardrails.guardrail_hooks.headroom import HeadroomRequestContext from litellm.types.proxy.litellm_pre_call_utils import SecretFields from litellm.types.router import ( ConfigurableClientsideParamsCustomAuth, @@ -231,6 +232,7 @@ OPTION_NAMES: Final = ( ) AGENTIC_LOOP_STATE_NAMES: Final = ( + "_headroom_request_context", "_agentic_loop_depth", "_agentic_loop_fingerprints", "_agentic_loop_api_surface", @@ -529,6 +531,7 @@ TYPE_HINT_NAMESPACE: Final[Mapping[str, object]] = { "Logging": Logging, "SecretFields": SecretFields, "CompactionState": CompactionState, + "HeadroomRequestContext": HeadroomRequestContext, "RouterWeights": RouterWeights, "AttemptedFallbackTargets": AttemptedFallbackTargets, } From 7b259690e7eac9720ee21067a71152912f1080ef Mon Sep 17 00:00:00 2001 From: tin-berri Date: Fri, 30 Oct 2026 12:36:11 -0700 Subject: [PATCH 31/74] fix(proxy): don't send llm_exceptions alert for BudgetExceededError (#49290) * fix(proxy): don't send llm_exceptions alert for BudgetExceededError * test(proxy): verify budget alert suppression through failure hooks --------- Co-authored-by: Arvind Kumar --- litellm/proxy/utils.py | 2 + ..._utils_model_creation_and_error_logging.py | 49 +++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 00dbcd0f9a8..cc72dd03a40 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -633,6 +633,8 @@ def _is_client_error_exception(exc: Exception) -> bool: return exc.status_code < 500 if isinstance(exc, ProxyException): return not (exc.code.isdigit() and int(exc.code) >= 500) + if isinstance(exc, litellm.BudgetExceededError): + return exc.status_code < 500 return False diff --git a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py index 3cdd2edb2fb..0a483840336 100644 --- a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py +++ b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py @@ -1844,6 +1844,26 @@ class TestPostCallFailureHookLLMExceptionAlerting: alerting_handler = await self._alerted(HTTPException(status_code=429, detail="rate limited")) assert alerting_handler.called is False + @pytest.mark.asyncio + @pytest.mark.parametrize("status_code, should_alert", ((400, False), (422, False), (429, False), (500, True))) + async def test_budget_exceeded_alerts_only_for_server_status( + self, status_code: int, should_alert: bool, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(litellm, "budget_exceeded_status_code", status_code) + exc: Final = litellm.BudgetExceededError(current_cost=1.0, max_budget=0.5) + assert (await self._alerted(exc)).called is should_alert + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "exc", + ( + litellm.AuthenticationError(message="invalid provider key", model="test-model", llm_provider="openai"), + litellm.RateLimitError(message="provider quota exceeded", model="test-model", llm_provider="openai"), + ), + ) + async def test_provider_client_status_still_alerts(self, exc: Exception) -> None: + assert (await self._alerted(exc)).called is True + class TestPostCallFailureHookProxyExceptionLogging: """A guardrail block raises a ProxyException; on an LLM route it must still @@ -4109,3 +4129,32 @@ def test_handle_exception_on_proxy_logs_bug_report_only_for_unmapped_500(caplog) assert provider_result.code == internal_result.code == "500" assert ISSUE_URL_BASE in caplog.text assert ISSUE_URL_BASE not in internal_result.message + + +def test_is_client_error_exception_budget_exceeded_is_client_error() -> None: + from litellm.exceptions import BudgetExceededError + from litellm.proxy.utils import _is_client_error_exception + + exc: Final = BudgetExceededError(current_cost=100.56, max_budget=100.0) + assert _is_client_error_exception(exc) is True + + +def test_is_client_error_exception_budget_exceeded_5xx_is_not_client_error() -> None: + from litellm.exceptions import BudgetExceededError + from litellm.proxy.utils import _is_client_error_exception + + exc: Final = BudgetExceededError(current_cost=100.56, max_budget=100.0) + exc.status_code = 500 + assert _is_client_error_exception(exc) is False + + +def test_is_client_error_exception_existing_behaviour_unchanged() -> None: + from litellm.proxy._types import ProxyException + from litellm.proxy.utils import _is_client_error_exception + + assert _is_client_error_exception(HTTPException(status_code=400, detail="x")) is True + assert _is_client_error_exception(HTTPException(status_code=500, detail="x")) is False + assert _is_client_error_exception( + ProxyException(message="x", type="x", param=None, code="429") + ) is True + assert _is_client_error_exception(ValueError("x")) is False From 67954b5c3a4b00a1da3c86b18bca8452194002ac Mon Sep 17 00:00:00 2001 From: moe-berri Date: Fri, 30 Oct 2026 12:38:21 -0700 Subject: [PATCH 32/74] fix(proxy): apply key and team router_settings to /v1/images/generations (#49286) * fix(proxy): apply key and team router_settings to /v1/images/generations Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(proxy): format router settings helper Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): resolve key model_group_alias once on image generations Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/common_request_processing.py | 93 ++++++---- litellm/proxy/image_endpoints/endpoints.py | 18 ++ ...st_key_router_settings_image_generation.py | 162 ++++++++++++++++++ .../proxy/image_endpoints/test_endpoints.py | 148 +++++++++++++++- 4 files changed, 389 insertions(+), 32 deletions(-) create mode 100644 tests/integration/routing/test_key_router_settings_image_generation.py diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 762f552dadc..9b2813a8cd8 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -151,7 +151,7 @@ from litellm.router_utils.common_utils import resolve_model_group_alias, resolve from litellm.types.extraction import ExtractionResponse from litellm.types.guardrails import GuardrailEventHooks from litellm.types.router import RouterRateLimitError -from litellm.types.router_weights import validate_router_weights +from litellm.types.router_weights import RouterWeights, validate_router_weights _LateResponseT = TypeVar("_LateResponseT", bound=Response) _LlmCallT = TypeVar("_LlmCallT") @@ -894,6 +894,61 @@ async def _resolve_per_request_model_group_alias( return target +async def hierarchical_router_settings_updates( + requested_model: object, + user_api_key_dict: UserAPIKeyAuth, + llm_router: Router, + proxy_config: ProxyConfig, + proxy_logging_obj: ProxyLogging, + apply_model_group_alias: bool, +) -> Mapping[str, object]: + # Apply hierarchical router_settings (Key > Team) + # Global router_settings are already on the Router object itself. + # This sits with the other alias rewrites, and ahead of the guardrail + # merge and the pre-call hooks, so everything that keys off the model + # group -- model-level guardrails, per-model budgets and rate limits, + # the logging object -- sees the group that will actually serve. + from litellm.proxy.proxy_server import prisma_client + + router_settings: Final[Mapping[str, object] | None] = _ROUTER_SETTINGS_ADAPTER.validate_python( + await proxy_config._get_hierarchical_router_settings( + user_api_key_dict=user_api_key_dict, + prisma_client=prisma_client, + proxy_logging_obj=proxy_logging_obj, + ) + ) + if router_settings is None: + return MappingProxyType({}) + + # If router_settings found (from key or team), apply them + # Pass settings as per-request overrides instead of creating a new Router + # This avoids expensive Router instantiation on each request + def validated_router_weights() -> RouterWeights | None: + try: + return validate_router_weights(router_settings.get("weights")) + except ValidationError: + verbose_proxy_logger.warning("Ignoring invalid saved router weights; update team/key router_settings") + return None + + router_weights: Final = validated_router_weights() + alias_target: Final = ( + await _resolve_per_request_model_group_alias( + requested_model=requested_model, + router_settings=router_settings, + user_api_key_dict=user_api_key_dict, + llm_router=llm_router, + ) + if apply_model_group_alias + else None + ) + settings_updates: Final[Mapping[str, object]] = { + "router_settings_override": router_settings, + "_router_weights": router_weights, + **({"model": alias_target} if alias_target is not None else {}), + } + return MappingProxyType(settings_updates) + + _REQUEST_MODEL: Final[TypeAdapter[str | list[str] | None]] = TypeAdapter(str | list[str] | None) _DECODED_JSON: Final = TypeAdapter(object) _ROUTER_SETTINGS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) @@ -2495,41 +2550,17 @@ class ProxyBaseLLMRequestProcessing: ): self.data["model"] = user_api_key_dict.aliases[self.data["model"]] - # Apply hierarchical router_settings (Key > Team) - # Global router_settings are already on the Router object itself. - # This sits with the other alias rewrites, and ahead of the guardrail - # merge and the pre-call hooks, so everything that keys off the model - # group -- model-level guardrails, per-model budgets and rate limits, - # the logging object -- sees the group that will actually serve. if llm_router is not None and proxy_config is not None: - from litellm.proxy.proxy_server import prisma_client - - router_settings: Final = await proxy_config._get_hierarchical_router_settings( - user_api_key_dict=user_api_key_dict, - prisma_client=prisma_client, - proxy_logging_obj=proxy_logging_obj, - ) - - # If router_settings found (from key or team), apply them - # Pass settings as per-request overrides instead of creating a new Router - # This avoids expensive Router instantiation on each request - if router_settings is not None: - self.data["router_settings_override"] = router_settings - try: - self.data["_router_weights"] = validate_router_weights(router_settings.get("weights")) - except ValidationError: - self.data["_router_weights"] = None - verbose_proxy_logger.warning( - "Ignoring invalid saved router weights; update team/key router_settings" - ) - alias_target: Final = await _resolve_per_request_model_group_alias( + self.data.update( + await hierarchical_router_settings_updates( requested_model=self.data.get("model"), - router_settings=router_settings, user_api_key_dict=user_api_key_dict, llm_router=llm_router, + proxy_config=proxy_config, + proxy_logging_obj=proxy_logging_obj, + apply_model_group_alias=True, ) - if alias_target is not None: - self.data["model"] = alias_target + ) aliased_model: Final = self.data.get("model") if llm_router is not None and route_type in ROUTER_SCHEDULED_ROUTE_TYPES and isinstance(aliased_model, str): diff --git a/litellm/proxy/image_endpoints/endpoints.py b/litellm/proxy/image_endpoints/endpoints.py index 28fb9baf2aa..6244e5025ce 100644 --- a/litellm/proxy/image_endpoints/endpoints.py +++ b/litellm/proxy/image_endpoints/endpoints.py @@ -8,8 +8,10 @@ from typing import Final, get_type_hints import orjson from fastapi import APIRouter, Depends, File, HTTPException, Request, Response, UploadFile, status from fastapi.responses import ORJSONResponse +from pydantic import TypeAdapter import litellm +from litellm.constants import MODEL_GROUP_ALIAS_RESOLVED_SCOPE_KEY from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_str_from_messages, ) @@ -17,6 +19,7 @@ from litellm.proxy._types import * from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth from litellm.proxy.common_request_processing import ( ProxyBaseLLMRequestProcessing, + hierarchical_router_settings_updates, log_llm_api_exception, resolve_litellm_call_id, ) @@ -36,6 +39,7 @@ from litellm.types.images.main import ImageEditRequestParams from litellm.types.llms.openai import ChatCompletionUserMessage router: Final = APIRouter() +_IMAGE_GENERATION_MODEL_ADAPTER: Final[TypeAdapter[object]] = TypeAdapter(object) IMAGE_EDIT_NUMERIC_FORM_FIELDS: Final = numeric_form_fields(get_type_hints(ImageEditRequestParams)) IMAGE_EDIT_OPTIONAL_FIELD_DEFAULTS: Final = MappingProxyType({"prompt": None, "image": None}) @@ -132,6 +136,19 @@ async def image_generation( if data["model"] in litellm.model_alias_map: data["model"] = litellm.model_alias_map[data["model"]] + if llm_router is not None: + requested_model: Final = _IMAGE_GENERATION_MODEL_ADAPTER.validate_python(data.get("model")) + data.update( + await hierarchical_router_settings_updates( + requested_model=requested_model, + user_api_key_dict=user_api_key_dict, + llm_router=llm_router, + proxy_config=proxy_config, + proxy_logging_obj=proxy_logging_obj, + apply_model_group_alias=request.scope.get(MODEL_GROUP_ALIAS_RESOLVED_SCOPE_KEY) is not True, + ) + ) + ### CALL HOOKS ### - modify incoming data / reject request before calling the model prompt_value: Final = data.get("prompt") if prompt_value is not None: @@ -156,6 +173,7 @@ async def image_generation( route_type="aimage_generation", llm_router=llm_router, user_model=user_model, + user_api_key_dict=user_api_key_dict, ) response = await llm_call diff --git a/tests/integration/routing/test_key_router_settings_image_generation.py b/tests/integration/routing/test_key_router_settings_image_generation.py new file mode 100644 index 00000000000..84415c72f2f --- /dev/null +++ b/tests/integration/routing/test_key_router_settings_image_generation.py @@ -0,0 +1,162 @@ +from __future__ import annotations + +import json +import uuid +from types import MappingProxyType +from typing import Final + +from integration._support.client import Gateway, Scenario, gateway_from_environment +from integration._support.wire import Reply, Request, Wire, wire_server + +FRESH_CONNECTION: Final = MappingProxyType({"Connection": "close"}) +IMAGE_RESPONSE: Final = { + "created": 1, + "background": None, + "data": [ + { + "b64_json": None, + "revised_prompt": None, + "url": "https://images.example/generated.png", + "provider_specific_fields": None, + } + ], + "output_format": None, + "quality": None, + "size": None, + "usage": { + "total_tokens": 0, + "input_tokens": 0, + "input_tokens_details": {"image_tokens": 0, "text_tokens": 0}, + "output_tokens": 0, + }, +} + + +def _primary_unavailable(_: Request) -> Reply: + return Reply( + status=503, + body=json.dumps({"error": {"message": "primary unavailable", "type": "server_error"}}).encode(), + ) + + +def _image_reply(_: Request) -> Reply: + return Reply( + body=json.dumps( + { + "created": 1, + "data": [{"url": "https://images.example/generated.png"}], + } + ).encode() + ) + + +def _image_model(scenario: Scenario, group: str, wire: Wire) -> str: + return scenario.model( + name=group, + model="openai/dall-e-3", + api_base=f"{wire.url}/v1", + api_key="integration-provider-key", + num_retries=0, + ) + + +def _assert_image_fallback( + gateway: Gateway, + key: str, + primary: Wire, + fallback: Wire, + primary_model: str, +) -> None: + response: Final = gateway.request( + "POST", + "/v1/images/generations", + {"model": primary_model, "prompt": "draw a tree"}, + key=key, + headers=FRESH_CONNECTION, + ) + + assert response.status_code == 200, response.text + assert response.json() == IMAGE_RESPONSE, response.text + assert tuple((request.method, request.target) for request in primary.drain()) == ( + ("POST", "/v1/images/generations"), + ), response.text + assert tuple((request.method, request.target) for request in fallback.drain()) == ( + ("POST", "/v1/images/generations"), + ), response.text + + +def test_key_router_settings_fallback_routes_image_generation() -> None: + with ( + wire_server(_primary_unavailable) as primary_wire, + wire_server(_image_reply) as fallback_wire, + gateway_from_environment() as gateway, + gateway.scenario() as scenario, + ): + primary_model: Final = _image_model(scenario, f"image-primary-{uuid.uuid4().hex}", primary_wire) + fallback_model: Final = _image_model(scenario, f"image-fallback-{uuid.uuid4().hex}", fallback_wire) + key: Final = scenario.key(router_settings={"fallbacks": [{primary_model: [fallback_model]}], "num_retries": 0}) + + _assert_image_fallback(gateway, key, primary_wire, fallback_wire, primary_model) + + +def test_team_router_settings_fallback_routes_image_generation() -> None: + with ( + wire_server(_primary_unavailable) as primary_wire, + wire_server(_image_reply) as fallback_wire, + gateway_from_environment() as gateway, + gateway.scenario() as scenario, + ): + primary_model: Final = _image_model(scenario, f"image-primary-{uuid.uuid4().hex}", primary_wire) + fallback_model: Final = _image_model(scenario, f"image-fallback-{uuid.uuid4().hex}", fallback_wire) + team: Final = scenario.team( + router_settings={"fallbacks": [{primary_model: [fallback_model]}], "num_retries": 0} + ) + key: Final = scenario.key(team_id=team) + + _assert_image_fallback(gateway, key, primary_wire, fallback_wire, primary_model) + + +def test_key_model_group_alias_resolves_once_for_image_generation() -> None: + with ( + wire_server(_image_reply) as primary_wire, + wire_server(_image_reply) as target_wire, + wire_server(_image_reply) as chained_target_wire, + gateway_from_environment() as gateway, + gateway.scenario() as scenario, + ): + primary_model: Final = _image_model(scenario, f"image-primary-{uuid.uuid4().hex}", primary_wire) + target_model: Final = _image_model(scenario, f"image-target-{uuid.uuid4().hex}", target_wire) + chained_target_model: Final = _image_model( + scenario, f"image-chained-target-{uuid.uuid4().hex}", chained_target_wire + ) + alias_name: Final = f"image-alias-{uuid.uuid4().hex}" + key: Final = scenario.key( + router_settings={ + "model_group_alias": { + alias_name: target_model, + target_model: chained_target_model, + } + } + ) + + response: Final = gateway.request( + "POST", + "/v1/images/generations", + {"model": alias_name, "prompt": "draw a tree"}, + key=key, + headers=FRESH_CONNECTION, + ) + primary_requests: Final = tuple((request.method, request.target) for request in primary_wire.drain()) + target_requests: Final = tuple((request.method, request.target) for request in target_wire.drain()) + chained_target_requests: Final = tuple( + (request.method, request.target) for request in chained_target_wire.drain() + ) + + assert alias_name not in {primary_model, target_model, chained_target_model}, response.text + assert response.status_code == 200, response.text + assert response.json() == IMAGE_RESPONSE, response.text + assert primary_requests == (), response.text + assert target_requests == (("POST", "/v1/images/generations"),), ( + f"{response.text}; chained target requests: {chained_target_requests}" + ) + assert chained_target_requests == (), response.text diff --git a/tests/unit/proxy/image_endpoints/test_endpoints.py b/tests/unit/proxy/image_endpoints/test_endpoints.py index f4aebecc11e..6a2c7e3ed41 100644 --- a/tests/unit/proxy/image_endpoints/test_endpoints.py +++ b/tests/unit/proxy/image_endpoints/test_endpoints.py @@ -3,7 +3,7 @@ import copy import logging from collections.abc import Iterator, Mapping from types import SimpleNamespace -from typing import Any, Dict +from typing import Any, Dict, Final, cast import orjson import pytest @@ -12,10 +12,17 @@ from fastapi.testclient import TestClient from starlette.requests import Request from starlette.responses import Response +import litellm from litellm._logging import verbose_proxy_logger +from litellm.models.team import LiteLLM_TeamTableCachedObj from litellm.proxy._types import ProxyException, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.image_endpoints import endpoints +from litellm.proxy.proxy_server import PrismaClient, ProxyConfig +from litellm.proxy.utils import ProxyLogging +from litellm.types.utils import ImageObject, ImageResponse +from litellm.utils import custom_llm_setup @pytest.mark.asyncio @@ -410,3 +417,142 @@ async def test_failure_before_the_provider_call_bills_the_callers_litellm_call_i assert raised.value.headers["x-litellm-call-id"] == call_id assert [data["litellm_call_id"] for data in hook_request_data] == [call_id] + + +class _ImageRouterHandler(litellm.CustomLLM): + def __init__(self) -> None: + self.calls: list[str] = [] + + async def aimage_generation( + self, + model: str, + prompt: str, + model_response: ImageResponse, + api_key: str | None, + api_base: str | None, + optional_params: Mapping[str, object], + logging_obj: object, + timeout: object | None = None, + client: object | None = None, + ) -> ImageResponse: + self.calls.append(model) + if model.endswith("failing"): + raise litellm.RateLimitError(message="primary failed", llm_provider="image-regression", model=model) + return ImageResponse(created=1, data=[ImageObject(url="https://images.example/generated.png")]) + + +def _image_generation_client( + monkeypatch: pytest.MonkeyPatch, + user_api_key_dict: UserAPIKeyAuth, + team_settings: Mapping[str, object] | None = None, +) -> tuple[TestClient, _ImageRouterHandler]: + from litellm.proxy import proxy_server + + handler: Final = _ImageRouterHandler() + cache: Final = UserApiKeyCache() + monkeypatch.setattr(litellm, "provider_list", [*litellm.provider_list]) + monkeypatch.setattr(litellm, "_custom_providers", [*litellm._custom_providers]) + monkeypatch.setattr(litellm, "custom_provider_map", [{"provider": "image-regression", "custom_handler": handler}]) + custom_llm_setup() + if team_settings is not None: + team_id: Final = user_api_key_dict.team_id + assert team_id is not None + cache.set_cache( + key=f"team_id:{team_id}", + value=LiteLLM_TeamTableCachedObj(team_id=team_id, router_settings=dict(team_settings)), + local_only=True, + model_type=LiteLLM_TeamTableCachedObj, + ) + monkeypatch.setattr(proxy_server, "prisma_client", cast(PrismaClient, object())) + + router: Final = litellm.Router( + model_list=[ + {"model_name": "A", "litellm_params": {"model": "image-regression/failing"}}, + {"model_name": "B", "litellm_params": {"model": "image-regression/working"}}, + ], + num_retries=0, + ) + monkeypatch.setattr(litellm, "model_alias_map", {}) + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "proxy_config", ProxyConfig()) + monkeypatch.setattr(proxy_server, "proxy_logging_obj", ProxyLogging(user_api_key_cache=cache)) + monkeypatch.setattr(proxy_server, "user_api_key_cache", cache) + monkeypatch.setattr(proxy_server, "user_model", None) + monkeypatch.setattr(proxy_server, "version", "test-version") + + app: Final = FastAPI() + app.include_router(endpoints.router) + app.dependency_overrides[user_api_key_auth] = lambda: user_api_key_dict + return TestClient(app, raise_server_exceptions=False), handler + + +def test_image_generation_uses_key_router_settings_fallbacks(monkeypatch: pytest.MonkeyPatch) -> None: + client, handler = _image_generation_client( + monkeypatch, + UserAPIKeyAuth(router_settings={"fallbacks": [{"A": ["B"]}]}), + ) + + response: Final = client.post("/v1/images/generations", json={"model": "A", "prompt": "draw a tree"}) + + assert response.status_code == 200, response.text + assert response.json() == ImageResponse( + created=1, + data=[ImageObject(url="https://images.example/generated.png")], + ).model_dump(), response.text + assert handler.calls == ["failing", "working"], response.text + + +def test_image_generation_uses_team_router_settings_fallbacks(monkeypatch: pytest.MonkeyPatch) -> None: + team_id: Final = "image-regression-team" + client, handler = _image_generation_client( + monkeypatch, + UserAPIKeyAuth(team_id=team_id), + {"fallbacks": [{"A": ["B"]}]}, + ) + + response: Final = client.post("/v1/images/generations", json={"model": "A", "prompt": "draw a tree"}) + + assert response.status_code == 200, response.text + assert response.json() == ImageResponse( + created=1, + data=[ImageObject(url="https://images.example/generated.png")], + ).model_dump(), response.text + assert handler.calls == ["failing", "working"], response.text + + +def test_image_generation_without_router_settings_does_not_use_fallback(monkeypatch: pytest.MonkeyPatch) -> None: + client, handler = _image_generation_client(monkeypatch, UserAPIKeyAuth()) + + response: Final = client.post("/v1/images/generations", json={"model": "A", "prompt": "draw a tree"}) + + assert response.status_code == 500, response.text + assert handler.calls == ["failing"], response.text + + +def test_image_generation_uses_key_model_group_alias(monkeypatch: pytest.MonkeyPatch) -> None: + client, handler = _image_generation_client( + monkeypatch, + UserAPIKeyAuth(router_settings={"model_group_alias": {"alias": "B"}}), + ) + + response: Final = client.post("/v1/images/generations", json={"model": "alias", "prompt": "draw a tree"}) + + assert response.status_code == 200, response.text + assert response.json() == ImageResponse( + created=1, + data=[ImageObject(url="https://images.example/generated.png")], + ).model_dump(), response.text + assert handler.calls == ["working"], response.text + + +def test_image_generation_uses_key_num_retries(monkeypatch: pytest.MonkeyPatch) -> None: + client, handler = _image_generation_client( + monkeypatch, + UserAPIKeyAuth(router_settings={"num_retries": 2}), + ) + + response: Final = client.post("/v1/images/generations", json={"model": "A", "prompt": "draw a tree"}) + + assert response.status_code == 500, response.text + assert handler.calls == ["failing", "failing", "failing"], response.text From 8f968b6c8e2ed2a7c3a1bdbc4d75dc733308e14e Mon Sep 17 00:00:00 2001 From: moe-berri Date: Fri, 30 Oct 2026 12:51:48 -0700 Subject: [PATCH 33/74] feat(openrouter): record upstream provider in logging payload and spend logs (#49287) * test: cover OpenRouter upstream provider logging Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test: scope streaming provider metadata guard Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(openrouter): preserve upstream provider in logs Co-authored-by: BeWayllar <124474371+BeWayllar@users.noreply.github.com> Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): omit upstream_provider from spend metadata when unset * refactor(openrouter): set upstream provider hidden params without mutation --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: BeWayllar <124474371+BeWayllar@users.noreply.github.com> --- .../streaming_chunk_builder_utils.py | 34 ++- .../litellm_core_utils/streaming_handler.py | 23 +- .../llms/openrouter/chat/transformation.py | 30 ++- litellm/proxy/_types.py | 1 + .../spend_tracking/spend_tracking_utils.py | 21 ++ litellm/types/utils.py | 1 + .../test_openrouter_upstream_provider.py | 216 ++++++++++++++++++ .../test_litellm_logging.py | 7 +- .../test_streaming_chunk_builder_utils.py | 65 ++++++ .../test_openrouter_chat_transformation.py | 117 ++++++++++ .../test_spend_tracking_utils.py | 64 +++++- 11 files changed, 564 insertions(+), 15 deletions(-) create mode 100644 tests/integration/spend/test_openrouter_upstream_provider.py diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 0ddfa1210c2..4857973d51d 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -329,6 +329,20 @@ class ChunkProcessor: models[0] if models else None, ) + @staticmethod + def _get_upstream_provider(chunks: Sequence[object]) -> str | None: + def _provider_from_chunk(chunk: object) -> str | None: + hidden_params: Final = get_hidden_params(chunk) + if hidden_params is None: + return None + provider: Final[object] = hidden_params.get("upstream_provider") + return provider if isinstance(provider, str) and provider else None + + return next( + (provider for chunk in chunks if (provider := _provider_from_chunk(chunk)) is not None), + None, + ) + @staticmethod def apply_provider_assembled_streaming_metadata( response: ModelResponse, @@ -506,10 +520,22 @@ class ChunkProcessor: chunks, first_chunk_model, ) - if provider_response_model is not None: - response._hidden_params = dict( # pyright: ignore[reportPrivateUsage] # ModelResponse exposes no public hidden-params setter - response._hidden_params, # pyright: ignore[reportPrivateUsage] # ModelResponse exposes no public hidden-params getter - provider_response_model=provider_response_model, + upstream_provider: Final = self._get_upstream_provider(chunks) + provider_hidden_params: Final[Mapping[str, str]] = MappingProxyType( + { + key: value + for key, value in ( + ("provider_response_model", provider_response_model), + ("upstream_provider", upstream_provider), + ) + if value is not None + } + ) + if provider_hidden_params: + response_hidden_params: Final[Mapping[str, builtins.object]] = get_hidden_params(response) or {} + set_hidden_params( + response, + dict(response_hidden_params, **provider_hidden_params), ) return response diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 9710956c97e..484edd869cb 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -277,6 +277,7 @@ class _VertexChunkLike(Protocol): class _ParsedChunkHiddenParams(LiteLLMBaseModel): provider_specific_fields: Mapping[str, object] | None = None + upstream_provider: str | None = None def _provider_response_model(chunk: object) -> str | None: @@ -293,9 +294,17 @@ def _parsed_provider_hidden_params(hidden: object) -> _ParsedChunkHiddenParams | return None +def _provider_upstream_provider(chunk: object) -> str | None: + parsed: Final = _parsed_provider_hidden_params(get_hidden_params(chunk)) + if parsed is None or not parsed.upstream_provider: + return None + return parsed.upstream_provider + + def _provider_hidden_params( chunk: object, provider_response_model: str | None, + upstream_provider: str | None, ) -> Mapping[str, object] | None: hidden: Final = get_hidden_params(chunk) parsed: Final = _parsed_provider_hidden_params(hidden) @@ -308,6 +317,10 @@ def _provider_hidden_params( for key, value in ( ("provider_response_model", provider_response_model), ("provider_specific_fields", provider_specific_fields), + ( + "upstream_provider", + parsed.upstream_provider if parsed is not None and parsed.upstream_provider else upstream_provider, + ), ) if value is not None } @@ -360,6 +373,7 @@ class CustomStreamWrapper: self.system_fingerprint: str | None = None self._provider_response_model: str | None = None + self._upstream_provider: str | None = None self.received_finish_reason: str | None = None self.intermittent_finish_reason: str | None = None # finish reasons that show up mid-stream self.special_tokens = [ @@ -1653,10 +1667,17 @@ class CustomStreamWrapper: provider_response_model: Final = _provider_response_model(chunk) if provider_response_model is not None: self._provider_response_model = provider_response_model + upstream_provider: Final = _provider_upstream_provider(chunk) + if self._upstream_provider is None and upstream_provider is not None: + self._upstream_provider = upstream_provider prebuilt_delta: Final = self._prebuilt_plain_delta(chunk) plain_delta_created: bool = prebuilt_delta is not None # rebind-ok: the first chunk builds its delta below model_response = self.model_response_creator( - hidden_params=_provider_hidden_params(chunk, self._provider_response_model), + hidden_params=_provider_hidden_params( + chunk, + self._provider_response_model, + self._upstream_provider, + ), delta=prebuilt_delta, ) response_obj: dict[str, Any] = {} diff --git a/litellm/llms/openrouter/chat/transformation.py b/litellm/llms/openrouter/chat/transformation.py index cca6815399f..69f3d76c034 100644 --- a/litellm/llms/openrouter/chat/transformation.py +++ b/litellm/llms/openrouter/chat/transformation.py @@ -14,7 +14,7 @@ import httpx from pydantic import ConfigDict, TypeAdapter from litellm.litellm_core_utils.core_helpers import finite_cost -from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params +from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, get_hidden_params, set_hidden_params from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam @@ -212,19 +212,27 @@ class OpenrouterConfig(OpenAIGPTConfig): # OpenRouter returns cost information in the usage object when usage.include=true try: response_json: Final = _JSON_OBJECT.validate_python(raw_response.json()) + response_provider: Final = response_json.get("provider") + if isinstance(response_provider, str) and response_provider: + set_hidden_params( + model_response, + {**(get_hidden_params(model_response) or {}), "upstream_provider": response_provider}, + ) if "usage" in response_json and response_json["usage"]: response_cost: Final = _JSON_OBJECT.validate_python(response_json["usage"]).get("cost") if response_cost is not None: # Store cost in hidden params for the cost calculator to use if not hasattr(model_response, HIDDEN_PARAMS_ATTR): set_hidden_params(model_response, {}) - hidden_params: Final = cast( # cast-ok: preserve mapping operations on dynamic response metadata - dict[str, object], getattr(model_response, HIDDEN_PARAMS_ATTR) + cost_hidden_params: Final = ( + cast( # cast-ok: preserve mapping operations on dynamic response metadata + dict[str, object], getattr(model_response, HIDDEN_PARAMS_ATTR) + ) ) - if "additional_headers" not in hidden_params: - hidden_params["additional_headers"] = {} + if "additional_headers" not in cost_hidden_params: + cost_hidden_params["additional_headers"] = {} additional_headers: Final = cast( # cast-ok: preserve mapping operations on response metadata - dict[str, object], hidden_params["additional_headers"] + dict[str, object], cost_hidden_params["additional_headers"] ) reported_cost: Final = ( finite_cost(float(response_cost)) if isinstance(response_cost, str | int | float) else None @@ -282,7 +290,7 @@ class OpenRouterChatCompletionStreamingHandler(BaseModelResponseIterator): for choice in chunk["choices"]: choice["delta"]["reasoning_content"] = choice["delta"].get("reasoning") new_choices.append(choice) - return ModelResponseStream( + model_response: Final = ModelResponseStream( id=chunk["id"], object="chat.completion.chunk", created=chunk["created"], @@ -290,6 +298,12 @@ class OpenRouterChatCompletionStreamingHandler(BaseModelResponseIterator): model=chunk["model"], choices=new_choices, ) + typed_chunk: Final[Mapping[str, object]] = cast( # cast-ok: stream chunks are decoded JSON objects + Mapping[str, object], chunk + ) + upstream_provider: Final[object] = typed_chunk.get("provider") + if isinstance(upstream_provider, str) and upstream_provider: + set_hidden_params(model_response, {"upstream_provider": upstream_provider}) except KeyError as e: raise OpenRouterException( message=f"KeyError: {e}, Got unexpected response from OpenRouter: {chunk}", @@ -298,3 +312,5 @@ class OpenRouterChatCompletionStreamingHandler(BaseModelResponseIterator): ) except Exception as e: raise e + else: + return model_response diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index f165355da5a..73b12399af4 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -3758,6 +3758,7 @@ class SpendLogsMetadata(TypedDict): kubernetes_pod_routing: ReadOnly[StandardLoggingKubernetesPodRouting | None] internal_call_origin: InternalCallOrigin | None billed_retry_attempt: ReadOnly[NotRequired[bool | None]] + upstream_provider: ReadOnly[NotRequired[str | None]] litellm_roi_estimator: ReadOnly[NotRequired[bool | None]] is_silent_experiment: ReadOnly[NotRequired[bool | None]] guardrail_information: list[StandardLoggingGuardrailInformation] | None diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index f8d5cba4229..306e70f7029 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -11,6 +11,7 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Final, Literal, Protocol, cast, runtime_checkable from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError +from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_proxy_logger @@ -194,6 +195,7 @@ _STAMPED_METADATA_KEYS: Final = frozenset( "autorouter_savings_estimate", "autorouter_baseline_observation", "used_client_oauth_token", + "upstream_provider", KUBERNETES_POD_ROUTING_KEY, "litellm_roi_estimator", "is_silent_experiment", @@ -203,6 +205,10 @@ _STAMPED_METADATA_KEYS: Final = frozenset( ) +class _UpstreamProviderMetadata(TypedDict, total=False): + upstream_provider: ReadOnly[str] + + def _stored_user_email(metadata: Mapping[str, object]) -> str | None: email: Final = metadata.get("user_api_key_user_email") return email if isinstance(email, str) and email.strip() else None @@ -221,6 +227,7 @@ def _get_spend_logs_metadata( model_map_information: StandardLoggingModelInformation | None = None, cold_storage_object_key: str | None = None, litellm_overhead_time_ms: float | None = None, + upstream_provider: str | None = None, cost_breakdown: CostBreakdown | None = None, litellm_call_id: str | None = None, autorouter_savings: float | None = None, @@ -232,6 +239,9 @@ def _get_spend_logs_metadata( kubernetes_pod_routing: StandardLoggingKubernetesPodRouting | None = None, is_silent_experiment: bool = False, ) -> SpendLogsMetadata: + upstream_provider_metadata: Final[_UpstreamProviderMetadata] = ( + {"upstream_provider": upstream_provider} if upstream_provider is not None else {} + ) if metadata is None: return SpendLogsMetadata( user_api_key=None, @@ -265,6 +275,7 @@ def _get_spend_logs_metadata( eval_information=None, cold_storage_object_key=cold_storage_object_key, litellm_overhead_time_ms=None, + **upstream_provider_metadata, attempted_retries=None, max_retries=None, attempted_fallbacks=None, @@ -290,6 +301,7 @@ def _get_spend_logs_metadata( **MappingProxyType( {key: metadata.get(key) for key in SpendLogsMetadata.__annotations__ if key not in _STAMPED_METADATA_KEYS} ), + **upstream_provider_metadata, autorouter_savings=autorouter_savings, autorouter_savings_estimate=autorouter_savings_estimate, autorouter_baseline_observation=autorouter_baseline_observation, @@ -706,6 +718,14 @@ def get_logging_payload( if standard_logging_payload is not None: hidden_params: Final = standard_logging_payload.get("hidden_params", {}) litellm_overhead_time_ms = hidden_params.get("litellm_overhead_time_ms") + upstream_provider_value: Final[object] = ( + standard_logging_payload["hidden_params"]["upstream_provider"] + if standard_logging_payload is not None + and "hidden_params" in standard_logging_payload + and "upstream_provider" in standard_logging_payload["hidden_params"] + else None + ) + upstream_provider: Final = upstream_provider_value if isinstance(upstream_provider_value, str) else None logged_provider: Final = ( kwargs.get("custom_llm_provider") @@ -807,6 +827,7 @@ def get_logging_payload( else None ), litellm_overhead_time_ms=litellm_overhead_time_ms, + upstream_provider=upstream_provider, cost_breakdown=( standard_logging_payload.get("cost_breakdown", None) if standard_logging_payload is not None else None ), diff --git a/litellm/types/utils.py b/litellm/types/utils.py index f8115571dfd..db865c7488a 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3460,6 +3460,7 @@ class StandardLoggingHiddenParams(TypedDict): batch_models: list[str] | None batch_successful_requests: ReadOnly[int | None] batch_failed_requests: ReadOnly[int | None] + upstream_provider: ReadOnly[NotRequired[str | None]] litellm_model_name: str | None # the model name sent to the provider by litellm usage_object: dict | None diff --git a/tests/integration/spend/test_openrouter_upstream_provider.py b/tests/integration/spend/test_openrouter_upstream_provider.py new file mode 100644 index 00000000000..fa17a364aea --- /dev/null +++ b/tests/integration/spend/test_openrouter_upstream_provider.py @@ -0,0 +1,216 @@ +import json +import uuid +from collections.abc import Callable +from typing import Final, Literal + +import httpx +import pytest +from integration._support.client import JSON_OBJECT, Gateway, Scenario, eventually, object_value, string_value +from integration._support.database import read_rows +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue + +pytestmark: Final = pytest.mark.shares_worker_state + +_BACKEND: Final = "openai/gpt-4o-mini" +_PROMPT: Final = "upstream provider trace" +_PROVIDER: Final = "serving-site-a" +_ANSWER: Final = "hello" +_USAGE: Final[dict[str, JsonValue]] = { + "prompt_tokens": 5, + "completion_tokens": 1, + "total_tokens": 6, + "cost": 0.00015, +} + + +def _reply( + request_id: str, + *, + response_provider: bool, + stream_provider_mode: Literal["all", "last", "none"], +) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + assert request.method == "POST" + assert request.target.split("?", 1)[0].endswith("/chat/completions"), request.target + body: Final = JSON_OBJECT.validate_json(request.body) + assert body["messages"] == [{"role": "user", "content": _PROMPT}], body + if body.get("stream") is True: + common: Final = { + "id": request_id, + "object": "chat.completion.chunk", + "created": 1, + "model": _BACKEND, + } + all_provider: Final[dict[str, JsonValue]] = {"provider": _PROVIDER} + first_provider: Final = all_provider if stream_provider_mode == "all" else {} + finish_provider: Final = ( + all_provider if stream_provider_mode in ("all", "last") else {} + ) + usage_provider: Final = all_provider if stream_provider_mode == "all" else {} + chunks: Final[tuple[dict[str, JsonValue], ...]] = ( + { + **common, + **first_provider, + "choices": [{"index": 0, "delta": {"role": "assistant", "content": _ANSWER}, "finish_reason": None}], + }, + { + **common, + **finish_provider, + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + }, + {**common, **usage_provider, "choices": [], "usage": _USAGE}, + ) + return Reply( + content_type="text/event-stream", + chunks=tuple(f"data: {json.dumps(chunk)}\n\n".encode() for chunk in chunks) + + (b"data: [DONE]\n\n",), + ) + + provider: Final[dict[str, JsonValue]] = {"provider": _PROVIDER} if response_provider else {} + return Reply( + body=json.dumps( + { + "id": request_id, + "object": "chat.completion", + "created": 1, + "model": _BACKEND, + **provider, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": _ANSWER}, + "finish_reason": "stop", + } + ], + "usage": _USAGE, + } + ).encode() + ) + + return respond + + +def _stream_chunks(response: httpx.Response) -> tuple[dict[str, JsonValue], ...]: + return tuple( + JSON_OBJECT.validate_json(line.removeprefix("data: ")) + for line in response.text.splitlines() + if line.startswith("data: ") and line != "data: [DONE]" + ) + + +def _send( + gateway: Gateway, + scenario: Scenario, + wire_url: str, + request_id: str, + *, + openrouter: bool, + stream: bool, +) -> httpx.Response: + model: Final = scenario.model( + model=f"{'openrouter' if openrouter else 'openai'}/{_BACKEND}", + api_base=wire_url, + api_key="synthetic-provider-key", + ) + key: Final = scenario.key(models=[model]) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [{"role": "user", "content": _PROMPT}], + "stream": stream, + **({"stream_options": {"include_usage": True}} if stream else {}), + }, + key=key, + ) + assert response.status_code == 200, response.text + returned_id: Final = ( + string_value(_stream_chunks(response)[0]["id"]) + if stream + else string_value(JSON_OBJECT.validate_json(response.content)["id"]) + ) + assert returned_id == request_id, response.text + return response + + +def _spend_metadata(request_id: str) -> dict[str, JsonValue]: + rows: Final = eventually( + lambda: read_rows( + 'SELECT metadata FROM "LiteLLM_SpendLogs" WHERE request_id=%s', + (request_id,), + ), + lambda values: len(values) == 1, + seconds=70, + ) + return object_value(rows[0]["metadata"]) + + +@pytest.mark.parametrize( + ("stream", "stream_provider_mode"), + [ + pytest.param(False, "none", id="nonstream"), + pytest.param(True, "all", id="stream-every-chunk"), + pytest.param(True, "last", id="stream-later-chunk"), + ], +) +def test_openrouter_provider_is_persisted_in_spend_metadata_regression( + gateway: Gateway, stream: bool, stream_provider_mode: Literal["all", "last", "none"] +) -> None: + request_id: Final = f"chatcmpl-{uuid.uuid4().hex}" + with ( + wire_server( + _reply( + request_id, + response_provider=True, + stream_provider_mode=stream_provider_mode, + ) + ) as wire, + gateway.scenario() as scenario, + ): + _send(gateway, scenario, wire.url, request_id, openrouter=True, stream=stream) + metadata: Final = _spend_metadata(request_id) + + assert metadata.get("upstream_provider") == _PROVIDER, metadata + + +def test_openrouter_client_response_keeps_provider_in_body_guard(gateway: Gateway) -> None: + request_id: Final = f"chatcmpl-{uuid.uuid4().hex}" + with ( + wire_server( + _reply( + request_id, + response_provider=True, + stream_provider_mode="none", + ) + ) as wire, + gateway.scenario() as scenario, + ): + response: Final = _send(gateway, scenario, wire.url, request_id, openrouter=True, stream=False) + + body: Final = JSON_OBJECT.validate_json(response.content) + choices: Final = body["choices"] + assert isinstance(choices, list) + message: Final = object_value(object_value(choices[0])["message"]) + assert body["provider"] == _PROVIDER + assert "upstream_provider" not in body + assert message["content"] == _ANSWER + + +def test_non_openrouter_spend_metadata_has_no_provider_guard(gateway: Gateway) -> None: + request_id: Final = f"chatcmpl-{uuid.uuid4().hex}" + with ( + wire_server( + _reply( + request_id, + response_provider=False, + stream_provider_mode="none", + ) + ) as wire, + gateway.scenario() as scenario, + ): + _send(gateway, scenario, wire.url, request_id, openrouter=False, stream=False) + metadata: Final = _spend_metadata(request_id) + + assert "upstream_provider" not in metadata, metadata diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index d0e31cdd57a..f7c2082b683 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -12574,9 +12574,10 @@ def test_get_hidden_params(): assert result["api_base"] is None assert result["response_cost"] is None assert result["additional_headers"] is None + assert result.get("upstream_provider") is None # assert all fields in StandardLoggingHiddenParams are present - assert all(field in result for field in StandardLoggingHiddenParams.__annotations__) + assert all(field in result for field in StandardLoggingHiddenParams.__annotations__ if field != "upstream_provider") # Test with valid params hidden_params = { @@ -12584,6 +12585,7 @@ def test_get_hidden_params(): "cache_key": "test-cache", "api_base": "https://api.test.com", "response_cost": 0.001, + "upstream_provider": "serving-site-a", "additional_headers": { "x-ratelimit-limit-requests": "2000", "x-ratelimit-remaining-requests": "1999", @@ -12596,8 +12598,9 @@ def test_get_hidden_params(): assert result["response_cost"] == 0.001 assert result["additional_headers"] is not None assert result["additional_headers"]["x_ratelimit_limit_requests"] == 2000 + assert result.get("upstream_provider") == "serving-site-a" # assert all fields in StandardLoggingHiddenParams are present - assert all(field in result for field in StandardLoggingHiddenParams.__annotations__) + assert all(field in result for field in StandardLoggingHiddenParams.__annotations__ if field != "upstream_provider") @pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") diff --git a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py index 9df45b92686..bbf1897b3a7 100644 --- a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -910,6 +910,71 @@ def test_stream_chunk_builder_dict_snapshot_preserves_hidden_provider_fields(): assert response._hidden_params["provider_specific_fields"]["traffic_type"] == "default" +def test_stream_chunk_builder_finds_upstream_provider_in_later_chunk_regression() -> None: + first: Final = ModelResponseStream( + id="chatcmpl-provider", + created=1, + model="gpt-4.1-mini", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + finish_reason=None, + index=0, + delta=Delta(content="hello", role="assistant"), + ) + ], + ) + later: Final = ModelResponseStream( + id="chatcmpl-provider", + created=1, + model="gpt-4.1-mini", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + finish_reason="stop", + index=0, + delta=Delta(content=" world", role=None), + ) + ], + ) + first_hidden_params: Final = {"origin": "first"} + later_hidden_params: Final = {"upstream_provider": "serving-site-a"} + chunks: Final = [ + {**first.model_dump(), "_hidden_params": first_hidden_params}, + {**later.model_dump(), "_hidden_params": later_hidden_params}, + ] + + response: Final = stream_chunk_builder(chunks=chunks) + + assert response is not None + assert response._hidden_params.get("upstream_provider") == "serving-site-a" + assert first_hidden_params.get("upstream_provider") is None + assert later_hidden_params.get("upstream_provider") == "serving-site-a" + + +def test_stream_chunk_builder_preserves_source_provider_metadata_guard() -> None: + chunk: Final = ModelResponseStream( + id="chatcmpl-provider", + created=1, + model="gpt-4.1-mini", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + finish_reason="stop", + index=0, + delta=Delta(content="hello", role="assistant"), + ) + ], + ) + hidden_params: Final = {"upstream_provider": "serving-site-a"} + chunk_dict: Final = {**chunk.model_dump(), "_hidden_params": hidden_params} + + response: Final = stream_chunk_builder(chunks=[chunk_dict]) + + assert response is not None + assert chunk_dict["_hidden_params"]["upstream_provider"] == "serving-site-a" + + def test_stream_chunk_builder_propagates_vertex_ai_metadata_from_chunks(): """Vertex AI metadata on streaming chunks must appear on assembled response.""" grounding_metadata = [{"webSearchQueries": ["weather in SF"]}] diff --git a/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py b/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py index 888c47505a4..3d659641f54 100644 --- a/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py +++ b/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py @@ -872,3 +872,120 @@ def test_openrouter_streaming_reasoning_is_preserved() -> None: assembled: Final = litellm.stream_chunk_builder(chunks=list(stream_chunks), messages=messages) assert assembled is not None assert assembled.choices[0].message.reasoning_content == "Consider the evidence. Apply the rule." + + +def _transform_openrouter_response(body: dict[str, object]) -> litellm.ModelResponse: + return OpenrouterConfig().transform_response( + model="openrouter/openai/gpt-4o-mini", + raw_response=httpx.Response(200, json=body), + model_response=litellm.ModelResponse(), + logging_obj=MagicMock(), + request_data={}, + messages=[{"role": "user", "content": "hi"}], + optional_params={}, + litellm_params={}, + encoding=None, + ) + + +def test_openrouter_nonstream_provider_without_usage_is_preserved_regression() -> None: + response: Final = _transform_openrouter_response( + { + "id": "gen-provider-no-usage", + "object": "chat.completion", + "created": 1, + "model": "openai/gpt-4o-mini", + "provider": "serving-site-a", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "hi"}, + "finish_reason": "stop", + } + ], + } + ) + + assert response._hidden_params.get("upstream_provider") == "serving-site-a" + + +@pytest.mark.parametrize( + ("provider_fields", "case"), + [ + pytest.param({}, "missing", id="missing"), + pytest.param({"provider": None}, "null", id="null"), + pytest.param({"provider": ""}, "empty", id="empty"), + pytest.param({"provider": 123}, "non-string", id="non-string"), + ], +) +def test_openrouter_invalid_provider_keeps_reported_cost_guard( + provider_fields: dict[str, object], case: str +) -> None: + response: Final = _transform_openrouter_response( + { + "id": f"gen-invalid-provider-{case}", + "object": "chat.completion", + "created": 1, + "model": "openai/gpt-4o-mini", + **provider_fields, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "hi"}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 10, + "completion_tokens": 20, + "total_tokens": 30, + "cost": 0.00015, + }, + } + ) + + assert response._hidden_params.get("upstream_provider") is None + assert response._hidden_params["additional_headers"][OPENROUTER_COST_HEADER] == 0.00015 + + +def test_openrouter_streaming_chunk_preserves_provider_regression() -> None: + chunk: Final = { + "id": "gen-provider-stream", + "object": "chat.completion.chunk", + "created": 1, + "model": "openai/gpt-4o-mini", + "provider": "serving-site-a", + "choices": [{"index": 0, "delta": {"content": "hi"}, "finish_reason": None}], + } + + response: Final = OpenRouterChatCompletionStreamingHandler( + streaming_response=None, sync_stream=True + ).chunk_parser(chunk) + + assert response._hidden_params.get("upstream_provider") == "serving-site-a" + + +@pytest.mark.parametrize( + ("provider_fields", "case"), + [ + pytest.param({}, "missing", id="missing"), + pytest.param({"provider": None}, "null", id="null"), + pytest.param({"provider": ""}, "empty", id="empty"), + pytest.param({"provider": 123}, "non-string", id="non-string"), + ], +) +def test_openrouter_streaming_chunk_ignores_invalid_provider_guard(provider_fields: dict[str, object], case: str) -> None: + chunk: Final = { + "id": f"gen-invalid-provider-{case}", + "object": "chat.completion.chunk", + "created": 1, + "model": "openai/gpt-4o-mini", + **provider_fields, + "choices": [{"index": 0, "delta": {"content": "hi"}, "finish_reason": None}], + } + + response: Final = OpenRouterChatCompletionStreamingHandler( + streaming_response=None, sync_stream=True + ).chunk_parser(chunk) + + assert response._hidden_params.get("upstream_provider") is None diff --git a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py index 1ea445fda0d..a91cdd512fe 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py @@ -2394,6 +2394,65 @@ def test_get_logging_payload_includes_overhead_in_spend_logs_metadata(): ) +@patch("litellm.proxy.proxy_server.master_key", None) +@patch("litellm.proxy.proxy_server.general_settings", {}) +def test_get_logging_payload_carries_upstream_provider_from_hidden_params_regression() -> None: + standard_logging_payload: Final = _make_standard_logging_payload_with_usage_object(usage_object={}) + payload_with_provider: Final = { + **dict(standard_logging_payload), + "hidden_params": { + **dict(standard_logging_payload["hidden_params"]), + "upstream_provider": "serving-site-a", + }, + } + payload: Final = get_logging_payload( + kwargs={ + "model": "gpt-5.6", + "call_type": "responses", + "litellm_params": {"metadata": {"user_api_key": "test-key"}}, + "standard_logging_object": payload_with_provider, + }, + response_obj={ + "id": "resp-test", + "usage": { + "input_tokens": 1000, + "output_tokens": 10, + "total_tokens": 1010, + }, + }, + start_time=datetime.datetime(2025, 1, 1, tzinfo=timezone.utc), + end_time=datetime.datetime(2025, 1, 1, 0, 0, 1, tzinfo=timezone.utc), + ) + + assert json.loads(payload["metadata"]).get("upstream_provider") == "serving-site-a" + + +@patch("litellm.proxy.proxy_server.master_key", None) +@patch("litellm.proxy.proxy_server.general_settings", {}) +def test_get_logging_payload_omits_upstream_provider_when_hidden_param_absent_guard() -> None: + standard_logging_payload: Final = _make_standard_logging_payload_with_usage_object(usage_object={}) + payload: Final = get_logging_payload( + kwargs={ + "model": "gpt-5.6", + "call_type": "responses", + "litellm_params": {"metadata": {"user_api_key": "test-key"}}, + "standard_logging_object": standard_logging_payload, + }, + response_obj={ + "id": "resp-test", + "usage": { + "input_tokens": 1000, + "output_tokens": 10, + "total_tokens": 1010, + }, + }, + start_time=datetime.datetime(2025, 1, 1, tzinfo=timezone.utc), + end_time=datetime.datetime(2025, 1, 1, 0, 0, 1, tzinfo=timezone.utc), + ) + + assert "upstream_provider" not in json.loads(payload["metadata"]) + + @patch("litellm.proxy.proxy_server.master_key", None) @patch("litellm.proxy.proxy_server.general_settings", {}) def test_get_logging_payload_strips_null_bytes_from_request_tags(): @@ -6911,7 +6970,10 @@ def test_spend_logs_payload(model_id: str | None): ) assert len(payload["request_id"]) > 0 - expected_metadata_keys: Final = SpendLogsMetadata.__annotations__.keys() - {"is_silent_experiment"} + expected_metadata_keys: Final = SpendLogsMetadata.__annotations__.keys() - { + "is_silent_experiment", + "upstream_provider", + } assert "metadata" in payload assert isinstance(payload["metadata"], str) From f4d1f45d87dcba757edd1feae71300146fbb1570 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Fri, 30 Oct 2026 13:05:12 -0700 Subject: [PATCH 34/74] fix(proxy): raise live budget counters for failure spend rows (#47741) * fix(proxy): raise live budget counters for failure spend rows A stream that breaks mid-flight writes its recovered cost to the spend log, but the failure hook had already released the reservation and never touched the live counters, so an over-budget key kept passing until the counter was next repaired from the database Every charged failure row with a cost now raises the same counters the success path raises. The error waits at most a second for them, so a hung Redis cannot hold it back Fixes #35568 * test(proxy): cover failure spend counters across budgets, endpoints and outages Integration cells for team, user, end-user, tag and window budgets, concurrent streams, billed retries, native /v1/messages, Bedrock guardrail cost, a Redis stall on two workers and disable_spend_updates * test(proxy): check stalled failure counters and use the real spend writer - The Redis stall cell now reads the burst key's live counter and requires at least the full failure spend - Unit tests drop the charging writer double, bare dicts, list mutation and wall-clock timestamps - Partial-stream test models are frozen * test(proxy): move the stalled counter bound reason into the assert message --------- Co-authored-by: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> --- .../proxy/hooks/proxy_track_cost_callback.py | 124 ++++++- litellm/proxy/proxy_server.py | 2 +- tests/integration/spend/_partial_stream.py | 173 +++++++++ .../test_failure_spend_live_counters_chaos.py | 239 ++++++++++++ .../spend/test_partial_stream_live_budget.py | 350 ++++++++++++++++++ .../hooks/test_proxy_track_cost_callback.py | 225 +++++++++++ 6 files changed, 1103 insertions(+), 10 deletions(-) create mode 100644 tests/integration/spend/_partial_stream.py create mode 100644 tests/integration/spend/test_failure_spend_live_counters_chaos.py create mode 100644 tests/integration/spend/test_partial_stream_live_budget.py diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 0d333d4fd08..2951bc02d8c 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -12,6 +12,7 @@ from pydantic import TypeAdapter, ValidationError import litellm from litellm._logging import verbose_proxy_logger from litellm.batches.batch_utils import batch_cost_is_final +from litellm.caching.redis_batch import flush_post_call_redis_batches from litellm.constants import BACKGROUND_INTERACTION_COST_POLLING_ENABLED, BILLED_RETRY_ATTEMPT_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import ( @@ -134,6 +135,8 @@ _CAPTURED_IDENTITY_CALL_TYPES: Final[frozenset[str]] = frozenset( ) ) +FAILURE_SPEND_COUNTER_WAIT_SECONDS: Final = 1.0 + _OBJECT_MAPPING: Final = TypeAdapter(ObjectMapping) @@ -235,6 +238,12 @@ def _proxy_spend_writer() -> DBSpendUpdateWriter: return proxy_logging_obj.db_spend_update_writer +def _proxy_spend_counters() -> "_IncrementSpendCounters": + from litellm.proxy.proxy_server import increment_spend_counters + + return increment_spend_counters + + DeploymentLookup: TypeAlias = Callable[[str], Deployment | None] @@ -361,11 +370,16 @@ class _ProxyDBLogger(CustomLogger): message_logging: bool = True, spend_writer: Callable[[], DBSpendUpdateWriter] = _proxy_spend_writer, deployment_lookup: DeploymentLookup = _proxy_deployment, + spend_counters: Callable[[], "_IncrementSpendCounters"] = _proxy_spend_counters, + failure_counter_wait_seconds: float = FAILURE_SPEND_COUNTER_WAIT_SECONDS, ) -> None: super().__init__(turn_off_message_logging=turn_off_message_logging, message_logging=message_logging) self.spend_event_producer = spend_event_producer self._spend_writer: Final = spend_writer self._deployment_lookup: Final = deployment_lookup + self._spend_counters: Final = spend_counters + self._failure_counter_wait_seconds: Final = failure_counter_wait_seconds + self._pending_increments: Final[set[asyncio.Task[None]]] = set() # mutable-ok: tasks leave when done async def async_log_success_event( self, kwargs: ObjectMapping, response_obj: object, start_time: datetime, end_time: datetime @@ -590,9 +604,11 @@ class _ProxyDBLogger(CustomLogger): completed_cost: bool = False, request_tags: list[str] | None = None, ) -> None: - """One spend row (and one key/team/user spend deduction) per ``(response_cost, kwargs)`` pair.""" - for row_cost, row_kwargs in spend_rows: - if completed_cost: + """One spend row (and one key/team/user spend deduction) per ``(response_cost, kwargs)`` pair, then the + live spend counters raised by every charged row that cost something. A completed cost settles through the + success path's write, which raises the counters against the kept reservation.""" + if completed_cost: + for row_cost, row_kwargs in spend_rows: from litellm.proxy.proxy_server import increment_spend_counters, proxy_logging_obj await _update_database_and_spend_counters( @@ -623,12 +639,16 @@ class _ProxyDBLogger(CustomLogger): router=get_llm_router(), ), ) - continue + return + + token: Final = LiteLLMProxyRequestSetup.get_logged_api_key(user_api_key_dict) + end_user_id: Final = None if litellm.disable_end_user_cost_tracking else user_api_key_dict.end_user_id + charged: Final = [ await self._spend_writer().update_database( - token=LiteLLMProxyRequestSetup.get_logged_api_key(user_api_key_dict), + token=token, response_cost=row_cost, user_id=user_api_key_dict.user_id, - end_user_id=None if litellm.disable_end_user_cost_tracking else user_api_key_dict.end_user_id, + end_user_id=end_user_id, team_id=user_api_key_dict.team_id, kwargs=dict(row_kwargs), completion_response=original_exception, @@ -637,6 +657,91 @@ class _ProxyDBLogger(CustomLogger): org_id=user_api_key_dict.org_id, project_id=user_api_key_dict.project_id, ) + for row_cost, row_kwargs in spend_rows + ] + counted_rows: Final = tuple( + row for row, row_charged in zip(spend_rows, charged, strict=True) if row_charged and row[0] > 0 + ) + if counted_rows and not ProxyUpdateSpend.disable_spend_updates(): + await self._increment_failure_spend_counters( + user_api_key_dict, token, end_user_id, counted_rows, start_time + ) + + async def _increment_failure_spend_counters( + self, + user_api_key_dict: UserAPIKeyAuth, + token: str | None, + end_user_id: str | None, + counted_rows: Sequence[tuple[float, Mapping[str, object]]], + start_time: datetime, + ) -> None: + """The counters the success path raises for the same spend, sent before the client sees the error. A + counter store slower than ``failure_counter_wait_seconds`` finishes them in the background instead of + holding the error back. The reservation was already released, so every counter takes the full row cost.""" + increment: Final = asyncio.create_task( + self._raise_failure_spend_counters(user_api_key_dict, token, end_user_id, counted_rows, start_time) + ) + self._pending_increments.add(increment) + increment.add_done_callback(self._pending_increments.discard) + finished, _ = await asyncio.wait((increment,), timeout=self._failure_counter_wait_seconds) + if not finished: + verbose_proxy_logger.warning( + "Spend counters for failure spend rows took over %ss, finishing them in the background", + self._failure_counter_wait_seconds, + ) + + async def _raise_failure_spend_counters( + self, + user_api_key_dict: UserAPIKeyAuth, + token: str | None, + end_user_id: str | None, + counted_rows: Sequence[tuple[float, Mapping[str, object]]], + start_time: datetime, + ) -> None: + for row_cost, row_kwargs in counted_rows: + await self._raise_row_spend_counters( + user_api_key_dict, token, end_user_id, row_cost, row_kwargs, start_time + ) + await flush_post_call_redis_batches() + + async def _raise_row_spend_counters( + self, + user_api_key_dict: UserAPIKeyAuth, + token: str | None, + end_user_id: str | None, + row_cost: float, + row_kwargs: Mapping[str, object], + start_time: datetime, + ) -> None: + standard_logging_object: Final = row_kwargs.get("standard_logging_object") + sl_object: Final = ( + cast("StandardLoggingPayload", standard_logging_object) # cast-ok: the failure snapshot dict + if isinstance(standard_logging_object, dict) + else None + ) + metadata: Final = _str_keyed_fields(_str_keyed_fields(row_kwargs.get("litellm_params")).get("metadata")) + served_model_id: Final = _str_keyed_fields(metadata.get("model_info")).get("id") or _str_keyed_fields( + standard_logging_object + ).get("model_id") + try: + await self._spend_counters()( + token=token, + team_id=user_api_key_dict.team_id, + user_id=user_api_key_dict.user_id, + response_cost=row_cost, + org_id=user_api_key_dict.org_id, + end_user_id=end_user_id, + tags=_get_request_tags_for_cost_tracking(sl_object=sl_object, metadata=dict(metadata)), + request_started_at=start_time, + model_access_groups=debitable_model_access_groups( + attributed=get_request_model_access_groups(row_kwargs), + served_model_id=served_model_id if isinstance(served_model_id, str) else None, + router=get_llm_router(), + ), + project_id=user_api_key_dict.project_id, + ) + except Exception: # noqa: BLE001 # one row's counter failure must not keep the other rows' counters from landing + verbose_proxy_logger.exception("Failed to raise spend counters for a failure spend row") async def _PROXY_track_cost_callback( self, @@ -1060,7 +1165,7 @@ def _metadata_keys(metadata: object) -> tuple[str, ...]: return tuple(sorted(str(key) for key in metadata)) -def _get_budget_reservation_from_metadata(metadata: dict) -> dict | None: +def _get_budget_reservation_from_metadata(metadata: dict) -> dict[str, object] | None: return budget_reservation_from_metadata(metadata) @@ -1095,6 +1200,7 @@ class _IncrementSpendCounters(Protocol): tags: list[str] | None = None, request_started_at: datetime | None = None, model_access_groups: Sequence[str] | None = None, + project_id: str | None = None, ) -> None: ... @@ -1111,7 +1217,7 @@ async def _update_database_and_spend_counters( start_time: datetime | None, end_time: datetime | None, response_cost: float, - budget_reservation: dict | None, + budget_reservation: dict[str, object] | None, request_tags: list[str] | None = None, model_access_groups: Sequence[str] | None = None, project_id: str | None = None, @@ -1174,7 +1280,7 @@ async def _update_database_and_spend_counters_in_batch( start_time: datetime | None, end_time: datetime | None, response_cost: float, - budget_reservation: dict | None, + budget_reservation: dict[str, object] | None, request_tags: list[str] | None, model_access_groups: Sequence[str] | None, project_id: str | None, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 7ea1c246da5..ea6b51b228b 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -3510,7 +3510,7 @@ async def increment_spend_counters( user_id: str | None, response_cost: float | None, org_id: str | None = None, - budget_reservation: dict | None = None, + budget_reservation: dict[str, object] | None = None, end_user_id: str | None = None, tags: list[str] | None = None, request_started_at: datetime | None = None, diff --git a/tests/integration/spend/_partial_stream.py b/tests/integration/spend/_partial_stream.py new file mode 100644 index 00000000000..e35320a8d8e --- /dev/null +++ b/tests/integration/spend/_partial_stream.py @@ -0,0 +1,173 @@ +import json +import threading +import uuid +from collections.abc import Callable, Iterable, Iterator +from hashlib import sha256 +from typing import Final + +import httpx +from integration._support.client import Gateway, Scenario, eventually, held +from integration._support.wire import Reply, Request, Wire +from openai.types.chat import ChatCompletionChunk +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter + +JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +BACKEND: Final = "integration-partial-stream" +PIECE: Final = "partial stream billed tokens " +DELIVERED_PIECES: Final = 12 +_FAILURE_ROWS: Final = 'SELECT spend FROM "LiteLLM_SpendLogs" WHERE api_key=%s AND status=%s ORDER BY "startTime"' +ROWS: Final = 'SELECT status, spend FROM "LiteLLM_SpendLogs" WHERE api_key=%s ORDER BY "startTime"' + + +class _SpendRow(BaseModel): + model_config = ConfigDict(frozen=True) + + spend: float + + +class StatusRow(BaseModel): + model_config = ConfigDict(frozen=True) + + status: str + spend: float + + +class _BudgetError(BaseModel): + model_config = ConfigDict(frozen=True) + + type: str + + +class _ErrorResponse(BaseModel): + model_config = ConfigDict(frozen=True) + + error: _BudgetError + + +def _chunk(stream_id: str, delta: dict[str, JsonValue], finish_reason: str | None = None) -> bytes: + body: Final = { + "id": stream_id, + "object": "chat.completion.chunk", + "created": 1700000000, + "model": BACKEND, + "choices": [{"index": 0, "delta": delta, "finish_reason": finish_reason}], + } + return f"data: {json.dumps(body)}\n\n".encode() + + +def completion() -> bytes: + return json.dumps( + { + "id": f"chatcmpl-{uuid.uuid4().hex}", + "object": "chat.completion", + "created": 1700000000, + "model": BACKEND, + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + ).encode() + + +def peer(delivered: int | None, gate: threading.Event | None = None) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + if request.method != "POST": + return Reply(status=404) + if JSON_OBJECT.validate_json(request.body).get("stream") is True: + stream_id: Final = f"chatcmpl-{uuid.uuid4().hex}" + return Reply( + content_type="text/event-stream", + chunks=( + *(_chunk(stream_id, {"role": "assistant", "content": PIECE}) for _ in range(DELIVERED_PIECES)), + _chunk(stream_id, {}, "stop"), + b"data: [DONE]\n\n", + ), + abort_after=delivered, + pause_between_chunks=0.02, + gate_before_first=gate, + ) + return Reply(body=completion()) + + return respond + + +def priced_model(scenario: Scenario, wire: Wire) -> str: + return scenario.model( + model=f"openai/{BACKEND}", + api_base=f"{wire.url}/v1", + api_key="synthetic-openai-key", + input_cost_per_token=0.001, + output_cost_per_token=0.002, + ) + + +def stream(gateway: Gateway, model: str, key: str, **fields: JsonValue) -> httpx.Response: + return gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "max_tokens": 5, + "stream": True, + "messages": [{"role": "user", "content": "partial stream"}], + **fields, + }, + key=key, + ) + + +def broken_stream(gateway: Gateway, model: str, key: str, **fields: JsonValue) -> None: + response: Final = stream(gateway, model, key, **fields) + assert response.status_code == 200, response.text + events: Final = tuple( + line.removeprefix("data: ") + for line in response.text.splitlines() + if line.startswith("data: ") and line != "data: [DONE]" + ) + assert "error" in JSON_OBJECT.validate_json(events[-1]), response.text + delivered: Final = "".join(_streamed_text(ChatCompletionChunk.model_validate_json(event) for event in events[:-1])) + assert delivered == PIECE * DELIVERED_PIECES, response.text + + +def _streamed_text(chunks: Iterable[ChatCompletionChunk]) -> Iterator[str]: + for chunk in chunks: + yield from (choice.delta.content or "" for choice in chunk.choices) + + +def chat(gateway: Gateway, model: str, key: str, **fields: JsonValue) -> httpx.Response: + return gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "max_tokens": 5, + "messages": [{"role": "user", "content": f"after {uuid.uuid4().hex}"}], + **fields, + }, + key=key, + ) + + +def failure_spend(gateway: Gateway, key: str, rows: int = 1) -> tuple[float, ...]: + digest: Final = sha256(key.encode()).hexdigest() + eventually(lambda: gateway.rows(_FAILURE_ROWS, (digest, "failure")), lambda found: len(found) >= rows, seconds=70) + found: Final = held( + lambda: gateway.rows(_FAILURE_ROWS, (digest, "failure")), lambda latest: len(latest) == rows, holding=2 + ) + return tuple(_SpendRow.model_validate(row).spend for row in found) + + +def chats_sent(wire: Wire) -> int: + return sum(1 for request in wire.drain() if request.method == "POST") + + +def recovered_cost(gateway: Gateway, scenario: Scenario, model: str) -> float: + key: Final = scenario.key(models=[model], max_budget=100) + broken_stream(gateway, model, key) + (spend,) = failure_spend(gateway, key) + assert spend > 0 + return spend + + +def assert_budget_refusal(response: httpx.Response) -> None: + assert response.status_code == 422, response.text + assert _ErrorResponse.model_validate_json(response.content).error.type == "budget_exceeded", response.text diff --git a/tests/integration/spend/test_failure_spend_live_counters_chaos.py b/tests/integration/spend/test_failure_spend_live_counters_chaos.py new file mode 100644 index 00000000000..ecdd5aadb8e --- /dev/null +++ b/tests/integration/spend/test_failure_spend_live_counters_chaos.py @@ -0,0 +1,239 @@ +import json +import threading +import uuid +from concurrent.futures import ThreadPoolExecutor +from hashlib import sha256 +from pathlib import Path +from typing import Final + +import httpx +import pytest +import yaml +from integration._support.client import Gateway, eventually +from integration._support.process import owned_proxy_process +from integration._support.redis_process import owned_redis +from integration._support.timing import assert_not_delayed_by, timed +from integration._support.wire import Reply, Request, Wire, wire_server +from integration.spend._partial_stream import ( + DELIVERED_PIECES, + assert_budget_refusal, + broken_stream, + chat, + chats_sent, + failure_spend, + peer, + priced_model, + recovered_cost, +) +from pydantic import JsonValue +from redis import Redis + +from litellm.constants import REDIS_SOCKET_TIMEOUT + +_BURST: Final = 12 +_SPREAD: Final = 6 +_ATTACK_MARKER: Final = "synthetic-attack-marker" +_POLICY_ID: Final = "synthetic-priced-policy" +_POLICY_APPLY_PATH: Final = f"/guardrail/{_POLICY_ID}/version/DRAFT/apply" + + +def _proxy_config(directory: Path, name: str, sections: dict[str, JsonValue]) -> Path: + base: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + path: Final = directory / f"{name}.yaml" + path.write_text(yaml.safe_dump({**base, **sections})) + return path + + +def _redis_counter_config(directory: Path) -> Path: + base: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + return _proxy_config( + directory, + "redis-counters", + { + "router_settings": { + **base["router_settings"], + "redis_host": "os.environ/REDIS_HOST", + "redis_port": "os.environ/REDIS_PORT", + } + }, + ) + + +def _spread_requests(gateway: Gateway, model: str, key: str) -> tuple[httpx.Response, ...]: + return tuple( + gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "max_tokens": 5, "messages": [{"role": "user", "content": f"spread {uuid.uuid4().hex}"}]}, + key=key, + headers={"connection": "close"}, + ) + for _ in range(_SPREAD) + ) + + +@pytest.mark.timeout(600) +def test_broken_streams_through_a_redis_stall_log_once_and_the_budget_holds_on_every_worker( + gateway: Gateway, tmp_path: Path +) -> None: + release: Final = threading.Event() + with ( + owned_redis(tmp_path) as counters, + wire_server(peer(DELIVERED_PIECES)) as wire, + wire_server(peer(DELIVERED_PIECES, gate=release)) as held_wire, + owned_proxy_process( + gateway, + tmp_path, + {"REDIS_HOST": counters.host, "REDIS_PORT": str(counters.port)}, + config=_redis_counter_config(tmp_path), + workers=2, + ) as owned, + owned.gateway.scenario() as scenario, + ): + scenario.cleanups.callback(release.set) + candidate: Final = owned.gateway + model: Final = priced_model(scenario, wire) + stalled_model: Final = priced_model(scenario, held_wire) + cost: Final = recovered_cost(candidate, scenario, model) + control: Final = timed(lambda: broken_stream(candidate, model, scenario.key(models=[model]))) + burst_key: Final = scenario.key(models=[stalled_model]) + chats_sent(held_wire) + + with ThreadPoolExecutor(max_workers=_BURST) as pool: + pending: Final = tuple( + pool.submit(broken_stream, candidate, stalled_model, burst_key) for _ in range(_BURST) + ) + eventually(held_wire.received.qsize, lambda held: held >= _BURST, seconds=30) + with counters.paused(): + release.set() + drained: Final = timed(lambda: tuple(future.result() for future in pending)) + + assert_not_delayed_by((drained.seconds,), control=(control.seconds,), injected_delay=REDIS_SOCKET_TIMEOUT) + assert failure_spend(candidate, burst_key, rows=_BURST) == pytest.approx((cost,) * _BURST) + spent: Final = cost * _BURST + with Redis(host=counters.host, port=counters.port) as live_counters: + landed: Final = eventually( + lambda: float(live_counters.get(f"spend:key:{sha256(burst_key.encode()).hexdigest()}") or 0), + lambda counted: counted >= spent - 1e-9, + seconds=30, + return_last_on_timeout=True, + ) + assert spent - 1e-9 <= landed <= 2 * spent + 1e-9, ( + f"one broken stream costs {cost}. The cold counter is seeded from the database, and a spend flush" + " during the stall can land these rows there first, so up to twice their cost is expected, as for" + " completed streams held past a flush" + ) + + budgeted: Final = scenario.key(models=[model], max_budget=cost / 2) + broken_stream(candidate, model, budgeted) + before_kill: Final = _spread_requests(candidate, model, budgeted) + owned.kill_worker(min(owned.worker_pids())) + after_kill: Final = _spread_requests(candidate, model, budgeted) + + for refused in (*before_kill, *after_kill): + assert_budget_refusal(refused) + + +def _policy(request: Request) -> Reply: + if request.method != "POST" or request.target != _POLICY_APPLY_PATH: + return Reply(status=404) + if _ATTACK_MARKER.encode() not in request.body: + return Reply(body=json.dumps({"action": "NONE", "outputs": [], "assessments": []}).encode()) + return Reply( + body=json.dumps( + { + "action": "GUARDRAIL_INTERVENED", + "outputs": [{"text": "blocked by synthetic policy"}], + "assessments": [ + {"contentPolicy": {"filters": [{"type": "VIOLENCE", "confidence": "HIGH", "action": "BLOCKED"}]}} + ], + "usage": {"contentPolicyUnits": 1}, + } + ).encode() + ) + + +def _policy_config(directory: Path, policy: Wire) -> Path: + return _proxy_config( + directory, + "priced-policy", + { + "guardrails": [ + { + "guardrail_name": "priced-bedrock-policy", + "litellm_params": { + "guardrail": "bedrock", + "mode": "pre_call", + "default_on": True, + "guardrailIdentifier": _POLICY_ID, + "guardrailVersion": "DRAFT", + "aws_region_name": "us-east-1", + "aws_access_key_id": "AKIASYNTHETICGUARDRAIL", + "aws_secret_access_key": "synthetic-guardrail-secret", + "aws_bedrock_runtime_endpoint": policy.url, + }, + } + ] + }, + ) + + +def _attack(gateway: Gateway, model: str, key: str) -> httpx.Response: + blocked: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": f"{_ATTACK_MARKER} {uuid.uuid4().hex}"}]}, + key=key, + ) + assert blocked.status_code == 400, blocked.text + return blocked + + +@pytest.mark.timeout(480) +def test_guardrail_cost_on_blocked_requests_counts_toward_the_budget(gateway: Gateway, tmp_path: Path) -> None: + with ( + wire_server(_policy) as policy, + wire_server(peer(None)) as wire, + owned_proxy_process(gateway, tmp_path, {}, config=_policy_config(tmp_path, policy), workers=2) as owned, + owned.gateway.scenario() as scenario, + ): + candidate: Final = owned.gateway + model: Final = priced_model(scenario, wire) + meter: Final = scenario.key(models=[model], max_budget=100) + _attack(candidate, model, meter) + (cost,) = failure_spend(candidate, meter) + assert cost > 0 + key: Final = scenario.key(models=[model], max_budget=cost * 1.5) + chats_sent(wire) + + _attack(candidate, model, key) + _attack(candidate, model, key) + after: Final = chat(candidate, model, key) + + assert_budget_refusal(after) + assert chats_sent(wire) == 0 + + +@pytest.mark.timeout(480) +def test_broken_streams_leave_the_budget_alone_while_spend_updates_are_disabled( + gateway: Gateway, tmp_path: Path +) -> None: + base: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config: Final = _proxy_config( + tmp_path, "spend-updates-off", {"general_settings": {**base["general_settings"], "disable_spend_updates": True}} + ) + with ( + wire_server(peer(DELIVERED_PIECES)) as wire, + gateway.scenario() as metering, + owned_proxy_process(gateway, tmp_path, {}, config=config) as owned, + owned.gateway.scenario() as scenario, + ): + cost: Final = recovered_cost(gateway, metering, priced_model(metering, wire)) + candidate: Final = owned.gateway + model: Final = priced_model(scenario, wire) + key: Final = scenario.key(models=[model], max_budget=cost / 2) + + broken_stream(candidate, model, key) + after: Final = chat(candidate, model, key) + + assert after.status_code == 200, after.text diff --git a/tests/integration/spend/test_partial_stream_live_budget.py b/tests/integration/spend/test_partial_stream_live_budget.py new file mode 100644 index 00000000000..89fb66c9bbf --- /dev/null +++ b/tests/integration/spend/test_partial_stream_live_budget.py @@ -0,0 +1,350 @@ +import json +import threading +import uuid +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor +from hashlib import sha256 +from typing import Final + +import httpx +import pytest +from integration._support.client import Gateway, eventually, held +from integration._support.wire import Reply, Request, wire_server +from integration.spend._partial_stream import ( + BACKEND, + DELIVERED_PIECES, + JSON_OBJECT, + PIECE, + ROWS, + StatusRow, + assert_budget_refusal, + broken_stream, + chat, + chats_sent, + completion, + failure_spend, + peer, + priced_model, + recovered_cost, + stream, +) +from pydantic import JsonValue + +_CONCURRENT_STREAMS: Final = 6 +_MESSAGES_BACKEND: Final = "claude-integration-partial-stream" +_UNCACHED: Final[dict[str, JsonValue]] = {"cache": {"no-cache": True}} + + +@pytest.mark.timeout(240) +def test_a_broken_stream_that_spends_past_the_budget_blocks_the_next_request(gateway: Gateway) -> None: + with wire_server(peer(DELIVERED_PIECES)) as wire, gateway.scenario() as scenario: + model: Final = priced_model(scenario, wire) + cost: Final = recovered_cost(gateway, scenario, model) + key: Final = scenario.key(models=[model], max_budget=cost / 2) + chats_sent(wire) + + broken_stream(gateway, model, key) + after: Final = chat(gateway, model, key) + + assert_budget_refusal(after) + assert failure_spend(gateway, key, rows=2) == pytest.approx((cost, 0.0)) + assert chats_sent(wire) == 1 + + +@pytest.mark.timeout(240) +def test_broken_stream_spend_counts_once_toward_the_budget(gateway: Gateway) -> None: + with wire_server(peer(DELIVERED_PIECES)) as wire, gateway.scenario() as scenario: + model: Final = priced_model(scenario, wire) + cost: Final = recovered_cost(gateway, scenario, model) + key: Final = scenario.key(models=[model], max_budget=cost * 1.5) + + broken_stream(gateway, model, key) + broken_stream(gateway, model, key) + after: Final = chat(gateway, model, key) + + assert_budget_refusal(after) + assert failure_spend(gateway, key, rows=3) == pytest.approx((cost, cost, 0.0)) + + +@pytest.mark.timeout(240) +def test_a_stream_that_breaks_before_any_chunk_leaves_the_budget_untouched(gateway: Gateway) -> None: + with wire_server(peer(0)) as wire, gateway.scenario() as scenario: + model: Final = priced_model(scenario, wire) + key: Final = scenario.key(models=[model], max_budget=0.05) + + refused: Final = stream(gateway, model, key) + after: Final = chat(gateway, model, key) + + assert refused.status_code == 500, refused.text + + assert after.status_code == 200, after.text + assert failure_spend(gateway, key) == (0.0,) + + +def _broken_stream_then_refusal( + gateway: Gateway, model: str, spender: str, other: str, **fields: JsonValue +) -> httpx.Response: + broken_stream(gateway, model, spender, **fields) + return chat(gateway, model, other, **fields) + + +@pytest.mark.timeout(240) +def test_a_broken_stream_on_one_team_key_blocks_the_teams_other_keys(gateway: Gateway) -> None: + with wire_server(peer(DELIVERED_PIECES)) as wire, gateway.scenario() as scenario: + model: Final = priced_model(scenario, wire) + cost: Final = recovered_cost(gateway, scenario, model) + team: Final = scenario.team(models=[model], max_budget=cost / 2) + + after: Final = _broken_stream_then_refusal( + gateway, model, scenario.key(team_id=team, models=[model]), scenario.key(team_id=team, models=[model]) + ) + + assert_budget_refusal(after) + + +@pytest.mark.timeout(240) +def test_a_broken_stream_on_one_of_a_users_keys_blocks_their_other_keys(gateway: Gateway) -> None: + with wire_server(peer(DELIVERED_PIECES)) as wire, gateway.scenario() as scenario: + model: Final = priced_model(scenario, wire) + cost: Final = recovered_cost(gateway, scenario, model) + user: Final = scenario.user(max_budget=cost / 2) + + after: Final = _broken_stream_then_refusal( + gateway, model, scenario.key(user_id=user, models=[model]), scenario.key(user_id=user, models=[model]) + ) + + assert_budget_refusal(after) + + +@pytest.mark.timeout(240) +def test_a_broken_stream_for_an_end_user_blocks_their_next_request_on_any_key(gateway: Gateway) -> None: + end_user: Final = f"integration-end-user-{uuid.uuid4().hex}" + with wire_server(peer(DELIVERED_PIECES)) as wire, gateway.scenario() as scenario: + model: Final = priced_model(scenario, wire) + cost: Final = recovered_cost(gateway, scenario, model) + gateway.post("/customer/new", {"user_id": end_user, "max_budget": cost / 2}) + scenario.cleanups.callback(gateway.post, "/customer/delete", {"user_ids": [end_user]}) + + after: Final = _broken_stream_then_refusal( + gateway, model, scenario.key(models=[model]), scenario.key(models=[model]), user=end_user + ) + + assert_budget_refusal(after) + + +@pytest.mark.timeout(240) +def test_a_broken_stream_under_a_tag_blocks_the_next_request_with_that_tag(gateway: Gateway) -> None: + tag: Final = f"integration-partial-stream-{uuid.uuid4().hex}" + with wire_server(peer(DELIVERED_PIECES)) as wire, gateway.scenario() as scenario: + model: Final = priced_model(scenario, wire) + cost: Final = recovered_cost(gateway, scenario, model) + gateway.post("/tag/new", {"name": tag, "max_budget": cost / 2}) + scenario.cleanups.callback(gateway.post, "/tag/delete", {"name": tag}) + key: Final = scenario.key(models=[model]) + + after: Final = _broken_stream_then_refusal(gateway, model, key, key, metadata={"tags": [tag]}) + untagged: Final = chat(gateway, model, key) + + assert_budget_refusal(after) + assert untagged.status_code == 200, untagged.text + + +@pytest.mark.timeout(240) +def test_a_broken_stream_fills_the_keys_budget_window(gateway: Gateway) -> None: + with wire_server(peer(DELIVERED_PIECES)) as wire, gateway.scenario() as scenario: + model: Final = priced_model(scenario, wire) + cost: Final = recovered_cost(gateway, scenario, model) + key: Final = scenario.key(models=[model], budget_limits=[{"budget_duration": "1d", "max_budget": cost / 2}]) + + after: Final = _broken_stream_then_refusal(gateway, model, key, key) + + assert_budget_refusal(after) + + +@pytest.mark.timeout(240) +def test_concurrent_broken_streams_each_count_once_toward_the_budget(gateway: Gateway) -> None: + with wire_server(peer(DELIVERED_PIECES)) as wire, gateway.scenario() as scenario: + model: Final = priced_model(scenario, wire) + cost: Final = recovered_cost(gateway, scenario, model) + key: Final = scenario.key(models=[model], max_budget=cost * (_CONCURRENT_STREAMS + 0.5)) + + with ThreadPoolExecutor(max_workers=_CONCURRENT_STREAMS) as pool: + tuple(pool.map(lambda _: broken_stream(gateway, model, key), range(_CONCURRENT_STREAMS))) + within_budget: Final = chat(gateway, model, key) + broken_stream(gateway, model, key) + over_budget: Final = chat(gateway, model, key) + + assert within_budget.status_code == 200, within_budget.text + assert_budget_refusal(over_budget) + assert sorted(failure_spend(gateway, key, rows=_CONCURRENT_STREAMS + 2)) == pytest.approx( + [0.0] + [cost] * (_CONCURRENT_STREAMS + 1) + ) + + +@pytest.mark.timeout(240) +def test_completed_streams_count_once_toward_the_budget(gateway: Gateway) -> None: + with wire_server(peer(None)) as wire, gateway.scenario() as scenario: + model: Final = priced_model(scenario, wire) + meter: Final = scenario.key(models=[model], max_budget=100) + assert stream(gateway, model, meter, **_UNCACHED).status_code == 200 + (metered,) = eventually( + lambda: gateway.rows(ROWS, (sha256(meter.encode()).hexdigest(),)), + lambda found: len(found) == 1, + seconds=70, + ) + cost: Final = StatusRow.model_validate(metered).spend + key: Final = scenario.key(models=[model], max_budget=cost * 1.5) + + first: Final = stream(gateway, model, key, **_UNCACHED) + second: Final = stream(gateway, model, key, **_UNCACHED) + assert (first.status_code, second.status_code) == (200, 200), (first.text, second.text) + assert _rows(gateway, key, 2) == (StatusRow(status="success", spend=cost),) * 2 + third: Final = stream(gateway, model, key, **_UNCACHED) + + assert_budget_refusal(third) + + +def _hung_then_served(release: threading.Event, hung_model: str) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + if request.method != "POST": + return Reply(status=404) + if JSON_OBJECT.validate_json(request.body).get("model") == hung_model: + return Reply(chunks=(completion(),), gate_before_first=release) + return Reply(body=completion()) + + return respond + + +def _timed_out_spend(gateway: Gateway, model: str, key: str) -> float: + timed_out: Final = chat(gateway, model, key, messages=[{"role": "user", "content": "hung"}]) + assert timed_out.status_code in (408, 500), timed_out.text + digest: Final = sha256(key.encode()).hexdigest() + landed: Final = eventually(lambda: gateway.rows(ROWS, (digest,)), lambda found: len(found) >= 3, seconds=70) + rows: Final = tuple( + StatusRow.model_validate(row) + for row in held(lambda: gateway.rows(ROWS, (digest,)), lambda latest: len(latest) == len(landed), holding=2) + ) + assert {row.status for row in rows} == {"failure"}, rows + return sum(row.spend for row in rows) + + +@pytest.mark.timeout(240) +def test_retries_the_provider_billed_before_a_timeout_count_toward_the_budget(gateway: Gateway) -> None: + hung_backend: Final = f"{BACKEND}-hung" + release: Final = threading.Event() + with wire_server(_hung_then_served(release, hung_backend)) as wire, gateway.scenario() as scenario: + scenario.cleanups.callback(release.set) + hung: Final = scenario.model( + model=f"openai/{hung_backend}", + api_base=f"{wire.url}/v1", + api_key="synthetic-openai-key", + input_cost_per_token=0.001, + output_cost_per_token=0.002, + timeout=1, + num_retries=2, + ) + served: Final = priced_model(scenario, wire) + billed: Final = _timed_out_spend(gateway, hung, scenario.key(models=[hung, served], max_budget=100)) + assert billed > 0 + key: Final = scenario.key(models=[hung, served], max_budget=billed / 2) + + assert _timed_out_spend(gateway, hung, key) == pytest.approx(billed) + after: Final = chat(gateway, served, key) + + assert_budget_refusal(after) + + +def _messages_event(kind: str, fields: dict[str, JsonValue]) -> bytes: + return f"event: {kind}\ndata: {json.dumps({'type': kind, **fields})}\n\n".encode() + + +def _messages_peer(request: Request) -> Reply: + if request.method != "POST" or request.target != "/v1/messages": + return Reply(status=404) + message: Final[dict[str, JsonValue]] = { + "id": f"msg_{uuid.uuid4().hex}", + "type": "message", + "role": "assistant", + "model": _MESSAGES_BACKEND, + "stop_reason": None, + "stop_sequence": None, + "usage": {"input_tokens": 10, "output_tokens": 1}, + } + if JSON_OBJECT.validate_json(request.body).get("stream") is not True: + return Reply( + body=json.dumps( + {**message, "content": [{"type": "text", "text": "ok"}], "stop_reason": "end_turn"} + ).encode() + ) + return Reply( + content_type="text/event-stream", + chunks=( + _messages_event("message_start", {"message": {**message, "content": []}}), + _messages_event("content_block_start", {"index": 0, "content_block": {"type": "text", "text": ""}}), + *( + _messages_event("content_block_delta", {"index": 0, "delta": {"type": "text_delta", "text": PIECE}}) + for _ in range(DELIVERED_PIECES) + ), + _messages_event("content_block_stop", {"index": 0}), + _messages_event( + "message_delta", + {"delta": {"stop_reason": "end_turn", "stop_sequence": None}, "usage": {"output_tokens": 60}}, + ), + _messages_event("message_stop", {}), + ), + abort_after=DELIVERED_PIECES + 2, + pause_between_chunks=0.02, + ) + + +def _messages(gateway: Gateway, model: str, key: str, *, streamed: bool) -> httpx.Response: + return gateway.request( + "POST", + "/v1/messages", + { + "model": model, + "max_tokens": 5, + "stream": streamed, + "messages": [{"role": "user", "content": f"partial stream {uuid.uuid4().hex}"}], + }, + key=key, + ) + + +def _broken_messages_stream(gateway: Gateway, model: str, key: str) -> None: + response: Final = _messages(gateway, model, key, streamed=True) + assert response.status_code == 200, response.text + assert response.text.count(PIECE) == DELIVERED_PIECES, response.text + assert "message_stop" not in response.text, response.text + + +def _rows(gateway: Gateway, key: str, count: int) -> tuple[StatusRow, ...]: + digest: Final = sha256(key.encode()).hexdigest() + eventually(lambda: gateway.rows(ROWS, (digest,)), lambda found: len(found) >= count, seconds=70) + return tuple( + StatusRow.model_validate(row) + for row in held(lambda: gateway.rows(ROWS, (digest,)), lambda latest: len(latest) == count, holding=2) + ) + + +@pytest.mark.timeout(240) +def test_a_broken_messages_stream_counts_toward_the_budget(gateway: Gateway) -> None: + with wire_server(_messages_peer) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"anthropic/{_MESSAGES_BACKEND}", + api_base=wire.url, + api_key="synthetic-anthropic-key", + input_cost_per_token=0.001, + output_cost_per_token=0.002, + ) + meter: Final = scenario.key(models=[model], max_budget=100) + _broken_messages_stream(gateway, model, meter) + (metered,) = _rows(gateway, meter, 1) + assert metered.spend > 0, metered + key: Final = scenario.key(models=[model], max_budget=metered.spend / 2) + + _broken_messages_stream(gateway, model, key) + after: Final = _messages(gateway, model, key, streamed=False) + + assert_budget_refusal(after) + assert _rows(gateway, key, 2) == (metered, StatusRow(status="failure", spend=0.0)) diff --git a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py index 8d4f8cd87ad..e482b6552a4 100644 --- a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py @@ -14,6 +14,7 @@ from fastapi import HTTPException import litellm from litellm._logging import verbose_proxy_logger +from litellm.caching.dual_cache import DualCache from litellm.litellm_core_utils.internal_call_metadata import MODEL_ACCESS_GROUP_METADATA_KEY from litellm.proxy._types import ProxyException, SpendLogsPayload, UserAPIKeyAuth from litellm.proxy.collector import SpendEventConsumer @@ -36,6 +37,7 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payloa from litellm.proxy.utils import ( REJECTED_BEFORE_REQUEST_SETUP, REJECTED_BEFORE_REQUEST_SETUP_KEY, + BilledFailedAttempt, ProxyUpdateSpend, ) from litellm.types.router import Deployment, LiteLLM_Params @@ -2676,6 +2678,229 @@ async def test_async_post_call_failure_hook_records_recovered_partial_spend(): assert mock_update_database.call_args[1]["response_cost"] == 3.5e-05 +@pytest.fixture() +def live_spend_counters(monkeypatch: pytest.MonkeyPatch) -> DualCache: + counters: Final = DualCache() + monkeypatch.setattr("litellm.proxy.proxy_server.spend_counter_cache", counters) + monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", DualCache()) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None) + return counters + + +def _failure_with_billed_attempts() -> Mapping[str, object]: + usage: Final = Usage(prompt_tokens=24, completion_tokens=0, total_tokens=24) + return { + "combined_usage_object": usage, + "response_cost": 0.5, + "billed_failed_attempts": ( + BilledFailedAttempt(attempted_retries=0, deployment_id="dep-a", usage=usage, response_cost=0.25), + BilledFailedAttempt(attempted_retries=1, deployment_id="dep-b", usage=usage, response_cost=0.5), + ), + } + + +_LIVE_COUNTERS: Final = { + "spend:key:hashed-key": 0.2, + "spend:team:team-1": 0.3, + "spend:team_member:user-1:team-1": 0.35, + "spend:user:user-1": 0.4, + "spend:end_user:end-user-1": 0.45, + "spend:org:org-1": 0.5, + "spend:project:project-1": 0.55, + "spend:tag:batch-jobs": 0.6, + "spend:model_access_group:premium": 0.65, +} + + +async def _fail_request(logger: ProxyDBLogger, failure: Mapping[str, object]) -> None: + await logger.async_post_call_failure_hook( + request_data={ + "model": "gpt-4.1", + "messages": [{"role": "user", "content": "Hello"}], + "metadata": {"tags": ["batch-jobs"], MODEL_ACCESS_GROUP_METADATA_KEY: ["premium"]}, + "proxy_server_request": {"request_id": "rid"}, + **failure, + }, + original_exception=Exception("stream broke mid-flight"), + user_api_key_dict=UserAPIKeyAuth( + api_key="hashed-key", + team_id="team-1", + user_id="user-1", + end_user_id="end-user-1", + org_id="org-1", + project_id="project-1", + ), + ) + + +async def _counter_values(counters: DualCache) -> dict[str, float]: + return {key: await counters.async_get_cache(key) for key in _LIVE_COUNTERS} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("failure", "charged"), + [ + ( + { + "combined_usage_object": Usage(prompt_tokens=30, completion_tokens=40, total_tokens=70), + "response_cost": 0.125, + }, + 0.125, + ), + (_failure_with_billed_attempts(), 1.25), + ({}, 0.0), + ], +) +async def test_failure_spend_rows_raise_the_live_spend_counters_by_their_cost( + live_spend_counters: DualCache, failure: Mapping[str, object], charged: float +): + """#35568: budget checks read the live counters, so a failure row's cost has to land there as well as in the + spend log, or an over-budget key keeps passing until the counter is next repaired from the database.""" + for key, spend in _LIVE_COUNTERS.items(): + await live_spend_counters.async_set_cache(key, spend) + + await _fail_request(ProxyDBLogger(), failure) + + assert await _counter_values(live_spend_counters) == pytest.approx( + {key: spend + charged for key, spend in _LIVE_COUNTERS.items()} + ) + + +@pytest.mark.asyncio +async def test_a_completed_cost_failure_raises_the_live_spend_counters_once(live_spend_counters: DualCache): + """A failure whose cost was already finalized settles through the success path's write, which raises the + counters itself.""" + for key, spend in _LIVE_COUNTERS.items(): + await live_spend_counters.async_set_cache(key, spend) + + await _fail_request( + ProxyDBLogger(), {"standard_logging_object": {"cost_finalized": True, "response_cost": 0.125}} + ) + + assert await _counter_values(live_spend_counters) == pytest.approx( + {key: spend + 0.125 for key, spend in _LIVE_COUNTERS.items()} + ) + + +@pytest.mark.asyncio +async def test_failure_spend_rows_leave_the_live_spend_counters_alone_while_spend_updates_are_disabled( + live_spend_counters: DualCache, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {"disable_spend_updates": True}) + for key, spend in _LIVE_COUNTERS.items(): + await live_spend_counters.async_set_cache(key, spend) + + await _fail_request(ProxyDBLogger(), _failure_with_billed_attempts()) + + assert await _counter_values(live_spend_counters) == pytest.approx(_LIVE_COUNTERS) + + +@pytest.mark.asyncio +async def test_a_failing_spend_counter_store_is_still_sent_every_failure_spend_row(): + counters: Final = AsyncMock(side_effect=RuntimeError("redis unavailable")) + + await _fail_request(ProxyDBLogger(spend_counters=lambda: counters), _failure_with_billed_attempts()) + + assert [call.kwargs["response_cost"] for call in counters.call_args_list] == [0.5, 0.25, 0.5] + + +@pytest.mark.asyncio +@pytest.mark.timeout(30) +async def test_a_slow_spend_counter_store_lets_the_failure_out_and_raises_the_counters_once_it_answers(): + store_answers: Final = asyncio.Event() + raised_cost: Final[asyncio.Future[object]] = asyncio.get_running_loop().create_future() + + async def slow_counter_store(**counter_args: object) -> None: + await store_answers.wait() + raised_cost.set_result(counter_args["response_cost"]) + + logger: Final = ProxyDBLogger(spend_counters=lambda: slow_counter_store, failure_counter_wait_seconds=0) + + await _fail_request( + logger, + { + "combined_usage_object": Usage(prompt_tokens=30, completion_tokens=40, total_tokens=70), + "response_cost": 0.125, + }, + ) + assert not raised_cost.done() + + store_answers.set() + assert await raised_cost == 0.125 + + +@pytest.mark.asyncio +async def test_a_failure_spend_row_lands_in_the_key_budget_window_as_of_the_request_start( + live_spend_counters: DualCache, monkeypatch: pytest.MonkeyPatch +): + """The window row's one-time seed from the spend log stops at the request start, which keeps the failure row + it already logged from being counted twice.""" + from litellm.proxy.proxy_server import proxy_logging_obj + + window_queue: Final = proxy_logging_obj.db_spend_update_writer.window_spend_update_queue + await window_queue.flush_and_get_aggregated_window_spend_transactions() + key_cache: Final = DualCache() + monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", key_cache) + request_start: Final = datetime(2026, 10, 20, 12, 0, tzinfo=timezone.utc) + reset_at: Final = request_start + timedelta(days=10) + await key_cache.async_set_cache( + "hashed-key", + UserAPIKeyAuth( + token="hashed-key", + budget_limits=[{"budget_duration": "30d", "max_budget": 100.0, "reset_at": reset_at.isoformat()}], + ), + ) + + await _fail_request( + ProxyDBLogger(), + { + "combined_usage_object": Usage(prompt_tokens=30, completion_tokens=40, total_tokens=70), + "response_cost": 0.125, + "start_time": request_start, + }, + ) + + assert [ + (row["entity_type"], row["entity_id"], row["spend"], row["started_at"]) + for row in await window_queue.flush_and_get_aggregated_window_spend_transactions() + ] == [("key", "hashed-key", pytest.approx(0.125), "2026-10-20T12:00:00.000000")] + + +@pytest.mark.asyncio +async def test_a_billed_attempt_on_a_deployment_the_router_dropped_debits_every_attributed_access_group( + live_spend_counters: DualCache, monkeypatch: pytest.MonkeyPatch +): + """Its spend row names the dropped deployment, which narrows nothing, so the counters must not narrow it to the + final attempt's groups either.""" + monkeypatch.setattr( + "litellm.proxy.proxy_server.llm_router", + litellm.Router( + model_list=[ + { + "model_name": "gpt-4.1", + "litellm_params": {"model": "openai/gpt-4.1", "api_key": "sk-test"}, + "model_info": {"id": "dep-b", "access_groups": ["pool"]}, + } + ] + ), + ) + + await _fail_request( + ProxyDBLogger(), + { + **_failure_with_billed_attempts(), + "metadata": {MODEL_ACCESS_GROUP_METADATA_KEY: ["pool", "premium"], "model_info": {"id": "dep-b"}}, + }, + ) + + assert { + group: await live_spend_counters.async_get_cache(f"spend:model_access_group:{group}") + for group in ("premium", "pool") + } == pytest.approx({"premium": 0.25, "pool": 1.25}) + + @pytest.mark.asyncio async def test_track_cost_callback_enriches_user_id_for_mcp_style_metadata(): """MCP tool calls may only carry user_api_key; user/team rollups still need user_id.""" From 632f71b850d157efbfa75aa4c1f7a2066576446c Mon Sep 17 00:00:00 2001 From: tin-berri Date: Fri, 30 Oct 2026 13:19:58 -0700 Subject: [PATCH 35/74] fix(router): recover streamed context window errors before output (#49293) --- litellm/responses/streaming_iterator.py | 2 +- litellm/router.py | 81 ++++++- ..._mantle_context_overflow_fallbacks_wire.py | 10 +- .../test_streaming_iterator_error_events.py | 19 +- tests/unit/test_router/test_router.py | 15 +- .../unit/test_router/test_router_fallbacks.py | 201 ++++++++++++++++++ 6 files changed, 303 insertions(+), 25 deletions(-) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index fb91ba6a618..d42f1af380e 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -312,7 +312,7 @@ def map_stream_error_to_exception( error_body: Final = {"message": error_message, "type": error_type, "code": error_code} provider_exception: Final = BaseLLMException( status_code=status_code, - message=f"Error code: {status_code} - {{'error': {error_body}}}", + message=f"Error code: {status_code} - {json.dumps({'error': error_body})}", body=error_body, headers=headers, ) diff --git a/litellm/router.py b/litellm/router.py index 233e7581a40..538cd751fb7 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -897,6 +897,23 @@ def _stream_pre_content_error( ) +def _context_window_stream_fallback_error( + error: "MidStreamFallbackError | litellm.ContextWindowExceededError", + model: object, + llm_provider: str, + has_output: bool, +) -> "MidStreamFallbackError": + if isinstance(error, litellm.ContextWindowExceededError): + if has_output: + raise error + return _stream_pre_content_error(error, model if isinstance(model, str) else "", llm_provider) + if isinstance(error.original_exception, litellm.ContextWindowExceededError) and ( + has_output or error.generated_content or error.emitted_disqualifying_content + ): + raise error.original_exception + return error + + def _deployment_num_retries(deployment: "Deployment | None") -> int | None: """The deployment's own num_retries litellm_param, an int or a digit string the way the config loader leaves it.""" configured: Final = getattr(deployment.litellm_params, "num_retries", None) if deployment is not None else None @@ -3845,7 +3862,19 @@ class Router(DeprecatedRouterMembers): try: async for item in _achunks_with_first(model_response, first_chunk): yield item - except MidStreamFallbackError as e: + except (MidStreamFallbackError, litellm.ContextWindowExceededError) as stream_error: + e: Final = ( + _context_window_stream_fallback_error( + stream_error, + getattr(model_response, "model", ""), + model_response.custom_llm_provider or "", + bool(getattr(model_response, "response_uptil_now", "")) + or _stream_chunks_have_generated_content(model_response.chunks), + ) + if isinstance(stream_error, litellm.ContextWindowExceededError) + or isinstance(stream_error.original_exception, litellm.ContextWindowExceededError) + else stream_error + ) with anyio.CancelScope(shield=True): await close_model_response() await held_slot.aclose() @@ -3917,7 +3946,9 @@ class Router(DeprecatedRouterMembers): initial_kwargs["messages"] = messages self._update_kwargs_before_fallbacks(model=model_group, kwargs=initial_kwargs) fallback_response = await self.async_function_with_fallbacks_common_utils( - e=e, + e=e.original_exception + if isinstance(e.original_exception, litellm.ContextWindowExceededError) + else e, disable_fallbacks=fallbacks_disabled_for_request(initial_kwargs), fallbacks=fallbacks, context_window_fallbacks=context_window_fallbacks, @@ -3985,10 +4016,10 @@ class Router(DeprecatedRouterMembers): if fallback_response is not None and hasattr(fallback_response, "aclose"): try: await fallback_response.aclose() - except BaseException as e: + except BaseException as close_error: verbose_router_logger.debug( "stream_with_fallbacks: error closing fallback_response: %s", - e, + close_error, ) wrapped_response: Final = FallbackStreamWrapper(stream_with_fallbacks()) @@ -4348,6 +4379,10 @@ class Router(DeprecatedRouterMembers): def __aiter__(self): return self + @property + def yielded_first_chunk(self) -> bool: + return self._yielded_first_chunk + async def __anext__(self): try: chunk: Final = await self._async_generator.__anext__() @@ -4375,6 +4410,7 @@ class Router(DeprecatedRouterMembers): # the very key that created the container (#30210). if self.completed_response is None and getattr(chunk, "type", None) in _RESPONSES_TERMINAL_EVENT_TYPES: self.completed_response = chunk + self._yielded_first_chunk = True return chunk async def aclose(self): @@ -4401,7 +4437,18 @@ class Router(DeprecatedRouterMembers): yield item for held_event in held_lifecycle_events: yield held_event - except MidStreamFallbackError as e: + except (MidStreamFallbackError, litellm.ContextWindowExceededError) as stream_error: + e: Final = ( + _context_window_stream_fallback_error( + stream_error, + source_iterator.model or "", + source_iterator.custom_llm_provider or "", + wrapper.yielded_first_chunk, + ) + if isinstance(stream_error, litellm.ContextWindowExceededError) + or isinstance(stream_error.original_exception, litellm.ContextWindowExceededError) + else stream_error + ) if response_id_exposed: self._raise_released_stream_error(e, initial_kwargs) async with contextlib.aclosing( @@ -4572,7 +4619,11 @@ class Router(DeprecatedRouterMembers): # The content-policy dispatch branch matches on the trigger's own type, so a refusal's # MidStreamFallbackError envelope is unwrapped here or the wrong fallback list is consulted. fallback_trigger: Final[Exception] = ( - e.original_exception if isinstance(e.original_exception, litellm.ContentPolicyViolationError) else e + e.original_exception + if isinstance( + e.original_exception, (litellm.ContentPolicyViolationError, litellm.ContextWindowExceededError) + ) + else e ) fallback_response = await self.async_function_with_fallbacks_common_utils( # rebind-ok: set on success e=fallback_trigger, @@ -4658,7 +4709,19 @@ class Router(DeprecatedRouterMembers): try: for item in _chunks_with_first(model_response, first_chunk): yield item - except MidStreamFallbackError as e: + except (MidStreamFallbackError, litellm.ContextWindowExceededError) as stream_error: + e: Final = ( + _context_window_stream_fallback_error( + stream_error, + getattr(model_response, "model", ""), + model_response.custom_llm_provider or "", + bool(getattr(model_response, "response_uptil_now", "")) + or _stream_chunks_have_generated_content(model_response.chunks), + ) + if isinstance(stream_error, litellm.ContextWindowExceededError) + or isinstance(stream_error.original_exception, litellm.ContextWindowExceededError) + else stream_error + ) if fallbacks_disabled_for_request(initial_kwargs) or ( not e.is_pre_first_chunk and (e.generated_content or _stream_chunks_have_generated_content(model_response.chunks)) @@ -4683,7 +4746,9 @@ class Router(DeprecatedRouterMembers): router_self._update_kwargs_before_fallbacks(model=model_group, kwargs=initial_kwargs) fallback_response = run_async_function( router_self.async_function_with_fallbacks_common_utils, - e=e, + e=e.original_exception + if isinstance(e.original_exception, litellm.ContextWindowExceededError) + else e, disable_fallbacks=fallbacks_disabled_for_request(initial_kwargs), fallbacks=fallbacks, context_window_fallbacks=context_window_fallbacks, diff --git a/tests/integration/providers/test_bedrock_mantle_context_overflow_fallbacks_wire.py b/tests/integration/providers/test_bedrock_mantle_context_overflow_fallbacks_wire.py index 37aa96c2467..23b75a5a8cf 100644 --- a/tests/integration/providers/test_bedrock_mantle_context_overflow_fallbacks_wire.py +++ b/tests/integration/providers/test_bedrock_mantle_context_overflow_fallbacks_wire.py @@ -419,12 +419,11 @@ def test_context_window_fallback_is_not_consulted_on_streamed_messages(cwf_rig: _assert_no_fallback(cwf_rig, "overflow", prompt) -def test_context_window_fallback_does_not_fire_on_streamed_responses(cwf_rig: Rig) -> None: +def test_context_window_fallback_fires_on_streamed_responses(cwf_rig: Rig) -> None: prompt: Final = _prompt("cwf responses stream") response, text = _consumed(cwf_rig.gateway, "/v1/responses", _responses(cwf_rig.models["cwf"], prompt, True)) assert response.status_code == 200, text - assert _GENERIC in str(_only_failed(text)["message"]), text - _assert_no_fallback(cwf_rig, "overflow", prompt) + _assert_fell_back(cwf_rig, "overflow", prompt, text) def test_plain_fallback_fires_for_the_overflow_on_chat_completions(fb_rig: Rig) -> None: @@ -467,12 +466,11 @@ def test_streamed_messages_overflow_on_a_plain_fallback_deployment_returns_400_w _assert_no_fallback(fb_rig, "overflow", prompt) -def test_plain_fallback_does_not_fire_on_streamed_responses(fb_rig: Rig) -> None: +def test_plain_fallback_fires_on_streamed_responses(fb_rig: Rig) -> None: prompt: Final = _prompt("fb responses stream") response, text = _consumed(fb_rig.gateway, "/v1/responses", _responses(fb_rig.models["fb"], prompt, True)) assert response.status_code == 200, text - assert _GENERIC in str(_only_failed(text)["message"]), text - _assert_no_fallback(fb_rig, "overflow", prompt) + _assert_fell_back(fb_rig, "overflow", prompt, text) def test_streamed_messages_overflow_on_an_openai_fallback_deployment_returns_the_error_event_without_falling_back( diff --git a/tests/unit/responses/test_streaming_iterator_error_events.py b/tests/unit/responses/test_streaming_iterator_error_events.py index 7445070bd3a..14ecb185f06 100644 --- a/tests/unit/responses/test_streaming_iterator_error_events.py +++ b/tests/unit/responses/test_streaming_iterator_error_events.py @@ -166,21 +166,24 @@ def test_maybe_raise_for_error_event_maps_context_length_code_to_400(): assert not isinstance(exc_info.value, MidStreamFallbackError) -def test_maybe_raise_for_error_event_raises_context_window_exceeded_directly(): +@pytest.mark.parametrize( + "message", + ( + "This model's maximum context length is 128000 tokens. However, your messages resulted in 130000 tokens.", + "Your input exceeds the context window of this model. Please adjust your input and try again.", + ), +) +def test_maybe_raise_for_error_event_raises_context_window_exceeded_directly(message: str) -> None: """A context-length error event maps to ContextWindowExceededError exactly like the non-streaming path and, being a non-retriable client error, is raised directly rather than wrapped for mid-stream fallback, preserving the direct-SDK 400 contract from issue #15785.""" - iterator = _make_iterator() - chunk = _make_error_chunk( - "invalid_request_error", - "context_length_exceeded", - "This model's maximum context length is 128000 tokens. However, your messages resulted in 130000 tokens.", - ) + iterator: Final = _make_iterator() + chunk: Final = _make_error_chunk("invalid_request_error", "context_length_exceeded", message) with pytest.raises(litellm.ContextWindowExceededError) as exc_info: iterator._maybe_raise_for_error_event(chunk) assert exc_info.value.status_code == 400 assert not isinstance(exc_info.value, MidStreamFallbackError) - assert "maximum context length" in str(exc_info.value) + assert message in str(exc_info.value) CONTENT_POLICY_MESSAGE = "This content was flagged for possible cybersecurity risk. The response was halted mid-stream." diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index a6436e47507..96df8d3e1f0 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -5140,7 +5140,10 @@ async def test_set_response_headers_is_the_only_complexity_header_source_for_pro @pytest.mark.asyncio -async def test_acompletion_streaming_iterator_adopts_fallback_response_headers(): +@pytest.mark.parametrize("close_fails", [False, True]) +async def test_acompletion_streaming_iterator_adopts_fallback_response_headers( + close_fails: bool, +) -> None: """LIT-6767: after a successful pre-first-chunk fallback, the wrapper must describe the deployment that served the stream, with no value left over from the attempt that failed.""" @@ -5187,6 +5190,7 @@ async def test_acompletion_streaming_iterator_adopts_fallback_response_headers() class FallbackStream: def __init__(self): + self.close_attempted: bool = False self._response_headers = {"x-request-id": "req-FALLBACK"} self._hidden_params = { "model_id": "fallback-deployment", @@ -5204,6 +5208,11 @@ async def test_acompletion_streaming_iterator_adopts_fallback_response_headers() except StopIteration: raise StopAsyncIteration from None + async def aclose(self) -> None: + self.close_attempted = True + if close_fails: + raise RuntimeError("fallback transport failed during close") + fallback_stream = FallbackStream() with patch.object( router, @@ -5220,12 +5229,14 @@ async def test_acompletion_streaming_iterator_adopts_fallback_response_headers() # the very first chunk the fallback produces must already be published under # the fallback's identity: the proxy commits response headers once that chunk # is buffered, so adopting any later is adopting too late - await result.__anext__() + first_chunk: Final = await result.__anext__() + assert first_chunk.choices[0].delta.content == "OK" assert result._response_headers == {"x-request-id": "req-FALLBACK"} assert result._hidden_params["model_id"] == "fallback-deployment" async for _ in result: pass + assert fallback_stream.close_attempted assert result._response_headers == {"x-request-id": "req-FALLBACK"} assert result._hidden_params["model_id"] == "fallback-deployment" assert result._hidden_params["api_base"] == "https://fallback.example" diff --git a/tests/unit/test_router/test_router_fallbacks.py b/tests/unit/test_router/test_router_fallbacks.py index 88d89696297..bccd6290f44 100644 --- a/tests/unit/test_router/test_router_fallbacks.py +++ b/tests/unit/test_router/test_router_fallbacks.py @@ -1006,3 +1006,204 @@ async def test_unknown_default_fallback_raises_after_primary_failure( assert route.call_count == 1 assert "missing" in str(exc_info.value) + + +def _context_error_response_stream( + *, event_type: str = "error", preceding_event: dict[str, object] | None = None, completed: bool = False +) -> httpx.Response: + response: Final = { + "id": "resp_backup" if completed else "resp_primary", + "object": "response", + "created_at": 1700000000, + "model": "backup" if completed else "primary", + "status": "completed" if completed else "in_progress", + "output": [], + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + "temperature": 1.0, + "top_p": 1.0, + } + error: Final = { + "type": "invalid_request_error", + "code": "context_length_exceeded", + "message": "Your input exceeds the context window of this model. Please adjust your input and try again.", + } + terminal: Final = ( + {"type": "response.completed", "response": response, "sequence_number": 2} + if completed + else {"type": event_type, "error": error, "sequence_number": 2} + if event_type == "error" + else {"type": event_type, "response": {**response, "status": "failed", "error": error}, "sequence_number": 2} + ) + events: Final = ( + {"type": "response.created", "response": response, "sequence_number": 0}, + *((preceding_event,) if preceding_event is not None else ()), + terminal, + ) + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content="".join(f"data: {json.dumps(event)}\n\n" for event in events).encode(), + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("event_type", ("error", "response.failed")) +@pytest.mark.parametrize("context_fallback", (False, True)) +async def test_responses_context_error_uses_the_configured_fallback_before_output( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, event_type: str, context_fallback: bool +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = _fallback_router(fallbacks=[{"primary": ["missing"]}] if context_fallback else None) + route: Final = respx_mock.post("https://fallback-migration.local/v1/responses").mock( + side_effect=( + _context_error_response_stream(event_type=event_type), + _context_error_response_stream(completed=True), + ) + ) + stream: Final = await router.aresponses( + model="primary", + input="oversized prompt", + stream=True, + context_window_fallbacks=[{"primary": ["backup"]}] if context_fallback else None, + ) + events: Final = [event async for event in stream] + assert tuple(json.loads(call.request.content)["model"] for call in route.calls) == ("primary", "backup") + assert tuple(event.type for event in events) == ("response.created", "response.completed") + assert events[0].response.id == events[-1].response.id + assert events[-1].response.model == "backup" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("barrier", ("text", "tool", "disabled", "unconfigured")) +async def test_responses_context_error_does_not_restart_committed_or_disabled_streams( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, barrier: str +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = _fallback_router(fallbacks=[] if barrier == "unconfigured" else None) + preceding: Final = { + "text": { + "type": "response.output_text.delta", + "delta": "already visible", + "item_id": "msg_primary", + "output_index": 0, + "content_index": 0, + "sequence_number": 1, + }, + "tool": { + "type": "response.output_item.added", + "item": { + "type": "function_call", "id": "fc_primary", "call_id": "call_primary", + "name": "lookup", "arguments": "", + }, + "output_index": 0, + "sequence_number": 1, + }, + }.get(barrier) + route: Final = respx_mock.post("https://fallback-migration.local/v1/responses").mock( + return_value=_context_error_response_stream(preceding_event=preceding) + ) + stream: Final = await router.aresponses( + model="primary", input="oversized prompt", stream=True, disable_fallbacks=barrier == "disabled" + ) + with pytest.raises(litellm.ContextWindowExceededError): + async for _ in stream: + pass + assert route.call_count == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sync_mode", (False, True)) +@pytest.mark.parametrize("emission", ("role", "text", "tool")) +async def test_chat_context_error_falls_back_after_role_only_but_not_after_output( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, sync_mode: bool, emission: str +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = _fallback_router(fallbacks=[{"primary": ["missing"]}]) + delta: Final = ( + {"role": "assistant", "tool_calls": [{ + "index": 0, "id": "call_primary", "type": "function", "function": {"name": "lookup", "arguments": ""} + }]} + if emission == "tool" + else {"role": "assistant", "content": "visible" if emission == "text" else ""} + ) + first_chunk: Final = { + "id": "chatcmpl-primary", "object": "chat.completion.chunk", "created": 1700000000, "model": "primary", + "choices": [{"index": 0, "delta": delta}], + } + error: Final = {"error": { + "type": "invalid_request_error", "code": "context_length_exceeded", + "message": "This model's maximum context length is 128000 tokens. Your messages resulted in 130000 tokens.", + }} + backup: Final = { + "id": "chatcmpl-backup", "object": "chat.completion.chunk", "created": 1700000000, "model": "backup", + "choices": [{"index": 0, "delta": {"role": "assistant", "content": "backup answer"}, "finish_reason": "stop"}], + } + route: Final = respx_mock.post("https://fallback-migration.local/v1/chat/completions").mock( + side_effect=( + httpx.Response(200, content=f"data: {json.dumps(first_chunk)}\n\ndata: {json.dumps(error)}\n\n".encode()), + httpx.Response(200, content=f"data: {json.dumps(backup)}\n\ndata: [DONE]\n\n".encode()), + ) + ) + request: Final = { + "model": "primary", "messages": [{"role": "user", "content": "oversized prompt"}], "stream": True, + "context_window_fallbacks": [{"primary": ["backup"]}], + } + stream: Final = router.completion(**request) if sync_mode else await router.acompletion(**request) + if emission != "role": + with pytest.raises(litellm.ContextWindowExceededError): + tuple(stream) if sync_mode else tuple([chunk async for chunk in stream]) + assert route.call_count == 1 + return + chunks: Final = tuple(stream) if sync_mode else tuple([chunk async for chunk in stream]) + assert tuple(json.loads(call.request.content)["model"] for call in route.calls) == ("primary", "backup") + assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks if chunk.choices) == "backup answer" + + +@pytest.mark.parametrize( + "wrapped,has_output,generated_content,disqualifying,recoverable", + ( + (False, False, "", False, True), + (False, True, "", False, False), + (True, False, "", False, True), + (True, True, "", False, False), + (True, False, "already visible", False, False), + (True, False, "", True, False), + ), +) +def test_context_stream_fallback_obeys_all_output_receipts( + wrapped: bool, has_output: bool, generated_content: str, disqualifying: bool, recoverable: bool +) -> None: + from litellm.exceptions import MidStreamFallbackError + from litellm.router import _context_window_stream_fallback_error + + original: Final = litellm.ContextWindowExceededError(message="too long", model="primary", llm_provider="openai") + error: Final = ( + MidStreamFallbackError( + message="too long", model="primary", llm_provider="openai", original_exception=original, + is_pre_first_chunk=True, generated_content=generated_content, emitted_disqualifying_content=disqualifying, + ) + if wrapped + else original + ) + if not recoverable: + with pytest.raises(litellm.ContextWindowExceededError) as raised: + _context_window_stream_fallback_error(error, "primary", "openai", has_output) + assert raised.value is original + return + recovered: Final = _context_window_stream_fallback_error(error, "primary", "openai", has_output) + assert recovered.original_exception is original + assert recovered.is_pre_first_chunk is True + if wrapped: + assert recovered is error + + +def test_context_stream_fallback_preserves_other_errors_for_existing_continuation_policy() -> None: + from litellm.exceptions import MidStreamFallbackError + from litellm.router import _context_window_stream_fallback_error + + error: Final = MidStreamFallbackError( + message="provider unavailable", model="primary", llm_provider="openai", generated_content="already visible" + ) + assert _context_window_stream_fallback_error(error, "primary", "openai", True) is error From fbd126e5182040a5ed8b97d54d41b5febc94452b Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 30 Oct 2026 13:25:47 -0700 Subject: [PATCH 36/74] test: fix integration tests that fail on test order, races and intended changes (#49310) Seven integration tests fail on main for reasons that are not product bugs. Two are order or race bugs in the tests, one reads rows other tests wrote, one passed on an error body, and three still expect behavior that #48644, #39864 and #39540 changed on purpose --- .../compatibility/test_missing_body_param_status.py | 2 +- .../database/migrations/test_request_log_indexes.py | 9 ++++++--- .../integration/observability/test_guardrail_effects.py | 7 +++++-- .../providers/test_gemini_tts_voice_mapping_wire.py | 4 ++-- .../providers/test_hosted_vllm_stream_stall_wire.py | 6 +++++- tests/integration/routing/test_observed_routing.py | 2 +- .../security/test_hide_secrets_tool_compatibility.py | 8 +++++++- 7 files changed, 27 insertions(+), 11 deletions(-) diff --git a/tests/integration/compatibility/test_missing_body_param_status.py b/tests/integration/compatibility/test_missing_body_param_status.py index 005e581c643..d47503e7f08 100644 --- a/tests/integration/compatibility/test_missing_body_param_status.py +++ b/tests/integration/compatibility/test_missing_body_param_status.py @@ -1199,7 +1199,7 @@ def test_anthropic_messages_explicit_null_reaches_upstream(gateway: Gateway) -> "anthropic_messages", deployment_params={"max_tokens": 32}, provider_model="openai/gpt-4o-mini", - response_route="acompletion", + response_route="aresponses", ) response: Final = _post( gateway, diff --git a/tests/integration/database/migrations/test_request_log_indexes.py b/tests/integration/database/migrations/test_request_log_indexes.py index d06f0b13cff..5741c85470b 100644 --- a/tests/integration/database/migrations/test_request_log_indexes.py +++ b/tests/integration/database/migrations/test_request_log_indexes.py @@ -1033,13 +1033,14 @@ def test_the_entry_size_check_draws_the_line_exactly_where_postgres_does( create_index: Final = sql.SQL('CREATE INDEX "entry_probe_idx" ON "entry_probe" ({})').format( sql.SQL(", ").join(sql.Identifier(column) for column in index_columns) ) + fitting_key: Final = secrets.token_hex(longest_fitting_key)[:longest_fitting_key] with psycopg.connect(scratch_database, autocommit=True) as conn: conn.execute('CREATE TABLE "entry_probe" ("s" TEXT, "k" TEXT, "t" TIMESTAMP(3))') conn.execute( 'INSERT INTO "entry_probe" ("s", "k", "t") VALUES (%s, %s, %s), (%s, %s, %s)', ( s_value, - secrets.token_hex(longest_fitting_key)[:longest_fitting_key], + fitting_key, t_value, s_value, secrets.token_hex(longest_fitting_key)[: longest_fitting_key + 1], @@ -1051,7 +1052,8 @@ def test_the_entry_size_check_draws_the_line_exactly_where_postgres_does( ) with pytest.raises(psycopg.errors.ProgramLimitExceeded, match="exceeds btree version 4 maximum 2704"): conn.execute(create_index) - conn.execute('DELETE FROM "entry_probe" WHERE octet_length("k") > %s', (longest_fitting_key,)) + conn.execute('TRUNCATE "entry_probe"') + conn.execute('INSERT INTO "entry_probe" ("s", "k", "t") VALUES (%s, %s, %s)', (s_value, fitting_key, t_value)) conn.execute(create_index) (finding,) = findings @@ -1075,7 +1077,8 @@ def test_a_value_postgres_compresses_under_the_limit_is_not_counted_while_random ) with pytest.raises(psycopg.errors.ProgramLimitExceeded, match="index row size 3016 exceeds"): conn.execute(create_index) - conn.execute('DELETE FROM "compress_probe" WHERE "k" <> %s', ("a" * 3000,)) + conn.execute('TRUNCATE "compress_probe"') + conn.execute('INSERT INTO "compress_probe" ("k") VALUES (%s)', ("a" * 3000,)) conn.execute(create_index) stored: Final = conn.execute('SELECT octet_length("k"), pg_column_size("k") FROM "compress_probe"').fetchone() diff --git a/tests/integration/observability/test_guardrail_effects.py b/tests/integration/observability/test_guardrail_effects.py index 4f5c04bddcc..1220a65528c 100644 --- a/tests/integration/observability/test_guardrail_effects.py +++ b/tests/integration/observability/test_guardrail_effects.py @@ -23,7 +23,7 @@ import yaml from integration._support.client import Gateway, eventually, object_value from integration._support.database import read_rows from integration._support.mcp import mcp_peer, register_mcp, tool_names -from integration._support.process import OwnedProxy, owned_proxy, owned_proxy_process +from integration._support.process import OwnedProxy, OwnedState, owned_proxy, owned_proxy_process from integration._support.wire import Reply, Request, Wire, wire_server from integration._support.ports import allocate_port from openai import APIError, AsyncOpenAI, OpenAI @@ -3263,7 +3263,10 @@ def test_client_disconnect_during_the_body_upload_runs_no_endpoint_and_writes_no with ( wire_server(_openai_upstream) as upstream, owned_proxy_process( - gateway, tmp_path, {"OPENAI_BASE_URL": upstream.url + "/v1", "OPENAI_API_KEY": "synthetic-openai-key"} + gateway, + tmp_path, + {"OPENAI_BASE_URL": upstream.url + "/v1", "OPENAI_API_KEY": "synthetic-openai-key"}, + state=OwnedState.PRIVATE, ) as owned, owned.gateway.scenario() as scenario, ): diff --git a/tests/integration/providers/test_gemini_tts_voice_mapping_wire.py b/tests/integration/providers/test_gemini_tts_voice_mapping_wire.py index 90ff32e01dd..20472dc2399 100644 --- a/tests/integration/providers/test_gemini_tts_voice_mapping_wire.py +++ b/tests/integration/providers/test_gemini_tts_voice_mapping_wire.py @@ -501,7 +501,7 @@ def test_vertex_batch_file_rows_carry_the_mapped_voice(gateway: Gateway) -> None assert tuple(text_in(row) for row in rows) == (first, second) -def test_vertex_live_default_mode_sends_one_setup_without_the_client_voice(gateway: Gateway) -> None: +def test_vertex_live_sends_one_setup_carrying_the_mapped_session_update_voice(gateway: Gateway) -> None: with gateway.scenario() as scenario: handle: Final = live_scenario(scenario) key: Final = scenario.key() @@ -511,7 +511,7 @@ def test_vertex_live_default_mode_sends_one_setup_without_the_client_voice(gatew assert types[0] == "session.created" and types[-1] == "response.done", frames setups: Final = live_setups(gateway, handle.scenario_id) assert len(setups) == 1, setups - assert voice_in(setups[0]) == NO_VOICE, setups[0] + assert voice_in(setups[0]) == "Umbriel", setups[0] @dataclass(frozen=True, slots=True) diff --git a/tests/integration/providers/test_hosted_vllm_stream_stall_wire.py b/tests/integration/providers/test_hosted_vllm_stream_stall_wire.py index fad7898871d..018c6a653d1 100644 --- a/tests/integration/providers/test_hosted_vllm_stream_stall_wire.py +++ b/tests/integration/providers/test_hosted_vllm_stream_stall_wire.py @@ -279,7 +279,11 @@ def cooldown_rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Cooldown upstream, directory, {"PROXY_CONFIG_RELOAD_INTERVAL_SECONDS": "2"}, - config=cooldown_config(directory, router_settings(cache, allowed_fails=0), prometheus=False), + config=cooldown_config( + directory, + router_settings(cache, allowed_fails=0, allowed_fails_policy={"TimeoutErrorAllowedFails": 0}), + prometheus=False, + ), ) ) yield _CooldownRig(owned.gateway, cache) diff --git a/tests/integration/routing/test_observed_routing.py b/tests/integration/routing/test_observed_routing.py index ca5f248e0de..64ab799fe3a 100644 --- a/tests/integration/routing/test_observed_routing.py +++ b/tests/integration/routing/test_observed_routing.py @@ -593,7 +593,7 @@ def test_health_routing_fallbacks_preserve_error_specific_targets_and_request_co "choices": [{"index": 0, "delta": {"role": "assistant", "content": ""}}], }, response_text assert {key: events[1]["error"][key] for key in events[1]["error"] if key != "message"} == { - "type": None, + "type": "internal_server_error", "param": None, "code": "500", }, response_text diff --git a/tests/integration/security/test_hide_secrets_tool_compatibility.py b/tests/integration/security/test_hide_secrets_tool_compatibility.py index d8b900de6d0..a89678aef8e 100644 --- a/tests/integration/security/test_hide_secrets_tool_compatibility.py +++ b/tests/integration/security/test_hide_secrets_tool_compatibility.py @@ -140,9 +140,15 @@ def _key(rig: Rig, *, permissions: dict[str, JsonValue] | None = None) -> Iterat ) +def _without_cooldowns(configuration: dict[str, JsonValue]) -> None: + settings: Final = configuration["router_settings"] + assert isinstance(settings, dict) + settings["disable_cooldowns"] = True + + @pytest.fixture(scope="module") def caller(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Caller]: - with tool_rig(tmp_path_factory.mktemp("hide-secrets-compat"), _peer) as rig: + with tool_rig(tmp_path_factory.mktemp("hide-secrets-compat"), _peer, configure=_without_cooldowns) as rig: with _key(rig) as key: yield Caller(rig, key) From 0382f134e74e027d923c0cde7ad013174990b9b4 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Fri, 30 Oct 2026 13:39:01 -0700 Subject: [PATCH 37/74] fix(ui): add Meta Model API to the Add Model provider dropdown (#49314) * test(ui): cover separate Meta Model API provider onboarding Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ui): add Meta Model API provider onboarding Co-authored-by: lostmartian Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(ui): align Meta coverage with provider keys and listed registry Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ui): show Meta model placeholder Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: lostmartian --- .../provider_create_fields.json | 28 +++++++++++++++++++ .../public_endpoints/test_public_endpoints.py | 23 ++++++++++++++- .../components/provider_info_helpers.test.tsx | 16 +++++++++++ .../src/components/provider_info_helpers.tsx | 6 +++- 4 files changed, 71 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index df3af4ad487..d849b9fbaac 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -2377,6 +2377,34 @@ ], "default_model_placeholder": "gpt-3.5-turbo" }, + { + "provider": "META", + "provider_display_name": "Meta Model API", + "litellm_provider": "meta", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": null, + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": false, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "meta/muse-spark-1.1" + }, { "provider": "MILVUS", "provider_display_name": "Milvus", diff --git a/tests/unit/proxy/public_endpoints/test_public_endpoints.py b/tests/unit/proxy/public_endpoints/test_public_endpoints.py index 48672c7a305..3fe5345d008 100644 --- a/tests/unit/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/unit/proxy/public_endpoints/test_public_endpoints.py @@ -28,6 +28,28 @@ from litellm.types.proxy.public_endpoints.public_endpoints import ProviderCreate from litellm.types.utils import LlmProviders +def test_meta_model_api_fields_are_separate_from_meta_llama() -> None: + app: Final = FastAPI() + app.include_router(router) + response: Final = TestClient(app).get("/public/providers/fields") + assert response.status_code == 200 + providers: Final = response.json() + meta: Final = next((p for p in providers if p["litellm_provider"] == "meta"), None) + llama: Final = next(p for p in providers if p["litellm_provider"] == "meta_llama") + assert meta is not None + assert meta["provider"] == "META" + assert meta["provider_display_name"] == "Meta Model API" + assert meta["default_model_placeholder"] == "meta/muse-spark-1.1" + fields: Final = {field["key"]: field for field in meta["credential_fields"]} + assert set(fields) == {"api_base", "api_key"} + assert fields["api_base"]["field_type"] == "text" + assert fields["api_key"]["field_type"] == "password" + assert fields["api_base"]["required"] is False + assert fields["api_key"]["required"] is False + assert llama["provider"] == "LLAMA" + assert llama["provider_display_name"] == "Meta Llama" + + def test_bigmodel_fields_require_private_key_and_share_default_base_with_adapter() -> None: from litellm.llms.bigmodel.common_utils import BIGMODEL_API_BASE @@ -568,7 +590,6 @@ ADD_MODEL_UNLISTED_PROVIDERS: Final = frozenset( "libertai", "litellm_agent", "manus", - "meta", "modelscope", "mongodb", "nano-gpt", diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx index 02c31ecc082..4c483434831 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx @@ -20,6 +20,22 @@ const BUNDLED_MODEL_MAP: Record = JSON.pa ); describe("provider_info_helpers", () => { + it("resolves Meta Model API separately from Meta Llama and isolates their bundled models", () => { + expect(resolveLitellmProviderSlug("META")).toBe("meta"); + expect(getPlaceholder(Providers.META)).toBe("meta/muse-spark-1.1"); + expect(getProviderLogoAndName("meta").displayName).toBe("Meta Model API"); + const metaModels = Object.keys(BUNDLED_MODEL_MAP).filter( + (key) => BUNDLED_MODEL_MAP[key]?.litellm_provider === "meta", + ); + const llamaModels = Object.keys(BUNDLED_MODEL_MAP).filter( + (key) => BUNDLED_MODEL_MAP[key]?.litellm_provider === "meta_llama", + ); + expect(metaModels).toContain("meta/muse-spark-1.1"); + expect(llamaModels.length).toBeGreaterThan(0); + expect(getProviderModels("META", BUNDLED_MODEL_MAP).sort()).toEqual(metaModels.sort()); + expect(getProviderModels("LLAMA", BUNDLED_MODEL_MAP).sort()).toEqual(llamaModels.sort()); + }); + it("resolves BigModel metadata, name, logo and model placeholder", () => { const metadata = PROVIDER_CREATE_FIELDS.find((provider) => provider.provider === "BIGMODEL"); expect(provider_map.BIGMODEL).toBe(metadata?.litellm_provider); diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx index 6448bd936cc..a8491f656a0 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx @@ -140,6 +140,7 @@ export enum Providers { LLAMAFILE = "Llamafile", LM_STUDIO = "Lm Studio", LLAMA = "Meta Llama", + META = "Meta Model API", MARITALK = "Maritalk", MICROSOFT_365_COPILOT = "Microsoft 365 Copilot", MiniMax = "MiniMax", @@ -270,6 +271,7 @@ export const provider_map: Record = { LEMONADE: "lemonade", LLAMAFILE: "llamafile", LLAMA: "meta_llama", + META: "meta", LM_STUDIO: "lm_studio", MICROSOFT_365_COPILOT: "microsoft_365_copilot", MARITALK: "maritalk", @@ -337,7 +339,7 @@ export const provider_map: Record = { ZeroGPU: "zerogpu", }; -const standaloneSubproviderSlugs = new Set(["bedrock_mantle"]); +const standaloneSubproviderSlugs = new Set(["bedrock_mantle", "meta_llama"]); export const resolveLitellmProviderSlug = (provider: string): string => provider_map[provider] ?? provider.toLowerCase(); @@ -393,6 +395,7 @@ export const providerLogoMap: Partial> = { [Providers.LAMBDA_AI]: lambdaLogo.src, [Providers.LM_STUDIO]: lmstudioLogo.src, [Providers.LLAMA]: metaLlamaLogo.src, + [Providers.META]: metaLlamaLogo.src, [Providers.MiniMax]: minimaxLogo.src, [Providers.MICROSOFT_365_COPILOT]: microsoft365CopilotLogo.src, [Providers.MistralAI]: mistralLogo.src, @@ -491,6 +494,7 @@ const providerPlaceholderMap: Partial> = { [Providers.FalAI]: "fal_ai/fal-ai/flux-pro/v1.1-ultra", [Providers.Google_AI_Studio]: "gemini-pro", [Providers.JinaAI]: "jina_ai/", + [Providers.META]: "meta/muse-spark-1.1", [Providers.MICROSOFT_365_COPILOT]: "microsoft_365_copilot/chat", [Providers.NVIDIA_RIVA]: "nvidia_riva/nvidia/parakeet-ctc-1_1b-asr", [Providers.Oracle]: "oci/xai.grok-4", From 516c796e86a5b1028b2073cd74ca341a174fa289 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Fri, 30 Oct 2026 13:46:27 -0700 Subject: [PATCH 38/74] fix(rerank): preserve token usage in spend logs (#49308) --- litellm/litellm_core_utils/litellm_logging.py | 11 +- litellm/llms/bedrock/rerank/transformation.py | 6 +- .../llms/hosted_vllm/rerank/transformation.py | 4 +- .../llms/together_ai/rerank/transformation.py | 4 +- litellm/types/rerank.py | 46 +++++++- .../test_litellm_logging.py | 100 ++++++++++++++++++ .../test_hosted_vllm_rerank_transformation.py | 35 ++++++ .../jina_ai/rerank/test_transformation.py | 5 +- 8 files changed, 200 insertions(+), 11 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 26949d9bbfb..924603dc7e8 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -165,7 +165,7 @@ from litellm.types.llms.openai import ( ) from litellm.types.mcp import MCPPostCallResponseObject from litellm.types.prompts.init_prompts import PromptSpec -from litellm.types.rerank import RerankResponse +from litellm.types.rerank import RerankResponse, get_rerank_usage from litellm.types.router import RouterRateLimitError, SilentExperimentFlag from litellm.types.utils import ( DEPLOYMENT_SCOPED_PRICING_FIELDS, @@ -355,6 +355,7 @@ def _get_provider_request_id(original_exception: Exception) -> str | None: # Cache custom pricing keys as frozenset for O(1) lookups instead of looping through 49 keys _CUSTOM_PRICING_KEYS: Final[frozenset[str]] = frozenset(CustomPricingLiteLLMParams.model_fields.keys()) _METADATA_MAPPING: Final = TypeAdapter(Mapping[str, object]) +_RESPONSE_MAPPING: Final = TypeAdapter(Mapping[str, object]) _MODEL_INFO_CUSTOM_PRICING_KEYS: Final[frozenset[str]] = _CUSTOM_PRICING_KEYS | DEPLOYMENT_SCOPED_PRICING_FIELDS _UNSERIALIZABLE_METADATA_KEYS: Final[frozenset[str]] = frozenset( ("user_api_key_auth", "user_api_key_budget_reservation") @@ -6817,7 +6818,11 @@ class StandardLoggingPayloadSetup: total_tokens=0, ) - usage: Final = response_obj.get("usage", None) or {} + usage: Final = response_obj.get("usage", None) + if usage is None: + rerank_usage: Final = get_rerank_usage(_RESPONSE_MAPPING.validate_python(response_obj)) + if rerank_usage is not None: + return Usage(**rerank_usage) if usage is None or (not isinstance(usage, dict) and not isinstance(usage, Usage)): return Usage( prompt_tokens=0, @@ -6853,7 +6858,7 @@ class StandardLoggingPayloadSetup: return _empty _raw: Final = response_obj.get("usage", None) if _raw is None: - return _empty + return dict(get_rerank_usage(_RESPONSE_MAPPING.validate_python(response_obj)) or _empty) if isinstance(_raw, ResponseAPIUsage): return ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(_raw).model_dump() if isinstance(_raw, dict): diff --git a/litellm/llms/bedrock/rerank/transformation.py b/litellm/llms/bedrock/rerank/transformation.py index 442903fc05e..c2f5035528d 100644 --- a/litellm/llms/bedrock/rerank/transformation.py +++ b/litellm/llms/bedrock/rerank/transformation.py @@ -6,6 +6,8 @@ Why separate file? Make it easy to see how transformation works from typing import Final +from pydantic import TypeAdapter + from litellm._uuid import uuid from litellm.types.llms.bedrock import ( BedrockRerankBedrockRerankingConfiguration, @@ -90,7 +92,9 @@ class BedrockRerankConfig: example input: {"results":[{"index":0,"relevanceScore":0.6847912669181824},{"index":1,"relevanceScore":0.5980774760246277}]} """ - _billed_units = RerankBilledUnits(**response.get("usage", {"search_units": 1})) # by default 1 search unit + _billed_units: Final = TypeAdapter(RerankBilledUnits).validate_python( + response.get("usage", {"search_units": 1}) + ) _tokens: Final = RerankTokens(**response.get("usage", {})) rerank_meta: Final = RerankResponseMeta(billed_units=_billed_units, tokens=_tokens) diff --git a/litellm/llms/hosted_vllm/rerank/transformation.py b/litellm/llms/hosted_vllm/rerank/transformation.py index 0b77a04c69a..1b7da6f6635 100644 --- a/litellm/llms/hosted_vllm/rerank/transformation.py +++ b/litellm/llms/hosted_vllm/rerank/transformation.py @@ -244,5 +244,7 @@ class HostedVLLMRerankConfig(BaseRerankConfig): return RerankResponse( id=response.get("id") or str(uuid.uuid4()), results=rerank_results, - meta=rerank_meta, + meta=TypeAdapter(RerankResponseMeta).validate_python(response["meta"]) + if "meta" in response and response["meta"] is not None + else rerank_meta, ) diff --git a/litellm/llms/together_ai/rerank/transformation.py b/litellm/llms/together_ai/rerank/transformation.py index f1c3ca9e87d..eb7ec53416d 100644 --- a/litellm/llms/together_ai/rerank/transformation.py +++ b/litellm/llms/together_ai/rerank/transformation.py @@ -6,6 +6,8 @@ Why separate file? Make it easy to see how transformation works from typing import Final +from pydantic import TypeAdapter + from litellm._uuid import uuid from litellm.types.rerank import ( RerankBilledUnits, @@ -19,7 +21,7 @@ from litellm.types.rerank import ( class TogetherAIRerankConfig: def _transform_response(self, response: dict) -> RerankResponse: - _billed_units: Final = RerankBilledUnits(**response.get("usage", {})) + _billed_units: Final = TypeAdapter(RerankBilledUnits).validate_python(response.get("usage", {})) _tokens: Final = RerankTokens(**response.get("usage", {})) rerank_meta: Final = RerankResponseMeta(billed_units=_billed_units, tokens=_tokens) diff --git a/litellm/types/rerank.py b/litellm/types/rerank.py index ffdcf53fd7e..ca2767a7174 100644 --- a/litellm/types/rerank.py +++ b/litellm/types/rerank.py @@ -4,10 +4,10 @@ https://docs.cohere.com/reference/rerank """ -from collections.abc import Sequence -from typing import Literal, overload +from collections.abc import Mapping, Sequence +from typing import Final, Literal, overload -from pydantic import ConfigDict, PrivateAttr +from pydantic import ConfigDict, PrivateAttr, TypeAdapter, ValidationError from typing_extensions import ReadOnly, Required, TypedDict from litellm.types.llms.base import LiteLLMBaseModel @@ -57,6 +57,8 @@ class OptionalRerankParams(TypedDict, total=False): class RerankBilledUnits(TypedDict, total=False): search_units: int | None total_tokens: int | None + input_tokens: ReadOnly[int | None] + output_tokens: ReadOnly[int | None] class RerankTokens(TypedDict, total=False): @@ -65,11 +67,47 @@ class RerankTokens(TypedDict, total=False): class RerankResponseMeta(TypedDict, total=False): - api_version: dict | None + api_version: dict[str, object] | None billed_units: RerankBilledUnits | None tokens: RerankTokens | None +class RerankUsage(TypedDict): + prompt_tokens: ReadOnly[int] + completion_tokens: ReadOnly[int] + total_tokens: ReadOnly[int] + + +_RERANK_META: Final = TypeAdapter(RerankResponseMeta) + + +def get_rerank_usage(response: Mapping[str, object]) -> RerankUsage | None: + if "results" not in response or response.get("meta") is None: + return None + try: + meta: Final = _RERANK_META.validate_python(response.get("meta")) + except ValidationError: + return None + tokens: Final = meta.get("tokens") or RerankTokens() + billed: Final = meta.get("billed_units") or RerankBilledUnits() + input_tokens: Final = next( + (count for count in (tokens.get("input_tokens"), billed.get("input_tokens")) if count is not None), None + ) + output_tokens: Final = next( + (count for count in (tokens.get("output_tokens"), billed.get("output_tokens")) if count is not None), None + ) + total_tokens: Final = billed.get("total_tokens") + if input_tokens is None and output_tokens is None and total_tokens is None: + return None + completion_tokens: Final = output_tokens or 0 + prompt_tokens: Final = input_tokens if input_tokens is not None else max((total_tokens or 0) - completion_tokens, 0) + return RerankUsage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=total_tokens if total_tokens is not None else prompt_tokens + completion_tokens, + ) + + class RerankResponseDocument(TypedDict): text: str diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index f7c2082b683..e952127a32c 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -59,6 +59,7 @@ from litellm.proxy.hooks.max_iterations_limiter import PROXY_MaxIterationsHandle from litellm.proxy.hooks.parallel_request_limiter_v3 import PROXY_MaxParallelRequestsHandler_v3 from litellm.proxy.utils import InternalUsageCache from litellm.types.llms.openai import ResponseAPIUsage, ResponseCompletedEvent, ResponsesAPIResponse +from litellm.types.rerank import RerankResponse from litellm.types.utils import ( CallTypes, CostBreakdown, @@ -4514,6 +4515,105 @@ def test_logging_preserves_missing_usage_without_accepting_request_metadata( assert (payload["prompt_tokens"], payload["completion_tokens"], payload["total_tokens"]) == (0, 0, 0) +@pytest.mark.parametrize( + ("meta", "expected"), + [ + ({"tokens": {"input_tokens": 7, "output_tokens": 3}}, (7, 3, 10)), + ({"tokens": {}, "billed_units": {"total_tokens": 42}}, (42, 0, 42)), + ({"tokens": {"output_tokens": 3}, "billed_units": {"total_tokens": 12}}, (9, 3, 12)), + ({"tokens": {"input_tokens": 7, "output_tokens": 3}, "billed_units": {"total_tokens": 0}}, (7, 3, 0)), + ({"billed_units": {"input_tokens": 7, "output_tokens": 3}}, (7, 3, 10)), + ( + { + "tokens": {"input_tokens": 7, "output_tokens": 3}, + "billed_units": {"input_tokens": 90, "output_tokens": 10, "total_tokens": 12}, + }, + (7, 3, 12), + ), + ( + { + "tokens": {"input_tokens": 0, "output_tokens": 0}, + "billed_units": {"input_tokens": 90, "output_tokens": 10, "total_tokens": 12}, + }, + (0, 0, 12), + ), + ( + { + "tokens": {"input_tokens": None, "output_tokens": None}, + "billed_units": {"input_tokens": 7, "output_tokens": 3, "total_tokens": None}, + }, + (7, 3, 10), + ), + ({"tokens": {"input_tokens": 0}}, (0, 0, 0)), + (None, None), + ({}, None), + ({"tokens": None, "billed_units": None}, None), + ({"billed_units": {"search_units": 3}}, None), + ], +) +def test_rerank_metadata_reaches_standard_and_spend_logs( + logging_obj: Logging, meta: dict[str, object] | None, expected: tuple[int, int, int] | None +) -> None: + from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload + + now: Final = datetime_unit_test(2026, 1, 1, 12, 0, 0) + native: Final = {"id": "rerank-usage", "results": [{"index": 0, "relevance_score": 0.9}], "meta": meta} + response: Final = RerankResponse.model_validate(native) + counts: Final = expected if expected is not None else (0, 0, 0) + expected_usage: Final = dict(zip(("prompt_tokens", "completion_tokens", "total_tokens"), counts)) + assert response.model_dump() == native + usage: Final = StandardLoggingPayloadSetup.get_usage_from_response_obj(response.model_dump()) + assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == counts + assert StandardLoggingPayloadSetup.get_usage_as_dict(response.model_dump()) == ( + expected_usage if expected is not None else {} + ) + kwargs: Final = {"model": "rerank-model", "call_type": "arerank", "litellm_params": {"metadata": {}}} + standard: Final = get_standard_logging_object_payload( + kwargs=kwargs, + init_response_obj=response, + start_time=now, + end_time=now, + logging_obj=logging_obj, + status="success", + ) + assert standard is not None + assert (standard["prompt_tokens"], standard["completion_tokens"], standard["total_tokens"]) == counts + assert standard["metadata"]["usage_object"] == (expected_usage if expected is not None else {}) + spend: Final = get_logging_payload( + kwargs={**kwargs, "standard_logging_object": standard}, response_obj=response, start_time=now, end_time=now + ) + assert (spend["prompt_tokens"], spend["completion_tokens"], spend["total_tokens"]) == counts + + +@pytest.mark.parametrize("combined", [None, Usage(prompt_tokens=11, completion_tokens=2, total_tokens=13)]) +@pytest.mark.parametrize("explicit_usage", [{}, {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}]) +def test_rerank_metadata_does_not_override_explicit_or_combined_usage( + combined: Usage | None, explicit_usage: dict[str, int] +) -> None: + response: Final = {"results": [], "meta": {"tokens": {"input_tokens": 42}}, "usage": explicit_usage} + expected: Final = combined.model_dump() if combined is not None else explicit_usage + assert StandardLoggingPayloadSetup.get_usage_as_dict(response, combined) == expected + usage: Final = StandardLoggingPayloadSetup.get_usage_from_response_obj(response, combined) + assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == ( + expected.get("prompt_tokens", 0), + expected.get("completion_tokens", 0), + expected.get("total_tokens", 0), + ) + + +@pytest.mark.parametrize( + "response", + [ + {"meta": {"tokens": {"input_tokens": 42}}}, + {"results": [], "meta": {"tokens": {"input_tokens": "invalid"}}}, + ], +) +def test_rerank_metadata_ignores_unrelated_and_invalid_responses(response: dict[str, object]) -> None: + assert StandardLoggingPayloadSetup.get_usage_as_dict(response) == {} + usage: Final = StandardLoggingPayloadSetup.get_usage_from_response_obj(response) + assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (0, 0, 0) + + def test_append_system_prompt_messages(): """ Test append_system_prompt_messages prepends system message from kwargs to messages list. diff --git a/tests/unit/llms/hosted_vllm/test_hosted_vllm_rerank_transformation.py b/tests/unit/llms/hosted_vllm/test_hosted_vllm_rerank_transformation.py index 9a62fcf6f0f..7461611e0fa 100644 --- a/tests/unit/llms/hosted_vllm/test_hosted_vllm_rerank_transformation.py +++ b/tests/unit/llms/hosted_vllm/test_hosted_vllm_rerank_transformation.py @@ -148,6 +148,41 @@ class TestHostedVLLMRerankTransform: self.config._transform_response(response_dict) +@pytest.mark.parametrize( + "meta", + [ + { + "api_version": {"version": "2"}, + "tokens": {"input_tokens": 7, "output_tokens": 3}, + "billed_units": {"total_tokens": 12}, + }, + {"tokens": {"input_tokens": 0}, "billed_units": {"input_tokens": 0, "output_tokens": 0}}, + {"tokens": None, "billed_units": None}, + {}, + ], +) +@pytest.mark.parametrize("legacy_usage", [{}, {"usage": {"total_tokens": 99}}]) +def test_hosted_vllm_rerank_preserves_native_metadata_over_legacy_usage( + meta: dict[str, object], legacy_usage: dict[str, object] +) -> None: + response: Final = HostedVLLMRerankConfig()._transform_response( + {"id": "rerank-native", "results": [{"index": 0, "relevance_score": 0.9}], "meta": meta, **legacy_usage} + ) + assert response.meta == meta + assert response.model_dump() == { + "id": "rerank-native", + "results": [{"index": 0, "relevance_score": 0.9}], + "meta": meta, + } + + +def test_hosted_vllm_rerank_null_metadata_uses_legacy_usage() -> None: + response: Final = HostedVLLMRerankConfig()._transform_response( + {"results": [], "meta": None, "usage": {"total_tokens": 42}} + ) + assert response.meta == {"billed_units": {"total_tokens": 42}, "tokens": {"input_tokens": 42}} + + class TestGetOptionalRerankParamsInstruction: """`instruction` is threaded through get_optional_rerank_params only when set.""" diff --git a/tests/unit/llms/jina_ai/rerank/test_transformation.py b/tests/unit/llms/jina_ai/rerank/test_transformation.py index 9ec02a4548b..b752f7b4649 100644 --- a/tests/unit/llms/jina_ai/rerank/test_transformation.py +++ b/tests/unit/llms/jina_ai/rerank/test_transformation.py @@ -46,7 +46,10 @@ def test_transform_rerank_response_maps_results_and_usage(): ({"results": [], "usage": {}}, {"billed_units": {}, "tokens": {}}), ( {"results": [], "usage": {"total_tokens": 12, "input_tokens": 7, "output_tokens": 3, "unknown": 1}}, - {"billed_units": {"total_tokens": 12}, "tokens": {"input_tokens": 7, "output_tokens": 3}}, + { + "billed_units": {"total_tokens": 12, "input_tokens": 7, "output_tokens": 3}, + "tokens": {"input_tokens": 7, "output_tokens": 3}, + }, ), ], ) From a42c1c138a83cb15cf4586f967f105ff90c5c3e0 Mon Sep 17 00:00:00 2001 From: Graham Neubig Date: Fri, 30 Oct 2026 16:46:37 -0400 Subject: [PATCH 39/74] fix(exceptions): classify SGLang context window overflow (#48837) * fix(exceptions): classify SGLang context window overflow Co-authored-by: openhands * test(exceptions): use SGLang's real top-level error body in overflow regression Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: openhands Co-authored-by: Moe Khalil Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../exception_mapping_utils.py | 1 + .../test_exception_mapping_utils.py | 61 +++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 33b6d4e1857..85c232ad3f4 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -98,6 +98,7 @@ class ExceptionCheckers: known_exception_substrings: Final = [ "exceed context limit", "this model's maximum context length is", + "requested token count exceeds the model's maximum context length", "string too long. expected a string with maximum length", "model's maximum context limit", "is longer than the model's context length", diff --git a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py index efb9648cf13..2b1e4a5eed3 100644 --- a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py @@ -440,6 +440,67 @@ def test_gemini_context_window_error_mapping(error_message, should_raise_context ) +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.parametrize("streaming", [True, False]) +@pytest.mark.parametrize( + ("error_message", "expected_type"), + [ + ( + ( + "Requested token count exceeds the model's maximum context length of 131072 tokens. " + "You requested a total of 140000 tokens: 130000 tokens from the input messages and " + "10000 tokens for the completion. Please reduce the number of tokens in the input " + "messages or the completion to fit within the limit." + ), + litellm.ContextWindowExceededError, + ), + ( + ( + "requested token count exceeds the model's maximum context length of 8192 tokens. " + "you requested a total of 9000 tokens" + ), + litellm.ContextWindowExceededError, + ), + ("max_tokens must be a positive integer", litellm.BadRequestError), + ], + ids=["sglang-context-overflow", "lowercase-context-overflow", "unrelated-bad-request"], +) +@pytest.mark.asyncio +async def test_sglang_context_window_error_mapping( + sync_mode: bool, + streaming: bool, + error_message: str, + expected_type: type[litellm.BadRequestError], +) -> None: + def reject_request(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 400, + request=request, + json={"object": "error", "message": error_message, "type": "BadRequestError", "param": None, "code": 400}, + ) + + transport: Final = httpx.MockTransport(reject_request) + if sync_mode: + with ( + httpx.Client(transport=transport) as http_client, + OpenAI(api_key="sk-test", max_retries=0, http_client=http_client) as client, + pytest.raises(litellm.BadRequestError) as caught, + ): + _call_openai_api_sync("openai/sglang-model", "chat_completion", streaming, client) + else: + async with ( + httpx.AsyncClient(transport=transport) as async_http_client, + AsyncOpenAI(api_key="sk-test", max_retries=0, http_client=async_http_client) as async_client, + ): + with pytest.raises(litellm.BadRequestError) as caught: + await _call_openai_api_async("openai/sglang-model", "chat_completion", streaming, async_client) + + assert type(caught.value) is expected_type + assert caught.value.status_code == 400 + assert error_message in str(caught.value) + assert caught.value.llm_provider == "openai" + + def test_lemonade_context_window_error_mapping(): """Lemonade's llama.cpp backend should map context overflows to LiteLLM's standard error.""" From 467847b2e9da67dd1c4ef43ff7d95ca56c013fa3 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Fri, 30 Oct 2026 13:58:04 -0700 Subject: [PATCH 40/74] fix(ui): restore Z.AI credential fields in provider catalog (#49325) Carry the Z.AI catalog fix from #40088 and move its regression into the mapped unit test file. Co-authored-by: Chaitanya Laxman Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../provider_create_fields.json | 28 ++++++++++++++++++ .../public_endpoints/test_public_endpoints.py | 29 ++++++++++++++++++- 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index d849b9fbaac..01a822c5d93 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -4285,6 +4285,34 @@ ], "default_model_placeholder": "gpt-3.5-turbo" }, + { + "provider": "ZAI", + "provider_display_name": "Z.AI (Zhipu AI)", + "litellm_provider": "zai", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://api.z.ai/api/paas/v4", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "zai/glm-4.5" + }, { "provider": "ZeroGPU", "provider_display_name": "ZeroGPU", diff --git a/tests/unit/proxy/public_endpoints/test_public_endpoints.py b/tests/unit/proxy/public_endpoints/test_public_endpoints.py index 3fe5345d008..72755633376 100644 --- a/tests/unit/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/unit/proxy/public_endpoints/test_public_endpoints.py @@ -408,6 +408,34 @@ def test_cognition_provider_fields(): assert fields_by_key["api_base"]["required"] is False +def test_zai_provider_fields(): + from litellm.llms.zai.chat.transformation import ZAI_API_BASE + + app_instance: Final = FastAPI() + app_instance.include_router(router) + test_client: Final = TestClient(app_instance) + + response: Final = test_client.get("/public/providers/fields") + assert response.status_code == 200 + providers: Final = response.json() + + zai: Final = next((p for p in providers if p["provider"] == "ZAI"), None) + assert zai is not None, "Z.AI provider entry not found" + + assert zai["provider_display_name"] == "Z.AI (Zhipu AI)" + assert zai["litellm_provider"] == "zai" + assert zai["default_model_placeholder"].startswith("zai/") + + fields_by_key: Final = {f["key"]: f for f in zai["credential_fields"]} + + assert fields_by_key["api_key"]["required"] is True + assert fields_by_key["api_key"]["field_type"] == "password" + + assert fields_by_key["api_base"]["required"] is False + assert fields_by_key["api_base"]["field_type"] == "text" + assert fields_by_key["api_base"]["placeholder"] == ZAI_API_BASE + + def test_zerogpu_add_model_form_defaults_resolve_to_a_priced_zerogpu_model(): import litellm from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider @@ -611,7 +639,6 @@ ADD_MODEL_UNLISTED_PROVIDERS: Final = frozenset( "transcribe", "valkey", "xiaomi_mimo", - "zai", } ) From 7e37c207482dde6b17c19783e9d8aeafadaecd11 Mon Sep 17 00:00:00 2001 From: tavren27 <348969968+tavren27@users.noreply.github.com> Date: Fri, 30 Oct 2026 14:12:51 -0700 Subject: [PATCH 41/74] fix(responses): run the guardrails a Responses WebSocket frame selects (#48725) * fix(responses): run the guardrails a Responses WebSocket frame selects * fix(responses): name the field a malformed frame guardrail selection got wrong A frame whose guardrails or guardrail_config has the wrong shape now gets an error event naming the field, instead of pydantic's raw validation dump * fix(responses): accept every guardrail selection shape a Responses WebSocket frame sends, and run during_call guardrails A frame can pick guardrails as a list of names or name-to-config dicts, inside metadata or litellm_metadata, with policies validated as a list of strings. during_call guardrails now run before the turn goes upstream, and selection fields sent next to a nested response are rejected instead of dropped * fix(proxy): keep a key's configured guardrails when policies add guardrails A policy's guardrails were merged with dict.fromkeys, so a configured guardrail entry ({name: {extra_body}}) in key metadata raised unhashable type: 'dict' and every request naming policies failed with a 500 on HTTP and on the Responses WebSocket. The merge now keeps existing entries and skips policy guardrails already selected by name. Also types the WebSocket test guardrail's hooks. * fix(responses): run streaming-scoped guardrails on socket turns, keep guardrail fields on blocks, merge guardrail lists by name * fix(responses): check frame guardrail picks against the team permission, honor picks beside a nested response, prefer a guardrail entry with settings * style(responses): format the frame guard and give the pass-through guardrail test an enterprise license * fix(proxy): keep pass-through guardrail settings, gate frame policy picks, skip moderation-only hooks on socket turns * fix(responses): keep string metadata guardrail values as client metadata on socket turns * fix(responses): ask no team permission for string metadata named like a guardrail pick on socket turns * fix(proxy): socket guardrail picks read the key's team metadata when the proxy has no database --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/integrations/custom_guardrail.py | 43 +- litellm/proxy/auth/auth_checks.py | 81 +-- litellm/proxy/litellm_pre_call_utils.py | 67 +- .../passthrough_guardrails.py | 30 +- .../proxy/response_api_endpoints/endpoints.py | 294 ++++++++- litellm/proxy/utils.py | 6 +- litellm/responses/streaming_iterator.py | 36 +- .../integrations/test_custom_guardrail.py | 13 + ...st_auth_checks_object_access_and_lookup.py | 6 +- .../test_passthrough_guardrails.py | 19 + .../response_api_endpoints/test_endpoints.py | 615 +++++++++++++++++- .../unit/proxy/test_litellm_pre_call_utils.py | 82 +++ .../utils/helpers/test_guardrail_merge.py | 18 +- 13 files changed, 1134 insertions(+), 176 deletions(-) diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index a9fb5c29bad..736ebd58130 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -3,8 +3,9 @@ import copy import hashlib import os import secrets -from collections.abc import Mapping, Sequence +from collections.abc import Iterable, Mapping, Sequence from datetime import datetime +from itertools import chain from types import MappingProxyType from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, cast, get_args @@ -208,6 +209,46 @@ def guardrail_request_data_with_streaming( } +_GUARDRAIL_ENTRY_WITH_SETTINGS: Final = TypeAdapter(Mapping[str, object]) + + +def merged_guardrail_entries(*entry_lists: Iterable[object]) -> list[object]: + """Merges guardrail lists so each guardrail is named once: the first ``{name: settings}`` entry naming it wins over + any plain name in either order, else its first plain name stays""" + entries: Final = tuple(chain.from_iterable(entry_lists)) + settings_names: Final = tuple(_settings_entry_names(entry) for entry in entries) + named_with_settings: Final = frozenset[str]().union(*settings_names) + return [ + entry + for index, entry in enumerate(entries) + if not _superseded_guardrail_entry(entries, settings_names, named_with_settings, index) + ] + + +def guardrail_entry_names(entry: object) -> tuple[str, ...]: + if isinstance(entry, str): + return (entry,) + if isinstance(entry, dict): + return tuple(_GUARDRAIL_ENTRY_WITH_SETTINGS.validate_python(entry)) + return () + + +def _settings_entry_names(entry: object) -> frozenset[str]: + return frozenset(guardrail_entry_names(entry)) if isinstance(entry, dict) else frozenset() + + +def _superseded_guardrail_entry( + entries: tuple[object, ...], + settings_names: tuple[frozenset[str], ...], + named_with_settings: frozenset[str], + index: int, +) -> bool: + entry: Final = entries[index] + if isinstance(entry, str): + return entry in named_with_settings or entry in entries[:index] + return bool(settings_names[index]) and settings_names[index] <= frozenset[str]().union(*settings_names[:index]) + + def _request_is_streaming(data: object, event_type: GuardrailEventHooks | None = None) -> bool: if event_type in _REALTIME_STREAMING_HOOKS: return True diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 79a46ef355e..7d379a6f042 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -937,53 +937,58 @@ _GUARDRAIL_MODIFICATION_KEYS: Final[tuple] = ( ) -def _guardrail_modification_check(request_body: Mapping[str, object], team_object: LiteLLM_TeamTable | None) -> None: +def requests_guardrail_modification(request_body: Mapping[str, object]) -> bool: + """Whether a request body sets the plural ``guardrails`` list or a per-request toggle that influences whether + default-on guardrails run (``disable_global_guardrails``, ``disable_global_guardrail`` singular, and + ``opted_out_global_guardrails``), at the top level or in either metadata key""" + return ( + _requests_guardrail_modification_in(request_body.get("metadata")) + or _requests_guardrail_modification_in(request_body.get("litellm_metadata")) + or _requests_guardrail_modification_in(request_body) + ) + + +def _requests_guardrail_modification_in(container: object) -> bool: + coerced: Final = _guardrail_container_as_dict(container) + if coerced is None: + return False + return any(key in coerced for key in _GUARDRAIL_MODIFICATION_KEYS) + + +def _guardrail_container_as_dict(container: object) -> Mapping[str, object] | None: + """Accept dict or JSON-string (from multipart/form-data or extra_body). + + Without this, an attacker can smuggle guardrail keys past the check by + sending ``{"metadata": "{\\"disable_global_guardrails\\": true}"}`` — + ``isinstance(dict)`` on the string returns False, the check returns + no-modification, and ``add_litellm_data_to_request`` parses the string + to a dict downstream. + """ + if isinstance(container, Mapping): + return container + if isinstance(container, str): + parsed: Final = _safe_json_loads_obj(container) + return parsed if isinstance(parsed, dict) else None + return None + + +def guardrail_modification_check(request_body: Mapping[str, object], team_object: LiteLLM_TeamTable | None) -> None: """ Reject user-supplied metadata flags that would modify guardrail behavior - unless the team has explicit permission. Checked keys include the plural - ``guardrails`` list plus the per-request toggles that influence whether - default-on guardrails run (``disable_global_guardrails``, - ``disable_global_guardrail`` singular, and ``opted_out_global_guardrails``). + unless the team has explicit permission. User-supplied values for the bypass toggles are also silently ignored by ``_get_admin_metadata`` at read time; this check adds defense in depth by failing loudly at the auth layer so operators see an explicit 403 instead of a confusing silent-ignore. """ + if requests_guardrail_modification(request_body): + require_guardrail_modification_permission(team_object) + + +def require_guardrail_modification_permission(team_object: LiteLLM_TeamTable | None) -> None: from litellm.proxy.guardrails.guardrail_helpers import can_modify_guardrails - def _coerce_to_dict(container: object) -> dict | None: - """Accept dict or JSON-string (from multipart/form-data or extra_body). - - Without this, an attacker can smuggle guardrail keys past the check by - sending ``{"metadata": "{\\"disable_global_guardrails\\": true}"}`` — - ``isinstance(dict)`` on the string returns False, the check returns - no-modification, and ``add_litellm_data_to_request`` parses the string - to a dict downstream. - """ - if isinstance(container, dict): - return container - if isinstance(container, str): - parsed: Final = _safe_json_loads_obj(container) - return parsed if isinstance(parsed, dict) else None - return None - - def _user_requested_modification(container: object) -> bool: - coerced: Final = _coerce_to_dict(container) - if coerced is None: - return False - return any(key in coerced for key in _GUARDRAIL_MODIFICATION_KEYS) - - # Check both metadata keys — callers can populate either depending on the - # endpoint. Cover the top-level too so root-level injection is rejected. - modifies: Final = ( - _user_requested_modification(request_body.get("metadata")) - or _user_requested_modification(request_body.get("litellm_metadata")) - or _user_requested_modification(request_body) - ) - if not modifies: - return - if not can_modify_guardrails(team_object): raise HTTPException( status_code=403, @@ -1485,7 +1490,7 @@ async def common_checks( _enforce_user_param_check(general_settings, request, request_body, route) _global_proxy_budget_check(global_proxy_spend, skip_all_budget_checks, route) - _guardrail_modification_check(_typed_request_body(request_body), team_object) + guardrail_modification_check(_typed_request_body(request_body), team_object) # 10 [OPTIONAL] Organization RBAC checks organization_role_based_access_check(user_object=user_object, route=route, request_body=request_body) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index ae9a6d4b959..a9cffcc6566 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -48,7 +48,7 @@ from litellm.constants import ( VECTOR_STORE_CALLER_METADATA_KWARG, X_LITELLM_DISABLE_CALLBACKS, ) -from litellm.integrations.custom_guardrail import without_server_streaming_classification +from litellm.integrations.custom_guardrail import merged_guardrail_entries, without_server_streaming_classification from litellm.litellm_core_utils.core_helpers import is_codex_user_agent, normalize_drop_params from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( @@ -3948,30 +3948,14 @@ def add_guardrails_from_key_or_team_metadata( """ from litellm.proxy.utils import premium_user_check - # Initialize guardrails set (avoiding duplicates) - combined_guardrails: Final = set() - - # Add key-level guardrails first - if key_metadata and "guardrails" in key_metadata: - if isinstance(key_metadata["guardrails"], list) and len(key_metadata["guardrails"]) > 0: - premium_user_check() - combined_guardrails.update(key_metadata["guardrails"]) - - # Add team-level guardrails (set automatically handles duplicates) - if team_metadata and "guardrails" in team_metadata: - if isinstance(team_metadata["guardrails"], list) and len(team_metadata["guardrails"]) > 0: - premium_user_check() - combined_guardrails.update(team_metadata["guardrails"]) - - # Add project-level guardrails (set automatically handles duplicates) - if project_metadata and "guardrails" in project_metadata: - if isinstance(project_metadata["guardrails"], list) and len(project_metadata["guardrails"]) > 0: - premium_user_check() - combined_guardrails.update(project_metadata["guardrails"]) - - # Set combined guardrails in metadata as list - if combined_guardrails: - data[metadata_variable_name]["guardrails"] = list(combined_guardrails) + configured_guardrails: Final = tuple( + metadata["guardrails"] + for metadata in (key_metadata, team_metadata, project_metadata) + if metadata and isinstance(metadata.get("guardrails"), list) and len(metadata["guardrails"]) > 0 + ) + if configured_guardrails: + premium_user_check() + data[metadata_variable_name]["guardrails"] = merged_guardrail_entries(*configured_guardrails) _add_guardrails_from_key_or_team_metadata: Final = add_guardrails_from_key_or_team_metadata @@ -4066,14 +4050,10 @@ def _add_guardrails_from_policies_in_metadata( if metadata_variable_name not in data: data[metadata_variable_name] = {} - existing_guardrails = data[metadata_variable_name].get("guardrails", []) - if not isinstance(existing_guardrails, list): - existing_guardrails = [] - - # Combine existing guardrails with policy-resolved guardrails (no duplicates) - combined: Final = set(existing_guardrails) - combined.update(resolved_guardrails) - data[metadata_variable_name]["guardrails"] = list(combined) + existing_guardrails: Final = data[metadata_variable_name].get("guardrails", []) + data[metadata_variable_name]["guardrails"] = merged_guardrail_entries( + existing_guardrails if isinstance(existing_guardrails, list) else (), resolved_guardrails + ) # Store applied policies in metadata for tracking if "applied_policies" not in data[metadata_variable_name]: @@ -4308,10 +4288,11 @@ def _apply_resolved_guardrails_to_metadata( return existing_guardrails: Final = data[metadata_variable_name].get("guardrails", []) - existing_guardrails_list: Final = existing_guardrails if isinstance(existing_guardrails, list) else [] + existing_guardrails_list: Final = ( + _REQUEST_ENTRIES.validate_python(existing_guardrails) if isinstance(existing_guardrails, list) else () + ) - # Combine existing guardrails with policy-resolved guardrails (no duplicates) - combined: Final = list(dict.fromkeys((*existing_guardrails_list, *resolved_guardrails))) + combined: Final = merged_guardrail_entries(existing_guardrails_list, resolved_guardrails) data[metadata_variable_name]["guardrails"] = combined verbose_proxy_logger.debug("Policy engine: added guardrails to request metadata: %s", combined) @@ -4328,7 +4309,7 @@ def _policy_match_tags(request_data: Mapping[str, object], metadata_variable_nam async def add_guardrails_from_policy_engine( - data: dict, + data: dict[str, object], # mutable-ok: pops the request's policies and records the resolved guardrails in place metadata_variable_name: str, user_api_key_dict: UserAPIKeyAuth, ) -> None: @@ -4371,11 +4352,13 @@ async def add_guardrails_from_policy_engine( all_tags: Final = _policy_match_tags(data, metadata_variable_name) _team_alias: Final = user_api_key_dict.team_alias _key_alias: Final = user_api_key_dict.key_alias - context: Final = PolicyMatchContext( - team_alias=_team_alias if isinstance(_team_alias, str) else None, - key_alias=_key_alias if isinstance(_key_alias, str) else None, - model=data.get("model"), - tags=all_tags, + context: Final = PolicyMatchContext.model_validate( + { + "team_alias": _team_alias if isinstance(_team_alias, str) else None, + "key_alias": _key_alias if isinstance(_key_alias, str) else None, + "model": data.get("model"), + "tags": all_tags, + } ) verbose_proxy_logger.debug( diff --git a/litellm/proxy/pass_through_endpoints/passthrough_guardrails.py b/litellm/proxy/pass_through_endpoints/passthrough_guardrails.py index 1c5d270542f..bd8390d704e 100644 --- a/litellm/proxy/pass_through_endpoints/passthrough_guardrails.py +++ b/litellm/proxy/pass_through_endpoints/passthrough_guardrails.py @@ -10,6 +10,7 @@ Handles guardrail execution for passthrough endpoints with: from typing import Any, Final from litellm._logging import verbose_proxy_logger +from litellm.integrations.custom_guardrail import merged_guardrail_entries from litellm.proxy._types import ( PassThroughGuardrailsConfig, PassThroughGuardrailSettings, @@ -205,7 +206,7 @@ class PassthroughGuardrailHandler: def collect_guardrails( user_api_key_dict: UserAPIKeyAuth, passthrough_guardrails_config: PassThroughGuardrailsConfigInput | None, - ) -> dict[str, bool] | None: + ) -> list[object] | None: """ Collect guardrails for a passthrough endpoint. @@ -225,7 +226,7 @@ class PassthroughGuardrailHandler: passthrough_guardrails_config: List or Dict of guardrail names/settings Returns: - Dict of guardrail names to run (format: {guardrail_name: True}), or None + Guardrail entries to run, each a name or a ``{name: settings}`` entry, or None """ from litellm.proxy.litellm_pre_call_utils import ( add_guardrails_from_key_or_team_metadata, @@ -242,15 +243,6 @@ class PassthroughGuardrailHandler: verbose_proxy_logger.debug("Passthrough guardrails config is empty, skipping") return None - # Passthrough is enabled - collect guardrails - guardrails_to_run: Final[dict[str, bool]] = {} - - # Add passthrough-specific guardrails - for guardrail_name in normalized_config: - guardrails_to_run[guardrail_name] = True - verbose_proxy_logger.debug("Added passthrough-specific guardrail: %s", guardrail_name) - - # Add org/team/key level guardrails using shared helper temp_data: Final[dict[str, Any]] = {"metadata": {}} add_guardrails_from_key_or_team_metadata( key_metadata=user_api_key_dict.metadata, @@ -258,20 +250,12 @@ class PassthroughGuardrailHandler: data=temp_data, metadata_variable_name="metadata", ) + inherited_guardrails: Final[list[object]] = temp_data["metadata"].get("guardrails", []) + guardrails_to_run: Final = merged_guardrail_entries(normalized_config, inherited_guardrails) - # Merge inherited guardrails into guardrails_to_run - inherited_guardrails: Final = temp_data["metadata"].get("guardrails", []) - for guardrail_name in inherited_guardrails: - if guardrail_name not in guardrails_to_run: - guardrails_to_run[guardrail_name] = True - verbose_proxy_logger.debug("Added inherited guardrail (key/team level): %s", guardrail_name) + verbose_proxy_logger.debug("Collected guardrails for passthrough endpoint: %s", guardrails_to_run) - verbose_proxy_logger.debug( - "Collected guardrails for passthrough endpoint: %s", - list(guardrails_to_run.keys()), - ) - - return guardrails_to_run if guardrails_to_run else None + return guardrails_to_run or None @staticmethod def get_field_targeted_text( diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index ef59deb2e5e..089cb75a795 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -1,11 +1,13 @@ import asyncio import contextlib +import copy import json import uuid from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass from enum import Enum from functools import partial +from itertools import chain from types import MappingProxyType from typing import ( TYPE_CHECKING, @@ -35,7 +37,7 @@ from litellm.anthropic_interface.exceptions.exception_mapping_utils import Anthr from litellm.caching.dual_cache import DualCache from litellm.constants import EMPTY_MAPPING, MANAGED_BACKGROUND_RESPONSE_METADATA_KEY, TERMINAL_RESPONSE_STATUSES from litellm.exceptions import EndpointNotSupportedByModelError -from litellm.integrations.custom_guardrail import ModifyResponseException +from litellm.integrations.custom_guardrail import ModifyResponseException, guardrail_request_data_with_streaming from litellm.litellm_core_utils.core_helpers import as_str_mapping, get_metadata_variable_name_from_kwargs from litellm.litellm_core_utils.internal_call_metadata import is_background_response, requests_background from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -47,6 +49,11 @@ from litellm.llms.openai.responses.guardrail_translation.handler import ( build_blocked_response, ) from litellm.proxy._types import * +from litellm.proxy.auth.auth_checks import ( + get_team_object, + requests_guardrail_modification, + require_guardrail_modification_permission, +) from litellm.proxy.auth.auth_utils import carries_client_side_overrides from litellm.proxy.auth.user_api_key_auth import ( UserAPIKeyAuth, @@ -57,6 +64,7 @@ from litellm.proxy.common_request_processing import ( ProxyBaseLLMRequestProcessing, cost_headers, create_response, + proxy_exception_from_route_error, resolve_counted_tool_fields, resolve_litellm_call_id, ) @@ -79,6 +87,7 @@ from litellm.proxy.hooks.responses_id_security import ResponsesIDSecurity, decry from litellm.proxy.litellm_pre_call_utils import ( _CLIENT_PRICING_METADATA_FIELDS, # pyright: ignore[reportPrivateUsage] # the HTTP path's own strip list _UNTRUSTED_METADATA_CONTROL_FIELDS, # pyright: ignore[reportPrivateUsage] # the HTTP path's own strip list + add_guardrails_from_policy_engine, is_chat_completions_body, ) from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attribution import ( @@ -89,7 +98,12 @@ from litellm.proxy.utils import count_request_input_tokens, handle_exception_on_ from litellm.responses.litellm_completion_transformation.compaction_items import messages_since_newest_compaction from litellm.responses.litellm_completion_transformation.count_request import MODEL_FILE_ID_MAPPING_FIELD from litellm.responses.retrieve_stream_query import RETRIEVE_INCLUDE_QUERY_KEYS, RETRIEVE_STREAM_QUERY_KEYS -from litellm.responses.streaming_iterator import GuardedResponsesWsFrame, ResponsesWsFrameGuard, spawn_logging_task +from litellm.responses.streaming_iterator import ( + GuardedResponsesWsFrame, + ResponsesWsFrameGuard, + ResponsesWsFrameRejection, + spawn_logging_task, +) from litellm.router import Router from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.openai import ( @@ -2149,11 +2163,15 @@ _PII_TOKENS: Final = TypeAdapter(dict[str, str]) _GUARDRAIL_INFORMATION_KEY: Final = "standard_logging_guardrail_information" _GUARDED_FRAME_FIELDS: Final = ("input", "instructions", "tools") _CLIENT_METADATA_KEYS: Final = ("metadata", "litellm_metadata") -# the guardrail list is merged from key, team and the admission frame; a later frame cannot shrink it _CLIENT_GUARDRAIL_SELECTORS: Final = ("guardrails", "guardrail_config", "policies") +_METADATA_GUARDRAIL_SELECTORS: Final = ("guardrails", "guardrail_config") +_PROXY_ONLY_FRAME_FIELDS: Final = (*_CLIENT_GUARDRAIL_SELECTORS, "include_guardrail_response") +_UNFORWARDED_FRAME_FIELDS: Final = (*_GUARDED_FRAME_FIELDS, *_PROXY_ONLY_FRAME_FIELDS, "litellm_metadata") +_OBJECTS: Final = TypeAdapter(list[object]) +_TURN_COPIED_CONTAINERS: Final = (list, dict) -class _PreCallGuardrailWalk(Protocol): +class _GuardrailWalk(Protocol): async def pre_call_hook( self, *, @@ -2163,15 +2181,49 @@ class _PreCallGuardrailWalk(Protocol): guardrails_only: bool, ) -> dict[str, object]: ... # mutable-ok: ProxyLogging.pre_call_hook returns the guardrails' own dict + async def during_call_hook( + self, + *, + data: dict[str, object], # mutable-ok: ProxyLogging.during_call_hook hands the guardrails a dict they edit + user_api_key_dict: UserAPIKeyAuth | None, + call_type: Literal["aresponses"], + guardrails_only: bool, + ) -> object: ... + + +async def _key_team_object(user_api_key_dict: UserAPIKeyAuth) -> LiteLLM_TeamTable | None: + from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache + + team_id: Final = user_api_key_dict.team_id + if team_id is None or team_id == UI_TEAM_ID: + return None + team_from_key: Final = LiteLLM_TeamTable(team_id=team_id, metadata=user_api_key_dict.team_metadata) + if prisma_client is None: + return team_from_key + try: + return await get_team_object( + team_id=team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=user_api_key_dict.parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + ) + except HTTPException: + return team_from_key + def _responses_ws_frame_guard( admitted: Mapping[str, object], user_api_key_dict: UserAPIKeyAuth, - proxy_logging_obj: _PreCallGuardrailWalk, + proxy_logging_obj: _GuardrailWalk, record_refused_turn: Callable[[tuple[Mapping[str, object], ...], Exception], None], + *, + team_object: Callable[[UserAPIKeyAuth], Awaitable[LiteLLM_TeamTable | None]] = _key_team_object, ) -> ResponsesWsFrameGuard: """Socket admission skips guardrails (the first frame's ``input`` is not part of the admitted request), so every - ``response.create`` frame gets the pre_call guardrail walk the HTTP ``/v1/responses`` path runs on its body.""" + ``response.create`` frame gets the pre_call and during_call guardrail walks the HTTP ``/v1/responses`` path runs on + its body. during_call runs before the frame is forwarded, since a forwarded frame cannot be recalled, and like HTTP, + which sends the model call beside during_call, the frame carries the pre_call output and not during_call rewrites.""" metadata_key: Final = get_metadata_variable_name_from_kwargs(admitted) admitted_bucket: Final = admitted.get(metadata_key) bucket: Final = { @@ -2186,32 +2238,57 @@ def _responses_ws_frame_guard( # a blocked frame is billed as its own failed request; only the first turn settles the admission reservation rejected_frame_auth: Final = user_api_key_dict.model_copy(update={"budget_reservation": None}) + async def rejection( + exception: Exception, + data: dict[str, object], # mutable-ok: billed + ) -> ResponsesWsFrameRejection: + await _bill_rejected_frame(rejected_frame_auth, exception, data) + proxy_exception: Final = proxy_exception_from_route_error(exception) + return ResponsesWsFrameRejection(proxy_exception.message, proxy_exception.provider_specific_fields) + async def guard(frame: str) -> GuardedResponsesWsFrame: frame_obj: Final = _WS_FRAME.validate_json(frame) nested: Final = frame_obj.get("response") - if isinstance(nested, dict) and any(key in frame_obj for key in _GUARDED_FRAME_FIELDS): - raise ValueError( - "response.create carries input, instructions or tools both at the top level and under response" - ) body: Final = _WS_FRAME.validate_python(nested) if isinstance(nested, dict) else frame_obj - frame_data: Final = { - **{ - key: _without_untrusted_metadata(key, value) - for key, value in body.items() - if key != "type" and key not in _CLIENT_GUARDRAIL_SELECTORS + selection_sources: Final = (frame_obj, body) if isinstance(nested, dict) else (body,) + turn_data: Final = guardrail_request_data_with_streaming( + { + **{ + key: _without_untrusted_metadata(key, value) + for key, value in body.items() + if key != "type" and key not in _PROXY_ONLY_FRAME_FIELDS + }, + "model": model, + "litellm_call_id": str(uuid.uuid4()), + metadata_key: {**bucket}, }, - "model": model, - "litellm_call_id": str(uuid.uuid4()), - # shallow: the bucket carries proxy-owned objects (the OTEL parent span) that cannot be deep-copied - metadata_key: {**bucket}, - } + is_streaming=True, + ) + try: + _reject_guarded_field_beside_nested_response(frame_obj, nested) + await _check_frame_may_modify_guardrails(selection_sources, user_api_key_dict, team_object) + frame_data: Final = { + **turn_data, + metadata_key: await _metadata_with_frame_guardrails( + bucket, selection_sources, model, metadata_key, user_api_key_dict + ), + } + except Exception as e: + raise await rejection(e, turn_data) from e try: guarded_data: Final = await proxy_logging_obj.pre_call_hook( user_api_key_dict=user_api_key_dict, data=frame_data, call_type="aresponses", guardrails_only=True ) + # a guardrail may delete a field (the Responses translation pops ``instructions``), so start without them + guarded_fields: Final = { + key: copy.deepcopy(guarded_data[key]) for key in _GUARDED_FRAME_FIELDS if key in guarded_data + } + pii_tokens: Final = _recorded_pii_tokens(guarded_data.get("metadata")) + await proxy_logging_obj.during_call_hook( + data=guarded_data, user_api_key_dict=user_api_key_dict, call_type="aresponses", guardrails_only=True + ) except Exception as e: - await _bill_rejected_frame(rejected_frame_auth, e, frame_data) - raise + raise await rejection(e, frame_data) from e if guarded_data.get("model") != model: rerouted: Final = ValueError( f"a guardrail rerouted this turn to {guarded_data.get('model')!r}; an open Responses WebSocket" @@ -2219,25 +2296,186 @@ def _responses_ws_frame_guard( ) await _bill_rejected_frame(rejected_frame_auth, rerouted, frame_data) raise rerouted - # a guardrail may delete a field (the Responses translation pops ``instructions``), so start without them - guarded_fields: Final = {key: guarded_data[key] for key in _GUARDED_FRAME_FIELDS if key in guarded_data} - kept: Final = {key: value for key, value in body.items() if key not in _GUARDED_FRAME_FIELDS} rewritten: Final = ( - {**frame_obj, "response": {**kept, **guarded_fields}} + {**_forwarded_frame_fields(frame_obj), "response": {**_forwarded_frame_fields(body), **guarded_fields}} if isinstance(nested, dict) - else {**kept, **guarded_fields} + else {**_forwarded_frame_fields(body), **guarded_fields} ) guardrail_information: Final = _recorded_guardrail_information(guarded_data.get(metadata_key)) return GuardedResponsesWsFrame( frame=json.dumps(rewritten), guardrail_information=guardrail_information, - pii_tokens=_recorded_pii_tokens(guarded_data.get("metadata")), + pii_tokens=pii_tokens, bill_refusal=partial(record_refused_turn, guardrail_information), ) return guard +def _reject_guarded_field_beside_nested_response(frame: Mapping[str, object], nested: object) -> None: + """Only the nested ``response`` body runs through the guardrails, so a guarded field beside it would go upstream + unguarded""" + stray: Final = ( + next((key for key in _GUARDED_FRAME_FIELDS if key in frame), None) if isinstance(nested, dict) else None + ) + if stray is not None: + raise ValueError(f"response.create sends {stray} at the top level next to response; send it inside response") + + +async def _check_frame_may_modify_guardrails( + sources: tuple[Mapping[str, object], ...], + user_api_key_dict: UserAPIKeyAuth, + team_object: Callable[[UserAPIKeyAuth], Awaitable[LiteLLM_TeamTable | None]], +) -> None: + """A frame that picks guardrails or policies needs the team's permission to modify guardrails""" + if any( + "policies" in source or requests_guardrail_modification(_without_client_metadata(source)) for source in sources + ): + require_guardrail_modification_permission(await team_object(user_api_key_dict)) + + +def _without_client_metadata(source: Mapping[str, object]) -> Mapping[str, object]: + metadata: Final = source.get("metadata") + return {**source, "metadata": _metadata_picks("metadata", metadata)} if isinstance(metadata, dict) else source + + +def _forwarded_frame_fields(source: Mapping[str, object]) -> dict[str, object]: + return { + key: _without_guardrail_selection(value) if key == "metadata" else value + for key, value in source.items() + if key not in _UNFORWARDED_FRAME_FIELDS + } + + +class _FrameGuardrailSelection(TypedDict, total=False): + guardrails: ReadOnly[Sequence[str | Mapping[str, Mapping[str, object]]]] + guardrail_config: ReadOnly[Mapping[str, object]] + policies: ReadOnly[Sequence[str]] + + +_FRAME_GUARDRAIL_SELECTION: Final = TypeAdapter(_FrameGuardrailSelection) + + +async def _metadata_with_frame_guardrails( + bucket: Mapping[str, object], + sources: tuple[Mapping[str, object], ...], + model: object, + metadata_key: str, + user_api_key_dict: UserAPIKeyAuth, +) -> Mapping[str, object]: + """A frame selects guardrails the way an HTTP ``/v1/responses`` body does, adding to the key, team and policy + guardrails the socket was admitted with, so a frame can never shrink that list. A nested frame selects at its top + level and in its ``response`` body, the body last so its ``guardrail_config`` wins""" + selections: Final = tuple(chain.from_iterable(_frame_guardrail_selections(source) for source in sources)) + picked_policies: Final = tuple(selection["policies"] for selection in selections if "policies" in selection) + metadata: Final = ( + await _metadata_with_frame_policies( + bucket, tuple(chain.from_iterable(picked_policies)), model, metadata_key, user_api_key_dict + ) + if picked_policies + else bucket + ) + selected_guardrails: Final = tuple(chain.from_iterable(selection.get("guardrails", ()) for selection in selections)) + selected_config: Final = dict( + chain.from_iterable( + selection["guardrail_config"].items() for selection in selections if "guardrail_config" in selection + ) + ) + existing_guardrails: Final = metadata.get("guardrails") + existing_config: Final = metadata.get("guardrail_config") + return { + **metadata, + **( + { + "guardrails": [ + *(_OBJECTS.validate_python(existing_guardrails) if isinstance(existing_guardrails, list) else ()), + *selected_guardrails, + ] + } + if selected_guardrails + else {} + ), + **( + { + "guardrail_config": { + **(_WS_FRAME.validate_python(existing_config) if isinstance(existing_config, dict) else {}), + **selected_config, + } + } + if selected_config + else {} + ), + } + + +def _frame_guardrail_selections(body: Mapping[str, object]) -> tuple[_FrameGuardrailSelection, ...]: + """Like HTTP ``/v1/responses``, a frame selects guardrails at its top level and in its ``metadata`` and + ``litellm_metadata``, the top level last so its ``guardrail_config`` wins""" + return ( + *( + _frame_guardrail_selection((key,), _metadata_picks(key, value), _METADATA_GUARDRAIL_SELECTORS) + for key, value in body.items() + if key in _CLIENT_METADATA_KEYS and isinstance(value, dict) + ), + _frame_guardrail_selection((), body, _CLIENT_GUARDRAIL_SELECTORS), + ) + + +def _metadata_picks(key: str, metadata: object) -> Mapping[str, object]: + """OpenAI ``metadata`` values are strings, so a string there is the client's own metadata and never a pick""" + validated: Final = _WS_FRAME.validate_python(metadata) + if key != "metadata": + return validated + return { + name: item + for name, item in validated.items() + if name not in _METADATA_GUARDRAIL_SELECTORS or not isinstance(item, str) + } + + +def _frame_guardrail_selection( + location: tuple[str, ...], source: Mapping[str, object], selectors: tuple[str, ...] +) -> _FrameGuardrailSelection: + try: + return _FRAME_GUARDRAIL_SELECTION.validate_python({key: source[key] for key in selectors if key in source}) + except ValidationError as e: + raise ValueError(f"response.create {first_validation_error(e, prefix=location)}") from e + + +def _without_guardrail_selection(metadata: object) -> object: + if not isinstance(metadata, dict): + return metadata + return { + name: item + for name, item in _WS_FRAME.validate_python(metadata).items() + if name not in _METADATA_GUARDRAIL_SELECTORS or isinstance(item, str) + } + + +async def _metadata_with_frame_policies( + bucket: Mapping[str, object], + policies: Sequence[str], + model: object, + metadata_key: str, + user_api_key_dict: UserAPIKeyAuth, +) -> Mapping[str, object]: + selection: Final = { + "model": model, + "policies": list(policies), + metadata_key: {key: _turn_copy(value) for key, value in bucket.items()}, + } + await add_guardrails_from_policy_engine( + data=selection, metadata_variable_name=metadata_key, user_api_key_dict=user_api_key_dict + ) + return _WS_FRAME.validate_python(selection[metadata_key]) + + +def _turn_copy(value: object) -> object: + """Shallow, since the bucket carries proxy-owned objects (the OTEL parent span) that cannot be copied, and the + policy engine writes its lists in place""" + return copy.copy(value) if type(value) in _TURN_COPIED_CONTAINERS else value + + def _without_untrusted_metadata(key: str, value: object) -> object: """Like the HTTP path, drop the proxy-owned ``user_api_key_*`` and guardrail control keys a client may forge.""" if key not in _CLIENT_METADATA_KEYS or not isinstance(value, dict): diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index cc72dd03a40..0b0698faa9a 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -147,6 +147,7 @@ from litellm.exceptions import ( from litellm.integrations.custom_guardrail import ( CustomGuardrail, ModifyResponseException, + merged_guardrail_entries, ) from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.prometheus import PrometheusLogger @@ -3506,6 +3507,7 @@ class ProxyLogging: data: dict, user_api_key_dict: UserAPIKeyAuth | None, call_type: CallTypesLiteral, + guardrails_only: bool = False, ): caps: Final = ProxyLogging.callback_capabilities() if not caps.has_guardrail and not caps.has_moderation_override: @@ -3517,6 +3519,7 @@ class ProxyLogging: if ( isinstance(callback, CustomLogger) and not isinstance(callback, CustomGuardrail) + and (not guardrails_only or callback.enforces_request_content) and _overrides_moderation_hook(callback) and user_api_key_dict is not None ): @@ -9100,8 +9103,7 @@ def _merge_guardrails_with_existing(data: dict, model_level_guardrails: object) if not isinstance(model_level_guardrails, list): model_level_guardrails = [model_level_guardrails] if model_level_guardrails else [] - # Combine existing and model-level guardrails - metadata["guardrails"] = list(set(existing_guardrails + model_level_guardrails)) + metadata["guardrails"] = merged_guardrail_entries(existing_guardrails, model_level_guardrails) return modified_data diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index d42f1af380e..a8e55a5e94d 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -2277,6 +2277,29 @@ class GuardedResponsesWsFrame: ResponsesWsFrameGuard: TypeAlias = Callable[[str], Awaitable[GuardedResponsesWsFrame]] +class ResponsesWsFrameRejection(ValueError): + def __init__(self, message: str, provider_specific_fields: Mapping[str, object] | None = None) -> None: + super().__init__(message) + self.provider_specific_fields: Final = provider_specific_fields + + +def _frame_rejection_event(rejection: Exception) -> str: + provider_fields: Final = ( + rejection.provider_specific_fields if isinstance(rejection, ResponsesWsFrameRejection) else None + ) + return json.dumps( + { + "type": "error", + "error": { + "type": "invalid_request_error", + "message": str(rejection), + **({"provider_specific_fields": dict(provider_fields)} if provider_fields else {}), + }, + }, + default=str, + ) + + async def _reject_guarded_turn( websocket: ResponsesClientWebSocket, guarded: GuardedResponsesWsFrame, rejection: Exception ) -> None: @@ -2562,7 +2585,8 @@ class ResponsesWebSocketStreaming: return await self.frame_guard(message) except Exception as e: # noqa: BLE001 # any guardrail rejection maps to one client-facing error event verbose_logger.info("Responses WebSocket: pre_call guardrail rejected a response.create frame: %s", e) - await self._send_error_event("invalid_request_error", str(e)) + with contextlib.suppress(Exception): + await self.websocket.send_text(_frame_rejection_event(e)) return None async def _forward_guarded_frame( @@ -2995,14 +3019,6 @@ class ResponsesWebSocketStreaming: except Exception: # noqa: BLE001, S110 # client may already be gone pass - async def _send_error_event(self, error_type: str, message: str) -> None: - try: - await self.websocket.send_text( - json.dumps({"type": "error", "error": {"type": error_type, "message": message}}) - ) - except Exception: # noqa: BLE001, S110 # client may already be gone - pass - async def client_to_backend(self) -> None: """Forward response.create events from client to backend.""" try: @@ -3416,7 +3432,7 @@ class ManagedResponsesWebSocketHandler: return await self.frame_guard(raw_message) except Exception as e: # noqa: BLE001 # any guardrail rejection maps to one client-facing error event verbose_logger.info("ManagedResponsesWS: pre_call guardrail rejected a response.create frame: %s", e) - await self._send_error(str(e), error_type="invalid_request_error") + await self._send_frame(_frame_rejection_event(e)) return None @staticmethod diff --git a/tests/unit/integrations/test_custom_guardrail.py b/tests/unit/integrations/test_custom_guardrail.py index 0483888533e..c32c1b71f75 100644 --- a/tests/unit/integrations/test_custom_guardrail.py +++ b/tests/unit/integrations/test_custom_guardrail.py @@ -3800,3 +3800,16 @@ def test_default_on_guardrail() -> None: ) is True ) + + +def test_merged_guardrail_entries_names_each_guardrail_once_preferring_its_first_entry_with_settings(): + from litellm.integrations.custom_guardrail import merged_guardrail_entries + + assert merged_guardrail_entries(["a"], [{"a": {}, "b": {}}, {"a": {"x": 1}}], ["b", "c", "c"]) == [ + {"a": {}, "b": {}}, + "c", + ] + assert merged_guardrail_entries(["pii", {"pii": {"extra_body": {"threshold": 1}}}], ["model-guard"]) == [ + {"pii": {"extra_body": {"threshold": 1}}}, + "model-guard", + ] diff --git a/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py b/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py index 6b59f6d40dc..dc7bd8cb312 100644 --- a/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py +++ b/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py @@ -5884,7 +5884,7 @@ async def test_team_member_budget_check_reads_from_spend_counter(): class TestGuardrailModificationCheck: - """Defense-in-depth: `_guardrail_modification_check` must 403 when the + """Defense-in-depth: `guardrail_modification_check` must 403 when the caller's metadata attempts to modify any guardrail-related key and the team lacks the `modify_guardrails` permission. Checks both the historically-covered `guardrails` list and the bypass toggles that @@ -5892,11 +5892,11 @@ class TestGuardrailModificationCheck: """ def _call(self, request_body): - from litellm.proxy.auth.auth_checks import _guardrail_modification_check + from litellm.proxy.auth.auth_checks import guardrail_modification_check team_object = MagicMock() team_object.metadata = {} # no permission - return _guardrail_modification_check(request_body=request_body, team_object=team_object) + return guardrail_modification_check(request_body=request_body, team_object=team_object) def test_noop_when_no_guardrail_keys_present(self): # no-op — should return silently diff --git a/tests/unit/proxy/pass_through_endpoints/test_passthrough_guardrails.py b/tests/unit/proxy/pass_through_endpoints/test_passthrough_guardrails.py index 40403de846f..ef19482b30a 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_passthrough_guardrails.py +++ b/tests/unit/proxy/pass_through_endpoints/test_passthrough_guardrails.py @@ -73,6 +73,25 @@ class TestPassthroughGuardrailHandlerGetGuardrailNames: assert set(result) == {"pii-detection", "content-moderation"} +class TestPassthroughGuardrailHandlerCollectGuardrails: + def test_inherits_a_key_guardrail_with_its_settings(self, monkeypatch: pytest.MonkeyPatch): + from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.proxy._types import UserAPIKeyAuth + + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) + key = UserAPIKeyAuth( + api_key="hashed", metadata={"guardrails": [{"pii": {"extra_body": {"threshold": 1}}}, "toxic"]} + ) + + guardrails = PassthroughGuardrailHandler.collect_guardrails(key, ["passthrough-guard", "toxic"]) + + assert guardrails == ["passthrough-guard", "toxic", {"pii": {"extra_body": {"threshold": 1}}}] + request_data = {"metadata": {"guardrails": guardrails}} + assert CustomGuardrail(guardrail_name="pii").get_guardrail_dynamic_request_body_params(request_data) == { + "threshold": 1 + } + + class TestPassthroughGuardrailHandlerNormalizeConfig: """Tests for PassthroughGuardrailHandler.normalize_config method.""" diff --git a/tests/unit/proxy/response_api_endpoints/test_endpoints.py b/tests/unit/proxy/response_api_endpoints/test_endpoints.py index 1e4dded801e..07f85cb9628 100644 --- a/tests/unit/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/unit/proxy/response_api_endpoints/test_endpoints.py @@ -7,6 +7,7 @@ import contextlib import dataclasses import datetime import json +import re import unittest from collections.abc import Callable, Mapping from typing import Any, Final, Literal, cast @@ -22,12 +23,17 @@ from fastapi.testclient import TestClient from httpx import Response import litellm +from litellm.caching.caching import DualCache from litellm.constants import EMPTY_MAPPING -from litellm.proxy._types import ProxyException +from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.proxy._types import ProxyException, UserAPIKeyAuth from litellm.proxy.proxy_server import app +from litellm.proxy.utils import ProxyLogging from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator +from litellm.types.guardrails import GuardrailEventHooks from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse from litellm.types.router import RouterErrors +from litellm.types.utils import CallTypesLiteral from tests._master_key import MASTER_KEY @@ -2213,6 +2219,59 @@ class TestResponsesWSFirstFrameValidation: ws.close.assert_awaited_once_with(code=1011, reason="Internal server error") +class _FrameGuardrail(CustomGuardrail): + def __init__( + self, + name: str, + event_hook: GuardrailEventHooks = GuardrailEventHooks.pre_call, + rewrites: bool = False, + blocks: bool = False, + ): + super().__init__(guardrail_name=name, event_hook=event_hook, default_on=False) + self.inputs: list[object] = [] + self.configs: list[object] = [] + self.rewrites = rewrites + self.blocks = blocks + + async def async_pre_call_hook( + self, + user_api_key_dict: UserAPIKeyAuth, + cache: DualCache, + data: dict[str, object], + call_type: CallTypesLiteral, + ) -> dict[str, object]: + return self._scan(data) + + async def async_moderation_hook( + self, data: dict[str, object], user_api_key_dict: UserAPIKeyAuth, call_type: CallTypesLiteral + ) -> None: + data["input"] = self._scan(data)["input"] + + def _scan(self, data: dict[str, object]) -> dict[str, object]: + self.inputs.append(data["input"]) + self.configs.append(cast(dict[str, object], data["litellm_metadata"]).get("guardrail_config")) + if self.blocks: + raise HTTPException( + status_code=400, + detail={"error": f"Content blocked by {self.guardrail_name}", "guardrail_name": self.guardrail_name}, + ) + return {**data, "input": f"{data['input']} [{self.guardrail_name}]"} if self.rewrites else data + + +@contextlib.contextmanager +def _guardrails_on(*guardrails: CustomGuardrail): + with patch.object(litellm, "callbacks", list(guardrails)): + ProxyLogging._callback_capabilities_cache.clear() + try: + yield ProxyLogging(user_api_key_cache=DualCache()) + finally: + ProxyLogging._callback_capabilities_cache.clear() + + +def _key() -> UserAPIKeyAuth: + return UserAPIKeyAuth(api_key="hashed") + + class TestResponsesWSFirstFrameModelAuth: @pytest.mark.asyncio async def test_endpoint_enforces_auth_after_model_from_first_frame(self): @@ -2344,7 +2403,7 @@ class TestResponsesWSFirstFrameModelAuth: data["litellm_metadata"]["standard_logging_guardrail_information"] = [{"guardrail_name": "turn"}] return {**data, "input": data["input"] + " [guarded]"} - proxy_logging_obj = MagicMock() + proxy_logging_obj = MagicMock(during_call_hook=AsyncMock()) proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=pre_call_hook) admitted = { "model": "gpt-4o-mini", @@ -2375,7 +2434,7 @@ class TestResponsesWSFirstFrameModelAuth: data["litellm_metadata"]["standard_logging_guardrail_information"] = [{"guardrail_cost": 0.5}] raise ValueError("blocked") - proxy_logging_obj = MagicMock() + proxy_logging_obj = MagicMock(during_call_hook=AsyncMock()) proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=pre_call_hook) proxy_logging_obj.post_call_failure_hook = AsyncMock() spend_logger = MagicMock(spec=ProxyDBLogger) @@ -2413,7 +2472,7 @@ class TestResponsesWSFirstFrameModelAuth: async def pre_call_hook(user_api_key_dict, data, call_type, guardrails_only): return {**data, "input": data["input"] + " [guarded]", "tools": data["tools"][:1]} - proxy_logging_obj = MagicMock() + proxy_logging_obj = MagicMock(during_call_hook=AsyncMock()) proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=pre_call_hook) guard = _responses_ws_frame_guard( {"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj, MagicMock() @@ -2432,10 +2491,10 @@ class TestResponsesWSFirstFrameModelAuth: async def test_frame_guard_strips_the_proxy_owned_metadata_keys_a_client_sends(self): from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard - proxy_logging_obj = MagicMock() + proxy_logging_obj = MagicMock(during_call_hook=AsyncMock()) proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["data"]) guard = _responses_ws_frame_guard( - {"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj, MagicMock() + {"model": "gpt-4o-mini", "litellm_metadata": {}}, _key(), proxy_logging_obj, MagicMock() ) metadata = { "user_api_key_metadata": {"disable_global_guardrails": True}, @@ -2454,17 +2513,18 @@ class TestResponsesWSFirstFrameModelAuth: [ {"type": "response.create", "input": "forbidden", "guardrails": []}, {"type": "response.create", "input": "forbidden", "metadata": {"guardrails": []}}, + {"type": "response.create", "input": "forbidden", "litellm_metadata": {"guardrails": []}}, {"type": "response.create", "response": {"input": "forbidden", "guardrails": [], "policies": []}}, ], ) async def test_frame_guard_keeps_the_admitted_guardrail_list_when_a_frame_tries_to_replace_it(self, frame): from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard - proxy_logging_obj = MagicMock() + proxy_logging_obj = MagicMock(during_call_hook=AsyncMock()) proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["data"]) guard = _responses_ws_frame_guard( {"model": "gpt-4o-mini", "litellm_metadata": {"guardrails": ["key-required"]}}, - MagicMock(), + _key(), proxy_logging_obj, MagicMock(), ) @@ -2476,13 +2536,258 @@ class TestResponsesWSFirstFrameModelAuth: assert "guardrails" not in (scanned.get("metadata") or {}) assert scanned["litellm_metadata"]["guardrails"] == ["key-required"] + @pytest.mark.asyncio + @pytest.mark.parametrize( + "selected", + [ + {"guardrails": ["frame-pick"]}, + {"guardrails": [{"frame-pick": {"extra_body": {"threshold": 0.8}}}]}, + {"metadata": {"guardrails": ["frame-pick"], "tag": "kept"}}, + {"litellm_metadata": {"guardrails": ["frame-pick"]}}, + {"response": {"guardrails": ["frame-pick"]}}, + ], + ids=["top-level", "with-extra-body", "metadata", "litellm-metadata", "nested"], + ) + async def test_frame_guard_runs_the_guardrails_a_frame_selects_on_top_of_the_admitted_ones(self, selected: dict[str, object]): + from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard + + key_required = _FrameGuardrail("key-required") + frame_pick = _FrameGuardrail("frame-pick", rewrites=True) + unselected = _FrameGuardrail("unselected") + admitted = {"model": "gpt-4o-mini", "litellm_metadata": {"guardrails": ["key-required"]}} + nested = "response" in selected + body = {"input": "hi", "store": False, **(selected["response"] if nested else selected)} + frame = {"type": "response.create", "response": body} if nested else {"type": "response.create", **body} + + with _guardrails_on(key_required, frame_pick, unselected) as proxy_logging_obj: + guard = _responses_ws_frame_guard(admitted, _key(), proxy_logging_obj, MagicMock()) + selected_turn = await guard(json.dumps(frame)) + plain_turn = await guard(json.dumps({"type": "response.create", "input": "again"})) + + kept_metadata = {"metadata": {"tag": "kept"}} if "metadata" in selected else {} + forwarded = {"input": "hi [frame-pick]", "store": False, **kept_metadata} + assert json.loads(selected_turn.frame) == ( + {"type": "response.create", "response": forwarded} if nested else {"type": "response.create", **forwarded} + ) + assert json.loads(plain_turn.frame) == {"type": "response.create", "input": "again"} + assert key_required.inputs == ["hi", "again"] + assert frame_pick.inputs == ["hi"] + assert unselected.inputs == [] + assert admitted["litellm_metadata"] == {"guardrails": ["key-required"]} + + @pytest.mark.asyncio + async def test_frame_guard_forwards_string_metadata_named_like_a_pick_without_running_it(self): + from litellm.proxy._types import LiteLLM_TeamTable + from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard + + key_required = _FrameGuardrail("key-required") + frame_pick = _FrameGuardrail("frame-pick") + metadata = {"guardrails": "frame-pick", "guardrail_config": "strict", "trace": "t1"} + + async def locked_team(user_api_key_dict: UserAPIKeyAuth) -> LiteLLM_TeamTable: + return LiteLLM_TeamTable(team_id="team-1", metadata={"guardrails": {"modify_guardrails": False}}) + + with _guardrails_on(key_required, frame_pick) as proxy_logging_obj: + guard = _responses_ws_frame_guard( + {"model": "gpt-4o-mini", "litellm_metadata": {"guardrails": ["key-required"]}}, + _key(), + proxy_logging_obj, + MagicMock(), + team_object=locked_team, + ) + guarded = await guard(json.dumps({"type": "response.create", "input": "hi", "metadata": metadata})) + + assert json.loads(guarded.frame) == {"type": "response.create", "input": "hi", "metadata": metadata} + assert key_required.inputs == ["hi"] + assert frame_pick.inputs == [] + + @pytest.mark.asyncio + async def test_frame_guard_hands_a_selected_guardrail_its_config_with_the_top_level_winning(self): + from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard + + frame_pick = _FrameGuardrail("frame-pick") + frame = { + "type": "response.create", + "input": "hi", + "metadata": {"guardrail_config": {"frame-pick": {"threshold": 1}, "metadata-only": {"threshold": 3}}}, + "guardrails": ["frame-pick"], + "guardrail_config": {"frame-pick": {"threshold": 2}}, + } + + with _guardrails_on(frame_pick) as proxy_logging_obj: + guard = _responses_ws_frame_guard( + {"model": "gpt-4o-mini", "litellm_metadata": {"guardrail_config": {"key-required": {"threshold": 0}}}}, + _key(), + proxy_logging_obj, + MagicMock(), + ) + guarded = await guard(json.dumps(frame)) + + assert frame_pick.configs == [ + {"key-required": {"threshold": 0}, "frame-pick": {"threshold": 2}, "metadata-only": {"threshold": 3}} + ] + assert json.loads(guarded.frame) == {"type": "response.create", "input": "hi", "metadata": {}} + + @pytest.mark.asyncio + async def test_frame_guard_runs_streaming_scoped_guardrails_and_skips_non_streaming_ones_on_a_socket_turn(self): + from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard + + streaming_only = _FrameGuardrail("streaming-only") + streaming_only.apply_stream_scope("streaming") + streaming_during_call = _FrameGuardrail("streaming-during-call", event_hook=GuardrailEventHooks.during_call) + streaming_during_call.apply_stream_scope("streaming") + non_streaming_only = _FrameGuardrail("non-streaming-only") + non_streaming_only.apply_stream_scope("non_streaming") + frame = { + "type": "response.create", + "input": "hi", + "guardrails": ["streaming-only", "streaming-during-call", "non-streaming-only"], + } + + with _guardrails_on(streaming_only, streaming_during_call, non_streaming_only) as proxy_logging_obj: + guard = _responses_ws_frame_guard( + {"model": "gpt-4o-mini", "litellm_metadata": {}}, _key(), proxy_logging_obj, MagicMock() + ) + guarded = await guard(json.dumps(frame)) + + assert streaming_only.inputs == ["hi"] + assert streaming_during_call.inputs == ["hi"] + assert non_streaming_only.inputs == [] + assert json.loads(guarded.frame) == {"type": "response.create", "input": "hi"} + + @pytest.mark.asyncio + async def test_frame_guard_blocks_a_turn_with_the_message_a_selected_during_call_guardrail_raises(self): + from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard + + during_pick = _FrameGuardrail("during-pick", event_hook=GuardrailEventHooks.during_call, blocks=True) + + with _guardrails_on(during_pick) as proxy_logging_obj: + guard = _responses_ws_frame_guard( + {"model": "gpt-4o-mini", "litellm_metadata": {}}, _key(), proxy_logging_obj, MagicMock() + ) + unselected_turn = await guard(json.dumps({"type": "response.create", "input": "plain"})) + with pytest.raises(ValueError, match=r"^Content blocked by during-pick$"): + await guard(json.dumps({"type": "response.create", "input": "hi", "guardrails": ["during-pick"]})) + + assert json.loads(unselected_turn.frame) == {"type": "response.create", "input": "plain"} + assert during_pick.inputs == ["hi"] + + @pytest.mark.asyncio + async def test_frame_guard_runs_only_the_moderation_hooks_that_enforce_request_content(self): + from litellm.integrations.custom_logger import CustomLogger + from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard + + class _ModerationHook(CustomLogger): + def __init__(self, enforces_request_content: bool) -> None: + super().__init__() + self.enforces_request_content = enforces_request_content + self.inputs: list[object] = [] + + async def async_moderation_hook( + self, data: dict[str, object], user_api_key_dict: UserAPIKeyAuth, call_type: CallTypesLiteral + ) -> None: + self.inputs.append(data.get("input")) + + vendor_moderation = _ModerationHook(enforces_request_content=False) + content_enforcer = _ModerationHook(enforces_request_content=True) + + with _guardrails_on(vendor_moderation, content_enforcer) as proxy_logging_obj: + guard = _responses_ws_frame_guard( + {"model": "gpt-4o-mini", "litellm_metadata": {}}, _key(), proxy_logging_obj, MagicMock() + ) + await guard(json.dumps({"type": "response.create", "input": "hi"})) + + assert vendor_moderation.inputs == [] + assert content_enforcer.inputs == ["hi"] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("selection", "message"), + [ + ( + {"guardrails": "frame-pick"}, + "response.create guardrails: 'str' instances are not allowed as a Sequence value", + ), + ({"guardrails": [123]}, "response.create guardrails.0.str: Input should be a valid string"), + ({"guardrail_config": []}, "response.create guardrail_config: Input should be a valid dictionary"), + ( + {"policies": "frame-policy"}, + "response.create policies: 'str' instances are not allowed as a Sequence value", + ), + ({"policies": [1]}, "response.create policies.0: Input should be a valid string"), + ( + {"litellm_metadata": {"guardrails": "frame-pick"}}, + "response.create litellm_metadata.guardrails: 'str' instances are not allowed as a Sequence value", + ), + ( + {"litellm_metadata": {"guardrail_config": []}}, + "response.create litellm_metadata.guardrail_config: Input should be a valid dictionary", + ), + ], + ) + async def test_frame_guard_refuses_and_bills_a_frame_whose_guardrail_selection_is_malformed( + self, selection: dict[str, object], message: str + ): + from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger + from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard + + key_required = _FrameGuardrail("key-required") + spend_logger = MagicMock(spec=ProxyDBLogger) + spend_logger.async_post_call_failure_hook = AsyncMock() + + with _guardrails_on(key_required, spend_logger) as proxy_logging_obj: + guard = _responses_ws_frame_guard( + {"model": "gpt-4o-mini", "litellm_metadata": {"guardrails": ["key-required"]}}, + _key(), + proxy_logging_obj, + MagicMock(), + ) + with pytest.raises(ValueError, match=f"^{re.escape(message)}$"): + await guard(json.dumps({"type": "response.create", "input": "hi", **selection})) + + assert key_required.inputs == [] + spend_logger.async_post_call_failure_hook.assert_awaited_once() + + @pytest.mark.asyncio + async def test_frame_guard_runs_the_guardrails_of_the_policies_a_frame_selects(self): + from litellm.proxy.policy_engine.policy_registry import get_policy_registry + from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard + from litellm.types.proxy.policy_engine import Policy, PolicyGuardrails + + key_required = _FrameGuardrail("key-required") + policy_pick = _FrameGuardrail("policy-pick") + policy_registry = get_policy_registry() + policy_registry._policies = {} + policy_registry._policies_by_id = {} + policy_registry._initialized = False + try: + policy_registry.add_policy("frame-policy", Policy(guardrails=PolicyGuardrails(add=["policy-pick"]))) + with _guardrails_on(key_required, policy_pick) as proxy_logging_obj: + guard = _responses_ws_frame_guard( + {"model": "gpt-4o-mini", "litellm_metadata": {"guardrails": ["key-required"]}}, + _key(), + proxy_logging_obj, + MagicMock(), + ) + guarded = await guard( + json.dumps({"type": "response.create", "input": "hi", "policies": ["frame-policy"]}) + ) + finally: + policy_registry._policies = {} + policy_registry._policies_by_id = {} + policy_registry._initialized = False + + assert key_required.inputs == ["hi"] + assert policy_pick.inputs == ["hi"] + assert json.loads(guarded.frame) == {"type": "response.create", "input": "hi"} + @pytest.mark.asyncio async def test_frame_guard_copies_an_admitted_bucket_that_holds_the_otel_parent_span(self): import threading from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard - proxy_logging_obj = MagicMock() + proxy_logging_obj = MagicMock(during_call_hook=AsyncMock()) proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["data"]) span = threading.RLock() guard = _responses_ws_frame_guard( @@ -2510,7 +2815,7 @@ class TestResponsesWSFirstFrameModelAuth: data["litellm_metadata"]["standard_logging_guardrail_information"] = [{"guardrail_cost": 0.5}] return {**data, "model": "onprem-llama"} - proxy_logging_obj = MagicMock() + proxy_logging_obj = MagicMock(during_call_hook=AsyncMock()) proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=pre_call_hook) spend_logger = MagicMock(spec=ProxyDBLogger) spend_logger.async_post_call_failure_hook = AsyncMock() @@ -2543,7 +2848,7 @@ class TestResponsesWSFirstFrameModelAuth: data["litellm_metadata"]["standard_logging_guardrail_information"] = [{"guardrail_cost": 0.5}] raise ValueError("blocked") - proxy_logging_obj = MagicMock() + proxy_logging_obj = MagicMock(during_call_hook=AsyncMock()) proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=pre_call_hook) spend_logger = MagicMock(spec=ProxyDBLogger) spend_logger.async_post_call_failure_hook = AsyncMock() @@ -2571,7 +2876,7 @@ class TestResponsesWSFirstFrameModelAuth: async def pre_call_hook(user_api_key_dict, data, call_type, guardrails_only): return {key: value for key, value in data.items() if key != "instructions"} - proxy_logging_obj = MagicMock() + proxy_logging_obj = MagicMock(during_call_hook=AsyncMock()) proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=pre_call_hook) guard = _responses_ws_frame_guard( {"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj, MagicMock() @@ -2596,7 +2901,7 @@ class TestResponsesWSFirstFrameModelAuth: data["metadata"]["pii_tokens"] = {"": "alice@example.com"} return {**data, "input": "email please"} - proxy_logging_obj = MagicMock() + proxy_logging_obj = MagicMock(during_call_hook=AsyncMock()) proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=pre_call_hook) guard = _responses_ws_frame_guard( {"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj, MagicMock() @@ -2617,19 +2922,279 @@ class TestResponsesWSFirstFrameModelAuth: assert dict(guarded.pii_tokens) == {"": "alice@example.com"} @pytest.mark.asyncio - async def test_frame_guard_rejects_a_frame_that_carries_input_at_both_levels(self): + @pytest.mark.parametrize("field", ["input", "instructions", "tools"]) + async def test_frame_guard_rejects_and_bills_a_guarded_field_beside_a_nested_response(self, field: str): + from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard - proxy_logging_obj = MagicMock() + proxy_logging_obj = MagicMock(during_call_hook=AsyncMock()) + proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["data"]) + spend_logger = MagicMock(spec=ProxyDBLogger) + spend_logger.async_post_call_failure_hook = AsyncMock() + guard = _responses_ws_frame_guard( + {"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj, MagicMock() + ) + message = f"response.create sends {field} at the top level next to response; send it inside response" + + with ( + patch.object(litellm, "callbacks", [spend_logger]), + pytest.raises(ValueError, match=f"^{re.escape(message)}$"), + ): + await guard(json.dumps({"type": "response.create", field: "unguarded", "response": {"input": "hi"}})) + + proxy_logging_obj.pre_call_hook.assert_not_awaited() + assert spend_logger.async_post_call_failure_hook.await_args.kwargs["request_data"]["input"] == "hi" + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "top_level", + [ + {"guardrails": ["frame-pick"]}, + {"metadata": {"guardrails": ["frame-pick"], "tag": "kept"}}, + {"litellm_metadata": {"guardrails": ["frame-pick"]}}, + ], + ids=["top-level", "metadata", "litellm-metadata"], + ) + async def test_frame_guard_runs_the_guardrails_a_nested_frame_selects_beside_its_response(self, top_level: dict[str, object]): + from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard + + key_required = _FrameGuardrail("key-required") + frame_pick = _FrameGuardrail("frame-pick", rewrites=True) + unselected = _FrameGuardrail("unselected") + + with _guardrails_on(key_required, frame_pick, unselected) as proxy_logging_obj: + guard = _responses_ws_frame_guard( + {"model": "gpt-4o-mini", "litellm_metadata": {"guardrails": ["key-required"]}}, + _key(), + proxy_logging_obj, + MagicMock(), + ) + guarded = await guard(json.dumps({"type": "response.create", **top_level, "response": {"input": "hi"}})) + + kept_metadata = {"metadata": {"tag": "kept"}} if "metadata" in top_level else {} + assert json.loads(guarded.frame) == { + "type": "response.create", + **kept_metadata, + "response": {"input": "hi [frame-pick]"}, + } + assert key_required.inputs == ["hi"] + assert frame_pick.inputs == ["hi"] + assert unselected.inputs == [] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "frame", + [ + {"type": "response.create", "input": "hi", "guardrails": ["frame-pick"]}, + {"type": "response.create", "input": "hi", "metadata": {"guardrails": ["frame-pick"]}}, + {"type": "response.create", "input": "hi", "metadata": {"disable_global_guardrails": True}}, + {"type": "response.create", "input": "hi", "metadata": {"disable_global_guardrails": "true"}}, + {"type": "response.create", "response": {"input": "hi", "guardrails": ["frame-pick"]}}, + {"type": "response.create", "litellm_metadata": {"guardrails": ["frame-pick"]}, "response": {"input": "hi"}}, + {"type": "response.create", "input": "hi", "policies": ["frame-policy"]}, + {"type": "response.create", "response": {"input": "hi", "policies": ["frame-policy"]}}, + ], + ids=[ + "top-level", + "metadata", + "global-toggle", + "string-global-toggle", + "nested", + "beside-nested", + "policies", + "nested-policies", + ], + ) + async def test_frame_guard_refuses_guardrail_and_policy_picks_from_a_team_without_permission(self, frame: dict[str, object]): + from litellm.proxy._types import LiteLLM_TeamTable + from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger + from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard + + key_required = _FrameGuardrail("key-required") + frame_pick = _FrameGuardrail("frame-pick") + spend_logger = MagicMock(spec=ProxyDBLogger) + spend_logger.async_post_call_failure_hook = AsyncMock() + team_lookups: list[UserAPIKeyAuth] = [] + key = _key() + + async def locked_team(user_api_key_dict: UserAPIKeyAuth) -> LiteLLM_TeamTable: + team_lookups.append(user_api_key_dict) + return LiteLLM_TeamTable(team_id="team-1", metadata={"guardrails": {"modify_guardrails": False}}) + + with _guardrails_on(key_required, frame_pick, spend_logger) as proxy_logging_obj: + guard = _responses_ws_frame_guard( + {"model": "gpt-4o-mini", "litellm_metadata": {"guardrails": ["key-required"]}}, + key, + proxy_logging_obj, + MagicMock(), + team_object=locked_team, + ) + plain_turn = await guard(json.dumps({"type": "response.create", "input": "plain"})) + with pytest.raises(ValueError, match=r"^Your team does not have permission to modify guardrails\.$"): + await guard(json.dumps(frame)) + + assert json.loads(plain_turn.frame) == {"type": "response.create", "input": "plain"} + assert key_required.inputs == ["plain"] + assert frame_pick.inputs == [] + assert team_lookups == [key] + spend_logger.async_post_call_failure_hook.assert_awaited_once() + + @pytest.mark.asyncio + async def test_frame_guard_runs_a_frame_s_picks_for_a_team_allowed_to_modify_guardrails(self): + from litellm.proxy._types import LiteLLM_TeamTable + from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard + + frame_pick = _FrameGuardrail("frame-pick") + + async def open_team(user_api_key_dict: UserAPIKeyAuth) -> LiteLLM_TeamTable: + return LiteLLM_TeamTable(team_id="team-1", metadata={"guardrails": {"modify_guardrails": True}}) + + with _guardrails_on(frame_pick) as proxy_logging_obj: + guard = _responses_ws_frame_guard( + {"model": "gpt-4o-mini", "litellm_metadata": {}}, + _key(), + proxy_logging_obj, + MagicMock(), + team_object=open_team, + ) + await guard(json.dumps({"type": "response.create", "input": "hi", "guardrails": ["frame-pick"]})) + + assert frame_pick.inputs == ["hi"] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("modify_guardrails", "picked_inputs"), + [(True, ["hi"]), (False, [])], + ids=["open", "locked"], + ) + @pytest.mark.parametrize( + "team_lookup", + [ + lambda: patch( + "litellm.proxy.response_api_endpoints.endpoints.get_team_object", + AsyncMock(side_effect=HTTPException(status_code=404, detail={"error": "Team doesn't exist"})), + ), + lambda: patch("litellm.proxy.proxy_server.prisma_client", None), + ], + ids=["team-row-gone", "no-database"], + ) + async def test_frame_guard_reads_the_key_s_team_metadata_when_the_team_cannot_be_loaded( + self, + modify_guardrails: bool, + picked_inputs: list[str], + team_lookup: Callable[[], contextlib.AbstractContextManager[object]], + ): + from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard + + frame_pick = _FrameGuardrail("frame-pick") + key = UserAPIKeyAuth( + api_key="hashed", + team_id="deleted-team", + team_metadata={"guardrails": {"modify_guardrails": modify_guardrails}}, + ) + frame = json.dumps({"type": "response.create", "input": "hi", "guardrails": ["frame-pick"]}) + + with _guardrails_on(frame_pick) as proxy_logging_obj, team_lookup(): + guard = _responses_ws_frame_guard( + {"model": "gpt-4o-mini", "litellm_metadata": {}}, key, proxy_logging_obj, MagicMock() + ) + if modify_guardrails: + await guard(frame) + else: + with pytest.raises(ValueError, match=r"^Your team does not have permission to modify guardrails\.$"): + await guard(frame) + + assert frame_pick.inputs == picked_inputs + + @pytest.mark.asyncio + async def test_frame_guard_forwards_the_pre_call_output_and_not_a_during_call_rewrite_like_http(self): + from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard + + pre_pick = _FrameGuardrail("pre-pick", rewrites=True) + during_pick = _FrameGuardrail("during-pick", event_hook=GuardrailEventHooks.during_call, rewrites=True) + + with _guardrails_on(pre_pick, during_pick) as proxy_logging_obj: + guard = _responses_ws_frame_guard( + {"model": "gpt-4o-mini", "litellm_metadata": {}}, _key(), proxy_logging_obj, MagicMock() + ) + guarded = await guard( + json.dumps({"type": "response.create", "input": "hi", "guardrails": ["pre-pick", "during-pick"]}) + ) + + assert during_pick.inputs == ["hi [pre-pick]"] + assert json.loads(guarded.frame) == {"type": "response.create", "input": "hi [pre-pick]"} + + @pytest.mark.asyncio + async def test_frame_guard_forwards_a_nested_turn_without_top_level_fields_that_pick_no_guardrail(self): + from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard + + proxy_logging_obj = MagicMock(during_call_hook=AsyncMock()) proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["data"]) guard = _responses_ws_frame_guard( {"model": "gpt-4o-mini", "litellm_metadata": {}}, MagicMock(), proxy_logging_obj, MagicMock() ) - with patch.object(litellm, "callbacks", []), pytest.raises(ValueError, match="both at the top level"): - await guard(json.dumps({"type": "response.create", "input": "forbidden", "response": {}})) + with patch.object(litellm, "callbacks", []): + guarded = await guard( + json.dumps( + { + "type": "response.create", + "include_guardrail_response": True, + "guardrail_config": {"frame-pick": {"threshold": 1}}, + "metadata": {"trace": "t1"}, + "response": {"input": "hi"}, + } + ) + ) - proxy_logging_obj.pre_call_hook.assert_not_awaited() + assert json.loads(guarded.frame) == { + "type": "response.create", + "metadata": {"trace": "t1"}, + "response": {"input": "hi"}, + } + + @pytest.mark.asyncio + async def test_a_blocked_socket_turn_answers_with_the_guardrail_fields_http_returns(self): + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard + from litellm.responses.streaming_iterator import ManagedResponsesWebSocketHandler + + frame_pick = _FrameGuardrail("frame-pick", blocks=True) + websocket = MagicMock(send_text=AsyncMock()) + + with _guardrails_on(frame_pick) as proxy_logging_obj: + handler = ManagedResponsesWebSocketHandler( + websocket=websocket, + model="anthropic/claude-haiku-5-5", + logging_obj=Logging( + model="anthropic/claude-haiku-5-5", + messages=[], + stream=True, + call_type="aresponses", + start_time=0, + litellm_call_id="test-id", + function_id="test-func", + ), + frame_guard=_responses_ws_frame_guard( + {"model": "claude-managed", "litellm_metadata": {}}, _key(), proxy_logging_obj, MagicMock() + ), + ) + await handler._process_response_create( + json.dumps({"type": "response.create", "input": "hi", "guardrails": ["frame-pick"]}) + ) + + assert json.loads(websocket.send_text.await_args.args[0]) == { + "type": "error", + "error": { + "type": "invalid_request_error", + "message": "Content blocked by frame-pick", + "provider_specific_fields": { + "error": "Content blocked by frame-pick", + "guardrail_name": "frame-pick", + "guardrail_mode": "pre_call", + }, + }, + } @pytest.mark.asyncio async def test_frame_guard_keeps_the_guardrail_error_when_billing_the_rejected_frame_fails(self): @@ -2637,7 +3202,7 @@ class TestResponsesWSFirstFrameModelAuth: from litellm.proxy.hooks.proxy_track_cost_callback import ProxyDBLogger from litellm.proxy.response_api_endpoints.endpoints import _responses_ws_frame_guard - proxy_logging_obj = MagicMock() + proxy_logging_obj = MagicMock(during_call_hook=AsyncMock()) proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=ValueError("blocked")) spend_logger = MagicMock(spec=ProxyDBLogger) spend_logger.async_post_call_failure_hook = AsyncMock(side_effect=RuntimeError("db down")) @@ -2679,7 +3244,7 @@ class TestResponsesWSFirstFrameModelAuth: async def fake_llm_call(): return failure if provider_rejected else None - proxy_logging_obj = MagicMock() + proxy_logging_obj = MagicMock(during_call_hook=AsyncMock()) proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None) proxy_logging_obj.arelease_responses_websocket_slot = AsyncMock(return_value=None) user_api_key_dict = MagicMock() @@ -2768,7 +3333,7 @@ class TestResponsesWSFirstFrameModelAuth: return relay() row_written = asyncio.Event() - proxy_logging_obj = MagicMock() + proxy_logging_obj = MagicMock(during_call_hook=AsyncMock()) proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=pre_call_hook) proxy_logging_obj.post_call_failure_hook = AsyncMock(side_effect=lambda **kwargs: row_written.set()) proxy_logging_obj.arelease_responses_websocket_slot = AsyncMock(return_value=None) @@ -2858,7 +3423,7 @@ class TestResponsesWSFirstFrameModelAuth: return relay() - proxy_logging_obj = MagicMock() + proxy_logging_obj = MagicMock(during_call_hook=AsyncMock()) proxy_logging_obj.pre_call_hook = AsyncMock( side_effect=lambda user_api_key_dict, data, call_type, guardrails_only: data ) @@ -2948,7 +3513,7 @@ class TestResponsesWSFirstFrameModelAuth: return relay() - proxy_logging_obj = MagicMock() + proxy_logging_obj = MagicMock(during_call_hook=AsyncMock()) proxy_logging_obj.pre_call_hook = AsyncMock( side_effect=lambda user_api_key_dict, data, call_type, guardrails_only: data ) @@ -3022,7 +3587,7 @@ class TestResponsesWSFirstFrameModelAuth: return relay() - proxy_logging_obj = MagicMock() + proxy_logging_obj = MagicMock(during_call_hook=AsyncMock()) proxy_logging_obj.pre_call_hook = AsyncMock( side_effect=lambda user_api_key_dict, data, call_type, guardrails_only: { **data, @@ -3083,7 +3648,7 @@ class TestResponsesWSFirstFrameModelAuth: rejection = litellm.RateLimitError( message="origin deployment is cooling down", model="gpt-4o-mini", llm_provider="openai" ) - proxy_logging_obj = MagicMock() + proxy_logging_obj = MagicMock(during_call_hook=AsyncMock()) proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None) proxy_logging_obj.arelease_responses_websocket_slot = AsyncMock(return_value=None) user_api_key_dict = MagicMock() diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index c5136ff6e4a..3194124ad1c 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -5360,6 +5360,88 @@ async def test_add_guardrails_from_policy_engine_keeps_a_policy_added_guardrail_ assert [pipeline.mode for _policy_name, pipeline in data["metadata"]["_guardrail_pipelines"]] == ["post_call"] +@pytest.mark.asyncio +async def test_add_guardrails_from_policy_engine_keeps_configured_guardrails_and_skips_ones_already_selected(): + from litellm.proxy.policy_engine.attachment_registry import get_attachment_registry + from litellm.proxy.policy_engine.policy_registry import get_policy_registry + from litellm.types.proxy.policy_engine import Policy, PolicyAttachment, PolicyGuardrails + + configured = {"pii_blocker": {"extra_body": {"threshold": 0.9}}} + data = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hello"}], + "metadata": {"guardrails": [configured, "audit"]}, + } + policy_registry = get_policy_registry() + policy_registry._policies = { + "baseline": Policy(guardrails=PolicyGuardrails(add=["pii_blocker", "audit", "hipaa_audit"])), + } + policy_registry._initialized = True + attachment_registry = get_attachment_registry() + attachment_registry._attachments = [PolicyAttachment(policy="baseline", scope="*")] + attachment_registry._initialized = True + + try: + await add_guardrails_from_policy_engine( + data=data, + metadata_variable_name="metadata", + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + ) + finally: + policy_registry._policies = {} + policy_registry._initialized = False + attachment_registry._attachments = [] + attachment_registry._initialized = False + + assert data["metadata"]["guardrails"] == [configured, "audit", "hipaa_audit"] + + +def test_add_guardrails_from_auth_metadata_lists_each_key_and_team_guardrail_once_keeping_the_key_s_settings(): + from litellm.proxy.litellm_pre_call_utils import add_guardrails_from_auth_metadata + + configured = {"pii_blocker": {"extra_body": {"threshold": 0.9}}} + data = {"model": "gpt-4", "metadata": {}} + user_api_key_dict = UserAPIKeyAuth( + api_key="test-key", + metadata={"guardrails": [configured, "audit"]}, + team_metadata={"guardrails": ["audit", "pii_blocker", "toxic"]}, + ) + + with patch("litellm.proxy.utils.premium_user_check"): + add_guardrails_from_auth_metadata( + user_api_key_dict=user_api_key_dict, data=data, metadata_variable_name="metadata" + ) + + assert data["metadata"]["guardrails"] == [configured, "audit", "toxic"] + + +def test_add_guardrails_from_auth_metadata_keeps_a_selected_guardrail_s_settings_when_a_key_policy_adds_it(): + from litellm.proxy.litellm_pre_call_utils import add_guardrails_from_auth_metadata + from litellm.proxy.policy_engine.policy_registry import get_policy_registry + from litellm.types.proxy.policy_engine import Policy, PolicyGuardrails + + configured = {"pii_blocker": {"extra_body": {"threshold": 0.9}}} + data = {"model": "gpt-4", "metadata": {"guardrails": [configured, "audit"]}} + policy_registry = get_policy_registry() + policy_registry._policies = { + "baseline": Policy(guardrails=PolicyGuardrails(add=["pii_blocker", "audit", "hipaa_audit"])), + } + policy_registry._initialized = True + + try: + with patch("litellm.proxy.utils.premium_user_check"): + add_guardrails_from_auth_metadata( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key", metadata={"policies": ["baseline"]}), + data=data, + metadata_variable_name="metadata", + ) + finally: + policy_registry._policies = {} + policy_registry._initialized = False + + assert data["metadata"]["guardrails"] == [configured, "audit", "hipaa_audit"] + + @pytest.mark.asyncio async def test_add_guardrails_from_policy_engine_applies_inherited_parent_guardrail_when_child_condition_misses(): from litellm.proxy.policy_engine.attachment_registry import get_attachment_registry diff --git a/tests/unit/proxy/utils/helpers/test_guardrail_merge.py b/tests/unit/proxy/utils/helpers/test_guardrail_merge.py index e2c1a7c2c09..dd933c508cb 100644 --- a/tests/unit/proxy/utils/helpers/test_guardrail_merge.py +++ b/tests/unit/proxy/utils/helpers/test_guardrail_merge.py @@ -362,7 +362,17 @@ def test_merge_guardrails_with_existing_creates_metadata_when_missing(): } -def test_merge_guardrails_with_existing_raises_on_unhashable_guardrail(): - data = {"metadata": {"guardrails": [{"unhashable": True}]}} - with pytest.raises(TypeError): - _merge_guardrails_with_existing(data, ["g1"]) +def test_merge_guardrails_with_existing_keeps_a_selected_guardrail_s_settings_over_the_model_s_plain_entry(): + configured = {"pii-redact": {"extra_body": {"threshold": 0.9}}} + + result = _merge_guardrails_with_existing({"metadata": {"guardrails": [configured]}}, ["pii-redact", "toxic-filter"]) + + assert result["metadata"]["guardrails"] == [configured, "toxic-filter"] + + +def test_merge_guardrails_with_existing_keeps_a_selected_guardrail_s_settings_listed_after_the_key_s_plain_entry(): + configured = {"pii-redact": {"extra_body": {"threshold": 0.9}}} + + result = _merge_guardrails_with_existing({"metadata": {"guardrails": ["pii-redact", configured]}}, ["toxic-filter"]) + + assert result["metadata"]["guardrails"] == [configured, "toxic-filter"] From 35d3a1784b90cb2dd6e20afcb74ebf1f4bedbac4 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Fri, 30 Oct 2026 14:27:23 -0700 Subject: [PATCH 42/74] fix(proxy): preserve access windows when editing models (#49316) --- .../model_management_endpoints.py | 6 +-- .../test_model_management_endpoints.py | 51 ++++++++++++++++++- 2 files changed, 50 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index dbd77e45d57..2c7453ebb1c 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -1515,14 +1515,10 @@ def update_db_model( # convert to prisma compatible format - for key, value in merged_model_info.items(): - if isinstance(value, datetime.datetime): - merged_model_info[key] = value.isoformat() - prisma_compatible_model_dict: Final = PrismaCompatibleUpdateDBModel( model_name=merged_model_name, litellm_params=json.dumps(merged_litellm_params), - model_info=json.dumps(merged_model_info), + model_info=TypeAdapter(dict[str, object]).dump_json(merged_model_info).decode(), ) if updated_patch.blocked is not None: diff --git a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py index abaa802de2c..ab6cfcfebfe 100644 --- a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py @@ -4,8 +4,9 @@ import inspect import json import socket from collections.abc import AsyncIterator, Iterator, Mapping +from datetime import datetime, time, timezone from types import MappingProxyType, SimpleNamespace -from typing import Dict, Final, Optional +from typing import Dict, Final, Literal, Optional from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -46,7 +47,14 @@ from litellm.proxy.management_helpers.model_quota_policy import REQUIRE_BOTH_LIM from litellm.proxy.utils import PrismaClient from litellm.router import Router from litellm.router_api import ApplyResult, ChangeSet -from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo, updateDeployment, updateLiteLLMParams +from litellm.types.router import ( + Deployment, + LiteLLM_Params, + ModelAccessWindow, + ModelInfo, + updateDeployment, + updateLiteLLMParams, +) async def _passthrough_row(update_data): @@ -4048,6 +4056,45 @@ class TestUpdateDBModelBlocked: assert "blocked" not in result +@pytest.mark.parametrize("window_patch", ["omit", "echo", "replace", "clear"]) +def test_model_edit_preserves_or_explicitly_changes_access_windows( + window_patch: Literal["omit", "echo", "replace", "clear"], +) -> None: + from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model + + saved_window: Final = ModelAccessWindow( + start=time(13), end=time(17), timezone="Asia/Almaty", team_ids=("team-window", "team-shared") + ) + replacement_window: Final = ModelAccessWindow( + start=time(22, 0, 0, 123456), end=time(6, 0, 0, 654321), + timezone="America/New_York", team_ids=("team-other",), + ) + created_at: Final = datetime(2026, 1, 2, 3, 4, 5, 123456, tzinfo=timezone.utc) + requested_windows: Final = { + "omit": None, + "echo": (saved_window,), + "replace": (replacement_window,), + "clear": (), + }[window_patch] + expected_windows: Final = (saved_window,) if requested_windows is None else requested_windows + result: Final = update_db_model( + db_model=Deployment( + model_name="window-model", + litellm_params=LiteLLM_Params(model="openai/test-model"), + model_info=ModelInfo(id="window-deployment", access_windows=(saved_window,), created_at=created_at), + ), + updated_patch=updateDeployment( + litellm_params=updateLiteLLMParams(max_parallel_requests=7), + model_info=None if requested_windows is None else ModelInfo(access_windows=requested_windows), + ), + ) + + assert json.loads(result["litellm_params"])["max_parallel_requests"] == 7 + saved_info: Final = ModelInfo.model_validate_json(result["model_info"]) + assert saved_info.access_windows == expected_windows + assert saved_info.created_at == created_at + + class TestUpdateDBModelKeepsLegacyDropParams: def test_partial_patch_keeps_encrypted_string_drop_params(self, monkeypatch): from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper From ec4f66bb94ff706bb7f2fbb55261228baed29f77 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Fri, 30 Oct 2026 14:38:14 -0700 Subject: [PATCH 43/74] fix(router): budget classifier input by provider limits (#49306) --- litellm/decisions/call.py | 8 +- litellm/integrations/otel/routing.py | 5 + .../llms/azure_ai/decisions/transformation.py | 12 +- .../llms/base_llm/decisions/transformation.py | 21 +- .../llms/typesafe/decisions/transformation.py | 13 +- litellm/router_api.py | 3 + .../complexity_router/complexity_router.py | 119 ++++++---- .../complexity_router/jev_classifier.py | 157 ++++++++++++- litellm/types/utils.py | 10 + .../integrations/otel/test_otel_v2_logger.py | 5 + tests/unit/integrations/test_opentelemetry.py | 5 + .../test_base_decisions_transformation.py | 60 +++++ .../complexity_router/test_jev_classifier.py | 216 +++++++++++++++++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 10 + 14 files changed, 589 insertions(+), 55 deletions(-) diff --git a/litellm/decisions/call.py b/litellm/decisions/call.py index 3b73f65c917..ae01aa8d0f7 100644 --- a/litellm/decisions/call.py +++ b/litellm/decisions/call.py @@ -218,7 +218,7 @@ def supported_decisions_providers() -> tuple[str, ...]: ) -def _provider_config(model: str, custom_llm_provider: str) -> BaseDecisionsConfig: +def get_decisions_provider_config(model: str, custom_llm_provider: str) -> BaseDecisionsConfig: provider: Final = next((member for member in LlmProviders if member.value == custom_llm_provider), None) provider_config: Final = ( None if provider is None else ProviderConfigManager.get_provider_decisions_config(model, provider) @@ -408,7 +408,7 @@ def _provider_ir_request( return supported -def _outbound_headers( +def decisions_outbound_headers( extra_headers: Mapping[str, str] | None, custom_llm_provider: str, kwargs: Mapping[str, object] ) -> Mapping[str, str]: forwarded: Final = ProviderSpecificHeaderUtils.get_provider_specific_headers( @@ -466,7 +466,7 @@ def prepare_call( api_base=call_api_base, api_key=api_key, ) - provider_config: Final = _provider_config(upstream_model, provider) + provider_config: Final = get_decisions_provider_config(upstream_model, provider) if not upstream_model: raise litellm.BadRequestError( message="A model name is required for the Decisions API", @@ -478,7 +478,7 @@ def prepare_call( except ValueError as error: raise litellm.BadRequestError(message=str(error), model=model, llm_provider=provider) from error try: - headers: Final = _outbound_headers(extra_headers, provider, kwargs) + headers: Final = decisions_outbound_headers(extra_headers, provider, kwargs) request_safety_identifier: Final = _request_safety_identifier(safety_identifier, provider_config, kwargs) request: Final = _fields_without_unsupported_media( fields, provider_config, canonical_model, provider, kwargs diff --git a/litellm/integrations/otel/routing.py b/litellm/integrations/otel/routing.py index 82dd6237edc..339775855be 100644 --- a/litellm/integrations/otel/routing.py +++ b/litellm/integrations/otel/routing.py @@ -21,6 +21,11 @@ _ROUTING_SCALAR_FIELDS: Final = frozenset( "classifier_cost", "classifier_failure_reason", "classifier_error_type", + "classifier_input_budget_tokens", + "classifier_input_tokens", + "classifier_input_count_method", + "classifier_input_truncated", + "classifier_input_budget_status", "classifier_confidence", "classifier_primary_rule", "classifier_capability_boundary", diff --git a/litellm/llms/azure_ai/decisions/transformation.py b/litellm/llms/azure_ai/decisions/transformation.py index fb5330ed715..22c4b59f5e8 100644 --- a/litellm/llms/azure_ai/decisions/transformation.py +++ b/litellm/llms/azure_ai/decisions/transformation.py @@ -12,7 +12,7 @@ from litellm.llms.azure_ai.common_utils import ( AZURE_ENTRA_LITELLM_PARAM_KEYS, get_azure_ai_auth_headers, ) -from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig, DecisionsInputLimits _FOUNDRY_ROUTE_SUFFIX: Final = re.compile(r"(/api/projects/[^/]+)?(/openai/v1|/openai|/models|/v1)?$") _AZURE_AI_CREDENTIAL_PARAM_KEYS: Final = AZURE_ENTRA_LITELLM_PARAM_KEYS | frozenset({"azure_ad_token_provider"}) @@ -26,6 +26,16 @@ class AzureAIDecisionsConfig(BaseDecisionsConfig): api_base_env = ("AZURE_AI_API_BASE",) api_key_required = False + def get_input_limits( + self, model: str, custom_llm_provider: str, headers: Mapping[str, str] + ) -> DecisionsInputLimits: + limits: Final = super().get_input_limits(model, custom_llm_provider, headers) + return ( + limits + if limits.max_request_tokens is not None + else super().get_input_limits("Microsoft-Decision-1", custom_llm_provider, headers) + ) + def missing_api_base_message(self, custom_llm_provider: str) -> str: return "Missing AZURE_AI_API_BASE - set AZURE_AI_API_BASE or pass api_base, e.g. https://.services.ai.azure.com" diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index e8e4b77e757..efd3e3fe9b5 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -1,7 +1,7 @@ import json from abc import ABC from collections.abc import Callable, Mapping, Sequence -from dataclasses import replace +from dataclasses import dataclass, replace from itertools import chain from types import MappingProxyType from typing import Final @@ -511,6 +511,12 @@ def _is_decisions_model(model: str, custom_llm_provider: str) -> bool: return is_decisions_model_mode(model_mode) +@dataclass(frozen=True, slots=True) +class DecisionsInputLimits: + max_request_tokens: int | None = None + max_question_tokens: int | None = None + + class BaseDecisionsConfig(ABC): path: str = "/v1/systemone" api_key_env: tuple[str, ...] = () @@ -533,6 +539,19 @@ class BaseDecisionsConfig(ABC): def provider_reported_cost(self, response: DecisionsIRResponse) -> float | None: return None + def get_input_limits( + self, model: str, custom_llm_provider: str, headers: Mapping[str, str] + ) -> DecisionsInputLimits: + try: + limit: Final = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider).get( + "max_input_tokens" + ) + except Exception: # noqa: BLE001 # get_model_info raises bare Exception for an unmapped model + return DecisionsInputLimits() + return DecisionsInputLimits( + max_request_tokens=limit if isinstance(limit, int) and not isinstance(limit, bool) and limit > 0 else None + ) + def missing_api_base_message(self, custom_llm_provider: str) -> str: return f"api_base is required for Decisions provider '{custom_llm_provider}'" diff --git a/litellm/llms/typesafe/decisions/transformation.py b/litellm/llms/typesafe/decisions/transformation.py index 952c874bc8e..871596b7f00 100644 --- a/litellm/llms/typesafe/decisions/transformation.py +++ b/litellm/llms/typesafe/decisions/transformation.py @@ -6,7 +6,7 @@ from types import MappingProxyType from typing import Final, Literal from litellm.constants import TYPESAFE_DANGER_ZONE_HEADER_PREFIX -from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig, image_media_type +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig, DecisionsInputLimits, image_media_type from litellm.types.decisions import ( DecisionsAsset, DecisionsIRMessages, @@ -19,6 +19,7 @@ from litellm.types.decisions import ( ) _IMAGES_HEADER: Final = f"{TYPESAFE_DANGER_ZONE_HEADER_PREFIX}images" +_LONG_CONTEXT_HEADER: Final = f"{TYPESAFE_DANGER_ZONE_HEADER_PREFIX}long-context" _ASSET_FORMATS: Final[Mapping[str, Literal["png", "jpg"]]] = MappingProxyType( {"image/png": "png", "image/jpeg": "jpg", "image/jpg": "jpg"} ) @@ -84,6 +85,16 @@ class TypeSafeDecisionsConfig(BaseDecisionsConfig): def get_default_api_base(self) -> str | None: return "https://api.typesafe.ai" + def get_input_limits( + self, model: str, custom_llm_provider: str, headers: Mapping[str, str] + ) -> DecisionsInputLimits: + long_context_values: Final = tuple( + value.strip().lower() for name, value in headers.items() if name.lower() == _LONG_CONTEXT_HEADER + ) + if long_context_values and all(value == "true" for value in long_context_values): + return DecisionsInputLimits(max_request_tokens=262144, max_question_tokens=262144) + return DecisionsInputLimits(max_request_tokens=65536, max_question_tokens=32768) + def supports_images_in( self, decision_input: DecisionsIRState | DecisionsIRMessages, headers: Mapping[str, str] ) -> bool: diff --git a/litellm/router_api.py b/litellm/router_api.py index d62999cf261..19f12cec40d 100644 --- a/litellm/router_api.py +++ b/litellm/router_api.py @@ -1216,6 +1216,9 @@ class RoutingContext: def registry(self) -> RouterRegistry: return self._router.registry() + def deployment_params(self, request: Mapping[str, object], deployment: "Deployment | None") -> Mapping[str, object]: + return self._router._params_sent_to_deployment(request, deployment) + def settings(self) -> RouterSettings: return self._router.settings() diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 30f1021251f..97ad5ffa05b 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -126,10 +126,13 @@ from .jev_classifier import ( DEFAULT_JEV_INSTRUCTIONS, HttpJevClassifierClient, JevClassifierClient, + JevSystemOneRequest, JevVerdict, RouterJevClassifierClient, build_jev_request, + fit_jev_request, jev_classifier_cost, + jev_input_targets, ) from .llm_v2 import LLM_V2_PROMPT_VERSION, LLMV2Decision, LLMV2TaskContext, LLMV2Verdict, llm_v2_response_format from .stall_detector import detect_stalled_task @@ -1166,6 +1169,7 @@ class ClassificationOutcome(NamedTuple): heuristic_v2_forecast: StandardLoggingHeuristicV2Forecast | None = None classifier_failure_reason: ClassifierFailureReason | None = None classifier_error_type: str | None = None + classifier_input_telemetry: StandardLoggingRoutingDecision | None = None def _with_classifier_failure( @@ -1207,23 +1211,24 @@ def _with_classifier_forecast( decision: StandardLoggingRoutingDecision, outcome: ClassificationOutcome ) -> StandardLoggingRoutingDecision: """Attach validated forecasts and their applied policy to the routing decision.""" + with_input: Final[StandardLoggingRoutingDecision] = {**decision, **(outcome.classifier_input_telemetry or {})} if outcome.heuristic_v2_forecast is not None: - return {**decision, "heuristic_v2_forecast": outcome.heuristic_v2_forecast} + return {**with_input, "heuristic_v2_forecast": outcome.heuristic_v2_forecast} if outcome.jev_verdict is not None: forecasted_decision: Final[StandardLoggingRoutingDecision] = { - **decision, + **with_input, "classifier_probabilities": outcome.jev_verdict.probabilities, "classifier_confidence": outcome.jev_verdict.confidence, } return forecasted_decision if outcome.llm_v2_forecast is not None: - return _with_llm_v2_forecast(decision, outcome.llm_v2_forecast) + return _with_llm_v2_forecast(with_input, outcome.llm_v2_forecast) forecast: Final = outcome.capability_forecast if forecast is None: - return decision + return with_input verdict: Final = forecast.verdict enriched: Final[StandardLoggingRoutingDecision] = { - **decision, + **with_input, "classifier_crux": verdict.crux, "classifier_primary_rule": verdict.primary_rule, "classifier_capability_boundary": verdict.capability_boundary, @@ -1974,19 +1979,21 @@ class ComplexityRouter(CustomLogger): `score` because the score is only interpretable against the boundaries that mapped it to a tier. """ - decision: Final = StandardLoggingRoutingDecision( - router_model_name=self.model_name, - router_type="complexity", - routed_model=routed_model, - cause=cause, - conversation_continuing=conversation_continuing, - **( - {"classifier_failure_reason": classifier_failure_reason} - if classifier_failure_reason is not None - else {} - ), - **({"classifier_error_type": classifier_error_type} if classifier_error_type is not None else {}), + failure: Final[StandardLoggingRoutingDecision] = ( + {"classifier_failure_reason": classifier_failure_reason} if classifier_failure_reason is not None else {} ) + error: Final[StandardLoggingRoutingDecision] = ( + {"classifier_error_type": classifier_error_type} if classifier_error_type is not None else {} + ) + decision: Final[StandardLoggingRoutingDecision] = { + "router_model_name": self.model_name, + "router_type": "complexity", + "routed_model": routed_model, + "cause": cause, + "conversation_continuing": conversation_continuing, + **failure, + **error, + } if (baseline := self.savings_baseline) is not None: decision["savings_baseline_model"] = baseline.model if baseline.deployment_id is not None: @@ -2371,16 +2378,6 @@ class ComplexityRouter(CustomLogger): system_prompt, failure_reason="unsupported_input", ) - breaker: Final = self._classifier_circuit_breaker - permit: Final = breaker.acquire_permit() if breaker is not None else None - if breaker is not None and permit is None: - return self._classifier_failure_outcome( - "jev classifier circuit is open", - prompt, - system_prompt, - failure_reason="circuit_open", - signal=_CLASSIFIER_CIRCUIT_OPEN_SIGNAL, - ) criteria: Final[Mapping[str, str]] = ( MappingProxyType( { @@ -2395,13 +2392,40 @@ class ComplexityRouter(CustomLogger): ) ) timeout_s: Final = config.timeout_ms / 1000 - request: Final = build_jev_request( - prompt=self._classifier_context_payload(prompt, system_prompt, request_kwargs, messages), - system_prompt=None, - model=config.model, - instructions=config.instructions or DEFAULT_JEV_INSTRUCTIONS, - criteria=criteria, + from litellm.litellm_core_utils.token_counter import offload_token_count + + def build_request(current_prompt: str, caller_constraints: str | None) -> JevSystemOneRequest: + return build_jev_request( + prompt=self._classifier_context_payload( + current_prompt, caller_constraints, request_kwargs, messages, current_ask_for_history=prompt + ), + system_prompt=None, + model=config.model, + instructions=config.instructions or DEFAULT_JEV_INSTRUCTIONS, + criteria=criteria, + ) + + budgeted: Final = await offload_token_count(fit_jev_request)( + build_request, prompt, system_prompt, jev_input_targets(self.litellm_router_instance, config), _truncate ) + request: Final = budgeted.request + if request is None: + return self._classifier_failure_outcome( + "classifier input cannot fit the provider budget", + prompt, + system_prompt, + failure_reason="unsupported_input", + )._replace(classifier_input_telemetry=budgeted.telemetry) + breaker: Final = self._classifier_circuit_breaker + permit: Final = breaker.acquire_permit() if breaker is not None else None + if breaker is not None and permit is None: + return self._classifier_failure_outcome( + "jev classifier circuit is open", + prompt, + system_prompt, + failure_reason="circuit_open", + signal=_CLASSIFIER_CIRCUIT_OPEN_SIGNAL, + )._replace(classifier_input_telemetry=budgeted.telemetry) try: response: Final = await asyncio.wait_for(client.evaluate(request, timeout_s, request_kwargs), timeout_s) answer: Final = response.answers.get("tier") @@ -2449,6 +2473,7 @@ class ComplexityRouter(CustomLogger): cause="jev_classifier", classifier_cost=verdict.cost, jev_verdict=verdict, + classifier_input_telemetry=budgeted.telemetry, ) except asyncio.CancelledError: if breaker is not None and permit is not None: @@ -2463,7 +2488,7 @@ class ComplexityRouter(CustomLogger): system_prompt, failure_reason=_classifier_failure_reason(e), error_type=type(e).__name__, - ) + )._replace(classifier_input_telemetry=budgeted.telemetry) def _classifier_failure_outcome( self, @@ -2654,6 +2679,7 @@ class ComplexityRouter(CustomLogger): messages: Sequence[Mapping[str, object]] | None, *, encrypted_task: bool = False, + current_ask_for_history: str | None = None, ) -> str: include_assistant: Final = self.config.classifier_context_include_assistant_turns marker_pairs: Final = self._reminder_markers_for_request(request_kwargs or EMPTY_MAPPING) @@ -2661,7 +2687,7 @@ class ComplexityRouter(CustomLogger): prior_turns: Final = ( _extract_prior_turns( messages, - current_ask=prompt, + current_ask=prompt if current_ask_for_history is None else current_ask_for_history, window_size=self.config.classifier_context_window_size, budget_chars=self.config.classifier_context_budget_chars, per_turn_chars=self.config.classifier_context_per_turn_chars, @@ -4983,16 +5009,19 @@ class ComplexityRouter(CustomLogger): return PreRoutingHookResponse( model=fallback_model, messages=messages if has_original_messages else None, - routing_decision=self._build_routing_decision( - routed_model=fallback_model, - conversation_continuing=conversation_continuing, - cause=outcome.cause, - signals=outcome.signals, - classifier_failure_reason=outcome.classifier_failure_reason, - classifier_error_type=outcome.classifier_error_type, - escalation_keyword=escalation_keyword, - escalated=False, - tier_litellm_params=fallback_tier_params, + routing_decision=_with_classifier_forecast( + self._build_routing_decision( + routed_model=fallback_model, + conversation_continuing=conversation_continuing, + cause=outcome.cause, + signals=outcome.signals, + classifier_failure_reason=outcome.classifier_failure_reason, + classifier_error_type=outcome.classifier_error_type, + escalation_keyword=escalation_keyword, + escalated=False, + tier_litellm_params=fallback_tier_params, + ), + outcome, ), litellm_params=fallback_tier_params, ) diff --git a/litellm/router_strategy/complexity_router/jev_classifier.py b/litellm/router_strategy/complexity_router/jev_classifier.py index 87bd031490a..25ea813229c 100644 --- a/litellm/router_strategy/complexity_router/jev_classifier.py +++ b/litellm/router_strategy/complexity_router/jev_classifier.py @@ -1,4 +1,5 @@ -from collections.abc import Mapping +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass from datetime import datetime, timezone from types import MappingProxyType from typing import TYPE_CHECKING, Annotated, Final, Literal, NamedTuple, Protocol, TypeAlias @@ -25,10 +26,13 @@ from litellm.llms.pass_through.typesafe_logging_handler import TypeSafePassthrou from litellm.router_strategy.complexity_router.config import DEFAULT_JEV_INSTRUCTIONS as _DEFAULT_JEV_INSTRUCTIONS from litellm.types.decisions import DecisionsResponse from litellm.types.llms.base import LiteLLMBaseModel -from litellm.types.utils import AUTOROUTER_CLASSIFIER_CALL_ORIGIN +from litellm.types.utils import AUTOROUTER_CLASSIFIER_CALL_ORIGIN, StandardLoggingRoutingDecision if TYPE_CHECKING: from litellm.router import Router + from litellm.router_api import RoutingContext + from litellm.router_strategy.complexity_router.config import OpenSourceClassifierConfig + from litellm.types.router import Deployment JevProbability: TypeAlias = Annotated[float, Field(ge=0.0, le=1.0)] ClassifierProvider: TypeAlias = Literal["typesafe", "laya", "bespoke", "databricks"] @@ -52,6 +56,155 @@ class JevSystemOneRequest(LiteLLMBaseModel): questions: Mapping[str, JevChoiceQuestion] +@dataclass(frozen=True, slots=True) +class JevInputTarget: + model: str + max_tokens: int | None + + +@dataclass(frozen=True, slots=True) +class JevBudgetedRequest: + request: JevSystemOneRequest | None + telemetry: StandardLoggingRoutingDecision + + +def jev_input_targets(router: "Router", config: "OpenSourceClassifierConfig") -> tuple[JevInputTarget, ...]: + from litellm.router_api import RoutingContext + from litellm.types.router import Deployment + + if config.deployment_name is None: + provider: Final = "typesafe" if config.provider == "jev" else config.provider + return (_jev_input_target({"model": f"{provider}/{config.model}"}, None),) + context: Final = RoutingContext(router) + deployments: Final = context.registry().deployments(model_name=config.deployment_name) or () + return tuple( + _jev_deployment_input_target(context, Deployment.model_validate(deployment)) for deployment in deployments + ) + + +def _jev_deployment_input_target(context: "RoutingContext", deployment: "Deployment") -> JevInputTarget: + params: Final = context.deployment_params({}, deployment) + explicit_limit: Final = deployment.model_info.model_dump().get("max_input_tokens") + return _jev_input_target(params, explicit_limit) + + +def _jev_input_target(params: Mapping[str, object], explicit_limit: object) -> JevInputTarget: + from litellm.decisions.call import decisions_outbound_headers, get_decisions_provider_config + from litellm.litellm_core_utils.get_llm_provider_logic import resolve_model_and_provider_for_metadata + + qualified_model: Final = str(params.get("model") or "") + override: Final = params.get("custom_llm_provider") + try: + model, provider = resolve_model_and_provider_for_metadata( + qualified_model, override if isinstance(override, str) else None + ) + headers: Final = decisions_outbound_headers( + TypeAdapter[Mapping[str, str] | None](Mapping[str, str] | None).validate_python( + params.get("extra_headers") + ), + provider, + params, + ) + limits: Final = get_decisions_provider_config(model, provider).get_input_limits(model, provider, headers) + candidates: Final = tuple( + value + for value in (limits.max_request_tokens, limits.max_question_tokens, explicit_limit) + if isinstance(value, int) and not isinstance(value, bool) and value > 0 + ) + return JevInputTarget(f"{provider}/{model}", min(candidates) if candidates else None) + except Exception: # noqa: BLE001 # unavailable metadata must not turn an existing classifier into a request failure + return JevInputTarget(qualified_model, None) + + +def _jev_request_count(request: JevSystemOneRequest, models: Sequence[str], budget: int) -> tuple[int, str]: + from litellm._lazy_imports import get_token_counter_new + + payload: Final = request.model_dump_json(exclude={"model"}) + byte_count: Final = len(payload.encode("utf-8")) + if byte_count <= budget or litellm.disable_token_counter: + return byte_count, "utf8_bytes" + try: + counts: Final = tuple(get_token_counter_new()(model=model, text=payload) for model in models) + if counts and all(count > 0 for count in counts): + return max(counts), "estimated_tokens" + except Exception: # noqa: BLE001 # tokenization is best-effort; byte counting remains available + return byte_count, "utf8_bytes" + return byte_count, "utf8_bytes" + + +def fit_jev_request( + build_request: Callable[[str, str | None], JevSystemOneRequest], + prompt: str, + system_prompt: str | None, + targets: Sequence[JevInputTarget], + truncate: Callable[[str, int], str], +) -> JevBudgetedRequest: + known: Final = tuple(target for target in targets if target.max_tokens is not None) + if not known: + return JevBudgetedRequest( + build_request(prompt, system_prompt), + {"classifier_input_budget_status": "unknown_limits", "classifier_input_truncated": False}, + ) + budget: Final = max(1, int(min(target.max_tokens for target in known if target.max_tokens is not None) * 0.75)) + models: Final = tuple(dict.fromkeys(target.model for target in known)) + initial_cap: Final = max(120, budget * 8) + bounded_prompt: Final = truncate(prompt, initial_cap) + bounded_system: Final = truncate(system_prompt, initial_cap) if system_prompt is not None else None + return _fit_jev_components( + build_request, + bounded_prompt, + bounded_system, + models, + budget, + truncate, + len(known) != len(targets), + bounded_prompt != prompt or bounded_system != system_prompt, + ) + + +def _fit_jev_components( + build_request: Callable[[str, str | None], JevSystemOneRequest], + prompt: str, + system_prompt: str | None, + models: Sequence[str], + budget: int, + truncate: Callable[[str, int], str], + unknown_limits: bool, + truncated: bool, +) -> JevBudgetedRequest: + attempts: Final = len(prompt).bit_length() + len(system_prompt or "").bit_length() + 1 + components: tuple[str, str | None, bool, int] = ( # rebind-ok: bounded iterative fitting state + prompt, + system_prompt, + truncated, + attempts, + ) + while True: + current_prompt, current_system, was_truncated, remaining = components + request = build_request(current_prompt, current_system) + count, method = _jev_request_count(request, models, budget) + telemetry: StandardLoggingRoutingDecision = { + "classifier_input_budget_tokens": budget, + "classifier_input_tokens": count, + "classifier_input_count_method": method, + "classifier_input_truncated": was_truncated, + "classifier_input_budget_status": "unknown_limits" + if unknown_limits + else "truncated" + if was_truncated + else "within_budget", + } + if count <= budget: + return JevBudgetedRequest(request, telemetry) + if remaining == 0 or (len(current_system or "") <= 123 and len(current_prompt) <= 123): + return JevBudgetedRequest(None, {**telemetry, "classifier_input_budget_status": "untrimmable_input"}) + components = ( + (current_prompt, truncate(current_system, max(120, len(current_system) // 2)), True, remaining - 1) + if current_system is not None and len(current_system) > 123 + else (truncate(current_prompt, max(120, len(current_prompt) // 2)), current_system, True, remaining - 1) + ) + + class JevChoiceAnswer(LiteLLMBaseModel): model_config = ConfigDict(frozen=True, allow_inf_nan=False) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index db865c7488a..1a8f5adf154 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3323,6 +3323,11 @@ class StandardLoggingRoutingDecision(TypedDict, total=False): classifier_cost: float classifier_failure_reason: ReadOnly[ClassifierFailureReason] classifier_error_type: ReadOnly[str] + classifier_input_budget_tokens: ReadOnly[int] + classifier_input_tokens: ReadOnly[int] + classifier_input_count_method: ReadOnly[str] + classifier_input_truncated: ReadOnly[bool] + classifier_input_budget_status: ReadOnly[str] classifier_probabilities: ReadOnly[Mapping[str, float]] classifier_confidence: ReadOnly[float] heuristic_v2_forecast: ReadOnly[StandardLoggingHeuristicV2Forecast] @@ -3381,6 +3386,11 @@ DERIVED_ROUTING_DECISION_FIELDS: Final[frozenset[str]] = frozenset( "classifier_cost", "classifier_failure_reason", "classifier_error_type", + "classifier_input_budget_tokens", + "classifier_input_tokens", + "classifier_input_count_method", + "classifier_input_truncated", + "classifier_input_budget_status", "classifier_probabilities", "classifier_confidence", "heuristic_v2_forecast", diff --git a/tests/unit/integrations/otel/test_otel_v2_logger.py b/tests/unit/integrations/otel/test_otel_v2_logger.py index 1f9d3af71d9..ac1f5557bf6 100644 --- a/tests/unit/integrations/otel/test_otel_v2_logger.py +++ b/tests/unit/integrations/otel/test_otel_v2_logger.py @@ -207,6 +207,11 @@ def test_routing_diagnostics_reach_llm_spans_without_promoting_prompt_text(fail: "classifier_cost": 0.001, "classifier_failure_reason": "timeout", "classifier_error_type": "TimeoutError", + "classifier_input_budget_tokens": 24576, + "classifier_input_tokens": 23000, + "classifier_input_count_method": "estimated_tokens", + "classifier_input_truncated": True, + "classifier_input_budget_status": "truncated", "escalated": False, } payload: Final = _payload( diff --git a/tests/unit/integrations/test_opentelemetry.py b/tests/unit/integrations/test_opentelemetry.py index f0fca94c370..5f749e951b1 100644 --- a/tests/unit/integrations/test_opentelemetry.py +++ b/tests/unit/integrations/test_opentelemetry.py @@ -344,6 +344,11 @@ def test_v1_routing_diagnostics_add_scalar_attributes_and_preserve_legacy_metada "cause": "heuristic", "classifier_failure_reason": "timeout", "classifier_error_type": "TimeoutError", + "classifier_input_budget_tokens": 24576, + "classifier_input_tokens": 23000, + "classifier_input_count_method": "estimated_tokens", + "classifier_input_truncated": True, + "classifier_input_budget_status": "truncated", "classifier_cost": 0.001, } with provider.get_tracer(__name__).start_as_current_span("inference") as span: diff --git a/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py b/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py index 03459ba16c8..ec6f118837f 100644 --- a/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py +++ b/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py @@ -6,14 +6,19 @@ import httpx import pytest from pydantic import TypeAdapter, ValidationError +import litellm +from litellm.llms.azure_ai.decisions.transformation import AzureAIDecisionsConfig from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.decisions.transformation import ( + BaseDecisionsConfig, + DecisionsInputLimits, ir_to_systemone_request, ir_to_systemone_response, parse_systemone_response, systemone_request_to_ir, ) from litellm.llms.openai.decisions.transformation import openai_request_to_ir +from litellm.llms.typesafe.decisions.transformation import TypeSafeDecisionsConfig from litellm.types.decisions import ( MAX_DECISION_QUESTIONS, DecisionsIRRefusal, @@ -49,6 +54,61 @@ _SYSTEMONE_REQUEST: Final[Mapping[str, object]] = { } +@pytest.mark.parametrize("declared", (4096, 0, -1, True, None)) +def test_decision_input_limits_use_only_positive_catalog_limits( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, declared: int | None +) -> None: + monkeypatch.setitem( + litellm.model_cost, + "bespoke/test-input-limit", + {"litellm_provider": "bespoke", "mode": "evaluation", "max_input_tokens": declared}, + ) + + limits: Final = BaseDecisionsConfig().get_input_limits("test-input-limit", "bespoke", {}) + + assert limits == DecisionsInputLimits(max_request_tokens=4096 if declared == 4096 else None) + + +def test_unmapped_decision_model_limits_remain_unknown(local_model_cost_map: None) -> None: + assert BaseDecisionsConfig().get_input_limits("unmapped-input-limit", "databricks", {}) == DecisionsInputLimits() + + +@pytest.mark.parametrize( + ("headers", "long_context"), + ( + ({}, False), + ({"TypeSafe-Danger-Zone-Long-Context": "false"}, False), + ({"typesafe-danger-zone-long-context": " TRUE "}, True), + ({"TypeSafe-Danger-Zone-Long-Context": "true", "typesafe-danger-zone-long-context": "true"}, True), + ({"TypeSafe-Danger-Zone-Long-Context": "true", "typesafe-danger-zone-long-context": "false"}, False), + ({"typesafe-danger-zone-long-context": "true,false"}, False), + ), +) +def test_typesafe_limits_require_unambiguous_long_context_opt_in( + headers: Mapping[str, str], long_context: bool +) -> None: + """Provider contract from https://docs.typesafe.ai/models, verified 2026-10-30""" + limits: Final = TypeSafeDecisionsConfig().get_input_limits("jev-latest", "typesafe", headers) + + assert limits == ( + DecisionsInputLimits(max_request_tokens=262144, max_question_tokens=262144) + if long_context + else DecisionsInputLimits(max_request_tokens=65536, max_question_tokens=32768) + ) + + +@pytest.mark.parametrize("model", ("Microsoft-Decision-1", "unmapped-decision-deployment")) +def test_azure_decision_deployments_use_catalog_limits_without_typesafe_modes( + local_model_cost_map: None, model: str +) -> None: + catalog_limit: Final = litellm.get_model_info("Microsoft-Decision-1", "azure_ai")["max_input_tokens"] + limits: Final = AzureAIDecisionsConfig().get_input_limits( + model, "azure_ai", {"TypeSafe-Danger-Zone-Long-Context": "true"} + ) + + assert limits == DecisionsInputLimits(max_request_tokens=catalog_limit) + + def _openai_ir(raw: Mapping[str, object]) -> DecisionsIRRequest: return openai_request_to_ir(_OPENAI_BODY.validate_python(raw)) diff --git a/tests/unit/router_strategy/complexity_router/test_jev_classifier.py b/tests/unit/router_strategy/complexity_router/test_jev_classifier.py index 77d465526a4..a71f572254e 100644 --- a/tests/unit/router_strategy/complexity_router/test_jev_classifier.py +++ b/tests/unit/router_strategy/complexity_router/test_jev_classifier.py @@ -9,6 +9,7 @@ from unittest.mock import create_autospec import httpx import pytest import respx +import tiktoken import litellm from litellm._logging import verbose_router_logger @@ -16,12 +17,14 @@ from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler -from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter +from litellm.llms.typesafe.decisions.transformation import TypeSafeDecisionsConfig +from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter, _ClassifierCircuitBreaker from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig, JevClassifierConfig from litellm.router_strategy.complexity_router.jev_classifier import ( DEFAULT_JEV_INSTRUCTIONS, HttpJevClassifierClient, JevChoiceAnswer, + JevSystemOneRequest, JevSystemOneResponse, JevUsage, build_jev_request, @@ -1055,3 +1058,214 @@ def test_switching_to_deployed_classifier_does_not_copy_legacy_credentials() -> ) assert write.effective is not None assert write.effective["opensource_classifier_config"] == {"deployment_name": "registered"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("large_field", ["user", "system"]) +@pytest.mark.parametrize("disable_counter", [False, True]) +async def test_jev_bounds_complete_input_without_changing_forwarded_messages( + large_field: str, disable_counter: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_token_counter", disable_counter) + limit: Final = TypeSafeDecisionsConfig().get_input_limits("jev-latest", "typesafe", {}).max_question_tokens + assert limit is not None + rubric: Final = "Operator routing rubric " + "document the task category " * 500 + criterion: Final = ("Simple tasks " + "the classification criteria " * 15).rstrip() + large_text: Final = "BEGIN_SOURCE " + "budget 预算 function(x) return x; " * 18000 + " END_SOURCE" + ask: Final = large_text if large_field == "user" else "CURRENT_ASK Explain the example" + constraints: Final = large_text if large_field == "system" else "Use concise language" + messages: Final = [ + {"role": "system", "content": constraints}, + {"role": "user", "content": "PRIOR_TURN Explain the budget"}, + {"role": "user", "content": ask}, + ] + original: Final = deepcopy(messages) + handler: Final = AsyncHTTPHandler() + handler.client = httpx.AsyncClient() + classifier: Final = ComplexityRouter( + "bounded-classifier", litellm.Router(model_list=[]), + { + "classifier_type": "oss_classifier", + "opensource_classifier_config": {"instructions": rubric}, + "tier_definitions": [{"name": "SIMPLE", "description": criterion}, {"name": "REASONING"}], + "tiers": {"SIMPLE": "cheap", "REASONING": "deep"}, + "fallback_tier": "SIMPLE", + "session_affinity": False, + "deployment_affinity": False, + }, + jev_client=HttpJevClassifierClient("test", "https://budget.test", handler), + derive_savings_baseline=False, + ) + try: + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.post("https://budget.test/v1/systemone").respond( + json={"answers": {"tier": _answer().model_dump()}} + ) + result: Final = await classifier.async_pre_routing_hook( + model="bounded-classifier", request_kwargs={"messages": messages}, messages=messages + ) + sent: Final = JevSystemOneRequest.model_validate_json(route.calls[0].request.content) + complete_input: Final = sent.model_dump_json(exclude={"model"}) + measured: Final = len(tiktoken.get_encoding("o200k_base").encode(complete_input)) + assert measured < limit + assert sent.questions["tier"].instructions == rubric + assert sent.questions["tier"].criteria["SIMPLE"] == criterion + assert sent.state.count("BEGIN_SOURCE") == sent.state.count("END_SOURCE") == 1 + assert sent.state.count("PRIOR_TURN") == 1 + assert result is not None and result.routing_decision is not None + decision: Final = result.routing_decision + assert decision["cause"] == "jev_classifier" + assert decision["classifier_input_budget_status"] == "truncated" + assert decision["classifier_input_truncated"] is True + assert measured <= decision["classifier_input_tokens"] <= decision["classifier_input_budget_tokens"] < limit + assert messages == original + assert result.messages is None or result.messages == original + if disable_counter: + assert len(complete_input.encode()) < limit + assert decision["classifier_input_count_method"] == "utf8_bytes" + finally: + await GLOBAL_LOGGING_WORKER.flush() + await handler.client.aclose() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("mode", "default_header", "explicit_cap", "other_member", "expected_status"), + [ + ("false", None, None, None, "truncated"), + ("true", None, None, None, "within_budget"), + ("true", "false", None, None, "truncated"), + ("true", None, 8192, None, "truncated"), + ("true", None, 8192, "default", "truncated"), + ("true", None, None, "default", "truncated"), + ("false", None, None, "unknown", "unknown_limits"), + ], +) +async def test_jev_budget_uses_effective_headers_and_every_deployment( + mode: str, default_header: str | None, explicit_cap: int | None, + other_member: str | None, expected_status: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setitem(litellm.model_cost, "typesafe/jev-latest", deepcopy(litellm.model_cost["typesafe/jev-latest"])) + header: Final = "TypeSafe-Danger-Zone-Long-Context" + deployment: Final = { + "model_name": "budget-model", + "litellm_params": { + "model": "typesafe/jev-latest", "api_key": "test", "api_base": "https://budget.test", + "extra_headers": {header: mode}, + }, + "model_info": {"max_input_tokens": explicit_cap} if explicit_cap is not None else {}, + } + sibling: Final = { + "model_name": "budget-model", + "litellm_params": { + "model": "databricks/unknown-budget-endpoint" if other_member == "unknown" else "typesafe/jev-latest", + "api_key": "test", "api_base": "https://budget.test", + }, + } + router: Final = litellm.Router( + model_list=[deployment, sibling] if other_member else [deployment], + default_litellm_params={"extra_headers": {header: default_header}} if default_header else {}, + num_retries=0, + ) + classifier: Final = ComplexityRouter( + "bounded-classifier", router, + {"classifier_type": "oss_classifier", "opensource_classifier_config": {"deployment_name": "budget-model"}, + "tiers": {"SIMPLE": "cheap"}}, + derive_savings_baseline=False, + ) + ask: Final = "Explain this code " + "word001 word002 word003 " * 12000 + with respx.mock(assert_all_called=True) as upstream: + route: Final = upstream.route(method="POST").respond( + json={"answers": {"tier": _answer().model_dump()}} + ) + outcome: Final = await classifier.aclassify(ask) + assert outcome.cause == "jev_classifier" + telemetry: Final = outcome.classifier_input_telemetry + assert telemetry is not None and telemetry["classifier_input_budget_status"] == expected_status + assert telemetry["classifier_input_tokens"] <= telemetry["classifier_input_budget_tokens"] + sent: Final = JevSystemOneRequest.model_validate_json(route.calls[0].request.content) + if other_member is None: + assert route.calls[0].request.headers[header] == (default_header or mode) + if explicit_cap is not None: + assert telemetry["classifier_input_budget_tokens"] < explicit_cap + assert (ask in sent.state) is (expected_status == "within_budget") + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("oversized_field", "fallback"), + [("instructions", "heuristic"), ("instructions", "default_model"), ("criteria", "custom")], +) +async def test_jev_untrimmable_rubric_uses_configured_fallback_without_a_provider_call( + oversized_field: str, fallback: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setitem(litellm.model_cost, "typesafe/jev-latest", deepcopy(litellm.model_cost["typesafe/jev-latest"])) + router: Final = litellm.Router(model_list=[{ + "model_name": "budget-model", + "litellm_params": {"model": "typesafe/jev-latest", "api_key": "test", "api_base": "https://budget.test"}, + "model_info": {"max_input_tokens": 128 if oversized_field == "criteria" else 1024}, + }]) + rubric: Final = "word001 word002 word003 " * 2000 + classifier: Final = ComplexityRouter( + "bounded-classifier", router, + {"classifier_type": "oss_classifier", "opensource_classifier_config": { + "deployment_name": "budget-model", "instructions": rubric if oversized_field == "instructions" else "Choose", + }, "tiers": {"SIMPLE": "cheap", "REASONING": "deep"}, + **({"tier_definitions": [{"name": name, "description": "word001 " * 50} + for name in ("SIMPLE", "REASONING")], "fallback_tier": "SIMPLE"} + if fallback == "custom" else {"classifier_fallback": fallback, "default_model": "default"})}, + derive_savings_baseline=False, + ) + with respx.mock(assert_all_called=False) as upstream: + route: Final = upstream.route(method="POST").respond(json={"answers": {"tier": _answer().model_dump()}}) + outcome: Final = await classifier.aclassify("What is two plus two?") + assert route.call_count == 0 + assert outcome.cause == { + "heuristic": "heuristic_scorer", "default_model": "default_model_fallback", "custom": "classifier_fallback", + }[fallback] + assert outcome.classifier_failure_reason == "unsupported_input" + assert outcome.classifier_input_telemetry is not None + assert outcome.classifier_input_telemetry["classifier_input_budget_status"] == "untrimmable_input" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("fallback", ["heuristic", "default_model"]) +async def test_jev_open_breaker_preserves_input_budget_in_fallback_decision(fallback: str) -> None: + def clock() -> float: + return 100.0 + + transport: Final = create_autospec(httpx.AsyncBaseTransport, instance=True) + transport.handle_async_request.side_effect = TimeoutError("controlled classifier timeout") + handler: Final = AsyncHTTPHandler() + handler.client = httpx.AsyncClient(transport=transport) + classifier: Final = ComplexityRouter( + "breaker-budget", litellm.Router(model_list=[]), + {"classifier_type": "oss_classifier", "opensource_classifier_config": {}, + "tiers": {"SIMPLE": "cheap", "REASONING": "deep"}, "classifier_fallback": fallback, + "default_model": "default", "session_affinity": False, "deployment_affinity": False}, + jev_client=HttpJevClassifierClient("test", "https://budget.test", handler), + derive_savings_baseline=False, + ) + classifier._classifier_circuit_breaker = _ClassifierCircuitBreaker(30.0, clock=clock) + try: + first: Final = await classifier.async_pre_routing_hook( + model="breaker-budget", request_kwargs={}, messages=[{"role": "user", "content": "Explain budgets"}] + ) + second: Final = await classifier.async_pre_routing_hook( + model="breaker-budget", request_kwargs={}, + messages=[{"role": "user", "content": "Explain this code " + "word001 word002 word003 " * 12000}], + ) + assert first is not None and first.routing_decision is not None + assert first.routing_decision["classifier_failure_reason"] == "timeout" + assert second is not None and second.routing_decision is not None + decision: Final = second.routing_decision + assert decision["classifier_failure_reason"] == "circuit_open" + assert decision["cause"] == ("heuristic_scorer" if fallback == "heuristic" else "default_model_fallback") + assert decision.get("classifier_input_budget_status") == "truncated" + assert decision["classifier_input_truncated"] is True + assert 0 < decision["classifier_input_tokens"] <= decision["classifier_input_budget_tokens"] + transport.handle_async_request.assert_awaited_once() + finally: + await handler.client.aclose() diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 96c7c34621b..24736cdb34e 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -46205,6 +46205,16 @@ export interface components { * @enum {string} */ classifier_failure_reason?: "timeout" | "circuit_open" | "not_configured" | "unsupported_input" | "invalid_response" | "declined" | "classifier_error"; + /** Classifier Input Budget Status */ + classifier_input_budget_status?: string; + /** Classifier Input Budget Tokens */ + classifier_input_budget_tokens?: number; + /** Classifier Input Count Method */ + classifier_input_count_method?: string; + /** Classifier Input Tokens */ + classifier_input_tokens?: number; + /** Classifier Input Truncated */ + classifier_input_truncated?: boolean; /** Classifier Max Quality Gap */ classifier_max_quality_gap?: number; /** Classifier Model */ From e8f572b39bff31e8b59cc01e00d5adbb8e278405 Mon Sep 17 00:00:00 2001 From: jesus-berri Date: Fri, 30 Oct 2026 14:39:25 -0700 Subject: [PATCH 44/74] feat(sso): create the team_id_jwt_field team on UI SSO login when team_id_upsert is set (#49256) * feat(sso): create the team_id_jwt_field team on UI SSO login when team_id_upsert is set * fix(sso): keep the upserted team_id_jwt_field team out of result.team_ids so restricted_sso_group and the access-token group fallback are unchanged * fix(sso): only upsert the team_id_jwt_field team on UI SSO login when enable_jwt_auth is on The API JWT path only reaches team_id_upsert inside the enable_jwt_auth branch, so a deployment with a leftover litellm_jwtauth block and JWT auth off never created teams. Gate the UI SSO upsert the same way * fix(sso): read enable_jwt_auth through the proxy_server module so the basedpyright gate stays at its ceiling --- litellm/proxy/management_endpoints/types.py | 1 + litellm/proxy/management_endpoints/ui_sso.py | 67 +++- .../management/test_sso_sign_in.py | 110 ++++++- .../proxy/management_endpoints/test_ui_sso.py | 309 ++++++++++++++++-- 4 files changed, 448 insertions(+), 39 deletions(-) diff --git a/litellm/proxy/management_endpoints/types.py b/litellm/proxy/management_endpoints/types.py index 221a6221fab..d4fd7a31d13 100644 --- a/litellm/proxy/management_endpoints/types.py +++ b/litellm/proxy/management_endpoints/types.py @@ -93,5 +93,6 @@ def get_litellm_user_role(role_str: object) -> LitellmUserRoles | None: class CustomOpenID(OpenID): team_ids: list[str] + team_id_to_upsert: str | None = None user_role: LitellmUserRoles | None = None extra_fields: dict[str, object] | None = None diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 59c2df91d77..53079d84fdd 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -87,7 +87,7 @@ from litellm.proxy._types import ( TeamMemberAddRequest, UserAPIKeyAuth, ) -from litellm.proxy.auth.auth_checks import ExperimentalUIJWTToken, get_user_object +from litellm.proxy.auth.auth_checks import ExperimentalUIJWTToken, get_team_object, get_user_object from litellm.proxy.auth.auth_utils import ( # noqa: F401 # legacy module exports _get_request_ip_address, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export get_request_ip_address, @@ -1266,6 +1266,7 @@ def generic_response_convertor( sso_jwt_handler: JWTHandler | None = None, role_mappings: Optional["RoleMappings"] = None, team_mappings: Optional["TeamMappings"] = None, + jwt_auth_enabled: bool = False, ) -> CustomOpenID: generic_user_id_attribute_name: Final = os.getenv("GENERIC_USER_ID_ATTRIBUTE", "preferred_username") generic_user_display_name_attribute_name: Final = os.getenv("GENERIC_USER_DISPLAY_NAME_ATTRIBUTE", "sub") @@ -1286,14 +1287,15 @@ def generic_response_convertor( generic_user_email_attribute_name, ) - all_teams: Final = [] + token: Final = cast(dict[str, object], response) # cast-ok: the IdP userinfo JSON object fastapi-sso hands back + all_teams: Final = list[str]() if sso_jwt_handler is not None: - team_ids = sso_jwt_handler.get_all_jwt_team_ids(cast(dict, response)) + team_ids = sso_jwt_handler.get_all_jwt_team_ids(token) all_teams.extend(team_ids) if team_mappings is not None and team_mappings.team_ids_jwt_field is not None: team_ids_from_db_mapping: Final[list[str] | None] = get_nested_value( - data=cast(dict, response), + data=token, key_path=team_mappings.team_ids_jwt_field, default=[], ) @@ -1305,8 +1307,17 @@ def generic_response_convertor( team_ids_from_db_mapping, ) else: - team_ids = jwt_handler.get_all_jwt_team_ids(cast(dict, response)) + team_ids = jwt_handler.get_all_jwt_team_ids(token) all_teams.extend(team_ids) + team_id_jwt_field: Final = jwt_handler.litellm_jwtauth.team_id_jwt_field + upsert_candidate: Final[object] = ( + get_nested_value(data=token, key_path=team_id_jwt_field) + if jwt_auth_enabled and jwt_handler.litellm_jwtauth.team_id_upsert and team_id_jwt_field is not None + else None + ) + team_id_to_upsert: Final = ( + upsert_candidate if isinstance(upsert_candidate, str) and upsert_candidate.strip() else None + ) # Determine user role based on role_mappings if available # Only apply role_mappings for GENERIC SSO provider @@ -1375,6 +1386,7 @@ def generic_response_convertor( last_name=get_nested_value(response, generic_user_last_name_attribute_name), provider=get_nested_value(response, generic_provider_attribute_name), team_ids=all_teams, + team_id_to_upsert=team_id_to_upsert, user_role=user_role, extra_fields=extra_fields, ) @@ -1614,6 +1626,8 @@ async def get_generic_sso_response( # make generic sso provider from fastapi_sso.sso.base import DiscoveryDocument + from litellm.proxy import proxy_server + received_response: dict | None = None sso_assertion: SSOIdentityAssertion | None = None @@ -1665,6 +1679,7 @@ async def get_generic_sso_response( sso_jwt_handler=sso_jwt_handler, role_mappings=role_mappings, team_mappings=team_mappings, + jwt_auth_enabled=proxy_server.general_settings.get("enable_jwt_auth", False) is True, ) SSOProvider: Final = create_generic_sso_provider( @@ -1884,6 +1899,29 @@ async def find_existing_team_ids(team_ids: Sequence[str]) -> frozenset[str] | No return frozenset(row.team_id for row in rows) +async def create_sso_team(team_id: str) -> None: + """Create the ``team_id_jwt_field`` team on UI SSO login the way the API JWT path does under ``team_id_upsert``.""" + from litellm.proxy.proxy_server import prisma_client, user_api_key_cache + + if prisma_client is None: + return + await get_team_object( + team_id=team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + check_db_only=True, + team_id_upsert=True, + ) + + +async def _create_sso_team_or_warn(team_id: str, create_team: Callable[[str], Awaitable[object]]) -> None: + """A concurrent login may create the team first, so a failed create is not fatal: the join still runs.""" + try: + await create_team(team_id) + except Exception as e: # noqa: BLE001 # fail-safe: a failed team create must not fail the login or skip the join + verbose_proxy_logger.warning("SSO login could not create team %s, trying to join it anyway: %s", team_id, e) + + async def add_missing_team_member( user_info: NewUserResponse | LiteLLM_UserTable, sso_teams: Sequence[str], @@ -1892,21 +1930,30 @@ async def add_missing_team_member( [str, NewUserResponse | LiteLLM_UserTable], Awaitable[object] ] = create_team_member_add_task, existing_team_ids: Callable[[Sequence[str]], Awaitable[frozenset[str] | None]] = find_existing_team_ids, + team_id_to_upsert: str | None = None, + create_team: Callable[[str], Awaitable[object]] = create_sso_team, ) -> None: """ - Get missing teams (diff b/w user_info.team_ids and sso_teams) - Drop the IdP groups that are not teams with one lookup, since group claims often carry many of them + - Join the ``team_id_to_upsert`` team too, creating it when it is missing (``team_id_upsert``), the way the API JWT path does; + it stays out of ``result.team_ids`` so restricted_sso_group and the access-token group fallback see only the claimed groups - Add missing user to missing teams, one at a time in sso_teams order, so user.teams keeps the claim's order """ held_teams: Final = frozenset(user_info.teams or ()) - missing_teams: Final = tuple(team_id for team_id in dict.fromkeys(sso_teams) if team_id not in held_teams) + claimed_teams: Final = sso_teams if team_id_to_upsert is None else (*sso_teams, team_id_to_upsert) + missing_teams: Final = tuple(team_id for team_id in dict.fromkeys(claimed_teams) if team_id not in held_teams) if not missing_teams: return existing: Final = await existing_team_ids(missing_teams) teams_to_add: Final = ( - missing_teams if existing is None else tuple(team_id for team_id in missing_teams if team_id in existing) + missing_teams + if existing is None + else tuple(team_id for team_id in missing_teams if team_id in existing or team_id == team_id_to_upsert) ) for team_id in teams_to_add: + if team_id == team_id_to_upsert and (existing is None or team_id not in existing): + await _create_sso_team_or_warn(team_id, create_team) await add_team_member(team_id, user_info) @@ -3663,7 +3710,11 @@ class SSOAuthenticationHandler: verbose_proxy_logger.debug("User not found in LiteLLM DB, skipping team member addition") return sso_teams: Final = result.team_ids if isinstance(result, CustomOpenID) else [] - await add_missing_team_member(user_info=user_info, sso_teams=sso_teams) + await add_missing_team_member( + user_info=user_info, + sso_teams=sso_teams, + team_id_to_upsert=result.team_id_to_upsert if isinstance(result, CustomOpenID) else None, + ) @staticmethod def verify_user_in_restricted_sso_group( diff --git a/tests/integration/management/test_sso_sign_in.py b/tests/integration/management/test_sso_sign_in.py index d0b6e2579c8..93cfa00ea13 100644 --- a/tests/integration/management/test_sso_sign_in.py +++ b/tests/integration/management/test_sso_sign_in.py @@ -16,7 +16,7 @@ import yaml from pydantic import JsonValue from tests.integration._support.client import Gateway, gateway_from_environment -from tests.integration._support.database import read_rows +from tests.integration._support.database import read_rows, write_rows from tests.integration._support.process import OWNED_STATE_VARIABLE, OwnedState, owned_proxy from tests.integration._support.wire import Reply, Request, wire_server @@ -52,14 +52,29 @@ def _idp_reply(request: Request) -> Reply: "preferred_username": signed_in, "email": f"{signed_in}@example.com", "provider": IDP_PROVIDER, + "dept": _dept(signed_in), + "groups": [_group(signed_in)], } ).encode() ) return Reply(status=404, body=b'{"error": "not_found"}') +def _dept(subject: str) -> str: + return f"dept-{subject}" + + +def _group(subject: str) -> str: + return f"group-{subject}" + + @contextmanager -def _sso_gateway(directory: Path, litellm_settings: Mapping[str, JsonValue]) -> Iterator[Gateway]: +def _sso_gateway( + directory: Path, + litellm_settings: Mapping[str, JsonValue], + general_settings: Mapping[str, JsonValue] | None = None, + db_sso_settings: Mapping[str, JsonValue] | None = None, +) -> Iterator[Gateway]: config: Final = directory / "sso_proxy_config.yaml" config.write_text( yaml.safe_dump( @@ -68,6 +83,7 @@ def _sso_gateway(directory: Path, litellm_settings: Mapping[str, JsonValue]) -> "general_settings": { "master_key": "os.environ/LITELLM_MASTER_KEY", "database_url": "os.environ/DATABASE_URL", + **(general_settings or {}), }, "litellm_settings": dict(litellm_settings), } @@ -83,7 +99,33 @@ def _sso_gateway(directory: Path, litellm_settings: Mapping[str, JsonValue]) -> "OAUTHLIB_INSECURE_TRANSPORT": "1", } with owned_proxy(rig, directory, environment, config=config, remove_environment=("PROXY_BASE_URL",)) as gw: - yield gw + if db_sso_settings is None: + yield gw + return + with contextlib.ExitStack() as cleanups: + cleanups.callback( + write_rows, + 'DELETE FROM "LiteLLM_SSOConfig" WHERE id = %s', + ("sso_config",), + database_url=gw.database_url, + ) + stored: Final = _store_sso_settings(gw, {**_generic_sso_settings(environment), **db_sso_settings}) + assert stored.status_code == 200, stored.text + yield gw + + +def _generic_sso_settings(environment: Mapping[str, str]) -> Mapping[str, JsonValue]: + return { + "generic_client_id": environment["GENERIC_CLIENT_ID"], + "generic_client_secret": environment["GENERIC_CLIENT_SECRET"], + "generic_authorization_endpoint": environment["GENERIC_AUTHORIZATION_ENDPOINT"], + "generic_token_endpoint": environment["GENERIC_TOKEN_ENDPOINT"], + "generic_userinfo_endpoint": environment["GENERIC_USERINFO_ENDPOINT"], + } + + +def _store_sso_settings(gateway: Gateway, settings: Mapping[str, JsonValue]) -> httpx.Response: + return gateway.request("PATCH", "/update/sso_settings", settings) def _sign_in(gateway: Gateway, subject: str) -> httpx.Response: @@ -165,3 +207,65 @@ def test_free_sso_admission_preserves_existing_users_and_serializes_the_last_sea refused: Final = _sign_in(gateway, "free-sso-extra") assert (above_capacity.status_code, refused.status_code) == (303, 403) assert gateway.rows('SELECT COUNT(*) AS count FROM "LiteLLM_UserTable"', ()) == [{"count": 6}] + + +@pytest.mark.parametrize( + ("enable_jwt_auth", "team_id_upsert", "team_exists_before", "expect_member"), + [(True, True, False, True), (False, True, False, False), (True, False, False, False), (True, False, True, True)], + ids=[ + "upsert-creates-missing-team", + "upsert-without-jwt-auth-skips-missing-team", + "no-upsert-skips-missing-team", + "no-upsert-joins-existing-team", + ], +) +def test_sso_sign_in_upserts_the_team_id_jwt_field_team_only_when_team_id_upsert_is_on( + tmp_path: Path, enable_jwt_auth: bool, team_id_upsert: bool, team_exists_before: bool, expect_member: bool +) -> None: + subject: Final = f"sso-dept-user-{uuid.uuid4().hex[:12]}" + team_id: Final = _dept(subject) + jwtauth: Final = { + "enable_jwt_auth": enable_jwt_auth, + "litellm_jwtauth": {"team_id_jwt_field": "dept", "team_id_upsert": team_id_upsert}, + } + with _sso_gateway(tmp_path, {}, jwtauth) as gateway, contextlib.ExitStack() as cleanups: + cleanups.callback(gateway.request, "POST", "/team/delete", {"team_ids": [team_id]}) + cleanups.callback(_delete_user, gateway, subject) + if team_exists_before: + created: Final = gateway.request("POST", "/team/new", {"team_id": team_id}) + assert created.status_code == 200, created.text + + callback: Final = _sign_in(gateway, subject) + + assert callback.status_code == 303, f"{callback.status_code} {callback.text}" + teams: Final = gateway.rows('SELECT team_id FROM "LiteLLM_TeamTable" WHERE team_id = %s', (team_id,)) + user: Final = gateway.rows('SELECT teams FROM "LiteLLM_UserTable" WHERE user_id = %s', (subject,)) + assert (len(teams), user) == (int(expect_member), [{"teams": [team_id] if expect_member else []}]) + + +def test_sso_sign_in_with_db_team_mappings_creates_the_team_id_jwt_field_team_and_only_joins_groups( + tmp_path: Path, +) -> None: + subject: Final = f"sso-mapped-user-{uuid.uuid4().hex[:12]}" + team_id: Final = _dept(subject) + group_team_id: Final = _group(subject) + jwtauth: Final = { + "enable_jwt_auth": True, + "litellm_jwtauth": {"team_id_jwt_field": "dept", "team_id_upsert": True}, + } + team_mappings: Final = {"team_mappings": {"team_ids_jwt_field": "groups"}} + with ( + _sso_gateway(tmp_path, {}, jwtauth, db_sso_settings=team_mappings) as gateway, + contextlib.ExitStack() as cleanups, + ): + cleanups.callback(gateway.request, "POST", "/team/delete", {"team_ids": [team_id, group_team_id]}) + cleanups.callback(_delete_user, gateway, subject) + + callback: Final = _sign_in(gateway, subject) + + assert callback.status_code == 303, f"{callback.status_code} {callback.text}" + teams: Final = gateway.rows( + 'SELECT team_id FROM "LiteLLM_TeamTable" WHERE team_id IN (%s, %s)', (team_id, group_team_id) + ) + user: Final = gateway.rows('SELECT teams FROM "LiteLLM_UserTable" WHERE user_id = %s', (subject,)) + assert (teams, user) == ([{"team_id": team_id}], [{"teams": [team_id]}]) diff --git a/tests/unit/proxy/management_endpoints/test_ui_sso.py b/tests/unit/proxy/management_endpoints/test_ui_sso.py index 136cc86e67a..ba78fefd653 100644 --- a/tests/unit/proxy/management_endpoints/test_ui_sso.py +++ b/tests/unit/proxy/management_endpoints/test_ui_sso.py @@ -15,9 +15,15 @@ from fastapi import HTTPException, Request from pydantic import SecretStr import litellm +from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler -from litellm.proxy._types import LiteLLM_UserTable, NewUserResponse, ProxyException +from litellm.proxy._types import ( + LiteLLM_JWTAuth, + LiteLLM_UserTable, + NewUserResponse, + ProxyException, +) from litellm.proxy.auth.handle_jwt import JWTHandler from litellm.proxy.management_endpoints.sso import CustomMicrosoftSSO from litellm.proxy.management_endpoints.types import CustomOpenID @@ -287,6 +293,13 @@ def test_get_microsoft_callback_response(stubbed_graph_api): assert sso_assertion.issuer == "https://idp.example.com" + +def _jwt_handler_double() -> MagicMock: + handler: Final = MagicMock(spec=JWTHandler) + handler.litellm_jwtauth = LiteLLM_JWTAuth() + return handler + + GRAPH_ERROR_BODIES: Final = ( {"error": {"code": "Authentication_MissingOrMalformed", "message": "Access Token missing or malformed."}}, {"error": {"code": "InvalidAuthenticationToken", "message": "IDX14100: JWT is not well formed."}}, @@ -1158,7 +1171,7 @@ def test_generic_response_convertor_normalizes_email(): } # Mock JWT handler - mock_jwt_handler = MagicMock(spec=JWTHandler) + mock_jwt_handler = _jwt_handler_double() mock_jwt_handler.get_team_ids_from_jwt.return_value = [] result = generic_response_convertor( @@ -2226,7 +2239,7 @@ async def test_get_generic_sso_response_with_additional_headers(): # Arrange mock_request = MagicMock(spec=Request) - mock_jwt_handler = MagicMock(spec=JWTHandler) + mock_jwt_handler = _jwt_handler_double() mock_jwt_handler.get_team_ids_from_jwt.return_value = [] generic_client_id = "test_client_id" @@ -2299,7 +2312,7 @@ async def test_get_generic_sso_response_with_empty_headers(): # Arrange mock_request = MagicMock(spec=Request) - mock_jwt_handler = MagicMock(spec=JWTHandler) + mock_jwt_handler = _jwt_handler_double() mock_jwt_handler.get_team_ids_from_jwt.return_value = [] generic_client_id = "test_client_id" @@ -2359,8 +2372,8 @@ async def test_get_generic_sso_response_includes_token_claims_when_enabled(monke from litellm.proxy.management_endpoints.ui_sso import get_generic_sso_response mock_request = MagicMock(spec=Request) - mock_jwt_handler = MagicMock(spec=JWTHandler) - mock_sso_jwt_handler = MagicMock(spec=JWTHandler) + mock_jwt_handler = _jwt_handler_double() + mock_sso_jwt_handler = _jwt_handler_double() mock_sso_jwt_handler.get_all_jwt_team_ids.return_value = ["team-from-userinfo"] mock_sso_jwt_handler.get_team_ids_from_jwt.return_value = [] @@ -2435,8 +2448,8 @@ async def test_get_generic_sso_response_does_not_include_token_claims_when_disab from litellm.proxy.management_endpoints.ui_sso import get_generic_sso_response mock_request = MagicMock(spec=Request) - mock_jwt_handler = MagicMock(spec=JWTHandler) - mock_sso_jwt_handler = MagicMock(spec=JWTHandler) + mock_jwt_handler = _jwt_handler_double() + mock_sso_jwt_handler = _jwt_handler_double() mock_sso_jwt_handler.get_all_jwt_team_ids.return_value = ["team-from-userinfo"] mock_sso_jwt_handler.get_team_ids_from_jwt.return_value = [] access_token = pyjwt.encode({"upn": "token-user@example.com"}, "test-secret", algorithm="HS256") @@ -2578,7 +2591,7 @@ async def test_get_generic_sso_response_pkce_merges_token_claims_and_excludes_cr ): result, received_response, _, _ = await get_generic_sso_response( request=request, - jwt_handler=MagicMock(spec=JWTHandler), + jwt_handler=_jwt_handler_double(), generic_client_id="test-client", redirect_url="http://test.com/callback", sso_jwt_handler=None, @@ -2600,7 +2613,7 @@ async def test_get_generic_sso_response_ignores_opaque_and_empty_token_claims(mo from litellm.proxy.management_endpoints.ui_sso import get_generic_sso_response mock_request = MagicMock(spec=Request) - mock_jwt_handler = MagicMock(spec=JWTHandler) + mock_jwt_handler = _jwt_handler_double() userinfo = { "preferred_username": "userinfo-user", "email": "userinfo@example.com", @@ -4961,7 +4974,7 @@ class TestProcessSSOJWTAccessToken: @pytest.fixture def mock_jwt_handler(self): """Create a mock JWT handler for testing""" - mock_handler = MagicMock(spec=JWTHandler) + mock_handler = _jwt_handler_double() mock_handler.get_team_ids_from_jwt.return_value = ["team1", "team2", "team3"] return mock_handler @@ -5348,7 +5361,7 @@ class TestGenericResponseConvertorNestedAttributes: from litellm.proxy.management_endpoints.ui_sso import generic_response_convertor # Mock JWT handler - mock_jwt_handler = MagicMock(spec=JWTHandler) + mock_jwt_handler = _jwt_handler_double() mock_jwt_handler.get_team_ids_from_jwt.return_value = [] # Payload with nested attributes structure @@ -5419,7 +5432,7 @@ class TestGenericResponseConvertorUserRole: from litellm.proxy._types import LitellmUserRoles from litellm.proxy.management_endpoints.ui_sso import generic_response_convertor - mock_jwt_handler = MagicMock(spec=JWTHandler) + mock_jwt_handler = _jwt_handler_double() mock_jwt_handler.get_team_ids_from_jwt.return_value = [] sso_response = { @@ -5449,7 +5462,7 @@ class TestGenericResponseConvertorUserRole: """ from litellm.proxy.management_endpoints.ui_sso import generic_response_convertor - mock_jwt_handler = MagicMock(spec=JWTHandler) + mock_jwt_handler = _jwt_handler_double() mock_jwt_handler.get_team_ids_from_jwt.return_value = [] sso_response = { @@ -6813,7 +6826,9 @@ class TestAddMissingTeamMember: # Verify add_missing_team_member was called with correct args mock_add_member.assert_called_once_with( - user_info=new_user_info, sso_teams=["entra-group-1", "entra-group-2"] + user_info=new_user_info, + sso_teams=["entra-group-1", "entra-group-2"], + team_id_to_upsert=None, ) @pytest.mark.asyncio @@ -7897,7 +7912,7 @@ def test_generic_response_convertor_with_extra_attributes(monkeypatch): "custom_field3": {"nested": "data"}, } - mock_jwt_handler = MagicMock(spec=JWTHandler) + mock_jwt_handler = _jwt_handler_double() mock_jwt_handler.get_team_ids_from_jwt.return_value = [] result = generic_response_convertor( @@ -7931,7 +7946,7 @@ def test_generic_response_convertor_without_extra_attributes(monkeypatch): "custom_field2": "value2", } - mock_jwt_handler = MagicMock(spec=JWTHandler) + mock_jwt_handler = _jwt_handler_double() mock_jwt_handler.get_team_ids_from_jwt.return_value = [] result = generic_response_convertor( @@ -7959,7 +7974,7 @@ def test_generic_response_convertor_extra_attributes_with_nested_paths(monkeypat "org_info": {"department": "Engineering", "manager": "Jane Smith"}, } - mock_jwt_handler = MagicMock(spec=JWTHandler) + mock_jwt_handler = _jwt_handler_double() mock_jwt_handler.get_team_ids_from_jwt.return_value = [] result = generic_response_convertor( @@ -7986,7 +8001,7 @@ def test_generic_response_convertor_extra_attributes_missing_field(monkeypatch): "email": "user@example.com", } - mock_jwt_handler = MagicMock(spec=JWTHandler) + mock_jwt_handler = _jwt_handler_double() mock_jwt_handler.get_team_ids_from_jwt.return_value = [] result = generic_response_convertor( @@ -8865,7 +8880,7 @@ class TestPKCEStateCookieBinding: ): await get_generic_sso_response( request=mock_request, - jwt_handler=MagicMock(spec=JWTHandler), + jwt_handler=_jwt_handler_double(), generic_client_id="cid", redirect_url="https://proxy.example.com/sso/callback", sso_jwt_handler=None, @@ -8917,7 +8932,7 @@ class TestPKCEStateCookieBinding: ): await get_generic_sso_response( request=mock_request, - jwt_handler=MagicMock(spec=JWTHandler), + jwt_handler=_jwt_handler_double(), generic_client_id="cid", redirect_url="https://proxy.example.com/sso/callback", sso_jwt_handler=None, @@ -8980,7 +8995,7 @@ class TestPKCEStateCookieBinding: }, ), ): - jwt_handler = MagicMock(spec=JWTHandler) + jwt_handler = _jwt_handler_double() jwt_handler.get_team_ids_from_jwt.return_value = [] result, _, _, _ = await get_generic_sso_response( request=mock_request, @@ -9087,7 +9102,7 @@ async def test_debug_sso_callback_renders_full_jwt_claims(): patch("litellm.proxy.proxy_server.general_settings", {}), patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), patch("litellm.proxy.proxy_server.user_api_key_cache", MagicMock()), - patch("litellm.proxy.proxy_server.jwt_handler", MagicMock(spec=JWTHandler)), + patch("litellm.proxy.proxy_server.jwt_handler", _jwt_handler_double()), ): # Microsoft / Google envs may leak in from other tests — ensure only # the generic path runs. @@ -9173,7 +9188,7 @@ async def test_debug_sso_callback_handles_missing_raw_response(): patch("litellm.proxy.proxy_server.general_settings", {}), patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), patch("litellm.proxy.proxy_server.user_api_key_cache", MagicMock()), - patch("litellm.proxy.proxy_server.jwt_handler", MagicMock(spec=JWTHandler)), + patch("litellm.proxy.proxy_server.jwt_handler", _jwt_handler_double()), ): for var in ("GENERIC_CLIENT_ID", "GOOGLE_CLIENT_ID"): os.environ.pop(var, None) @@ -9224,7 +9239,7 @@ async def _render_debug_page(provider_env, id_jag_registered, force_inert=False) patch("litellm.proxy.proxy_server.general_settings", {}), # test-quality-ok: debug endpoint reads proxy globals patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), # test-quality-ok: debug endpoint reads proxy DB patch("litellm.proxy.proxy_server.user_api_key_cache", MagicMock()), # test-quality-ok: debug endpoint reads proxy globals - patch("litellm.proxy.proxy_server.jwt_handler", MagicMock(spec=JWTHandler)), # test-quality-ok: debug endpoint reads proxy globals + patch("litellm.proxy.proxy_server.jwt_handler", _jwt_handler_double()), # test-quality-ok: debug endpoint reads proxy globals ] if force_inert: stack.append( @@ -9589,7 +9604,7 @@ async def test_pkce_arm_captures_sso_assertion(): }, ), ): - jwt_handler = MagicMock(spec=JWTHandler) + jwt_handler = _jwt_handler_double() jwt_handler.get_team_ids_from_jwt.return_value = [] result, received_response, _, sso_assertion = await get_generic_sso_response( request=mock_request, @@ -9615,7 +9630,7 @@ async def test_verify_and_process_arm_captures_sso_assertion(): id_token = _ema_id_token() mock_request = MagicMock(spec=Request) - mock_jwt_handler = MagicMock(spec=JWTHandler) + mock_jwt_handler = _jwt_handler_double() mock_jwt_handler.get_team_ids_from_jwt.return_value = [] mock_sso_instance = MagicMock() @@ -9680,7 +9695,7 @@ def _generic_sso_callback_request() -> Request: def _generic_sso_jwt_handler() -> MagicMock: - jwt_handler = MagicMock(spec=JWTHandler) + jwt_handler = _jwt_handler_double() jwt_handler.get_team_ids_from_jwt.return_value = [] return jwt_handler @@ -11144,3 +11159,241 @@ async def test_cli_sso_login_never_serves_the_admin_login_form_on_a_ui_disabled_ body = response.body.decode() assert "Admin UI is Disabled" in body assert 'name="username"' not in body + + +def _upsert_jwt_handler(**jwtauth: object) -> JWTHandler: + from litellm.caching.caching import DualCache + from litellm.proxy._types import LiteLLM_JWTAuth + + handler: Final = JWTHandler() + handler.update_environment( + prisma_client=None, user_api_key_cache=DualCache(), litellm_jwtauth=LiteLLM_JWTAuth(**jwtauth) + ) + return handler + + +@pytest.mark.parametrize( + ("jwtauth", "userinfo", "expected_team_ids", "expected_upsert"), + [ + ({"team_id_jwt_field": "dept", "team_id_upsert": True}, {"dept": "finance"}, ["finance"], "finance"), + ({"team_id_jwt_field": "dept", "team_id_upsert": False}, {"dept": "finance"}, ["finance"], None), + ({"team_id_jwt_field": "dept", "team_id_upsert": True}, {}, [], None), + ({"team_id_jwt_field": "dept", "team_id_upsert": True}, {"dept": ""}, [], None), + ({"team_id_jwt_field": "dept", "team_id_upsert": True}, {"dept": " "}, [" "], None), + ({"team_id_jwt_field": "dept", "team_id_upsert": True}, {"dept": ["a", "b"]}, ["a", "b"], None), + ( + {"team_id_jwt_field": "dept", "team_id_upsert": True, "team_id_default": "everyone"}, + {}, + [], + None, + ), + ( + {"team_ids_jwt_field": "groups", "team_id_upsert": True}, + {"groups": ["g1", "g2"]}, + ["g1", "g2"], + None, + ), + ( + {"team_ids_jwt_field": "groups", "team_id_jwt_field": "dept", "team_id_upsert": True}, + {"groups": ["g1"], "dept": "finance"}, + ["g1", "finance"], + "finance", + ), + ], + ids=[ + "upsert-on", + "upsert-off", + "claim-missing", + "blank-claim-not-upserted", + "whitespace-claim-not-upserted", + "list-valued-claim-join-only", + "default-team-not-upserted", + "plural-claim-join-only", + "mixed", + ], +) +def test_generic_sso_login_upserts_only_the_team_id_jwt_field_claim( + jwtauth: dict[str, object], + userinfo: dict[str, object], + expected_team_ids: list[str], + expected_upsert: str | None, +) -> None: + from litellm.proxy.management_endpoints.ui_sso import generic_response_convertor + + result: Final = generic_response_convertor( + response={"preferred_username": "alice", "email": "alice@example.com", **userinfo}, + jwt_handler=_upsert_jwt_handler(**jwtauth), + jwt_auth_enabled=True, + ) + + assert (result.team_ids, result.team_id_to_upsert) == (expected_team_ids, expected_upsert) + + +def test_generic_sso_login_does_not_upsert_the_team_id_jwt_field_claim_when_jwt_auth_is_disabled() -> None: + from litellm.proxy.management_endpoints.ui_sso import generic_response_convertor + + result: Final = generic_response_convertor( + response={"preferred_username": "alice", "email": "alice@example.com", "dept": "finance"}, + jwt_handler=_upsert_jwt_handler(team_id_jwt_field="dept", team_id_upsert=True), + jwt_auth_enabled=False, + ) + + assert (result.team_ids, result.team_id_to_upsert) == (["finance"], None) + + +def test_generic_sso_login_marks_the_team_id_jwt_field_team_for_upsert_without_adding_it_to_team_ids() -> None: + from litellm.proxy.management_endpoints.ui_sso import generic_response_convertor + + result: Final = generic_response_convertor( + response={"preferred_username": "alice", "email": "alice@example.com", "groups": ["g1"], "dept": "finance"}, + jwt_handler=_upsert_jwt_handler(team_id_jwt_field="dept", team_id_upsert=True), + team_mappings=TeamMappings(team_ids_jwt_field="groups"), + jwt_auth_enabled=True, + ) + + assert (result.team_ids, result.team_id_to_upsert) == (["g1"], "finance") + + +def test_generic_sso_login_keeps_team_ids_empty_for_a_dept_only_user_so_the_access_token_group_fallback_runs() -> None: + from litellm.proxy.management_endpoints.ui_sso import generic_response_convertor + + result: Final = generic_response_convertor( + response={"preferred_username": "alice", "email": "alice@example.com", "dept": "d1"}, + jwt_handler=_upsert_jwt_handler(team_id_jwt_field="dept", team_id_upsert=True), + team_mappings=TeamMappings(team_ids_jwt_field="groups"), + jwt_auth_enabled=True, + ) + + assert (result.team_ids, result.team_id_to_upsert) == ([], "d1") + + +def test_restricted_sso_group_still_rejects_a_user_whose_only_team_is_the_upserted_dept() -> None: + from litellm.proxy._types import ProxyException + from litellm.proxy.management_endpoints.ui_sso import SSOAuthenticationHandler, generic_response_convertor + + response: Final = {"preferred_username": "alice", "email": "alice@example.com", "dept": "X", "groups": []} + result: Final = generic_response_convertor( + response=response, + jwt_handler=_upsert_jwt_handler(team_id_jwt_field="dept", team_id_upsert=True), + team_mappings=TeamMappings(team_ids_jwt_field="groups"), + jwt_auth_enabled=True, + ) + + with pytest.raises(ProxyException) as denied: + SSOAuthenticationHandler.verify_user_in_restricted_sso_group( + general_settings={"ui_access_mode": {"type": "restricted_sso_group", "restricted_sso_group": "X"}}, + result=result, + received_response=response, + ) + + assert denied.value.code == "403" + + +def _recording_team_calls( + events: list[tuple[str, str]], +) -> tuple[Callable[[str, NewUserResponse | LiteLLM_UserTable], Awaitable[None]], Callable[[str], Awaitable[None]]]: + async def add(team_id: str, user_info: NewUserResponse | LiteLLM_UserTable) -> None: + events.append(("join", team_id)) + + async def create(team_id: str) -> None: + events.append(("create", team_id)) + + return add, create + + +async def _only_group_teams_exist(team_ids: Sequence[str]) -> frozenset[str] | None: + return frozenset(team_id for team_id in team_ids if team_id.startswith("g")) + + +@pytest.mark.asyncio +async def test_sso_login_joins_the_upsert_team_from_team_id_to_upsert_and_creates_it_before_the_join() -> None: + from litellm.proxy.management_endpoints.ui_sso import add_missing_team_member + + events: Final[list[tuple[str, str]]] = [] # mutable-ok: records call order across awaited tasks + add, create = _recording_team_calls(events) + + await add_missing_team_member( + NewUserResponse(user_id="alice", key="sk-x", teams=None), + ["g1", "g2"], + add_team_member=add, + existing_team_ids=_only_group_teams_exist, + team_id_to_upsert="finance", + create_team=create, + ) + + assert events == [("join", "g1"), ("join", "g2"), ("create", "finance"), ("join", "finance")] + + +@pytest.mark.asyncio +async def test_sso_login_joins_only_the_claimed_groups_when_team_id_upsert_is_off() -> None: + from litellm.proxy.management_endpoints.ui_sso import add_missing_team_member + + events: Final[list[tuple[str, str]]] = [] # mutable-ok: records call order across awaited tasks + add, create = _recording_team_calls(events) + + await add_missing_team_member( + NewUserResponse(user_id="alice", key="sk-x", teams=None), + ["g1", "g2"], + add_team_member=add, + existing_team_ids=_only_group_teams_exist, + create_team=create, + ) + + assert events == [("join", "g1"), ("join", "g2")] + + +@pytest.mark.asyncio +async def test_sso_login_does_not_create_the_team_id_jwt_field_team_when_it_already_exists() -> None: + from litellm.proxy.management_endpoints.ui_sso import add_missing_team_member + + events: Final[list[tuple[str, str]]] = [] # mutable-ok: records call order across awaited tasks + add, create = _recording_team_calls(events) + + async def every_team_exists(team_ids: Sequence[str]) -> frozenset[str] | None: + return frozenset(team_ids) + + await add_missing_team_member( + NewUserResponse(user_id="alice", key="sk-x", teams=None), + ["finance"], + add_team_member=add, + existing_team_ids=every_team_exists, + team_id_to_upsert="finance", + create_team=create, + ) + + assert events == [("join", "finance")] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "create_error", + [ + HTTPException(status_code=400, detail={"error": "Team id = finance already exists"}), + HTTPException(status_code=403, detail={"error": "team limit reached for this license"}), + ], + ids=["lost-the-create-race", "create-rejected"], +) +async def test_sso_login_still_joins_the_team_and_warns_when_creating_it_fails( + caplog: pytest.LogCaptureFixture, create_error: HTTPException +) -> None: + from litellm.proxy.management_endpoints.ui_sso import add_missing_team_member + + events: Final[list[tuple[str, str]]] = [] # mutable-ok: records call order across awaited tasks + add, _ = _recording_team_calls(events) + + async def failed_create(team_id: str) -> None: + raise create_error + + with caplog.at_level(logging.WARNING, logger=verbose_proxy_logger.name): + await add_missing_team_member( + NewUserResponse(user_id="alice", key="sk-x", teams=None), + ["finance"], + add_team_member=add, + existing_team_ids=_only_group_teams_exist, + team_id_to_upsert="finance", + create_team=failed_create, + ) + + warnings: Final = [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING] + assert events == [("join", "finance")] + assert len(warnings) == 1 and "finance" in warnings[0] and str(create_error.detail) in warnings[0] From 7e5a653c0129c5ba99e828d15e5e2ff1f5690e90 Mon Sep 17 00:00:00 2001 From: shrey Date: Fri, 30 Oct 2026 15:03:49 -0700 Subject: [PATCH 45/74] chore(cost-map): add Deepgram Aura-1 and Aura-2 text-to-speech rates (#49280) --- ...model_prices_and_context_window_backup.json | 18 ++++++++++++++++++ model_prices_and_context_window.json | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 1d18903bb07..843ab574da4 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -23078,6 +23078,24 @@ "output_cost_per_token_batches": 1e-06, "source": "https://developers.openai.com/api/docs/pricing" }, + "deepgram/aura": { + "input_cost_per_character": 1.5e-05, + "litellm_provider": "deepgram", + "metadata": { + "calculation": "$0.0150/1000 characters = $0.000015 per character", + "note": "Aura-1 text-to-speech voices (aura--), pay as you go" + }, + "source": "https://deepgram.com/pricing" + }, + "deepgram/aura-2": { + "input_cost_per_character": 3e-05, + "litellm_provider": "deepgram", + "metadata": { + "calculation": "$0.030/1000 characters = $0.00003 per character", + "note": "Aura-2 text-to-speech voices (aura-2--), pay as you go" + }, + "source": "https://deepgram.com/pricing" + }, "deepgram/base": { "input_cost_per_second": 0.00020833, "litellm_provider": "deepgram", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 1d18903bb07..843ab574da4 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -23078,6 +23078,24 @@ "output_cost_per_token_batches": 1e-06, "source": "https://developers.openai.com/api/docs/pricing" }, + "deepgram/aura": { + "input_cost_per_character": 1.5e-05, + "litellm_provider": "deepgram", + "metadata": { + "calculation": "$0.0150/1000 characters = $0.000015 per character", + "note": "Aura-1 text-to-speech voices (aura--), pay as you go" + }, + "source": "https://deepgram.com/pricing" + }, + "deepgram/aura-2": { + "input_cost_per_character": 3e-05, + "litellm_provider": "deepgram", + "metadata": { + "calculation": "$0.030/1000 characters = $0.00003 per character", + "note": "Aura-2 text-to-speech voices (aura-2--), pay as you go" + }, + "source": "https://deepgram.com/pricing" + }, "deepgram/base": { "input_cost_per_second": 0.00020833, "litellm_provider": "deepgram", From faf9ee18e962c0dde19de46fc9190ab70f34a70f Mon Sep 17 00:00:00 2001 From: tin-berri Date: Fri, 30 Oct 2026 15:05:32 -0700 Subject: [PATCH 46/74] fix(cost): preserve zero prices across batch, speech and search (#49294) * fix(cost): preserve zero cache-write prices in batch fallback * fix(cost): retain batch cache charges when ordinary input is free * fix(cost): preserve zero speech and Perplexity search rates * fix(cost): reject unset speech prices before zero defaults * test(cost): consolidate zero-price regression coverage --- CONTRIBUTING.md | 2 + litellm/cost_calculator.py | 13 +++- .../litellm_core_utils/llm_cost_calc/utils.py | 7 +- litellm/llms/perplexity/cost_calculator.py | 5 +- litellm/types/utils.py | 2 + litellm/utils.py | 9 ++- .../test_perplexity_cost_calculator.py | 51 ++++++++++++- tests/unit/test_cost_calculator.py | 73 ++++++++++++++----- ui/litellm-dashboard/src/lib/http/schema.d.ts | 15 ++++ 9 files changed, 147 insertions(+), 30 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 531a78d8982..e956b27d5da 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -276,6 +276,8 @@ When the provider reports its own usage or spend numbers, compare the proxy's nu Provider work (a new provider, a new parameter, a new usage or cost field) is the most common kind of contribution, and it is where PRs get reworked most often. Follow these rules +**Preserve zero prices.** In [`model_prices_and_context_window.json`](model_prices_and_context_window.json), an explicit `0.0` means that the priced operation is free. A missing field or `null` means no rate is specified, so the relevant calculator may use a fallback. For example, when no batch input rate is configured, batch cache writes use `cache_creation_input_token_cost` if set, otherwise the regular input rate, then apply the batch discount. A zero ordinary input rate does not make separately priced cache reads or writes free. Test `rate is None` before substituting a fallback, since `row.get("cache_creation_input_token_cost") or fallback` would discard a valid zero. Speech keeps character pricing ahead of token pricing even when the character rate is zero. If neither rate is configured, speech cost calculation raises an error. Its model lookup preserves unset rates instead of treating compatibility defaults as explicit zero prices. Perplexity keeps a configured legacy search rate ahead of the current search rate, including zero. When registering custom prices, `None` leaves an existing rate unchanged and `0.0` overrides it + **Find the existing machinery before you add plumbing.** LiteLLM already has generic paths that move data between the endpoints, the provider transformations, and cost tracking. A second, hand-rolled path for the same data is a common reason a provider PR gets rewritten before merge. [`ARCHITECTURE.md`](ARCHITECTURE.md) shows where the layers live, and the decision records in [`docs/adrs/`](docs/adrs/AGENTS.md) explain why they are shaped that way. For example, [the ADR on provider usage fields](docs/adrs/adr_001_provider_usage_fields_ride_as_usage_extras.md) covers how provider-specific usage fields reach cost tracking. If you cannot find a mechanism for what you need, say so in the PR or ask in [#pr-review on Slack](https://join.slack.com/t/litellmossslack/shared_invite/zt-3o7nkuyfr-p_kbNJj8taRfXGgQI1~YyA) before you build one **Bill a new provider usage field from the usage object.** Leave the field on the usage object as an extra and read it in that provider's `cost_calculator.py`. A provider without one gets a new `cost_calculator.py` and an arm for it in `cost_per_token` (`litellm/cost_calculator.py`), which picks the calculator by provider. If the provider's transform builds `Usage` from named fields, as the Anthropic, Gemini, and Bedrock Converse ones do, pass the new field there too. Streamed requests rebuild usage in `ChunkProcessor.calculate_usage` (`litellm/litellm_core_utils/streaming_chunk_builder_utils.py`), which does not copy extras yet the way the Responses-to-chat bridge does (minus a short exclusion list), so until it does, bill the field on streams by naming it there, as Anthropic's `inference_geo` is. A provider that reports its own charge for the request, as OpenRouter does in `usage.cost`, is billed at that charge from `_hidden_params` before any calculator runs, unless the deployment sets its own prices. When the field counts a built-in tool such as web search, add the provider to `get_cost_for_web_search_request` in `litellm/llms/__init__.py`, and add a gate to `litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py` only when no existing gate fires. Then show the field's cost in your proof on every endpoint it reaches, streaming included diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index ee102e82ca4..f5b9014d87c 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -623,7 +623,11 @@ def cost_per_token( ) if lyria_generation_cost is not None: return 0.0, lyria_generation_cost - speech_model_info = litellm.get_model_info(model=model_without_prefix, custom_llm_provider=custom_llm_provider) + speech_model_info: Final = cached_get_model_info_helper( + model=model_without_prefix, + custom_llm_provider=custom_llm_provider, + preserve_unset_costs=True, + ) cost_metric: Final = select_cost_metric_for_model(speech_model_info) prompt_cost: float = 0.0 completion_cost: float = 0.0 @@ -3084,7 +3088,7 @@ def batch_cost_calculator( batch_rates.cache_creation, batch_rates.cache_creation_rates_by_ttl, ) - elif input_cost_per_token: + elif input_cost_per_token is not None: details: Final = parse_prompt_tokens_details(usage) cache_read_tokens: Final = details["cache_hit_tokens"] cache_creation_tokens: Final = details["cache_creation_tokens"] @@ -3100,7 +3104,10 @@ def batch_cost_calculator( cache_read_cost_key: Final = get_service_tier_cost_key("cache_read_input_token_cost", None) total_prompt_cost += calculate_cost_component(model_info, cache_read_cost_key, cache_read_tokens) / 2 - cache_creation_cost: Final = model_info.get("cache_creation_input_token_cost") or input_cost_per_token + configured_cache_creation_cost: Final = model_info.get("cache_creation_input_token_cost") + cache_creation_cost: Final = ( + input_cost_per_token if configured_cache_creation_cost is None else configured_cache_creation_cost + ) total_prompt_cost += cache_creation_tokens * cache_creation_cost / 2 text_rate: Final = _batch_or_half(batch_rates.output, output_cost_per_token) image_rate: Final = _batch_or_half( diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 404dfab0a6d..50c26f4d4ca 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -28,6 +28,7 @@ from litellm.types.utils import ( DataResidency, ImageResponse, ModelInfo, + ModelInfoBase, PassthroughCallTypes, PromptTokensDetailsWrapper, ServiceTier, @@ -197,15 +198,15 @@ def get_billable_input_tokens(usage: Usage) -> int: def select_cost_metric_for_model( - model_info: ModelInfo, + model_info: ModelInfoBase, ) -> Literal["cost_per_character", "cost_per_token"]: """ Select 'cost_per_character' if model_info has 'input_cost_per_character' Select 'cost_per_token' if model_info has 'input_cost_per_token' """ - if model_info.get("input_cost_per_character"): + if model_info.get("input_cost_per_character") is not None: return "cost_per_character" - elif model_info.get("input_cost_per_token"): + elif model_info.get("input_cost_per_token") is not None: return "cost_per_token" else: raise ValueError( diff --git a/litellm/llms/perplexity/cost_calculator.py b/litellm/llms/perplexity/cost_calculator.py index c3d178771a5..c40a86b221b 100644 --- a/litellm/llms/perplexity/cost_calculator.py +++ b/litellm/llms/perplexity/cost_calculator.py @@ -100,8 +100,9 @@ def cost_per_token(model: str, usage: Usage, current_time: datetime | None = Non num_search_queries = getattr(usage.prompt_tokens_details, "web_search_requests", 0) or 0 # Check both possible keys for search cost (legacy and current) - search_cost_value: Final = model_info.get("search_queries_cost_per_query") or model_info.get( - "search_context_cost_per_query" + legacy_search_cost: Final = model_info.get("search_queries_cost_per_query") + search_cost_value: Final = ( + model_info.get("search_context_cost_per_query") if legacy_search_cost is None else legacy_search_cost ) if num_search_queries > 0 and search_cost_value is not None: # Handle both dict and float formats diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 1a8f5adf154..0c3440e88f7 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -464,6 +464,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): annotation_cost_per_page: float | None # for OCR models annotation_cost_per_page_batches: ReadOnly[float | None] search_context_cost_per_query: SearchContextCostPerQuery | None # Cost for using web search tool + search_queries_cost_per_query: ReadOnly[float | SearchContextCostPerQuery | None] web_search_billing_unit: ( Literal["per_query", "per_prompt"] | None ) # "per_query" (Gemini 3.x) or "per_prompt" (Gemini 2.x) @@ -4168,6 +4169,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): output_cost_per_video_per_second: float | None = None output_cost_per_audio_per_second: float | None = None search_context_cost_per_query: dict[str, Any] | None = None + search_queries_cost_per_query: float | SearchContextCostPerQuery | None = None google_maps_grounding_cost_per_query: float | None = None citation_cost_per_token: float | None = None cache_read_input_token_cost_above_272k_tokens: float | None = None diff --git a/litellm/utils.py b/litellm/utils.py index 560d0c31e4a..c2633d7eb81 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6889,6 +6889,8 @@ def cached_get_model_info_helper( model: str, custom_llm_provider: str | None, api_base: str | None = None, + *, + preserve_unset_costs: bool = False, ) -> ModelInfoBase: """ _get_model_info_helper wrapped with lru_cache @@ -6899,6 +6901,7 @@ def cached_get_model_info_helper( model=model, custom_llm_provider=custom_llm_provider, api_base=api_base, + preserve_unset_costs=preserve_unset_costs, ) @@ -6962,6 +6965,7 @@ def get_model_info_helper( api_key: str | None = None, *, catalog_only: bool = False, + preserve_unset_costs: bool = False, ) -> ModelInfoBase: """ Helper for 'get_model_info'. Separated out to avoid infinite loop caused by returning 'supported_openai_param's @@ -7079,7 +7083,7 @@ def get_model_info_helper( verbose_logger.debug("model not found in litellm.model_cost, tried: %s", potential_model_names) raise ModelNotMappedError(_model_not_mapped_message(model, custom_llm_provider)) _input_cost_per_token: float | None = _model_info.get("input_cost_per_token") - if _input_cost_per_token is None: + if _input_cost_per_token is None and not preserve_unset_costs: # default value to 0, be noisy about this verbose_logger.debug( "model=%s, custom_llm_provider=%s has no input_cost_per_token in model_cost_map. Defaulting to 0.", @@ -7089,7 +7093,7 @@ def get_model_info_helper( _input_cost_per_token = 0 _output_cost_per_token: float | None = _model_info.get("output_cost_per_token") - if _output_cost_per_token is None: + if _output_cost_per_token is None and not preserve_unset_costs: # default value to 0, be noisy about this verbose_logger.debug( "model=%s, custom_llm_provider=%s has no output_cost_per_token in model_cost_map. Defaulting to 0.", @@ -7371,6 +7375,7 @@ def get_model_info_helper( supports_regex_lookaround=_model_info.get("supports_regex_lookaround", None), supports_computer_use=_model_info.get("supports_computer_use", None), search_context_cost_per_query=_model_info.get("search_context_cost_per_query", None), + search_queries_cost_per_query=_model_info.get("search_queries_cost_per_query"), web_search_billing_unit=_model_info.get("web_search_billing_unit", None), search_free_requests=_model_info.get("search_free_requests", None), google_maps_grounding_cost_per_query=_model_info.get("google_maps_grounding_cost_per_query", None), diff --git a/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py b/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py index 380f85445e5..5e774b05207 100644 --- a/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py +++ b/tests/unit/llms/perplexity/test_perplexity_cost_calculator.py @@ -9,6 +9,7 @@ import json import logging import math import os +from collections.abc import Mapping from datetime import datetime, timezone from typing import Final from unittest.mock import patch @@ -119,7 +120,8 @@ class TestPerplexityCostCalculator: assert completion_cost == 0.008 assert prompt_cost + completion_cost == 0.008 - def test_uses_perplexity_provided_cost_when_normalized_to_float(self): + @pytest.mark.parametrize("reported_cost", (0.0, 0.008)) + def test_uses_perplexity_provided_cost_when_normalized_to_float(self, reported_cost: float): """ Regression: for Responses API / Agent API models, `ResponseAPIUsage.parse_cost` (litellm/types/llms/openai.py) already flattens Perplexity's @@ -135,12 +137,55 @@ class TestPerplexityCostCalculator: perplexity/openai/gpt-5.2 before rates existed). """ usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150) - usage.cost = 0.008 + usage.cost = reported_cost prompt_cost, completion_cost = perplexity_cost_per_token(model="sonar-pro", usage=usage) assert prompt_cost == 0.0 - assert completion_cost == 0.008 + assert completion_cost == reported_cost + + @pytest.mark.parametrize( + ("legacy_pricing", "expected_search_rate"), + ( + pytest.param({}, 0.01, id="missing"), + pytest.param({"search_queries_cost_per_query": None}, 0.01, id="null"), + pytest.param({"search_queries_cost_per_query": 0.0}, 0.0, id="free"), + pytest.param({"search_queries_cost_per_query": 0.004}, 0.004, id="paid"), + ), + ) + def test_search_cost_preserves_zero_and_defaults_only_when_unset( + self, + monkeypatch: pytest.MonkeyPatch, + legacy_pricing: Mapping[str, float | None], + expected_search_rate: float, + ) -> None: + model: Final = "perplexity/search-pricing-fixture" + monkeypatch.setitem( + litellm.model_cost, + model, + { + "litellm_provider": "perplexity", + "mode": "chat", + "input_cost_per_token": 2e-6, + "output_cost_per_token": 8e-6, + "output_cost_per_reasoning_token": 3e-6, + "citation_cost_per_token": 2e-6, + "search_context_cost_per_query": {"search_context_size_low": 0.01}, + **legacy_pricing, + }, + ) + usage: Final = Usage( + prompt_tokens=100, + completion_tokens=50, + prompt_tokens_details=PromptTokensDetailsWrapper(web_search_requests=2), + completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=10), + citation_tokens=25, + ) + + prompt_cost, completion_cost = litellm.cost_per_token(model=model, usage_object=usage) + + assert prompt_cost == pytest.approx((100 + 25) * 2e-6) + assert completion_cost == pytest.approx(40 * 8e-6 + 10 * 3e-6 + 2 * expected_search_rate) @pytest.mark.parametrize( "reported_cost", diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index f10baccd1cb..c536522e736 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -37,7 +37,7 @@ from litellm.types.llms.xai import RealtimeMeteredUsage from litellm.types.rerank import RerankResponse from litellm.types.utils import CacheCreationTokenDetails, CallTypes, Choices, CompletionTokensDetailsWrapper, EmbeddingResponse, ImageObject, ImageResponse, ImageUsage, ImageUsageInputTokensDetails, LiteLLMRealtimeStreamLoggingObject, Message, ModelInfo, ModelResponse, PromptTokensDetails, PromptTokensDetailsWrapper, Usage from litellm.types.videos.main import VideoObject -from litellm.utils import _invalidate_model_cost_lowercase_map, supports_prompt_caching +from litellm.utils import _invalidate_model_cost_lowercase_map, cached_get_model_info_helper, supports_prompt_caching import os import threading from unittest.mock import MagicMock, patch @@ -4598,23 +4598,40 @@ def test_batch_cost_calculator_prices_cache_creation_tokens_at_cache_write_rate( assert completion_cost_value == pytest.approx(200 * 15e-6 / 2) -def test_batch_cost_calculator_cache_creation_falls_back_to_input_rate(): +@pytest.mark.parametrize("input_rate", (3e-6, 0.0)) +@pytest.mark.parametrize( + ("cache_pricing", "expected_write_rate"), + [ + pytest.param({}, None, id="missing"), + pytest.param({"cache_creation_input_token_cost": None}, None, id="null"), + pytest.param({"cache_creation_input_token_cost": 0.0}, 0.0, id="free"), + pytest.param({"cache_creation_input_token_cost": 3.75e-6}, 3.75e-6, id="paid"), + ], +) +def test_batch_cost_calculator_preserves_cache_write_prices_and_defaults_only_when_unset( + cache_pricing: ModelInfo, + expected_write_rate: float | None, + input_rate: float, +) -> None: from litellm.cost_calculator import batch_cost_calculator - model_info: ModelInfo = { + model_info: Final[ModelInfo] = { "supported_openai_params": [], - "input_cost_per_token": 3e-6, + "input_cost_per_token": input_rate, "output_cost_per_token": 15e-6, "cache_read_input_token_cost": 3e-7, + **cache_pricing, } - prompt_cost, _ = batch_cost_calculator( + prompt_cost, completion_cost_value = batch_cost_calculator( usage=_batch_cache_usage(), - model="claude-sonnet-4-5-20250929", + model="batch-pricing-fixture", custom_llm_provider="anthropic", model_info=model_info, ) - assert prompt_cost == pytest.approx((1000 * 3e-6 + 8000 * 3e-7 + 2000 * 3e-6) / 2) + write_rate: Final = input_rate if expected_write_rate is None else expected_write_rate + assert prompt_cost == pytest.approx((1000 * input_rate + 8000 * 3e-7 + 2000 * write_rate) / 2) + assert completion_cost_value == pytest.approx(200 * 15e-6 / 2) def test_completion_cost_bills_interactions_api_response(): @@ -7817,20 +7834,42 @@ def test_vertex_ai_mistral_predict_cost(usage): assert predictive_cost > 0 +@pytest.mark.usefixtures("_local_model_cost_map") +@pytest.mark.parametrize("call_type", ("speech", "aspeech")) +@pytest.mark.parametrize("model", ("speech-fixture", "openai/speech-fixture", "azure/speech-fixture")) @pytest.mark.parametrize( - "model", ["openai/tts-1", "azure/tts-1", "openai/gpt-4o-mini-tts"] + ("pricing", "expected_cost"), + ( + pytest.param({}, None, id="missing"), + pytest.param({"input_cost_per_character": None, "input_cost_per_token": None}, None, id="null"), + pytest.param({"input_cost_per_token": 0.0}, 0.0, id="free-tokens"), + pytest.param({"input_cost_per_character": 0.0, "input_cost_per_token": 2e-6}, 0.0, id="free-characters"), + pytest.param({"input_cost_per_token": 2e-6}, 100 * 2e-6, id="missing-character-rate"), + pytest.param({"input_cost_per_character": None, "input_cost_per_token": 2e-6}, 100 * 2e-6, id="null-character-rate"), + pytest.param({"input_cost_per_character": 3e-6, "input_cost_per_token": 2e-6}, 1000 * 3e-6, id="paid-characters"), + ), ) -def test_completion_cost_tts(model): - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") - - cost = completion_cost( - model=model, - prompt="the quick brown fox jumped over the lazy dogs", - call_type="speech", +def test_registered_speech_prices_distinguish_unset_from_zero( + call_type: str, model: str, pricing: ModelInfo, expected_cost: float | None +) -> None: + provider: Final = "azure" if model.startswith("azure/") else "openai" + litellm.register_model( + {model: {"litellm_provider": provider, "mode": "audio_speech", **pricing}}, + persist_across_reloads=False, ) + compatibility_info: Final = cached_get_model_info_helper(model, provider) + assert compatibility_info["input_cost_per_token"] == (pricing.get("input_cost_per_token") or 0.0) + litellm.get_model_info(model) + response: Final = ModelResponse(usage=Usage(prompt_tokens=100, completion_tokens=0, total_tokens=100)) - assert cost > 0 + if expected_cost is None: + with pytest.raises(ValueError, match="does not have 'input_cost_per_character' or 'input_cost_per_token'"): + completion_cost(completion_response=response, model=model, prompt="a" * 1000, call_type=call_type) + litellm.register_model({model: {"input_cost_per_token": 0.0}}, persist_across_reloads=False) + + assert completion_cost( + completion_response=response, model=model, prompt="a" * 1000, call_type=call_type + ) == pytest.approx(expected_cost or 0.0) def test_completion_cost_anthropic(): diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 24736cdb34e..30d61f181cb 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -35894,6 +35894,8 @@ export interface components { search_context_cost_per_query?: { [key: string]: unknown; } | null; + /** Search Queries Cost Per Query */ + search_queries_cost_per_query?: number | components["schemas"]["SearchContextCostPerQuery"] | null; /** Stream Timeout */ stream_timeout?: number | string | null; /** Stream Timeout By Service Tier */ @@ -45445,6 +45447,17 @@ export interface components { } & { [key: string]: unknown; }; + /** SearchContextCostPerQuery */ + SearchContextCostPerQuery: { + /** Search Context Size High */ + search_context_size_high?: number; + /** Search Context Size Low */ + search_context_size_low?: number; + /** Search Context Size Medium */ + search_context_size_medium?: number; + } & { + [key: string]: unknown; + }; /** * SearchTool * @description Search tool configuration. @@ -51409,6 +51422,8 @@ export interface components { search_context_cost_per_query?: { [key: string]: unknown; } | null; + /** Search Queries Cost Per Query */ + search_queries_cost_per_query?: number | components["schemas"]["SearchContextCostPerQuery"] | null; /** Stream Timeout */ stream_timeout?: number | string | null; /** Stream Timeout By Service Tier */ From 298911d61bded3b81aa7d0face6e8538cb7114ed Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Fri, 30 Oct 2026 15:25:47 -0700 Subject: [PATCH 47/74] fix(batches): cost an unmanaged gs:// batch through the deployment that created it (#49301) * fix(batches): cost an unmanaged gs:// batch through the deployment that created it The create now stores the router deployment id as model_id inside the managed object row's file_object JSON and CheckBatchCost reads it before falling back to the first deployment matching the base model in the gs:// path. Rows without the field keep the old lookup * fix(batches): tolerate a non-string or malformed file_object when reading the stored deployment * fix(batches): read the stored creating deployment from the writer, not a lagging read replica * fix(batches): resolve the writer database once so the type gate count does not grow * fix(batches): decode a dict-shaped file_object when reading the stored deployment, accumulate wire samples, annotate test locals --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy/common_utils/check_batch_cost.py | 31 ++- .../proxy/hooks/managed_files.py | 50 ++++- ...ed_vertex_batch_deployment_routing_wire.py | 181 ++++++++++++++++++ .../proxy/test_managed_files_hook.py | 107 +++++++++++ .../common_utils/test_check_batch_cost.py | 47 +++++ 5 files changed, 408 insertions(+), 8 deletions(-) create mode 100644 tests/integration/spend/test_unmanaged_vertex_batch_deployment_routing_wire.py diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py index a1b4804ae29..a31e926256a 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py @@ -10,6 +10,7 @@ from dataclasses import replace as dataclasses_replace from datetime import datetime, timedelta from types import MappingProxyType from typing import TYPE_CHECKING, Final, List, Literal, Optional, Protocol, Tuple, cast +from pydantic import TypeAdapter, ValidationError from litellm_enterprise.proxy.common_utils.managed_object_attribution import ( ManagedObjectCreatorAttribution, @@ -50,6 +51,7 @@ def _error_names_resource(error: Exception, resource_id: str) -> bool: CHECK_BATCH_COST_USER_AGENT = "LiteLLM Proxy/CheckBatchCost" RECENT_BATCH_POLL_WINDOW: Final = timedelta(hours=48) GIVE_UP_POLLING_AFTER: Final = timedelta(days=2 * MANAGED_OBJECT_STALENESS_CUTOFF_DAYS) +_FILE_OBJECT_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) PROVIDER_TERMINAL_BATCH_STATUSES: Final[Tuple[str, ...]] = ( @@ -550,6 +552,10 @@ class CheckBatchCost: llm_provider: str, bare_model_name: str, ) -> Optional[Tuple[str, str]]: + stored_model_id: Final = self._get_stored_model_id(job) + if stored_model_id is not None and self._batch_deployment_exists(stored_model_id): + return stored_model_id, job.unified_object_id + deployment_id = self._get_deployment_id_for_bare_model(bare_model_name, llm_provider) if deployment_id is None: verbose_proxy_logger.info( @@ -662,23 +668,36 @@ class CheckBatchCost: ) @staticmethod - def _get_input_file_id(job: "ManagedObjectRow") -> Optional[str]: - from litellm.types.utils import LiteLLMBatch - + def _parse_file_object(job: "ManagedObjectRow") -> Optional[dict[str, object]]: file_object: Final = job.file_object try: - parsed_file_object: Final[object] = ( + return _FILE_OBJECT_ADAPTER.validate_python( json.loads(file_object) if isinstance(file_object, str) else file_object ) - except (json.JSONDecodeError, ValueError): + except (json.JSONDecodeError, ValueError, ValidationError): return None - if not isinstance(parsed_file_object, dict): + + @classmethod + def _get_input_file_id(cls, job: "ManagedObjectRow") -> Optional[str]: + from litellm.types.utils import LiteLLMBatch + + parsed_file_object: Final = cls._parse_file_object(job) + if parsed_file_object is None: return None try: return LiteLLMBatch.model_validate(parsed_file_object).input_file_id except Exception: return None + @classmethod + def _get_stored_model_id(cls, job: "ManagedObjectRow") -> Optional[str]: + """The deployment that created the batch, written into file_object by the managed + files hook at create time. Older rows have no such key and fall back to matching + the bare model name against the router.""" + parsed_file_object: Final = cls._parse_file_object(job) + model_id: Final = parsed_file_object.get("model_id") if parsed_file_object else None + return model_id if isinstance(model_id, str) and model_id else None + async def _track_completed_batch_cost( self, job: "ManagedObjectRow", diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index 5b54e7d72a0..0d46efbbb08 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -21,6 +21,7 @@ from typing import ( TypedDict, Union, cast, + runtime_checkable, ) from uuid import NAMESPACE_URL, uuid5 @@ -374,6 +375,12 @@ class _ManagedResourcePrismaClient(Protocol): def db(self) -> _ManagedResourceDatabase: ... +@runtime_checkable +class _WriterRoutedDatabase(Protocol): + @property + def writer(self) -> _ManagedResourceDatabase: ... + + class _SchedulerWithJobLookup(Protocol): def get_job(self, job_id: str) -> object: ... @@ -479,6 +486,11 @@ def _managed_object_table(prisma_client: _ManagedResourcePrismaClient) -> _Manag return prisma_client.db.litellm_managedobjecttable +def _writer_database(db: _ManagedResourceDatabase) -> _ManagedResourceDatabase: + """Reads that feed a write must see the create that just happened; a lagging read replica may not""" + return db.writer if isinstance(db, _WriterRoutedDatabase) else db + + _JSON_VALUE_ADAPTER: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) _POSTGRES_UNSTORABLE_CHARACTER: Final = "\x00" @@ -512,6 +524,25 @@ def _without_model_output( ) +def _with_model_id( + file_object: LiteLLMBatch | LiteLLMFineTuningJob | ResponsesAPIResponse, + model_id: str | None, +) -> LiteLLMBatch | LiteLLMFineTuningJob | ResponsesAPIResponse: + return file_object if model_id is None else file_object.model_copy(update={"model_id": model_id}) + + +async def _stored_model_id(managed_object_table: _ManagedObjectTableActions, unified_object_id: str) -> str | None: + row: Final = await managed_object_table.find_first(where={"unified_object_id": unified_object_id}) + if row is None: + return None + try: + parsed: Final = _JSON_VALUE_ADAPTER.validate_python(_decode_json_blob(row.file_object)) + except (json.JSONDecodeError, ValidationError): + return None + model_id: Final = parsed.get("model_id") if isinstance(parsed, dict) else None + return model_id if isinstance(model_id, str) and model_id else None + + def _storage_metadata_of(file_object: OpenAIFileObject | None) -> Mapping[str, str]: hidden_params: Final = cast( # cast-ok: _hidden_params is an untyped attribute the upload path sets "Mapping[str, object]", getattr(file_object, "_hidden_params", None) or {} @@ -697,9 +728,17 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): persist_attribution: bool = False, create_if_missing: bool = True, batch_processed: bool = False, + creating_model_id: str | None = None, ) -> None: """Persist a managed object row, caching it and upserting it in the DB. + creating_model_id is the router deployment that created the object. It is kept + inside the stored file_object JSON as `model_id`, because the raw provider id an + unmanaged batch stores as unified_object_id says nothing about which deployment + made it, and the cost poller needs that deployment to read the result back. A + later observe of the row (create_if_missing=False) keeps the stored value rather + than overwriting it with whichever deployment served the retrieve. + persist_attribution is set only by the batch and background response creates, the callers that can speak for the creator; it gates the api_key, request_tags, and, for a response, end_user_id columns that the cost poll jobs bill against, so a later poll or @@ -720,7 +759,14 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): It is written only in the upsert create branch. """ verbose_logger.info(f"Storing LiteLLM Managed {file_purpose} object with id={unified_object_id} in cache") - stored_object: Final = _without_model_output(file_object) + managed_database: Final = _managed_database(self.prisma_client) + managed_object_table: Final = managed_database.litellm_managedobjecttable + preserved_model_id: Final = ( + creating_model_id + if creating_model_id is not None or create_if_missing + else await _stored_model_id(_writer_database(managed_database).litellm_managedobjecttable, unified_object_id) + ) + stored_object: Final = _with_model_id(_without_model_output(file_object), preserved_model_id) litellm_managed_object = LiteLLM_ManagedObjectTable( unified_object_id=unified_object_id, model_object_id=model_object_id, @@ -752,7 +798,6 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): "status": file_object.status, "updated_by": user_api_key_dict.user_id, } - managed_object_table: Final = _managed_object_table(self.prisma_client) if not create_if_missing: await managed_object_table.update_many( where={"unified_object_id": unified_object_id}, @@ -2170,6 +2215,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): request_tags=request_tags_from_metadata(request_metadata if isinstance(request_metadata, dict) else {}), persist_attribution=is_batch_create, create_if_missing=is_batch_create, + creating_model_id=model_id if is_batch_create else None, ) if not is_batch_create: await self.store_batch_output_file_ownership( diff --git a/tests/integration/spend/test_unmanaged_vertex_batch_deployment_routing_wire.py b/tests/integration/spend/test_unmanaged_vertex_batch_deployment_routing_wire.py new file mode 100644 index 00000000000..91269e6557f --- /dev/null +++ b/tests/integration/spend/test_unmanaged_vertex_batch_deployment_routing_wire.py @@ -0,0 +1,181 @@ +"""Two Vertex deployments share a base model but live in different projects with different +buckets. An unmanaged gs:// batch created on the second one must be polled through that +deployment (LIT-10326). Before the fix the cost poller matched the base model name in the +gs:// path to the first deployment in the list and asked the wrong project for the job.""" + +import json +import re +from collections.abc import Callable +from contextlib import ExitStack +from pathlib import Path +from typing import Final +from uuid import uuid4 + +import pytest +import yaml +from integration._support.client import JSON_OBJECT, Gateway, eventually, object_value, string_value +from integration._support.process import owned_proxy_process +from integration._support.vertex import service_account_json +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue + +LOCATION: Final = "us-central1" +BASE_MODEL: Final = "gemini-2.5-flash" +TEAMS: Final = ("a", "b") +_JOB_PATH: Final = re.compile( + r"^/v1/projects/(?P[^/]+)/locations/(?P[^/]+)/batchPredictionJobs(?:/(?P\d+))?$" +) + + +def _project(team: str) -> str: + return f"scripted-project-{team}" + + +def _deployment(team: str) -> str: + return f"{BASE_MODEL}-team-{team}" + + +def _input_file(team: str) -> str: + return f"gs://scripted-bucket-{team}/litellm-vertex-files/publishers/google/models/{BASE_MODEL}/{uuid4().hex}/input.jsonl" + + +def _job_number(team: str) -> str: + return f"{TEAMS.index(team) + 1}{uuid4().int % 10**17:017d}" + + +def _vertex_peer(jobs: dict[str, tuple[str, str]]) -> Callable[[Request], Reply]: + """A Vertex double that remembers which project each job was created in. A retrieve + from any other project answers the 404 real Vertex gives (the job does not exist + there), a retrieve from the right one reports the job still running.""" + + def respond(request: Request) -> Reply: + path: Final = _JOB_PATH.match(request.target.split("?", 1)[0]) + assert path is not None, request.target + assert request.headers.get("authorization") == "Bearer scripted-token", request.headers + project, location, job = path.group("project"), path.group("location"), path.group("job") + if request.method == "POST": + team: Final = project.rsplit("-", 1)[-1] + uri: Final = re.search(r'"(gs://[^"]+)"', request.body.decode()) + assert uri is not None and uri.group(1).startswith(f"gs://scripted-bucket-{team}/"), (project, request.body) + created, created_uri = jobs.setdefault(project, (_job_number(team), uri.group(1))) + return Reply(body=json.dumps(_job(project, location, created, created_uri, "JOB_STATE_PENDING")).encode()) + assert request.method == "GET" and job is not None, (request.method, request.target) + known: Final = jobs.get(project) + if known is None or known[0] != job: + return Reply( + status=404, + body=json.dumps( + { + "error": { + "code": 404, + "message": f"The BatchPredictionJob does not exist: {job}", + "status": "NOT_FOUND", + } + } + ).encode(), + ) + return Reply(body=json.dumps(_job(project, location, job, known[1], "JOB_STATE_RUNNING")).encode()) + + return respond + + +def _job(project: str, location: str, job: str, input_uri: str, state: str) -> dict[str, JsonValue]: + return { + "name": f"projects/{project}/locations/{location}/batchPredictionJobs/{job}", + "displayName": "litellm-vertex-batch-scripted", + "model": f"publishers/google/models/{BASE_MODEL}", + "inputConfig": {"gcsSource": {"uris": [input_uri]}, "instancesFormat": "jsonl"}, + "outputConfig": {"predictionsFormat": "jsonl", "gcsDestination": {"outputUriPrefix": input_uri.rsplit("/", 1)[0]}}, + "state": state, + "createTime": "2026-10-30T20:00:00.000000Z", + "updateTime": "2026-10-30T20:00:00.000000Z", + } + + +def _owned_config(path: Path, gateway: Gateway, wire_url: str) -> Path: + config: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + path.write_text( + yaml.safe_dump( + { + **config, + "model_list": [ + { + "model_name": _deployment(team), + "litellm_params": { + "model": f"vertex_ai/{BASE_MODEL}", + "api_base": wire_url, + "vertex_project": _project(team), + "vertex_location": LOCATION, + "vertex_credentials": service_account_json(_project(team), gateway.upstream_url.rstrip("/")), + "gcs_bucket_name": f"scripted-bucket-{team}", + }, + } + for team in TEAMS + ], + "general_settings": { + **object_value(config.get("general_settings") or {}), + "track_unmanaged_batch_cost": True, + }, + "litellm_settings": { + **object_value(config.get("litellm_settings") or {}), + "enable_loadbalancing_on_batch_endpoints": True, + "user_url_allowed_hosts": ["127.0.0.1"], + }, + } + ) + ) + return path + + +def _create_batch(gateway: Gateway, key: str, team: str) -> str: + response: Final = gateway.request( + "POST", + "/v1/batches", + { + "input_file_id": _input_file(team), + "endpoint": "/v1/chat/completions", + "completion_window": "24h", + "model": _deployment(team), + }, + key=key, + ) + assert response.status_code == 200, response.text + return string_value(JSON_OBJECT.validate_json(response.content)["id"]) + + +def _retrieve_targets(requests: tuple[Request, ...], job: str) -> tuple[str, ...]: + return tuple(request.target for request in requests if request.method == "GET" and request.target.endswith(f"/{job}")) + + +@pytest.mark.timeout(240) +def test_unmanaged_batch_on_the_second_deployment_sharing_a_base_model_is_polled_through_its_own_project( + gateway: Gateway, tmp_path: Path +) -> None: + jobs: Final[dict[str, tuple[str, str]]] = {} # mutable-ok: the wire double records which project created each job + with ExitStack() as stack: + wire: Final = stack.enter_context(wire_server(_vertex_peer(jobs))) + config: Final = _owned_config(tmp_path / "two-vertex-teams.yaml", gateway, wire.url) + owned: Final = stack.enter_context(owned_proxy_process(gateway, tmp_path, {}, config=config)) + scenario: Final = stack.enter_context(owned.gateway.scenario()) + key: Final = scenario.key(models=[_deployment(team) for team in TEAMS]) + + batch_a: Final = _create_batch(owned.gateway, key, "a") + batch_b: Final = _create_batch(owned.gateway, key, "b") + assert (batch_a, batch_b) == (jobs[_project("a")][0], jobs[_project("b")][0]), (batch_a, batch_b, jobs) + + seen: Final[list[Request]] = [] # mutable-ok: drain empties the wire queue, so samples accumulate across retries + + def polled_so_far() -> tuple[Request, ...]: + seen.extend(wire.drain()) + return tuple(seen) + + polled: Final = eventually( + polled_so_far, + lambda requests: bool(_retrieve_targets(requests, batch_b)) and bool(_retrieve_targets(requests, batch_a)), + seconds=90, + ) + for team, batch in (("a", batch_a), ("b", batch_b)): + assert all( + target.startswith(f"/v1/projects/{_project(team)}/") for target in _retrieve_targets(polled, batch) + ), (team, _retrieve_targets(polled, batch)) + diff --git a/tests/unit/enterprise/proxy/test_managed_files_hook.py b/tests/unit/enterprise/proxy/test_managed_files_hook.py index f2df14cc8a3..89fff365c3f 100644 --- a/tests/unit/enterprise/proxy/test_managed_files_hook.py +++ b/tests/unit/enterprise/proxy/test_managed_files_hook.py @@ -1437,6 +1437,8 @@ def _make_object_store_instance(): mock_prisma = MagicMock() mock_prisma.db.litellm_managedobjecttable.upsert = AsyncMock() mock_prisma.db.litellm_managedobjecttable.update_many = AsyncMock() + mock_prisma.db.litellm_managedobjecttable.find_first = AsyncMock(return_value=None) + mock_prisma.db.writer = mock_prisma.db return ( PROXY_LiteLLMManagedFiles( @@ -2161,3 +2163,108 @@ async def test_store_unified_file_id_caches_the_storage_location_the_db_row_gets assert (cached.storage_backend, cached.storage_url) == ("litellm_db", "litellm_db://content-row-1") create_data = file_table.upsert.await_args.kwargs["data"]["create"] assert (create_data["storage_backend"], create_data["storage_url"]) == ("litellm_db", "litellm_db://content-row-1") + + +@pytest.mark.asyncio +async def test_create_stores_the_creating_deployment_in_file_object(): + """Regression (LIT-10326): an unmanaged gs:// batch stores the raw Vertex job id, which + says nothing about which of two deployments sharing a base model created it. The create + must keep the router's deployment id inside the stored file_object so the cost poller + reads the result through that deployment instead of the first base-model match.""" + managed_files, mock_prisma = _make_object_store_instance() + creator: Final = UserAPIKeyAuth(api_key="sk-team-b", user_id="bob", team_id="team-b", parent_otel_span=None) + + await managed_files.store_unified_object_id( + unified_object_id="projects/123/locations/us/batchPredictionJobs/456", + file_object=_make_batch_response(status="validating"), + litellm_parent_otel_span=None, + model_object_id="projects/123/locations/us/batchPredictionJobs/456", + file_purpose="batch", + user_api_key_dict=creator, + persist_attribution=True, + create_if_missing=True, + creating_model_id="deploy-team-b", + ) + + created: Final = mock_prisma.db.litellm_managedobjecttable.upsert.await_args.kwargs["data"]["create"] + assert json.loads(created["file_object"])["model_id"] == "deploy-team-b" + + +@pytest.mark.asyncio +async def test_poll_refresh_keeps_the_stored_creating_deployment(): + """A later observe of the row (a GET through whichever deployment the client named) + rewrites file_object; it must carry the create-time model_id forward, not drop it or + replace it with the deployment that served the retrieve.""" + managed_files, mock_prisma = _make_object_store_instance() + stored_row: Final = MagicMock() + stored_row.file_object = json.dumps({**json.loads(_make_batch_response(status="validating").model_dump_json()), "model_id": "deploy-team-b"}) + mock_prisma.db.litellm_managedobjecttable.find_first = AsyncMock(return_value=stored_row) + observer: Final = UserAPIKeyAuth(api_key="sk-team-b", user_id="bob", team_id="team-b", parent_otel_span=None) + + await managed_files.store_unified_object_id( + unified_object_id="projects/123/locations/us/batchPredictionJobs/456", + file_object=_make_batch_response(status="completed"), + litellm_parent_otel_span=None, + model_object_id="projects/123/locations/us/batchPredictionJobs/456", + file_purpose="batch", + user_api_key_dict=observer, + persist_attribution=False, + create_if_missing=False, + ) + + written: Final = json.loads(mock_prisma.db.litellm_managedobjecttable.update_many.await_args.kwargs["data"]["file_object"]) + assert written["model_id"] == "deploy-team-b" + assert written["status"] == "completed" + + +@pytest.mark.asyncio +async def test_poll_refresh_reads_the_stored_creating_deployment_from_the_writer(): + """With a read replica configured, a retrieve that lands before the replica has the create + must still find the create-time model_id, so the lookup goes to the writer. Prisma may hand + the Json column back already decoded, so the row here is a dict rather than JSON text.""" + managed_files, mock_prisma = _make_object_store_instance() + stored_row: Final = MagicMock() + stored_row.file_object = {**json.loads(_make_batch_response(status="validating").model_dump_json()), "model_id": "deploy-team-b"} + mock_prisma.db.litellm_managedobjecttable.find_first = AsyncMock(return_value=None) + mock_prisma.db.writer = MagicMock() + mock_prisma.db.writer.litellm_managedobjecttable.find_first = AsyncMock(return_value=stored_row) + observer: Final = UserAPIKeyAuth(api_key="sk-team-b", user_id="bob", team_id="team-b", parent_otel_span=None) + + await managed_files.store_unified_object_id( + unified_object_id="projects/123/locations/us/batchPredictionJobs/456", + file_object=_make_batch_response(status="completed"), + litellm_parent_otel_span=None, + model_object_id="projects/123/locations/us/batchPredictionJobs/456", + file_purpose="batch", + user_api_key_dict=observer, + persist_attribution=False, + create_if_missing=False, + ) + + written: Final = json.loads(mock_prisma.db.litellm_managedobjecttable.update_many.await_args.kwargs["data"]["file_object"]) + assert written["model_id"] == "deploy-team-b" + mock_prisma.db.litellm_managedobjecttable.find_first.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_batch_create_hook_passes_the_router_deployment_as_creating_model_id(): + """The post-call hook on a batch create hands the router's model_id to the object store; + a retrieve of the same batch hands None so it cannot overwrite the creator.""" + from litellm_enterprise.proxy.hooks.managed_files import BATCH_CREATE_HIDDEN_PARAM + + managed_files: Final = _make_managed_files_instance() + created: Final = _make_batch_response(model_id="deploy-team-b", output_file_id=None, status="validating") + created._hidden_params[BATCH_CREATE_HIDDEN_PARAM] = True + retrieved: Final = _make_batch_response(model_id="deploy-team-a", output_file_id=None, status="in_progress") + + with patch("litellm.proxy.proxy_server.llm_router", MagicMock()): + await managed_files.async_post_call_success_hook( + data={}, user_api_key_dict=_make_user_api_key_dict(), response=created + ) + await managed_files.async_post_call_success_hook( + data={}, user_api_key_dict=_make_user_api_key_dict(), response=retrieved + ) + + first, second = managed_files.store_unified_object_id.await_args_list + assert first.kwargs["creating_model_id"] == "deploy-team-b" + assert second.kwargs["creating_model_id"] is None diff --git a/tests/unit/proxy/common_utils/test_check_batch_cost.py b/tests/unit/proxy/common_utils/test_check_batch_cost.py index 4861b2687de..5b37cee88c5 100644 --- a/tests/unit/proxy/common_utils/test_check_batch_cost.py +++ b/tests/unit/proxy/common_utils/test_check_batch_cost.py @@ -1771,6 +1771,53 @@ class TestUnmanagedVertexRouting: assert result == ("deploy-ft", "8823717160934178816") + def _two_team_deployments(self): + """Two Vertex deployments sharing the base model, team A listed first.""" + return [ + { + "model_name": f"gemini-2.5-flash-team-{team}", + "litellm_params": {"model": "vertex_ai/gemini-2.5-flash", "custom_llm_provider": "vertex_ai"}, + "model_info": {"id": f"deploy-team-{team}"}, + } + for team in ("a", "b") + ] + + def test_flag_on_prefers_the_deployment_stored_on_the_row_over_the_first_base_model_match(self): + """Regression (LIT-10326): two deployments share gemini-2.5-flash with different + buckets. A batch created on team B's deployment stores deploy-team-b in file_object, + so the poller must read it back through team B, not the first match (team A).""" + router: Final = MagicMock() + router.registry.return_value.resolve.return_value = None + router.registry.return_value.deployments.return_value = self._two_team_deployments() + router.registry.return_value.deployment = MagicMock(return_value=self._vertex_deployment()) + instance: Final = self._instance(track_unmanaged=True, router=router) + job: Final = self._job( + file_object=json.dumps({**json.loads(_unmanaged_vertex_file_object()), "model_id": "deploy-team-b"}) + ) + + with patch(_IS_B64, return_value=False): + result: Final = instance._resolve_job_routing(job, MagicMock()) + + assert result == ("deploy-team-b", "8823717160934178816") + router.registry.return_value.deployment.assert_called_once_with("deploy-team-b") + + def test_flag_on_falls_back_to_base_model_match_when_the_stored_deployment_is_gone(self): + """The stored deployment was removed from the config after the batch was created: + the row still routes through today's base-model lookup instead of being skipped.""" + router: Final = MagicMock() + router.registry.return_value.resolve.return_value = None + router.registry.return_value.deployments.return_value = self._two_team_deployments() + router.registry.return_value.deployment = MagicMock(return_value=None) + instance: Final = self._instance(track_unmanaged=True, router=router) + job: Final = self._job( + file_object=json.dumps({**json.loads(_unmanaged_vertex_file_object()), "model_id": "deploy-removed"}) + ) + + with patch(_IS_B64, return_value=False): + result: Final = instance._resolve_job_routing(job, MagicMock()) + + assert result == ("deploy-team-a", "8823717160934178816") + def test_flag_on_skips_non_vertex_deployment_sharing_model_group(self): """Flag on, but the only deployment for the model group is a non-vertex_ai provider: must not be selected, even though the model group name matches.""" From 991ea7f7e8419feb9be44c1aa67fdc95289c1762 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 30 Oct 2026 15:47:06 -0700 Subject: [PATCH 48/74] refactor(ui): move policies, tool policies and guardrails monitor text onto i18n (#49343) Moves the page-authored English on /policies, /tool-policies and /guardrails-monitor into react-i18next t() calls backed by en.json, and adds a small data test for en.json. The rendered English is unchanged. --- .../_components/EvaluationSettingsModal.tsx | 36 +- .../_components/GuardrailDetail.tsx | 46 +- .../_components/GuardrailUsageBreakdown.tsx | 111 ++- .../_components/GuardrailsOverview.tsx | 91 +- .../_components/ScoreChart.tsx | 6 +- .../(dashboard)/guardrails-monitor/page.tsx | 4 +- .../policies/_components/AttachmentTable.tsx | 15 +- .../_components/AttachmentTableColumns.tsx | 76 +- .../policies/_components/PolicyTable.tsx | 15 +- .../_components/PolicyTableColumns.tsx | 65 +- .../policies/_components/TokenSelect.tsx | 6 +- .../_components/add_attachment_form.tsx | 240 ++--- .../policies/_components/add_policy_form.tsx | 215 +++-- .../_components/ai_suggestion_modal.tsx | 154 +-- .../_components/guardrail_selection_modal.tsx | 78 +- .../policies/_components/impact_popover.tsx | 28 +- .../_components/impact_preview_alert.tsx | 58 +- .../policies/_components/index.tsx | 157 +-- .../_components/pipeline_flow_builder.tsx | 530 +++++----- .../policies/_components/policy_info.tsx | 56 +- .../policies/_components/policy_templates.tsx | 31 +- .../_components/policy_test_panel.tsx | 99 +- .../_components/template_parameter_modal.tsx | 61 +- .../GuardrailsMonitor/CalcPopover.tsx | 7 +- .../GuardrailsMonitor/LogViewer.tsx | 38 +- .../GuardrailsMonitor/UnpricedNote.tsx | 26 +- .../GuardrailsMonitor/usageUnits.ts | 21 +- .../src/components/ToolDetail.tsx | 86 +- .../ToolPolicies/ToolPoliciesPanel.tsx | 51 +- .../ToolPolicies/ToolPoliciesTable.tsx | 82 +- .../ToolPolicies/ToolPoliciesTableColumns.tsx | 41 +- .../src/components/ToolPoliciesView.tsx | 6 +- .../policies/useDeletePolicyAttachment.ts | 6 +- ui/litellm-dashboard/src/i18n/locales.test.ts | 54 ++ ui/litellm-dashboard/src/i18n/locales/en.json | 911 ++++++++++++++++++ 35 files changed, 2455 insertions(+), 1052 deletions(-) create mode 100644 ui/litellm-dashboard/src/i18n/locales.test.ts diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/EvaluationSettingsModal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/EvaluationSettingsModal.tsx index 78091584cde..12ff10bc0c5 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/EvaluationSettingsModal.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/EvaluationSettingsModal.tsx @@ -1,6 +1,7 @@ import { useTerminology } from "@/contexts/TerminologyContext"; import { Play } from "lucide-react"; import React, { useEffect, useMemo, useState } from "react"; +import { useTranslation } from "react-i18next"; import { fetchAvailableModels, type ModelGroup } from "@/components/llm_calls/fetch_models"; import { SearchSelect } from "@/components/shared/SearchSelect"; import { Button } from "@/components/ui/button"; @@ -48,6 +49,7 @@ export function EvaluationSettingsModal({ accessToken, onRunEvaluation, }: EvaluationSettingsModalProps) { + const { t } = useTranslation(); const term = useTerminology(); const [prompt, setPrompt] = useState(DEFAULT_PROMPT); const [schema, setSchema] = useState(DEFAULT_SCHEMA); @@ -94,11 +96,11 @@ export function EvaluationSettingsModal({ !nextOpen && onClose()}> - Evaluation Settings + {t("guardrailsMonitor.evaluationSettings.title")} {guardrailName - ? `${term("Configure AI evaluation for")} ${guardrailName}` - : term("Configure AI evaluation for re-running on logs")} + ? `${term(t("guardrailsMonitor.evaluationSettings.descriptionPrefix"))} ${guardrailName}` + : term(t("guardrailsMonitor.evaluationSettings.descriptionGeneric"))} @@ -106,10 +108,10 @@ export function EvaluationSettingsModal({