From 5d69250cd04eee307c56990364ab0d2406bf40db Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 6 Nov 2026 08:53:19 -0800 Subject: [PATCH 01/39] fix(model-map): add verified Mistral retirement dates (#50294) * feat(gemini): add file-processing EAP catalog estimates * fix(model-map): add verified Mistral retirement dates Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 5 +++++ model_prices_and_context_window.json | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index aa358f4f6f3..4e81afb0ea6 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -44089,6 +44089,7 @@ "supports_vision": true }, "mistral/mistral-large-2512": { + "deprecation_date": "2027-04-30", "cache_read_input_token_cost": 5e-08, "input_cost_per_token": 5e-07, "litellm_provider": "mistral", @@ -44164,6 +44165,7 @@ "supports_vision": true }, "mistral/mistral-medium-3-5": { + "deprecation_date": "2027-04-30", "cache_read_input_token_cost": 1.5e-07, "input_cost_per_token": 1.5e-06, "litellm_provider": "mistral", @@ -64726,6 +64728,7 @@ "source": "https://docs.mistral.ai/models/model-cards/mistral-moderation-26-03" }, "mistral/voxtral-mini-2602": { + "deprecation_date": "2027-04-30", "input_cost_per_second": 5e-05, "litellm_provider": "mistral", "mode": "audio_transcription", @@ -64742,6 +64745,7 @@ "supports_audio_input": true }, "mistral/voxtral-mini-transcribe-realtime-2602": { + "deprecation_date": "2027-04-30", "input_cost_per_second": 0.0001, "litellm_provider": "mistral", "mode": "audio_transcription", @@ -64758,6 +64762,7 @@ "supports_audio_input": true }, "mistral/voxtral-mini-tts-2603": { + "deprecation_date": "2027-04-30", "input_cost_per_character": 1.6e-05, "litellm_provider": "mistral", "mode": "audio_speech", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index aa358f4f6f3..4e81afb0ea6 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -44089,6 +44089,7 @@ "supports_vision": true }, "mistral/mistral-large-2512": { + "deprecation_date": "2027-04-30", "cache_read_input_token_cost": 5e-08, "input_cost_per_token": 5e-07, "litellm_provider": "mistral", @@ -44164,6 +44165,7 @@ "supports_vision": true }, "mistral/mistral-medium-3-5": { + "deprecation_date": "2027-04-30", "cache_read_input_token_cost": 1.5e-07, "input_cost_per_token": 1.5e-06, "litellm_provider": "mistral", @@ -64726,6 +64728,7 @@ "source": "https://docs.mistral.ai/models/model-cards/mistral-moderation-26-03" }, "mistral/voxtral-mini-2602": { + "deprecation_date": "2027-04-30", "input_cost_per_second": 5e-05, "litellm_provider": "mistral", "mode": "audio_transcription", @@ -64742,6 +64745,7 @@ "supports_audio_input": true }, "mistral/voxtral-mini-transcribe-realtime-2602": { + "deprecation_date": "2027-04-30", "input_cost_per_second": 0.0001, "litellm_provider": "mistral", "mode": "audio_transcription", @@ -64758,6 +64762,7 @@ "supports_audio_input": true }, "mistral/voxtral-mini-tts-2603": { + "deprecation_date": "2027-04-30", "input_cost_per_character": 1.6e-05, "litellm_provider": "mistral", "mode": "audio_speech", From ddd162439f73630bb9c9dfcc07fe94d3ec4e2037 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Fri, 6 Nov 2026 08:55:44 -0800 Subject: [PATCH 02/39] fix(proxy): stop /tag/update 500 when the body carries the tag's budget_id (#50324) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/management_helpers/utils.py | 2 +- .../management/test_tag_budget_update.py | 20 +++++++++++++ .../test_management_helpers_utils.py | 29 +++++++++++++++++++ 3 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 tests/integration/management/test_tag_budget_update.py diff --git a/litellm/proxy/management_helpers/utils.py b/litellm/proxy/management_helpers/utils.py index ea19befccc6..1c4eada67fb 100644 --- a/litellm/proxy/management_helpers/utils.py +++ b/litellm/proxy/management_helpers/utils.py @@ -217,7 +217,7 @@ async def handle_budget_for_entity( ) # Get all budget field names - budget_params: Final = LiteLLM_BudgetTable.model_fields.keys() + budget_params: Final = LiteLLM_BudgetTable.model_fields.keys() - {"budget_id"} # Extract budget fields from data _json_data: Final = _BUDGET_DATA_MAPPING.validate_python( diff --git a/tests/integration/management/test_tag_budget_update.py b/tests/integration/management/test_tag_budget_update.py new file mode 100644 index 00000000000..c8ce1bdc8f4 --- /dev/null +++ b/tests/integration/management/test_tag_budget_update.py @@ -0,0 +1,20 @@ +from typing import Final +from uuid import uuid4 + +from tests.integration._support.client import Gateway, object_value, string_value + + +def test_tag_budget_can_be_updated_with_its_existing_budget_id(gateway: Gateway) -> None: + name: Final = f"integration-tag-{uuid4().hex}" + gateway.post("/tag/new", {"name": name, "max_budget": 10}) + with gateway.scenario() as scenario: + scenario.cleanups.callback(gateway.post, "/tag/delete", {"name": name}) + before: Final = object_value(gateway.post("/tag/info", {"names": [name]})[name]) + budget_id: Final = string_value(object_value(before["litellm_budget_table"])["budget_id"]) + scenario.cleanups.callback(gateway.post, "/budget/delete", {"id": budget_id}) + response: Final = gateway.request( + "POST", "/tag/update", {"name": name, "budget_id": budget_id, "max_budget": 20} + ) + assert response.status_code == 200, response.text + after: Final = object_value(gateway.post("/tag/info", {"names": [name]})[name]) + assert object_value(after["litellm_budget_table"])["max_budget"] == 20, after diff --git a/tests/unit/proxy/management_helpers/test_management_helpers_utils.py b/tests/unit/proxy/management_helpers/test_management_helpers_utils.py index 644f0656c7a..640e17aa58e 100644 --- a/tests/unit/proxy/management_helpers/test_management_helpers_utils.py +++ b/tests/unit/proxy/management_helpers/test_management_helpers_utils.py @@ -249,6 +249,35 @@ async def test_add_new_member_links_default_team_budget_id(): assert create_data["budget_id"] == test_default_budget_id +@pytest.mark.asyncio +async def test_handle_budget_for_entity_updates_with_existing_budget_id_in_request() -> None: + from litellm.proxy.management_helpers.utils import handle_budget_for_entity + from litellm.types.tag_management import TagUpdateRequest + + budget_update: Final = AsyncMock() + prisma_client: Final = cast( + PrismaClient, + SimpleNamespace(db=SimpleNamespace(litellm_budgettable=SimpleNamespace(update=budget_update))), + ) + with ( + patch("litellm.proxy.proxy_server.prisma_client", prisma_client), + patch("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin"), + ): + budget_id: Final = await handle_budget_for_entity( + data=TagUpdateRequest(name="budget-tag", budget_id="budget-1", max_budget=20), + existing_budget_id="budget-1", + user_api_key_dict=UserAPIKeyAuth(user_id="admin"), + prisma_client=prisma_client, + litellm_proxy_admin_name="admin", + ) + + assert budget_id == "budget-1" + budget_update.assert_awaited_once_with( + where={"budget_id": "budget-1"}, + data={"budget_id": "budget-1", "max_budget": 20.0, "updated_by": "admin"}, + ) + + @pytest.mark.parametrize( ("request_fields", "cleared_budget_fields", "should_update", "expected_max_budget"), cast( From 6e66b8281f33bd18821440e34f7b4bdff640ba51 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 6 Nov 2026 09:33:52 -0800 Subject: [PATCH 03/39] test(cost): sync Azure GPT flex price fields with the real cost map (#50410) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../integration/cost_calculation/cost_map.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/integration/cost_calculation/cost_map.py b/tests/integration/cost_calculation/cost_map.py index 8f11b3ce2f1..ffdb6752bf2 100644 --- a/tests/integration/cost_calculation/cost_map.py +++ b/tests/integration/cost_calculation/cost_map.py @@ -1603,32 +1603,44 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = { }, "azure/gpt-6-luna": { "cache_read_input_token_cost": 1.3e-08, + "cache_read_input_token_cost_flex": 6.5e-09, "cache_creation_input_token_cost": 1.625e-07, + "cache_creation_input_token_cost_flex": 8.125e-08, "cache_creation_input_token_cost_above_272k_tokens": 3.25e-07, + "cache_creation_input_token_cost_above_272k_tokens_flex": 1.625e-07, "input_cost_per_token": 1.3e-07, + "input_cost_per_token_flex": 6.5e-08, "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6.5e-07, + "output_cost_per_token_flex": 3.25e-07, "supports_function_calling": True, "supports_none_reasoning_effort": True, "input_cost_per_token_above_272k_tokens": 2.6e-07, + "input_cost_per_token_above_272k_tokens_flex": 1.3e-07, "output_cost_per_token_above_272k_tokens": 1.3e-06, + "output_cost_per_token_above_272k_tokens_flex": 4.875e-07, "cache_read_input_token_cost_above_272k_tokens": 2.6e-08, + "cache_read_input_token_cost_above_272k_tokens_flex": 1.3e-08, }, "azure/gpt-6-astra": { "cache_read_input_token_cost": 1.25e-06, + "cache_read_input_token_cost_flex": 6.25e-07, "cache_creation_input_token_cost": 1.5625e-05, + "cache_creation_input_token_cost_flex": 7.8125e-06, "cache_creation_input_token_cost_above_272k_tokens": 3.125e-05, "input_cost_per_token": 1.25e-05, + "input_cost_per_token_flex": 6.25e-06, "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 6.25e-05, + "output_cost_per_token_flex": 3.125e-05, "supports_function_calling": True, "supports_none_reasoning_effort": True, "input_cost_per_token_above_272k_tokens": 2.5e-05, @@ -1637,20 +1649,28 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = { }, "azure/gpt-6.1-sol": { "cache_read_input_token_cost": 2.4e-07, + "cache_read_input_token_cost_flex": 6e-08, "cache_creation_input_token_cost": 3e-06, + "cache_creation_input_token_cost_flex": 1.5e-06, "cache_creation_input_token_cost_above_272k_tokens": 6e-06, + "cache_creation_input_token_cost_above_272k_tokens_flex": 3e-06, "input_cost_per_token": 2.4e-06, + "input_cost_per_token_flex": 1.2e-06, "litellm_provider": "azure", "max_input_tokens": 922000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.2e-05, + "output_cost_per_token_flex": 6e-06, "supports_function_calling": True, "supports_none_reasoning_effort": True, "input_cost_per_token_above_272k_tokens": 4.8e-06, + "input_cost_per_token_above_272k_tokens_flex": 2.4e-06, "output_cost_per_token_above_272k_tokens": 2.4e-05, + "output_cost_per_token_above_272k_tokens_flex": 9e-06, "cache_read_input_token_cost_above_272k_tokens": 4.8e-07, + "cache_read_input_token_cost_above_272k_tokens_flex": 1.2e-07, }, } From 36c64ce85d1abb6db3a4c755d7c8af701e6d259e Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 6 Nov 2026 09:38:00 -0800 Subject: [PATCH 04/39] feat(bedrock): default to native structured output on Opus 5.5, Fable 5.1, Sonnet 5 and Haiku 5.5 (#50221) * feat(bedrock): enable native structured output by default for selected Claude models Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(bedrock): keep govcloud and bare model rows off native output Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(bedrock): retry rejected native schemas with the JSON tool on default-native models Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(bedrock): keep the rejection retry on the unforceable-tool native path Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(bedrock): retry rejected schemas on the Messages path for default-native models Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(bedrock): format Messages native output retry Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(bedrock): allow profile-only native output metadata Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(bedrock): only disable default-native output on schema-rejection retries Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(bedrock): type fixture parameters in native output retry tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(bedrock): retry with the JSON tool when Bedrock rejects a schema's grammar size Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(bedrock): drop strict tools after a native-output retry hits the grammar limit Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(bedrock): format native output fallback Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(bedrock): honor the deployment native output opt-out and keep wide batch schemas on the JSON tool A deployment's supports_bedrock_native_structured_output: false now keeps the JSON tool on rows that default to native output. Bedrock batch has no retry, so a record whose schema has more than 64 properties keeps the JSON tool there too. * fix(bedrock): retry a native structured output stream as a buffered stream when Bedrock denies the stream action A role allowed only bedrock:InvokeModel got a 403 on schema streams once these models went native, since native streams call converse-stream while the JSON tool path buffered through converse. The retry now sends that request again as a buffered JSON tool stream, the shape the base sent. The wire test moves its neither-key and opt-in cells to Opus 4.7, its Bedrock-key-only cells to Sonnet 5.5, and its unforceable-tool cells to the bare Opus 5.5 id, since this PR makes Opus 5.5 and Haiku 5.5 always native. The batch wire test now checks a thinking schema record on both sides of the contract the batch unit test already pins, the JSON tool on opt-in Sonnet 5.5 and native output on always-native Haiku 5.5, since its Haiku 5.5 and Opus 4.8 cells assumed those models were opt-in. * fix(bedrock): move the Claude 5 native output default to a key released proxies do not read Released proxies load the cost map from main and have none of the native output retries, so setting supports_native_structured_output on the Opus 5.5, Fable 5.1, Sonnet 5 and Haiku 5.5 profile rows turned working wide or min/max schemas into 400s on v1.108.1. Those rows keep supports_native_structured_output false and gain bedrock_native_structured_output_by_default true, which get_model_info reports as supports_native_structured_output, so this version behaves as before while released versions keep the JSON tool. * fix(bedrock): rename the Claude 5 native output default key to supports_bedrock_native_structured_output_by_default The cost map guard runs main's model catalog on the PR's cost map, and that catalog rejects any undeclared key that does not start with supports_. Declaring the old name on main first needs its own PR, so the key takes the supports_ prefix. Released proxies still never read it * fix(bedrock): honor registered and batch opt-outs, move Opus 5 and 4.8 to the default key A registered supports_bedrock_native_structured_output false now beats the native output default, and batch uploads read the deployment's model_info, so an opted-out deployment keeps the JSON tool on every record. The Opus 5 and Opus 4.8 profile rows move to supports_bedrock_native_structured_output_by_default, so released proxies on main's cost map keep the JSON tool on them. The 64 property batch cutoff now applies only to rows with that key, so models that were already native keep their batch output shape. * fix(proxy): report a deployment's Bedrock native output opt-out on /model/info /model/info merged the deployment's model_info over the backend row, so a deployment that set only supports_bedrock_native_structured_output: false still showed the backend's derived supports_native_structured_output: true. The generic flag is now derived from the merged info unless the deployment set it itself, through the same helper get_model_info uses. * fix(bedrock): keep native output when a denied schema stream is resent buffered A role without bedrock:InvokeModelWithResponseStream had its native output schema stream resent as a buffered JSON tool call. The resend now keeps the native params with fake_stream set, and falls back to the JSON tool only if Bedrock then rejects native output. * fix(bedrock): drop rejected strict tools on the JSON tool retry after a buffered native resend * fix(bedrock): send a buffered schema stream to /invoke and keep the by-default key on its deployment The shared HTTP handler now builds the non-streaming URL for a fake-streamed request, so the invoke route's buffered resend after a denied stream calls /invoke instead of invoke-with-response-stream. supports_bedrock_native_structured_output_by_default is deployment scoped, so setting it on one deployment no longer reaches sibling deployments of the same model * fix(bedrock): answer a buffered invoke stream from the non-streaming reply * fix(bedrock): keep the JSON tool retry buffered after a denied stream's schema is rejected * fix(bedrock): count only a true fake_stream and keep the deployment by-default key out of /model/info A fake_stream sent as "true" or 1 picked the non-streaming URL but kept the streaming reader, so Bedrock invoke streams answered 500 after three billed calls. Only a real True counts now, as on main. /model/info read supports_bedrock_native_structured_output_by_default from the deployment, while routing reads it only from the cost map, so it reported native output for deployments that use the JSON tool. --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../bedrock/chat/converse_transformation.py | 3 +- .../bedrock/chat/native_output_fallback.py | 157 +++++++++++++---- litellm/llms/bedrock/common_utils.py | 37 +++- litellm/llms/bedrock/files/transformation.py | 48 +++++- litellm/llms/custom_httpx/llm_http_handler.py | 8 +- litellm/main.py | 33 +++- ...odel_prices_and_context_window_backup.json | 57 +++++-- litellm/proxy/proxy_server.py | 21 ++- litellm/types/utils.py | 2 + litellm/utils.py | 12 +- model_prices_and_context_window.json | 57 +++++-- .../test_bedrock_batch_files_wire.py | 18 +- ...t_bedrock_native_structured_output_wire.py | 15 +- tests/unit/conftest.py | 5 +- ...mations_anthropic_claude_transformation.py | 14 +- .../chat/test_bedrock_converse_handler.py | 133 +++++++++++++++ .../chat/test_converse_transformation.py | 109 +++++++++--- .../chat/test_native_output_fallback.py | 116 +++++++++++++ .../test_bedrock_files_transformation.py | 72 +++++++- .../llms/bedrock/test_bedrock_common_utils.py | 158 +++++++----------- .../custom_httpx/test_llm_http_handler.py | 86 ++++++++++ .../proxy_server/test_routes_model_info.py | 45 +++++ .../unit/test_router_model_cost_isolation.py | 8 +- tests/unit/test_utils.py | 1 + tests/unit/test_utils_get_model_info.py | 25 ++- 25 files changed, 1024 insertions(+), 216 deletions(-) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index b93d40738ea..12c5031628b 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -960,7 +960,8 @@ class AmazonConverseConfig(BaseConfig): ) -> bool: """Check if the Bedrock model takes native structured outputs (outputConfig.textFormat) for this request. - ``supports_native_structured_output`` in ``litellm.model_cost`` turns it on for every request, and + ``supports_native_structured_output`` or ``supports_bedrock_native_structured_output_by_default`` in + ``litellm.model_cost`` turns it on for every request, and ``supports_bedrock_native_structured_output`` (the deployment's ``model_info`` first) also when Claude will think, which the json tool path would drop. When the request does not ask for thinking, a model that cannot force the json tool gets it unless that key is false. diff --git a/litellm/llms/bedrock/chat/native_output_fallback.py b/litellm/llms/bedrock/chat/native_output_fallback.py index 79742d0810c..7d5662e2024 100644 --- a/litellm/llms/bedrock/chat/native_output_fallback.py +++ b/litellm/llms/bedrock/chat/native_output_fallback.py @@ -9,7 +9,9 @@ from litellm.llms.bedrock.common_utils import ( NativeOutputFallback, bedrock_strict_tools_fallback, is_bedrock_native_output_rejection, + is_bedrock_stream_permission_denial, warn_bedrock_native_output_rejected, + warn_bedrock_native_output_stream_denied, warn_bedrock_strict_tools_dropped, ) from litellm.types.utils import ModelResponse @@ -25,22 +27,55 @@ NativeOutputFallbackFactory: TypeAlias = Callable[[], NativeOutputFallback | Non class _SelectedFallback: fallback: NativeOutputFallback native_output_retry: bool + buffered_native_output: bool = False _RetryFor: TypeAlias = Callable[[BaseLLMException], _SelectedFallback | None] +_NextRetryFor: TypeAlias = Callable[[BaseLLMException, _SelectedFallback], _SelectedFallback | None] -def _native_output_retry( - error: BaseLLMException, fallback: NativeOutputFallbackFactory | None, model: str +def _native_output_rejection_retry( + error: BaseLLMException, fallback: NativeOutputFallbackFactory, model: str ) -> NativeOutputFallback | None: - if fallback is None or not is_bedrock_native_output_rejection(error.status_code, error.message): - return None retry: Final = fallback() if retry is not None: warn_bedrock_native_output_rejected(model, error.message) return retry +def _buffered(retry: NativeOutputFallback) -> NativeOutputFallback: + return NativeOutputFallback( + optional_params={**retry.optional_params, "fake_stream": True}, litellm_params=retry.litellm_params + ) + + +def _buffered_stream_retry(fallback: NativeOutputFallbackFactory, model: str) -> NativeOutputFallback | None: + retry: Final = fallback() + if retry is None or retry.optional_params.get("fake_stream") is not True: + return None + warn_bedrock_native_output_stream_denied(model) + return retry + + +def _native_output_retry( + error: BaseLLMException, + fallback: NativeOutputFallbackFactory | None, + buffered_stream_fallback: NativeOutputFallbackFactory | None, + model: str, +) -> _SelectedFallback | None: + if fallback is not None and is_bedrock_native_output_rejection(error.status_code, error.message): + rejection_retry: Final = _native_output_rejection_retry(error, fallback, model) + return None if rejection_retry is None else _SelectedFallback(rejection_retry, native_output_retry=True) + if buffered_stream_fallback is not None and is_bedrock_stream_permission_denial(error.status_code, error.message): + stream_retry: Final = _buffered_stream_retry(buffered_stream_fallback, model) + return ( + None + if stream_retry is None + else _SelectedFallback(stream_retry, native_output_retry=True, buffered_native_output=True) + ) + return None + + def _strict_tools_retry( error: BaseLLMException, fallback: NativeOutputFallbackFactory | None, model: str ) -> NativeOutputFallback | None: @@ -55,12 +90,13 @@ def _strict_tools_retry( def _fallback_for( error: BaseLLMException, fallback: NativeOutputFallbackFactory | None, + buffered_stream_fallback: NativeOutputFallbackFactory | None, strict_tools_fallback: NativeOutputFallbackFactory | None, model: str, ) -> _SelectedFallback | None: - native_output_retry: Final = _native_output_retry(error, fallback, model) + native_output_retry: Final = _native_output_retry(error, fallback, buffered_stream_fallback, model) if native_output_retry is not None: - return _SelectedFallback(native_output_retry, native_output_retry=True) + return native_output_retry strict_tools_retry: Final = _strict_tools_retry(error, strict_tools_fallback, model) if strict_tools_retry is None: return None @@ -87,12 +123,81 @@ def _strict_tools_retry_after_native_output_retry( return composed_retry +def _next_retry( + error: BaseLLMException, + retry: _SelectedFallback, + fallback: NativeOutputFallbackFactory | None, + strict_tools_fallback: NativeOutputFallbackFactory | None, + model: str, +) -> _SelectedFallback | None: + if ( + retry.buffered_native_output + and fallback is not None + and is_bedrock_native_output_rejection(error.status_code, error.message) + ): + rejection_retry: Final = _native_output_rejection_retry(error, fallback, model) + return ( + None if rejection_retry is None else _SelectedFallback(_buffered(rejection_retry), native_output_retry=True) + ) + composed_retry: Final = _strict_tools_retry_after_native_output_retry(error, retry, strict_tools_fallback, model) + return None if composed_retry is None else _SelectedFallback(composed_retry, native_output_retry=False) + + +def _attempt_or_next_retry( + attempt: NativeOutputAttempt, retry: _SelectedFallback, next_retry_for: _NextRetryFor +) -> NativeOutputAttemptResult | _SelectedFallback: + try: + return attempt(retry.fallback) + except BaseLLMException as error: + next_retry: Final = next_retry_for(error, retry) + if next_retry is None: + raise + return next_retry + + +def _attempt_retries( + attempt: NativeOutputAttempt, retry: _SelectedFallback, next_retry_for: _NextRetryFor +) -> NativeOutputAttemptResult: + after_first_retry: Final = _attempt_or_next_retry(attempt, retry, next_retry_for) + if not isinstance(after_first_retry, _SelectedFallback): + return after_first_retry + after_second_retry: Final = _attempt_or_next_retry(attempt, after_first_retry, next_retry_for) + if not isinstance(after_second_retry, _SelectedFallback): + return after_second_retry + return attempt(after_second_retry.fallback) + + +async def _await_or_next_retry( + attempt: NativeOutputAttempt, retry: _SelectedFallback, next_retry_for: _NextRetryFor +) -> NativeOutputResponse | _SelectedFallback: + try: + result: Final = attempt(retry.fallback) + return await result if isinstance(result, Coroutine) else result + except BaseLLMException as error: + next_retry: Final = next_retry_for(error, retry) + if next_retry is None: + raise + return next_retry + + +async def _await_retries( + attempt: NativeOutputAttempt, retry: _SelectedFallback, next_retry_for: _NextRetryFor +) -> NativeOutputResponse: + after_first_retry: Final = await _await_or_next_retry(attempt, retry, next_retry_for) + if not isinstance(after_first_retry, _SelectedFallback): + return after_first_retry + after_second_retry: Final = await _await_or_next_retry(attempt, after_first_retry, next_retry_for) + if not isinstance(after_second_retry, _SelectedFallback): + return after_second_retry + last_attempt: Final = attempt(after_second_retry.fallback) + return await last_attempt if isinstance(last_attempt, Coroutine) else last_attempt + + async def _await_with_native_output_fallback( first_attempt: Coroutine[object, object, NativeOutputResponse], attempt: NativeOutputAttempt, retry_for: _RetryFor, - strict_tools_fallback: NativeOutputFallbackFactory | None, - model: str, + next_retry_for: _NextRetryFor, ) -> NativeOutputResponse: try: return await first_attempt @@ -100,17 +205,7 @@ async def _await_with_native_output_fallback( retry: Final = retry_for(error) if retry is None: raise - try: - second_attempt: Final = attempt(retry.fallback) - return await second_attempt if isinstance(second_attempt, Coroutine) else second_attempt - except BaseLLMException as error: - composed_retry: Final = _strict_tools_retry_after_native_output_retry( - error, retry, strict_tools_fallback, model - ) - if composed_retry is None: - raise - third_attempt: Final = attempt(composed_retry) - return await third_attempt if isinstance(third_attempt, Coroutine) else third_attempt + return await _await_retries(attempt, retry, next_retry_for) def call_with_native_output_fallback( @@ -118,11 +213,19 @@ def call_with_native_output_fallback( fallback: NativeOutputFallbackFactory | None, model: str, strict_tools_fallback: NativeOutputFallbackFactory | None = None, + buffered_stream_fallback: NativeOutputFallbackFactory | None = None, ) -> NativeOutputAttemptResult: - if fallback is None and strict_tools_fallback is None: + if fallback is None and strict_tools_fallback is None and buffered_stream_fallback is None: return attempt(None) retry_for: Final = partial( - _fallback_for, fallback=fallback, strict_tools_fallback=strict_tools_fallback, model=model + _fallback_for, + fallback=fallback, + buffered_stream_fallback=buffered_stream_fallback, + strict_tools_fallback=strict_tools_fallback, + model=model, + ) + next_retry_for: Final = partial( + _next_retry, fallback=fallback, strict_tools_fallback=strict_tools_fallback, model=model ) try: first_attempt: Final = attempt(None) @@ -130,15 +233,7 @@ def call_with_native_output_fallback( retry: Final = retry_for(error) if retry is None: raise - try: - return attempt(retry.fallback) - except BaseLLMException as retry_error: - composed_retry: Final = _strict_tools_retry_after_native_output_retry( - retry_error, retry, strict_tools_fallback, model - ) - if composed_retry is None: - raise - return attempt(composed_retry) + return _attempt_retries(attempt, retry, next_retry_for) if isinstance(first_attempt, Coroutine): - return _await_with_native_output_fallback(first_attempt, attempt, retry_for, strict_tools_fallback, model) + return _await_with_native_output_fallback(first_attempt, attempt, retry_for, next_retry_for) return first_attempt diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index f0627c7362c..0f8694a7e46 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -297,9 +297,16 @@ def _bedrock_model_supports(model: str, key: str, custom_llm_provider: str = "be def _bedrock_native_output_enabled( model: str, custom_llm_provider: str, model_info: Mapping[str, object] | None ) -> bool: - deployment_flag: Final = (model_info or {}).get("supports_native_structured_output") + deployment_info: Final = model_info or {} + deployment_flag: Final = deployment_info.get("supports_native_structured_output") if isinstance(deployment_flag, bool): return deployment_flag + return deployment_info.get("supports_bedrock_native_structured_output") is not False and ( + bedrock_model_always_takes_native_output(model, custom_llm_provider) + ) + + +def bedrock_model_always_takes_native_output(model: str, custom_llm_provider: str = "bedrock") -> bool: return _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider) @@ -443,6 +450,18 @@ def warn_bedrock_native_output_rejected(model: str, error_text: str) -> None: ) +def is_bedrock_stream_permission_denial(status_code: int, error_text: str) -> bool: + return status_code == 403 and "bedrock:InvokeModelWithResponseStream" in error_text + + +def warn_bedrock_native_output_stream_denied(model: str) -> None: + verbose_logger.warning( + "Bedrock denied bedrock:InvokeModelWithResponseStream for model=%s, so litellm sent the structured output " + "request again as a buffered stream. Allow that action to stream it.", + model, + ) + + @dataclass(frozen=True, slots=True) class NativeOutputFallback: optional_params: Mapping[str, object] @@ -543,6 +562,22 @@ def json_object_or_none(value: object) -> Mapping[str, object] | None: BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES: Final = 64 +def _response_format_json_schema(response_format: object) -> Mapping[str, object] | None: + json_schema: Final = json_object_or_none((json_object_or_none(response_format) or {}).get("json_schema")) + return json_object_or_none((json_schema or {}).get("schema")) + + +def bedrock_batch_record_takes_native_output( + model: str, response_format: object, model_info: Mapping[str, object] | None = None +) -> bool: + schema: Final = _response_format_json_schema(response_format) + return _bedrock_native_output_enabled(model, "bedrock", model_info) and ( + schema is None + or schema_property_count(schema) <= BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES + or not _bedrock_model_supports(model, "supports_bedrock_native_structured_output_by_default") + ) + + def bedrock_cleaned_output_schema( model: str, schema: Mapping[str, object], custom_llm_provider: str = "bedrock" ) -> Mapping[str, object] | None: diff --git a/litellm/llms/bedrock/files/transformation.py b/litellm/llms/bedrock/files/transformation.py index 4d137876890..ef54c85c131 100644 --- a/litellm/llms/bedrock/files/transformation.py +++ b/litellm/llms/bedrock/files/transformation.py @@ -71,6 +71,7 @@ from litellm.utils import get_llm_provider, get_optional_params from ..base_aws_llm import BaseAWSLLM from ..common_utils import ( BedrockError, + bedrock_batch_record_takes_native_output, deployment_model_info, merge_bedrock_aws_request_params, resolve_s3_encryption_key_id, @@ -167,6 +168,20 @@ _LITELLM_PARAMS_THE_MAPPER_TAKES: Final = frozenset({"allowed_openai_params"}) _MAPPED_PARAMS_THE_REQUEST_HANDLER_STRIPS: Final = frozenset({"json_mode"}) +_UPLOAD_PARAMS: Final = TypeAdapter(dict[str, object]) + + +def _upload_params(params: object) -> Mapping[str, object]: + return _UPLOAD_PARAMS.validate_python(params) if isinstance(params, Mapping) else {} + + +def _upload_deployment_model_info(litellm_params: object) -> Mapping[str, object]: + upload_params: Final = _upload_params(litellm_params) + return deployment_model_info(upload_params) or deployment_model_info( + _upload_params(upload_params.get("litellm_metadata")) + ) + + def _invoke_route_model(model: str) -> str: return f"invoke/{_strip_llm_routing_prefix(model).removeprefix('invoke/')}" @@ -1034,6 +1049,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): openai_request_body: Mapping[str, Any], model: str, provider: str | None = None, + model_info: Mapping[str, object] | None = None, ) -> dict[str, object]: """ Transform OpenAI request body to Bedrock-compatible modelInput @@ -1055,7 +1071,15 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): ) config: Final = AmazonAnthropicClaudeConfig() - litellm_params: Final = without_bedrock_native_output_opt_in({}) + litellm_params: Final = without_bedrock_native_output_opt_in( + { + "model_info": { + "supports_native_structured_output": bedrock_batch_record_takes_native_output( + _invoke_route_model(model), optional_params.get("response_format"), model_info + ) + } + } + ) mapped_params = get_optional_params( model=_invoke_route_model(model), custom_llm_provider="bedrock", @@ -1150,25 +1174,32 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): ] def transform_batch_input_lines( - self, batch_input_lines: Sequence[_BatchInputLine], target_model: str = "" + self, + batch_input_lines: Sequence[_BatchInputLine], + target_model: str = "", + model_info: Mapping[str, object] | None = None, ) -> tuple[_BedrockBatchRecord, ...]: return tuple( - self._transform_batch_input_line(idx, batch_input_line, target_model) + self._transform_batch_input_line(idx, batch_input_line, target_model, model_info) for idx, batch_input_line in enumerate(batch_input_lines) ) def _transform_batch_input_line( - self, idx: int, batch_input_line: _BatchInputLine, target_model: str + self, idx: int, batch_input_line: _BatchInputLine, target_model: str, model_info: Mapping[str, object] | None ) -> _BedrockBatchRecord: try: - return self._transform_openai_batch_record(idx, batch_input_line.record, target_model) + return self._transform_openai_batch_record(idx, batch_input_line.record, target_model, model_info) except _BATCH_RECORD_CONVERSION_ERRORS as e: raise _batch_line_error( batch_input_line.line_number, f"could not be converted to a Bedrock batch record: {e}" ) from e def _transform_openai_batch_record( - self, idx: int, openai_record: _OpenAIBatchRecord, target_model: str + self, + idx: int, + openai_record: _OpenAIBatchRecord, + target_model: str, + model_info: Mapping[str, object] | None = None, ) -> _BedrockBatchRecord: import litellm @@ -1192,6 +1223,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): openai_request_body=self._transform_batch_body_to_chat_body(openai_body, record_kind), model=model_for_transform, provider=provider, + model_info=model_info, ) ) return _BedrockBatchRecord( @@ -1211,7 +1243,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): model: str, create_file_data: CreateFileRequest, optional_params: dict, - litellm_params: dict, + litellm_params: dict[str, object], ) -> bytes | str | dict: """ Transform file request and return a pre-signed request for S3. @@ -1237,7 +1269,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): litellm_params_model: Final = litellm_params.get("model") target_model: Final = model or (litellm_params_model if isinstance(litellm_params_model, str) else "") bedrock_jsonl_content: Final = self.transform_batch_input_lines( - batch_input_lines, target_model=target_model + batch_input_lines, target_model=target_model, model_info=_upload_deployment_model_info(litellm_params) ) file_content = "\n".join(json.dumps(item) for item in bedrock_jsonl_content) elif isinstance(extracted_file_data_content, bytes): diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 169c21cf635..df82891f789 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -897,7 +897,7 @@ class BaseLLMHTTPHandler: fake_stream = ( fake_stream - or optional_params.pop("fake_stream", False) + or optional_params.pop("fake_stream", False) is True or provider_config.should_fake_stream(model=model, custom_llm_provider=custom_llm_provider, stream=stream) ) @@ -920,7 +920,7 @@ class BaseLLMHTTPHandler: api_key=api_key, model=model, optional_params=optional_params, - stream=stream, + stream=False if fake_stream else stream, litellm_params=litellm_params, ) @@ -1056,7 +1056,7 @@ class BaseLLMHTTPHandler: provider_config=provider_config, fake_stream=fake_stream, ) - if provider_config.has_custom_stream_wrapper is True: + if provider_config.has_custom_stream_wrapper is True and not fake_stream: return provider_config.get_sync_custom_stream_wrapper( model=model, custom_llm_provider=custom_llm_provider, @@ -1225,7 +1225,7 @@ class BaseLLMHTTPHandler: json_mode: bool | None = None, signed_json_body: bytes | None = None, ): - if provider_config.has_custom_stream_wrapper is True: + if provider_config.has_custom_stream_wrapper is True and not fake_stream: return await provider_config.get_async_custom_stream_wrapper( model=model, custom_llm_provider=custom_llm_provider, diff --git a/litellm/main.py b/litellm/main.py index 6b9964a450f..bdb35823757 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -4393,6 +4393,14 @@ def _converse_native_output_fallback( return None if AmazonConverseConfig.sends_native_text_format(fallback.optional_params) else fallback +def _buffered_stream_fallback( + first_attempt_params: Mapping[str, object], first_attempt_litellm_params: Mapping[str, object] +) -> NativeOutputFallback: + return NativeOutputFallback( + optional_params={**first_attempt_params, "fake_stream": True}, litellm_params=first_attempt_litellm_params + ) + + def _native_output_attempt_timeout( ctx: CompletionDispatchContext, fallback: NativeOutputFallback | None ) -> float | str | httpx.Timeout | None: @@ -4496,11 +4504,14 @@ def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResu ) converse_params: Final = _OPTIONAL_PARAMS_ADAPTER.validate_python(optional_params) + converse_uses_native_output: Final = AmazonConverseConfig().uses_native_output( + converse_model, converse_params, litellm_params + ) response = call_with_native_output_fallback( converse_attempt, ( partial(_converse_native_output_fallback, ctx, converse_params, dict(litellm_params)) - if AmazonConverseConfig().uses_native_output(converse_model, converse_params, litellm_params) + if converse_uses_native_output else None ), converse_model, @@ -4509,6 +4520,11 @@ def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResu if drops_unsupported_params(litellm_params.get("drop_params")) else None ), + buffered_stream_fallback=( + partial(_buffered_stream_fallback, converse_params, dict(litellm_params)) + if converse_uses_native_output + else None + ), ) elif bedrock_route == "converse_like": model = model.replace("converse_like/", "") @@ -4551,15 +4567,20 @@ def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResu ) invoke_params: Final = _OPTIONAL_PARAMS_ADAPTER.validate_python(optional_params) + invoke_uses_native_output: Final = isinstance( + provider_config, AmazonAnthropicClaudeConfig + ) and provider_config.uses_native_output(model, invoke_params, litellm_params) response = call_with_native_output_fallback( invoke_attempt, - ( - partial(_native_output_fallback, ctx, invoke_params, dict(litellm_params)) - if isinstance(provider_config, AmazonAnthropicClaudeConfig) - and provider_config.uses_native_output(model, invoke_params, litellm_params) + partial(_native_output_fallback, ctx, invoke_params, dict(litellm_params)) + if invoke_uses_native_output + else None, + model, + buffered_stream_fallback=( + partial(_buffered_stream_fallback, invoke_params, dict(litellm_params)) + if invoke_uses_native_output else None ), - model, ) return response diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 4e81afb0ea6..d9d996208d8 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1961,6 +1961,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2041,6 +2042,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2116,6 +2118,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2242,7 +2245,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2287,6 +2291,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2331,7 +2336,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2376,6 +2382,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2420,7 +2427,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2465,6 +2473,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2504,7 +2513,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2549,6 +2559,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2593,7 +2604,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2633,6 +2645,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2715,7 +2728,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -2759,7 +2773,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -2803,7 +2818,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -2847,7 +2863,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -2891,7 +2908,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -3023,6 +3041,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -3068,6 +3087,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -3113,6 +3133,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -3158,6 +3179,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -3203,6 +3225,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -75549,7 +75572,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -75589,7 +75613,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -75629,6 +75654,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -75702,6 +75728,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -77417,6 +77444,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -77474,6 +77502,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -77651,6 +77680,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -77703,6 +77733,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -77750,6 +77781,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -77863,6 +77895,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index f5369bc34b0..7f48d77ff09 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -221,7 +221,12 @@ from litellm.types.utils import ( is_server_derived_pricing_key, pricing_override_fields, ) -from litellm.utils import acredential_updates, cost_map_omits_token_price, with_named_credential +from litellm.utils import ( + acredential_updates, + cost_map_omits_token_price, + native_structured_output_support, + with_named_credential, +) if TYPE_CHECKING: from aiohttp import ClientSession @@ -16733,7 +16738,7 @@ def _enrich_model_info_with_litellm_data( stamped_model_info: Final = MappingProxyType( {**model_info, **_pricing_override_stamps(model_info, model.get("litellm_params") or MappingProxyType({}))} ) - model["model_info"] = { + merged_model_info: Final = { **stamped_model_info, **{ k: None if unpriced and k in ("input_cost_per_token", "output_cost_per_token") else v @@ -16741,6 +16746,18 @@ def _enrich_model_info_with_litellm_data( if k not in stamped_model_info or (stamped_model_info[k] is None and k in discovered_model_info) }, } + native_output_info: Final = { + **merged_model_info, + "supports_bedrock_native_structured_output_by_default": litellm_model_info.get( + "supports_bedrock_native_structured_output_by_default" + ), + } + model["model_info"] = ( + {**merged_model_info, "supports_native_structured_output": native_structured_output_support(native_output_info)} + if "supports_native_structured_output" in merged_model_info + and "supports_native_structured_output" not in stamped_model_info + else merged_model_info + ) # don't return the api key / vertex credentials # don't return the llm credentials model = remove_sensitive_info_from_deployment(model, excluded_keys={"litellm_credential_name"}) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 5a14dc95e7c..b082198d3f2 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -217,6 +217,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False): supports_native_streaming: bool | None supports_native_structured_output: bool | None supports_bedrock_native_structured_output: ReadOnly[bool | None] + supports_bedrock_native_structured_output_by_default: ReadOnly[bool | None] supports_parallel_function_calling: bool | None supports_web_search: bool | None supports_reasoning: bool | None @@ -4303,6 +4304,7 @@ DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset( "supports_regex_lookaround", "supports_assistant_prefill", "supports_bedrock_native_structured_output", + "supports_bedrock_native_structured_output_by_default", "supports_per_turn_control", "supports_pdf_input", "supports_continuation", diff --git a/litellm/utils.py b/litellm/utils.py index 8944ecb5648..13f22ddc36e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4167,6 +4167,13 @@ def _registration_base( return _RegistrationWrite(key=key, entry=dict(fallback.entry)) +def native_structured_output_support(model_info: Mapping[str, object]) -> bool | None: + if model_info.get("supports_bedrock_native_structured_output_by_default"): + return model_info.get("supports_bedrock_native_structured_output") is not False + declared: Final = model_info.get("supports_native_structured_output") + return declared if isinstance(declared, bool) else None + + def _base_fields_replaced_by(value: Mapping[str, object]) -> frozenset[str]: return ( frozenset({"explicit_cache_read_input_token_cost"}) @@ -7587,10 +7594,13 @@ def get_model_info_helper( supports_pdf_input=_model_info.get("supports_pdf_input", None), supports_embedding_image_input=_model_info.get("supports_embedding_image_input", None), supports_native_streaming=_model_info.get("supports_native_streaming", None), - supports_native_structured_output=_model_info.get("supports_native_structured_output", None), + supports_native_structured_output=native_structured_output_support(_model_info), supports_bedrock_native_structured_output=_model_info.get( "supports_bedrock_native_structured_output", None ), + supports_bedrock_native_structured_output_by_default=_model_info.get( + "supports_bedrock_native_structured_output_by_default", None + ), supports_web_search=_model_info.get("supports_web_search", None), supports_url_context=_model_info.get("supports_url_context", None), supports_continuation=_model_info.get("supports_continuation", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 4e81afb0ea6..d9d996208d8 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1961,6 +1961,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2041,6 +2042,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2116,6 +2118,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2242,7 +2245,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2287,6 +2291,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2331,7 +2336,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2376,6 +2382,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2420,7 +2427,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2465,6 +2473,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2504,7 +2513,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2549,6 +2559,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2593,7 +2604,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2633,6 +2645,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -2715,7 +2728,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -2759,7 +2773,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -2803,7 +2818,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -2847,7 +2863,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -2891,7 +2908,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -3023,6 +3041,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -3068,6 +3087,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -3113,6 +3133,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -3158,6 +3179,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -3203,6 +3225,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -75549,7 +75572,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -75589,7 +75613,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -75629,6 +75654,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_max_reasoning_effort": true, "supports_output_config": true, "supports_parallel_tool_use_config": true, @@ -75702,6 +75728,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -77417,6 +77444,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -77474,6 +77502,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -77651,6 +77680,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -77703,6 +77733,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -77750,6 +77781,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, @@ -77863,6 +77895,7 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true, "supports_native_structured_output": false, + "supports_bedrock_native_structured_output_by_default": true, "supports_bedrock_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_output_config": true, diff --git a/tests/integration/providers/test_bedrock_batch_files_wire.py b/tests/integration/providers/test_bedrock_batch_files_wire.py index 823bcda7e06..1657895109d 100644 --- a/tests/integration/providers/test_bedrock_batch_files_wire.py +++ b/tests/integration/providers/test_bedrock_batch_files_wire.py @@ -88,10 +88,12 @@ _SCHEMA: Final = { @pytest.mark.parametrize( - "backend", ["us.anthropic.claude-haiku-5-5", "us.anthropic.claude-opus-4-8"], ids=["haiku_5_5", "opus_4_8"] + ("backend", "native_output"), + [("us.anthropic.claude-sonnet-5-5", False), ("us.anthropic.claude-haiku-5-5", True)], + ids=["opt_in_native_output_model", "native_output_model"], ) -def test_json_schema_batch_record_with_thinking_uploads_the_json_tool_and_no_native_output( - gateway: Gateway, backend: str +def test_json_schema_batch_record_with_thinking_uploads_native_output_only_on_models_that_always_take_it( + gateway: Gateway, backend: str, native_output: bool ) -> None: record: Final = { "custom_id": "schema-record", @@ -125,7 +127,13 @@ def test_json_schema_batch_record_with_thinking_uploads_the_json_tool_and_no_nat (upload,) = wire.drain() (stored,) = tuple(JSON_OBJECT.validate_json(line) for line in upload.body.decode().splitlines() if line.strip()) model_input: Final = object_value(stored["modelInput"]) - assert "format" not in object_value(model_input.get("output_config", {})), model_input - (tool,) = (object_value(entry) for entry in list_value(model_input["tools"])) + output_format: Final = object_value(model_input.get("output_config", {})).get("format") + tools: Final = tuple(object_value(entry) for entry in list_value(model_input.get("tools", []))) + if native_output: + assert object_value(object_value(output_format)["schema"])["properties"] == _SCHEMA["properties"], model_input + assert not tools, model_input + return + assert output_format is None, model_input + (tool,) = tools assert tool["name"] == _JSON_TOOL, model_input assert object_value(tool["input_schema"])["properties"] == _SCHEMA["properties"], model_input diff --git a/tests/integration/providers/test_bedrock_native_structured_output_wire.py b/tests/integration/providers/test_bedrock_native_structured_output_wire.py index 2502bd90bd6..68942ed295b 100644 --- a/tests/integration/providers/test_bedrock_native_structured_output_wire.py +++ b/tests/integration/providers/test_bedrock_native_structured_output_wire.py @@ -18,9 +18,10 @@ from pydantic import JsonValue, TypeAdapter _HAIKU: Final = "us.anthropic.claude-haiku-5-5" _FABLE: Final = "us.anthropic.claude-fable-5" _SONNET: Final = "us.anthropic.claude-sonnet-4-6" -_OPUS: Final = "us.anthropic.claude-opus-5-5" -_OPT_IN: Final = _OPUS -_UNFORCEABLE_TOOL: Final = _OPUS +_NEITHER_KEY: Final = "us.anthropic.claude-opus-4-7" +_OPT_IN: Final = _NEITHER_KEY +_BEDROCK_KEY_ONLY: Final = "us.anthropic.claude-sonnet-5-5" +_UNFORCEABLE_TOOL: Final = "anthropic.claude-opus-5-5" _OPUS_4_8: Final = "us.anthropic.claude-opus-4-8" _DEEPSEEK: Final = "deepseek.v3.2" _AWS: Final[dict[str, JsonValue]] = { @@ -683,7 +684,7 @@ def test_a_model_that_cannot_force_the_json_tool_retries_a_rejected_native_outpu @pytest.mark.parametrize( ("backend", "model_info", "native"), [ - (_OPUS, None, False), + (_NEITHER_KEY, None, False), (_OPT_IN, {"supports_bedrock_native_structured_output": True}, True), (_OPUS_4_8, {"supports_bedrock_native_structured_output": False}, False), ], @@ -897,7 +898,7 @@ def test_messages_on_bedrock_invoke_follows_the_bedrock_native_key_without_think ) -> None: marker, marker_hex = _marker() with wire_server(_bedrock) as wire, gateway.scenario() as scenario: - model: Final = _deployment(scenario, wire, "invoke", _HAIKU) + model: Final = _deployment(scenario, wire, "invoke", _BEDROCK_KEY_ONLY) response: Final = gateway.request( "POST", "/v1/messages", @@ -945,9 +946,9 @@ def test_malformed_schema_on_a_native_model_reaches_bedrock_once_without_a_proxy ("route", "backend", "thinking", "schema", "native"), [ ("converse", _SONNET, False, _NUMERIC_FORMATS_SCHEMA, True), - ("invoke", _HAIKU, True, _NUMERIC_FORMATS_SCHEMA, True), + ("invoke", _BEDROCK_KEY_ONLY, True, _NUMERIC_FORMATS_SCHEMA, True), ("converse", _SONNET, False, _WIDE_SCHEMA, True), - ("converse", _HAIKU, True, _WIDE_SCHEMA, False), + ("converse", _BEDROCK_KEY_ONLY, True, _WIDE_SCHEMA, False), ], ids=["numeric_formats_old_key", "numeric_formats_new_key_invoke", "wide_old_key", "wide_new_key"], ) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 146a6a3ff1c..0a9dc0ab3cf 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -496,7 +496,7 @@ def declare_mid_conversation_system(monkeypatch: pytest.MonkeyPatch) -> Callable return declare -NativeStructuredOutputKey: TypeAlias = Literal["legacy", "bedrock"] | None +NativeStructuredOutputKey: TypeAlias = Literal["legacy", "bedrock", "default"] | None @pytest.fixture @@ -506,7 +506,8 @@ def set_native_structured_output( def set_flags(model_cost_key: str, native_key: NativeStructuredOutputKey) -> None: entry: Final = litellm.model_cost[model_cost_key] monkeypatch.setitem(entry, "supports_native_structured_output", native_key == "legacy") - monkeypatch.setitem(entry, "supports_bedrock_native_structured_output", native_key == "bedrock") + monkeypatch.setitem(entry, "supports_bedrock_native_structured_output", native_key in ("bedrock", "default")) + monkeypatch.setitem(entry, "supports_bedrock_native_structured_output_by_default", native_key == "default") litellm.get_model_info.cache_clear() return set_flags diff --git a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py index 053574d7dd6..4f17f8b10e1 100644 --- a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py +++ b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py @@ -570,6 +570,7 @@ def _invoke_params_for_schema( ("us.anthropic.claude-opus-4-8", "bedrock", {"thinking": {"type": "adaptive"}}, True), ("us.anthropic.claude-opus-4-8", "bedrock", {"reasoning_effort": "low"}, True), ("us.anthropic.claude-haiku-5-5", "bedrock", {"thinking": {"type": "disabled"}}, False), + ("us.anthropic.claude-haiku-5-5", "default", {"thinking": {"type": "disabled"}}, True), ], ids=[ "legacy-key", @@ -578,6 +579,7 @@ def _invoke_params_for_schema( "bedrock-key-adaptive", "bedrock-key-effort", "bedrock-key-thinking-disabled", + "default-key-thinking-disabled", ], ) def test_bedrock_native_output_key_applies_only_when_the_model_will_think( @@ -2047,9 +2049,9 @@ def _invoke_chat_params( ) @pytest.mark.parametrize("model", ["us.anthropic.claude-opus-4-8", "global.anthropic.claude-sonnet-5"]) def test_invoke_uses_opt_in_native_output_when_the_opt_in_drove_native_output( - local_model_cost_map, model, thinking_params, monkeypatch + set_native_structured_output, model, thinking_params ): - monkeypatch.setitem(litellm.model_cost[model], "supports_native_structured_output", False) + set_native_structured_output(model, "bedrock") optional_params: Final = _invoke_chat_params( model, None, {"response_format": _CITY_RESPONSE_FORMAT, **thinking_params} ) @@ -2084,8 +2086,12 @@ def test_invoke_uses_opt_in_native_output_is_false_when_native_output_did_not_co ) -def test_invoke_uses_default_native_output_for_schema_retry(local_model_cost_map: None) -> None: - model: Final = "us.anthropic.claude-opus-5" +@pytest.mark.parametrize( + "model", + ["us.anthropic.claude-opus-5", "us.anthropic.claude-sonnet-5"], + ids=["opus-5", "sonnet-5"], +) +def test_invoke_uses_default_native_output_for_schema_retry(local_model_cost_map: None, model: str) -> None: optional_params: Final = _invoke_chat_params( model, None, {"response_format": _CITY_RESPONSE_FORMAT, "thinking": {"type": "disabled"}} ) diff --git a/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py b/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py index 7d1b502df2b..d43cc771185 100644 --- a/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py +++ b/tests/unit/llms/bedrock/chat/test_bedrock_converse_handler.py @@ -599,6 +599,139 @@ async def test_native_output_retry_is_bounded_by_the_request_timeout_not_the_str assert _sent_paths(respx_mock) == ["converse-stream", "converse"] +# Bedrock's 403 for a role without the streaming action, seen live on us.anthropic.claude-sonnet-5 on 2026-11-06 +_STREAM_PERMISSION_DENIAL: Final = json.dumps( + { + "Message": "User: arn:aws:sts::123456789012:assumed-role/invoke-only/qa is not authorized to perform: " + "bedrock:InvokeModelWithResponseStream on resource: arn:aws:bedrock:us-east-1:123456789012:inference-profile/" + "us.anthropic.claude-opus-5-5 because no identity-based policy allows the " + "bedrock:InvokeModelWithResponseStream action" + } +).encode() + + +def _denies_streaming_then_answers(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/converse-stream"): + return httpx.Response(403, content=_STREAM_PERMISSION_DENIAL, request=request) + return httpx.Response(200, content=CONVERSE_ANSWER, headers={"content-type": "application/json"}, request=request) + + +@pytest.mark.asyncio +async def test_a_denied_native_output_stream_is_sent_again_buffered_with_native_output( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post(url__regex=_BEDROCK_RUNTIME_URL).mock(side_effect=_denies_streaming_then_answers) + router: Final = _bedrock_router( + "bedrock/converse/us.anthropic.claude-opus-5-5", timeout=60, stream_timeout=STREAM_TIMEOUT_SECONDS + ) + + response: Final = await router.acompletion( + model="bedrock", + messages=[{"role": "user", "content": "hi"}], + stream=True, + thinking={"type": "disabled"}, + response_format={"type": "json_schema", "json_schema": {"name": "answer", "schema": _ANSWER_SCHEMA}}, + ) + + assert "".join([chunk.choices[0].delta.content or "" async for chunk in response]) == "hi" + assert _sent_paths(respx_mock) == ["converse-stream", "converse"] + retry_body: Final = json.loads(respx_mock.calls[1].request.content) + assert "textFormat" in retry_body.get("outputConfig", {}) and "toolConfig" not in retry_body + + +_INVOKE_ANSWER: Final = json.dumps( + { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-opus-5-5", + "content": [{"type": "text", "text": '{"answer": "hi"}'}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 10, "output_tokens": 5}, + } +).encode() + + +def _denies_invoke_streaming_then_answers(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/invoke-with-response-stream"): + return httpx.Response(403, content=_STREAM_PERMISSION_DENIAL, request=request) + return httpx.Response(200, content=_INVOKE_ANSWER, headers={"content-type": "application/json"}, request=request) + + +@pytest.mark.asyncio +async def test_a_denied_native_output_stream_on_the_invoke_route_is_sent_again_buffered_to_invoke( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post(url__regex=_BEDROCK_RUNTIME_URL).mock(side_effect=_denies_invoke_streaming_then_answers) + router: Final = _bedrock_router( + "bedrock/invoke/us.anthropic.claude-opus-5-5", timeout=60, stream_timeout=STREAM_TIMEOUT_SECONDS + ) + + response: Final = await router.acompletion( + model="bedrock", + messages=[{"role": "user", "content": "hi"}], + stream=True, + thinking={"type": "disabled"}, + response_format={"type": "json_schema", "json_schema": {"name": "answer", "schema": _ANSWER_SCHEMA}}, + ) + + assert "".join([chunk.choices[0].delta.content or "" async for chunk in response]) == '{"answer": "hi"}' + assert _sent_paths(respx_mock) == ["invoke-with-response-stream", "invoke"] + retry_body: Final = json.loads(respx_mock.calls[1].request.content) + assert "format" in retry_body.get("output_config", {}) and "tools" not in retry_body + + +_INVOKE_JSON_TOOL_ANSWER: Final = json.dumps( + { + "id": "msg_2", + "type": "message", + "role": "assistant", + "model": "claude-opus-5-5", + "content": [{"type": "tool_use", "id": "toolu_1", "name": "json_tool_call", "input": {"answer": "hi"}}], + "stop_reason": "tool_use", + "stop_sequence": None, + "usage": {"input_tokens": 10, "output_tokens": 5}, + } +).encode() + + +def _denies_invoke_streaming_and_rejects_native_output(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/invoke-with-response-stream"): + return httpx.Response(403, content=_STREAM_PERMISSION_DENIAL, request=request) + if "format" in json.loads(request.content).get("output_config", {}): + return httpx.Response(400, content=_NATIVE_OUTPUT_REJECTION, request=request) + return httpx.Response( + 200, content=_INVOKE_JSON_TOOL_ANSWER, headers={"content-type": "application/json"}, request=request + ) + + +@pytest.mark.asyncio +async def test_a_rejected_schema_after_a_denied_invoke_stream_falls_back_to_the_json_tool_still_buffered( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post(url__regex=_BEDROCK_RUNTIME_URL).mock(side_effect=_denies_invoke_streaming_and_rejects_native_output) + router: Final = _bedrock_router( + "bedrock/invoke/us.anthropic.claude-opus-5-5", timeout=60, stream_timeout=STREAM_TIMEOUT_SECONDS + ) + + response: Final = await router.acompletion( + model="bedrock", + messages=[{"role": "user", "content": "hi"}], + stream=True, + thinking={"type": "disabled"}, + response_format={"type": "json_schema", "json_schema": {"name": "answer", "schema": _ANSWER_SCHEMA}}, + ) + + assert "".join([chunk.choices[0].delta.content or "" async for chunk in response]) == '{"answer": "hi"}' + assert _sent_paths(respx_mock) == ["invoke-with-response-stream", "invoke", "invoke"] + json_tool_body: Final = json.loads(respx_mock.calls[2].request.content) + assert "format" not in json_tool_body.get("output_config", {}) and "tools" in json_tool_body + + @pytest.mark.asyncio async def test_failed_native_output_retry_reports_the_request_timeout_it_was_bounded_by( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index b37d6977c10..4dcf312737d 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -5225,9 +5225,9 @@ def test_a_model_that_cannot_force_the_json_tool_gets_native_output_without_thin ("model", "thinking_params", "native"), [ ("us.anthropic.claude-opus-5-5", None, True), - ("us.anthropic.claude-opus-5-5", {"thinking": {"type": "adaptive"}}, False), + ("us.anthropic.claude-opus-5-5", {"thinking": {"type": "adaptive"}}, True), ("global.anthropic.claude-fable-5-1", None, True), - ("global.anthropic.claude-fable-5-1", {"reasoning_effort": "low"}, False), + ("global.anthropic.claude-fable-5-1", {"reasoning_effort": "low"}, True), ("us-gov.anthropic.claude-opus-5-5", None, False), ("us-gov.anthropic.claude-fable-5-1", None, False), ("us-gov-west-1/anthropic.claude-opus-5-5", None, False), @@ -5235,7 +5235,7 @@ def test_a_model_that_cannot_force_the_json_tool_gets_native_output_without_thin ("us-east-1/anthropic.claude-opus-5-5", None, True), ], ) -def test_models_whose_thinking_stops_under_native_output_keep_the_json_tool_only_when_thinking( +def test_native_output_follows_model_cost_map_for_bedrock_profiles( model: str, thinking_params: Mapping[str, object] | None, native: bool ) -> None: _, request = _converse_request_for_json_schema(model, False, thinking_params) @@ -5248,6 +5248,17 @@ def test_models_whose_thinking_stops_under_native_output_keep_the_json_tool_only @pytest.mark.parametrize( ("model", "thinking_params", "native"), [ + ("us.anthropic.claude-opus-5-5", {"thinking": {"type": "disabled"}}, True), + ("global.anthropic.claude-fable-5-1", {"thinking": {"type": "adaptive"}}, True), + ("global.anthropic.claude-fable-5-1", {"thinking": {"type": "disabled"}}, True), + ("us.anthropic.claude-sonnet-5", None, True), + ("us.anthropic.claude-sonnet-5", {"thinking": {"type": "adaptive"}}, True), + ("us.anthropic.claude-sonnet-5", {"thinking": {"type": "disabled"}}, True), + ("us.anthropic.claude-haiku-5-5", None, True), + ("us.anthropic.claude-haiku-5-5", {"thinking": {"type": "adaptive"}}, True), + ("us.anthropic.claude-haiku-5-5", {"thinking": {"type": "disabled"}}, True), + ("us-gov.anthropic.claude-sonnet-5", None, False), + ("us-gov.anthropic.claude-haiku-5-5", None, False), ("us.anthropic.claude-opus-5", None, True), ("us.anthropic.claude-opus-5", {"thinking": {"type": "adaptive"}}, True), ("us.anthropic.claude-opus-5", {"thinking": {"type": "disabled"}}, True), @@ -5258,6 +5269,17 @@ def test_models_whose_thinking_stops_under_native_output_keep_the_json_tool_only ("us-gov.anthropic.claude-opus-4-8", None, False), ], ids=[ + "opus-5-5-disabled", + "fable-5-1-adaptive", + "fable-5-1-disabled", + "sonnet-5-unset", + "sonnet-5-adaptive", + "sonnet-5-disabled", + "haiku-5-5-unset", + "haiku-5-5-adaptive", + "haiku-5-5-disabled", + "sonnet-5-us-gov", + "haiku-5-5-us-gov", "opus-5-unset", "opus-5-adaptive", "opus-5-disabled", @@ -5268,7 +5290,7 @@ def test_models_whose_thinking_stops_under_native_output_keep_the_json_tool_only "opus-4-8-us-gov", ], ) -def test_opus_5_and_4_8_cost_map_defaults_use_native_output_across_thinking_modes( +def test_model_cost_native_output_defaults_hold_across_thinking_modes( model: str, thinking_params: Mapping[str, object] | None, native: bool ) -> None: _, request = _converse_request_for_json_schema(model, False, thinking_params) @@ -5282,6 +5304,30 @@ def test_opus_5_and_4_8_cost_map_defaults_use_native_output_across_thinking_mode assert [tool["toolSpec"]["name"] for tool in request["toolConfig"]["tools"]] == ["json_tool_call"] +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize( + ("model", "thinking_params"), + [ + ("us.anthropic.claude-opus-5-5", {"thinking": {"type": "adaptive"}}), + ("global.anthropic.claude-fable-5-1", {"thinking": {"type": "adaptive"}}), + ("us.anthropic.claude-sonnet-5", {"thinking": {"type": "disabled"}}), + ("us.anthropic.claude-haiku-5-5", None), + ("us.anthropic.claude-opus-5", None), + ], +) +@pytest.mark.parametrize("stream", [False, True]) +def test_deployment_opt_out_keeps_the_json_tool_on_models_that_default_to_native_output( + model: str, thinking_params: Mapping[str, object] | None, stream: bool +) -> None: + optional_params, request = _converse_request_for_json_schema( + model, stream, thinking_params, model_info={"supports_bedrock_native_structured_output": False} + ) + + assert "outputConfig" not in request + assert [tool["toolSpec"]["name"] for tool in request["toolConfig"]["tools"]] == ["json_tool_call"] + assert optional_params.get("fake_stream", False) is stream + + def _converse_capital_answer(request: httpx.Request) -> httpx.Response: return httpx.Response( 200, @@ -5295,9 +5341,7 @@ def _converse_capital_answer(request: httpx.Request) -> httpx.Response: @pytest.mark.usefixtures("local_model_cost_map") -@pytest.mark.parametrize( - "model", ["us.anthropic.claude-opus-5-5", "global.anthropic.claude-sonnet-5-5", "global.anthropic.claude-fable-5-1"] -) +@pytest.mark.parametrize("model", ["global.anthropic.claude-sonnet-5-5"]) def test_streamed_json_schema_on_a_model_that_cannot_force_a_tool_still_calls_converse_without_streaming( model: str, ) -> None: @@ -5610,7 +5654,9 @@ def test_schema_past_a_native_output_size_limit_keeps_the_tool_path( @pytest.mark.parametrize( - ("native_key", "native_past_limit"), [("legacy", True), ("bedrock", False)], ids=["legacy-key", "bedrock-key"] + ("native_key", "native_past_limit"), + [("legacy", True), ("default", True), ("bedrock", False)], + ids=["legacy-key", "default-key", "bedrock-key"], ) def test_field_count_limit_sends_the_json_tool_only_on_the_bedrock_native_key( set_native_structured_output, native_key, native_past_limit @@ -10335,7 +10381,6 @@ def test_converse_reserves_the_tool_cache_point_within_the_cap(tool_cache_params assert request["toolConfig"]["tools"][-1] == cache_point - @pytest.mark.parametrize( "unsent_tool_params", [ @@ -10385,6 +10430,7 @@ def test_converse_leaves_the_cap_to_markers_when_a_marked_tool_is_not_sent( ] assert cache_point not in request.get("toolConfig", {}).get("tools", []) + def _converse_tool_turn(*result_parts: dict[str, object]) -> list[dict[str, object]]: return [ { @@ -10506,7 +10552,9 @@ def test_converse_translates_prompt_cache_breakpoint_on_an_unmapped_arn_only_whe messages=[ { "role": "user", - "content": [{"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit", **ttl}}], + "content": [ + {"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit", **ttl}} + ], } ], optional_params={}, @@ -10518,7 +10566,6 @@ def test_converse_translates_prompt_cache_breakpoint_on_an_unmapped_arn_only_whe assert user_content(unflagged_arn, {"ttl": "30m"}) == [{"text": "Hello"}] - def test_converse_prompt_cache_breakpoint_follows_the_arn_flag_over_its_base_model(local_model_cost_map): from litellm import Router @@ -10545,7 +10592,9 @@ def test_converse_prompt_cache_breakpoint_follows_the_arn_flag_over_its_base_mod messages=[ { "role": "user", - "content": [{"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit", **ttl}}], + "content": [ + {"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit", **ttl}} + ], } ], optional_params={}, @@ -10633,7 +10682,9 @@ def test_converse_prompt_cache_breakpoint_on_guarded_user_text_keeps_its_cache_p def test_converse_guarded_user_text_keeps_no_cache_point_on_a_model_not_known_to_take_one(local_model_cost_map): request: Final = litellm.AmazonConverseConfig()._transform_request( model="arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/unmapped1", - messages=[{"role": "user", "content": [{"type": "text", "text": "Hello", "cache_control": {"type": "ephemeral"}}]}], + messages=[ + {"role": "user", "content": [{"type": "text", "text": "Hello", "cache_control": {"type": "ephemeral"}}]} + ], optional_params={"guardrailConfig": {"guardrailIdentifier": "gr-1", "guardrailVersion": "DRAFT"}}, litellm_params={}, ) @@ -11638,9 +11689,9 @@ _OUTPUT_CONFIG_FORMAT: Final = { ) @pytest.mark.parametrize("model", ["us.anthropic.claude-opus-4-8", "global.anthropic.claude-sonnet-5"]) def test_uses_opt_in_native_output_when_the_opt_in_drove_native_output( - local_model_cost_map, monkeypatch, model, request_params + set_native_structured_output, model, request_params ): - monkeypatch.setitem(litellm.model_cost[model], "supports_native_structured_output", False) + set_native_structured_output(model, "bedrock") optional_params: Final = _converse_params(model, None, request_params) assert AmazonConverseConfig().uses_opt_in_native_output(model, optional_params, {"model_info": {"id": "d1"}}) @@ -11672,9 +11723,9 @@ def test_uses_opt_in_native_output_is_false_when_native_output_did_not_come_from def test_uses_opt_in_native_output_reads_the_base_model_of_an_application_inference_profile( - local_model_cost_map, monkeypatch + set_native_structured_output, ): - monkeypatch.setitem(litellm.model_cost["us.anthropic.claude-opus-4-8"], "supports_native_structured_output", False) + set_native_structured_output("us.anthropic.claude-opus-4-8", "bedrock") profile: Final = "arn:aws:bedrock:us-east-1:000000000000:application-inference-profile/profile-id" litellm_params: Final = {"base_model": "bedrock/us.anthropic.claude-opus-4-8"} optional_params: Final = _converse_params( @@ -11768,8 +11819,9 @@ def test_a_refused_native_schema_is_sent_again_only_when_the_retry_drops_native_ @pytest.mark.respx(assert_all_called=True) +@pytest.mark.parametrize("model", ["us.anthropic.claude-sonnet-5", "us.anthropic.claude-opus-5"]) def test_a_default_native_schema_rejection_retries_with_the_json_tool( - respx_mock: respx.Router, local_model_cost_map: None + respx_mock: respx.Router, local_model_cost_map: None, model: str ) -> None: route: Final = respx_mock.post(url__regex=r"/converse$").mock( return_value=httpx.Response(400, json={"message": _BEDROCK_SCHEMA_REFUSAL}) @@ -11777,7 +11829,7 @@ def test_a_default_native_schema_rejection_retries_with_the_json_tool( with pytest.raises(litellm.BadRequestError, match=re.escape(_BEDROCK_SCHEMA_REFUSAL)): completion( - model="bedrock/converse/us.anthropic.claude-opus-5", + model=f"bedrock/converse/{model}", messages=[{"role": "user", "content": "What is the capital of France?"}], response_format=CAPITAL_RESPONSE_FORMAT, thinking={"type": "disabled"}, @@ -11938,7 +11990,9 @@ def test_converse_sends_no_tools_when_drop_params_drops_every_hosted_mcp_tool(mo def _warnings_naming(caplog: pytest.LogCaptureFixture, param: str) -> tuple[str, ...]: return tuple( - r.getMessage() for r in caplog.records if r.levelno == logging.WARNING and f"Dropping {param}=" in r.getMessage() + r.getMessage() + for r in caplog.records + if r.levelno == logging.WARNING and f"Dropping {param}=" in r.getMessage() ) @@ -12102,9 +12156,15 @@ def test_converse_sends_only_the_header_betas_when_header_and_body_both_carry_be def _five_cache_hint_conversation() -> list[dict[str, object]]: return [ - {"role": "system", "content": [{"type": "text", "text": "system prompt", "cache_control": {"type": "ephemeral"}}]}, + { + "role": "system", + "content": [{"type": "text", "text": "system prompt", "cache_control": {"type": "ephemeral"}}], + }, {"role": "user", "content": [{"type": "text", "text": "turn 1", "cache_control": {"type": "ephemeral"}}]}, - {"role": "assistant", "content": [{"type": "text", "text": "answer 1", "cache_control": {"type": "ephemeral"}}]}, + { + "role": "assistant", + "content": [{"type": "text", "text": "answer 1", "cache_control": {"type": "ephemeral"}}], + }, {"role": "user", "content": [{"type": "text", "text": "turn 2", "cache_control": {"type": "ephemeral"}}]}, {"role": "assistant", "content": "answer 2"}, {"role": "user", "content": [{"type": "text", "text": "turn 3", "cache_control": {"type": "ephemeral"}}]}, @@ -12121,7 +12181,10 @@ def _five_cache_hint_conversation() -> list[dict[str, object]]: ], ) def test_converse_claude_request_over_the_cache_control_cap_drops_the_oldest_hints_only_under_drop_params( - drop_params: bool | str, expected_system_cache_points: int, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None + drop_params: bool | str, + expected_system_cache_points: int, + monkeypatch: pytest.MonkeyPatch, + local_model_cost_map: None, ) -> None: monkeypatch.setattr(litellm, "drop_params", False) messages: Final = _five_cache_hint_conversation() diff --git a/tests/unit/llms/bedrock/chat/test_native_output_fallback.py b/tests/unit/llms/bedrock/chat/test_native_output_fallback.py index 81832d7d8bf..c87451d9577 100644 --- a/tests/unit/llms/bedrock/chat/test_native_output_fallback.py +++ b/tests/unit/llms/bedrock/chat/test_native_output_fallback.py @@ -149,6 +149,122 @@ def test_a_rejection_reaches_the_caller_when_the_retry_would_send_native_output_ assert bedrock.sent == (None,) +# Bedrock's 403 for a role without the streaming action, seen live on us.anthropic.claude-sonnet-5 on 2026-11-06 +_STREAM_PERMISSION_DENIAL: Final = BedrockError( + status_code=403, + message='{"Message":"User: arn:aws:sts::123456789012:assumed-role/invoke-only/qa is not authorized to perform: ' + "bedrock:InvokeModelWithResponseStream on resource: arn:aws:bedrock:us-west-2:123456789012:inference-profile/" + "us.anthropic.claude-sonnet-5 because no identity-based policy allows the " + 'bedrock:InvokeModelWithResponseStream action"}', +) +_INVOKE_PERMISSION_DENIAL: Final = BedrockError( + status_code=403, + message='{"Message":"User: arn:aws:sts::123456789012:assumed-role/no-bedrock/qa is not authorized to perform: ' + 'bedrock:InvokeModel because no identity-based policy allows the bedrock:InvokeModel action"}', +) +_BUFFERED_NATIVE_STREAM: Final = NativeOutputFallback( + optional_params={"outputConfig": "native", "fake_stream": True}, litellm_params={} +) + + +def _buffered(fallback: NativeOutputFallback) -> NativeOutputFallback: + return NativeOutputFallback( + optional_params={**fallback.optional_params, "fake_stream": True}, litellm_params=fallback.litellm_params + ) + + +def _call_with_buffered_stream_fallback( + bedrock: ScriptedBedrock, fallback: NativeOutputFallback | None, buffered_stream_fallback: NativeOutputFallback +) -> object: + return _resolve( + call_with_native_output_fallback( + bedrock, + None if fallback is None else lambda: fallback, + _MODEL, + buffered_stream_fallback=lambda: buffered_stream_fallback, + ) + ) + + +@_MODES +def test_a_denied_native_stream_is_sent_again_once_as_a_buffered_native_request( + asynchronous: bool, caplog: pytest.LogCaptureFixture +) -> None: + bedrock: Final = ScriptedBedrock(errors=(_STREAM_PERMISSION_DENIAL, None), asynchronous=asynchronous) + + with caplog.at_level("WARNING", logger="LiteLLM"): + response: Final = _call_with_buffered_stream_fallback(bedrock, _FALLBACK, _BUFFERED_NATIVE_STREAM) + + assert isinstance(response, ModelResponse) and response.id == "answer-2" + assert bedrock.sent == (None, _BUFFERED_NATIVE_STREAM) + assert "bedrock:InvokeModelWithResponseStream" in caplog.text and _MODEL in caplog.text + + +@_MODES +def test_a_buffered_native_request_bedrock_rejects_is_sent_again_buffered_with_the_fallback_params( + asynchronous: bool, +) -> None: + bedrock: Final = ScriptedBedrock( + errors=(_STREAM_PERMISSION_DENIAL, _GRAMMAR_REJECTION, None), asynchronous=asynchronous + ) + + response: Final = _call_with_buffered_stream_fallback(bedrock, _FALLBACK, _BUFFERED_NATIVE_STREAM) + + assert isinstance(response, ModelResponse) and response.id == "answer-3" + assert bedrock.sent == (None, _BUFFERED_NATIVE_STREAM, _buffered(_FALLBACK)) + + +@_MODES +def test_strict_tools_bedrock_rejects_after_a_buffered_native_retry_are_dropped_on_the_buffered_json_tool_retry( + asynchronous: bool, +) -> None: + bedrock: Final = ScriptedBedrock( + errors=(_STREAM_PERMISSION_DENIAL, _GRAMMAR_REJECTION, _GRAMMAR_REJECTION, None), asynchronous=asynchronous + ) + + response: Final = _resolve( + call_with_native_output_fallback( + bedrock, + lambda: _NATIVE_RETRY_WITH_STRICT_TOOLS, + "us.anthropic.claude-sonnet-4-6", + strict_tools_fallback=lambda: _STRICT_FALLBACK, + buffered_stream_fallback=lambda: _BUFFERED_NATIVE_STREAM, + ) + ) + + assert isinstance(response, ModelResponse) and response.id == "answer-4" + assert bedrock.sent == ( + None, + _BUFFERED_NATIVE_STREAM, + _buffered(_NATIVE_RETRY_WITH_STRICT_TOOLS), + _buffered(_COMPOSED_FALLBACK), + ) + + +@_MODES +@pytest.mark.parametrize( + ("error", "buffered_stream_fallback"), + [ + ( + _STREAM_PERMISSION_DENIAL, + NativeOutputFallback(optional_params={"outputConfig": "native"}, litellm_params={}), + ), + (_INVOKE_PERMISSION_DENIAL, _BUFFERED_NATIVE_STREAM), + ], + ids=["retry_would_stream_again", "invoke_denied"], +) +def test_a_permission_denial_reaches_the_caller_unless_the_retry_buffers_the_stream( + asynchronous: bool, error: BedrockError, buffered_stream_fallback: NativeOutputFallback +) -> None: + bedrock: Final = ScriptedBedrock(errors=(error,), asynchronous=asynchronous) + + with pytest.raises(BedrockError) as raised: + _call_with_buffered_stream_fallback(bedrock, _FALLBACK, buffered_stream_fallback) + + assert raised.value is error + assert bedrock.sent == (None,) + + @pytest.mark.parametrize( ("error", "advises_opting_out"), [(_REJECTION, True), (_SCHEMA_REFUSAL, False)], diff --git a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py index 93bda5b444d..fdd6d5e61bc 100644 --- a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py +++ b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py @@ -21,7 +21,7 @@ from botocore.credentials import Credentials import litellm from litellm.constants import DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET -from litellm.llms.bedrock.common_utils import BedrockError +from litellm.llms.bedrock.common_utils import BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES, BedrockError from litellm.llms.bedrock.files.transformation import BedrockFilesConfig, BedrockJsonlFilesTransformation from tests.unit.proxy.conftest import httpx_transport @@ -2069,6 +2069,76 @@ class TestBedrockBatchAnthropicRowParams: assert ("format" in model_input.get("output_config", {})) is native_output assert (tool_names == {"json_tool_call"}) is not native_output + @pytest.mark.parametrize( + ("model", "native_over_the_limit"), + [(MODEL, True), ("bedrock/us.anthropic.claude-opus-5-5", False)], + ids=["already_native_model", "native_by_default_model"], + ) + @pytest.mark.parametrize("over_the_limit", [False, True], ids=["at_the_limit", "over_the_limit"]) + def test_only_native_by_default_models_send_json_schemas_wider_than_the_limit_through_the_json_tool( + self, local_model_cost_map, model, native_over_the_limit, over_the_limit + ): + native_output: Final = native_over_the_limit or not over_the_limit + field_names: Final = tuple( + f"field_{index:03d}" for index in range(BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES + over_the_limit) + ) + wide_schema: Final = { + "type": "object", + "properties": {name: {"type": "string"} for name in field_names}, + "required": list(field_names), + "additionalProperties": False, + } + model_input: Final = self._transform( + "/v1/chat/completions", + { + "messages": [{"role": "user", "content": "Describe a ceramic coffee mug"}], + "response_format": {"type": "json_schema", "json_schema": {"name": "wide", "schema": wide_schema}}, + }, + model=model, + ) + tool_names: Final = frozenset(tool["name"] for tool in model_input.get("tools", [])) + + assert ("format" in model_input.get("output_config", {})) is native_output + assert (tool_names == {"json_tool_call"}) is not native_output + + def test_a_deployment_opt_out_sends_uploaded_json_schema_rows_through_the_json_tool(self, local_model_cost_map): + from litellm.llms.bedrock.files.transformation import BedrockFilesConfig + + record: Final = { + "custom_id": "row-1", + "method": "POST", + "url": "/v1/chat/completions", + "body": { + "model": self.MODEL, + "messages": [{"role": "user", "content": "Weather in Paris?"}], + "response_format": { + "type": "json_schema", + "json_schema": {"name": "weather", "schema": self.PARAMETERS}, + }, + }, + } + request: Final = BedrockFilesConfig().transform_create_file_request( + model=self.MODEL, + create_file_data={ + "file": ("batch.jsonl", json.dumps(record).encode(), "application/jsonl"), + "purpose": "batch", + }, + optional_params={ + "aws_access_key_id": "test-key-id", + "aws_secret_access_key": "test-secret", + "aws_region_name": "us-west-2", + }, + litellm_params={ + "s3_bucket_name": "litellm-batch-bucket", + "litellm_metadata": {"model_info": {"supports_bedrock_native_structured_output": False}}, + }, + ) + assert isinstance(request, dict) + model_input: Final = json.loads(request["data"].splitlines()[0])["modelInput"] + + assert "format" not in model_input.get("output_config", {}) + assert [tool["name"] for tool in model_input["tools"]] == ["json_tool_call"] + def test_provider_native_params_still_pass_through(self): model_input = self._transform( "/v1/chat/completions", diff --git a/tests/unit/llms/bedrock/test_bedrock_common_utils.py b/tests/unit/llms/bedrock/test_bedrock_common_utils.py index 0c1c4f5475a..7cc4629e497 100644 --- a/tests/unit/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/unit/llms/bedrock/test_bedrock_common_utils.py @@ -1,11 +1,10 @@ - import asyncio, importlib, litellm, pytest import respx from httpx import Response from typing import Final -from litellm.llms.bedrock.common_utils import( +from litellm.llms.bedrock.common_utils import ( AmazonBedrockGlobalConfig, BedrockModelInfo, NativeOutputFallback, @@ -54,9 +53,7 @@ def test_bedrock_response_stream_shape_lazy_loads_once(): import litellm.llms.bedrock.common_utils as mod sentinel = MagicMock() - with patch.object( - mod, "_load_bedrock_response_stream_shape", return_value=sentinel - ) as mock_load: + with patch.object(mod, "_load_bedrock_response_stream_shape", return_value=sentinel) as mock_load: assert mod.get_bedrock_response_stream_shape() is sentinel assert mod.get_bedrock_response_stream_shape() is sentinel mock_load.assert_called_once() @@ -103,9 +100,7 @@ def test_bedrock_response_stream_shape_is_structure_shape(): from litellm.llms.bedrock.common_utils import get_bedrock_response_stream_shape loaded_shape = get_bedrock_response_stream_shape() - assert ( - loaded_shape is not None - ), "get_bedrock_response_stream_shape() is None — botocore may not be installed" + assert loaded_shape is not None, "get_bedrock_response_stream_shape() is None — botocore may not be installed" shape: StructureShape = loaded_shape assert isinstance(shape, StructureShape) assert shape.name == "ResponseStream" @@ -170,9 +165,7 @@ def test_deepseek_cris(): Test that DeepSeek models with cross-region inference prefix use converse route """ bedrock_model_info = BedrockModelInfo - bedrock_route = bedrock_model_info.get_bedrock_route( - model="bedrock/us.deepseek.r1-v1:0" - ) + bedrock_route = bedrock_model_info.get_bedrock_route(model="bedrock/us.deepseek.r1-v1:0") assert bedrock_route == "converse" @@ -228,7 +221,10 @@ def test_bare_opaque_arn_with_no_invoke_provider_routes_to_converse(arn_segment: model: Final = f"bedrock/arn:aws:bedrock:us-east-1:123412341234:{arn_segment}" assert BaseAWSLLM.get_bedrock_invoke_provider(model) is None assert BedrockModelInfo.get_bedrock_route(model) == "converse" - assert BedrockModelInfo.get_bedrock_route(f"bedrock/llama/arn:aws:bedrock:us-east-1:123412341234:{arn_segment}") == "invoke" + assert ( + BedrockModelInfo.get_bedrock_route(f"bedrock/llama/arn:aws:bedrock:us-east-1:123412341234:{arn_segment}") + == "invoke" + ) def test_govcloud_cross_region_inference_prefix(): @@ -238,27 +234,19 @@ def test_govcloud_cross_region_inference_prefix(): bedrock_model_info = BedrockModelInfo # Test us-gov prefix is stripped correctly for Claude models - base_model = bedrock_model_info.get_base_model( - model="bedrock/us-gov.anthropic.claude-haiku-4-5-20251001-v1:0" - ) + base_model = bedrock_model_info.get_base_model(model="bedrock/us-gov.anthropic.claude-haiku-4-5-20251001-v1:0") assert base_model == "anthropic.claude-haiku-4-5-20251001-v1:0" # Test us-gov prefix is stripped correctly for different Claude versions - base_model = bedrock_model_info.get_base_model( - model="bedrock/us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0" - ) + base_model = bedrock_model_info.get_base_model(model="bedrock/us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0") assert base_model == "anthropic.claude-sonnet-4-5-20250929-v1:0" # Test us-gov prefix is stripped correctly for Haiku models - base_model = bedrock_model_info.get_base_model( - model="bedrock/us-gov.anthropic.claude-3-haiku-20240307-v1:0" - ) + base_model = bedrock_model_info.get_base_model(model="bedrock/us-gov.anthropic.claude-3-haiku-20240307-v1:0") assert base_model == "anthropic.claude-3-haiku-20240307-v1:0" # Test us-gov prefix is stripped correctly for Meta models - base_model = bedrock_model_info.get_base_model( - model="bedrock/us-gov.meta.llama3-8b-instruct-v1:0" - ) + base_model = bedrock_model_info.get_base_model(model="bedrock/us-gov.meta.llama3-8b-instruct-v1:0") assert base_model == "meta.llama3-8b-instruct-v1:0" @@ -272,23 +260,14 @@ def test_context_window_suffix_stripped_for_cost_lookup(): """ from litellm.llms.bedrock.common_utils import get_bedrock_base_model - assert ( - get_bedrock_base_model("us.anthropic.claude-opus-4-6-v1[1m]") - == "anthropic.claude-opus-4-6-v1" - ) - assert ( - get_bedrock_base_model("us.anthropic.claude-sonnet-4-6[1m]") - == "anthropic.claude-sonnet-4-6" - ) + assert get_bedrock_base_model("us.anthropic.claude-opus-4-6-v1[1m]") == "anthropic.claude-opus-4-6-v1" + assert get_bedrock_base_model("us.anthropic.claude-sonnet-4-6[1m]") == "anthropic.claude-sonnet-4-6" assert ( get_bedrock_base_model("global.anthropic.claude-opus-4-5-20251101-v1:0[1m]") == "anthropic.claude-opus-4-5-20251101-v1:0" ) # Ensure models without suffix are unaffected - assert ( - get_bedrock_base_model("us.anthropic.claude-opus-4-6-v1") - == "anthropic.claude-opus-4-6-v1" - ) + assert get_bedrock_base_model("us.anthropic.claude-opus-4-6-v1") == "anthropic.claude-opus-4-6-v1" # Ensure :51k throughput suffix still works assert ( get_bedrock_base_model("anthropic.claude-3-5-sonnet-20241022-v2:0:51k") @@ -358,9 +337,7 @@ def test_output_config_effort_normalization_uses_model_info_ceiling(monkeypatch) ("us.anthropic.claude-opus-4-7", "xhigh"), ], ) -def test_bundled_bedrock_opus_model_info_declares_output_config_effort_ceiling( - model, expected_ceiling -): +def test_bundled_bedrock_opus_model_info_declares_output_config_effort_ceiling(model, expected_ceiling): from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap model_info = GetModelCostMap.load_local_model_cost_map()[model] @@ -379,54 +356,24 @@ def test_route_prefix_matched_as_path_segment_not_substring(): or a ``/`` boundary. """ # The bedrock_mantle/ provider prefix must NOT be read as the mantle/ route. - assert ( - BedrockModelInfo.get_bedrock_route("bedrock_mantle/openai.gpt-5.5") != "mantle" - ) - assert ( - BedrockModelInfo.get_bedrock_route("bedrock_mantle/openai.gpt-5.4") == "converse" - ) - assert ( - BedrockModelInfo._explicit_mantle_route("bedrock_mantle/openai.gpt-5.5") - is False - ) + assert BedrockModelInfo.get_bedrock_route("bedrock_mantle/openai.gpt-5.5") != "mantle" + assert BedrockModelInfo.get_bedrock_route("bedrock_mantle/openai.gpt-5.4") == "converse" + assert BedrockModelInfo._explicit_mantle_route("bedrock_mantle/openai.gpt-5.5") is False # A genuine mantle route still resolves, via the startswith branch... - assert ( - BedrockModelInfo.get_bedrock_route("mantle/anthropic.claude-mythos-preview") - == "mantle" - ) + assert BedrockModelInfo.get_bedrock_route("mantle/anthropic.claude-mythos-preview") == "mantle" # ...and via the mid-path "/mantle/" branch (after the bedrock/ provider prefix). - assert ( - BedrockModelInfo.get_bedrock_route( - "bedrock/mantle/anthropic.claude-mythos-preview" - ) - == "mantle" - ) + assert BedrockModelInfo.get_bedrock_route("bedrock/mantle/anthropic.claude-mythos-preview") == "mantle" def test_model_has_route_prefix_exercises_both_branches(): """``_model_has_route_prefix`` matches on ``startswith`` or a ``/`` boundary only.""" # startswith branch - assert ( - BedrockModelInfo._model_has_route_prefix( - "mantle/anthropic.claude-mythos-preview", "mantle/" - ) - is True - ) + assert BedrockModelInfo._model_has_route_prefix("mantle/anthropic.claude-mythos-preview", "mantle/") is True # f"/{prefix}" boundary branch - assert ( - BedrockModelInfo._model_has_route_prefix( - "bedrock/mantle/anthropic.claude-mythos-preview", "mantle/" - ) - is True - ) + assert BedrockModelInfo._model_has_route_prefix("bedrock/mantle/anthropic.claude-mythos-preview", "mantle/") is True # neither branch: the token only appears glued to another segment - assert ( - BedrockModelInfo._model_has_route_prefix( - "bedrock_mantle/openai.gpt-5.5", "mantle/" - ) - is False - ) + assert BedrockModelInfo._model_has_route_prefix("bedrock_mantle/openai.gpt-5.5", "mantle/") is False @pytest.mark.parametrize( @@ -476,16 +423,10 @@ def test_explicit_invoke_route_does_not_match_async_invoke(): """ async_invoke_model = "async_invoke/twelvelabs.marengo-embed-2-7-v1:0" assert BedrockModelInfo._explicit_invoke_route(async_invoke_model) is False - assert ( - BedrockModelInfo._explicit_invoke_route(f"bedrock/{async_invoke_model}") - is False - ) + assert BedrockModelInfo._explicit_invoke_route(f"bedrock/{async_invoke_model}") is False # ...while async_invoke/ is still detected as its own route. assert BedrockModelInfo._explicit_async_invoke_route(async_invoke_model) is True - assert ( - BedrockModelInfo._explicit_async_invoke_route(f"bedrock/{async_invoke_model}") - is True - ) + assert BedrockModelInfo._explicit_async_invoke_route(f"bedrock/{async_invoke_model}") is True def test_capability_lookups_fall_back_to_base_model_when_regional_entry_lacks_field(monkeypatch): @@ -605,7 +546,6 @@ def test_deployment_model_info_breakpoint_flag_covers_an_unmapped_arn(local_mode assert bedrock_model_accepts_cache_points(unflagged_arn) is True - _CONVERSE_REPLY: Final = { "output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}}, "stopReason": "end_turn", @@ -638,7 +578,11 @@ _MARKED_TOOL_CONVERSATION: Final = ( } ], }, - {"role": "tool", "tool_call_id": "call_1", "content": [{"type": "text", "text": "Sunny", "cache_control": _CACHE_MARK}]}, + { + "role": "tool", + "tool_call_id": "call_1", + "content": [{"type": "text", "text": "Sunny", "cache_control": _CACHE_MARK}], + }, {"role": "tool", "tool_call_id": "call_1_1", "content": "Cloudy"}, {"role": "assistant", "content": "Paris is sunny, Rome is cloudy.", "cache_control": _CACHE_MARK}, {"role": "user", "content": "Thanks"}, @@ -711,6 +655,7 @@ async def test_a_bedrock_deployment_sends_cache_points_by_its_own_flag_not_a_sib "flagless": not declared, } + async def test_a_routed_bedrock_deployment_sends_cache_points_when_register_model_opted_its_model_in( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None ) -> None: @@ -738,7 +683,9 @@ async def test_a_routed_bedrock_deployment_sends_cache_points_when_register_mode num_retries=0, ) - await router.acompletion(model="flagless", messages=[{"role": "system", "content": "stable"}, {"role": "user", "content": "hi"}]) + await router.acompletion( + model="flagless", messages=[{"role": "system", "content": "stable"}, {"role": "user", "content": "hi"}] + ) assert b"cachePoint" in converse.calls.last.request.content @@ -783,10 +730,13 @@ def test_a_deployments_prompt_caching_flag_written_as_a_string_or_number_decides ) assert ( - bedrock_model_accepts_cache_points(model, model_info={"id": "qa-deployment", "supports_prompt_caching": declared}) + bedrock_model_accepts_cache_points( + model, model_info={"id": "qa-deployment", "supports_prompt_caching": declared} + ) is expected ) + def test_merge_bedrock_aws_request_params_strips_caller_identity_when_deployment_has_static_credentials(): from litellm.llms.bedrock.common_utils import merge_bedrock_aws_request_params @@ -1491,6 +1441,7 @@ def _vcr_outcome_gate(request, vcr): yield record_vcr_outcome(request, vcr) + @pytest.fixture(scope="session") def event_loop(): try: @@ -1500,6 +1451,7 @@ def event_loop(): yield loop loop.close() + @pytest.fixture(scope="function") def setup_and_teardown(event_loop): import litellm @@ -1533,6 +1485,7 @@ def setup_and_teardown(event_loop): if pending: event_loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True)) + _SCALAR_DEFAULTS = { "num_retries": getattr(litellm, "num_retries", None), "set_verbose": getattr(litellm, "set_verbose", False), @@ -1547,6 +1500,7 @@ _SCALAR_DEFAULTS = { "cohere_key": getattr(litellm, "cohere_key", None), } + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestStripBedrockRoutingPrefix: """Tests for strip_bedrock_routing_prefix function.""" @@ -1577,6 +1531,7 @@ class TestStripBedrockRoutingPrefix: == "anthropic.claude-3-sonnet-20240229-v1:0" ) + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestStripBedrockThroughputSuffix: """Tests for strip_bedrock_throughput_suffix function.""" @@ -1604,6 +1559,7 @@ class TestStripBedrockThroughputSuffix: def test_strip_throughput_suffix(self, input_model, expected): assert strip_bedrock_throughput_suffix(input_model) == expected + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestExtractModelNameFromBedrockArn: """Tests for extract_model_name_from_bedrock_arn function.""" @@ -1624,6 +1580,7 @@ class TestExtractModelNameFromBedrockArn: arn = "ARN:aws:bedrock:us-east-1:123456789012:model/my-model" assert extract_model_name_from_bedrock_arn(arn) == "my-model" + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestGetBedrockCrossRegionInferenceRegions: """Tests for get_bedrock_cross_region_inference_regions function.""" @@ -1639,6 +1596,7 @@ class TestGetBedrockCrossRegionInferenceRegions: regions = get_bedrock_cross_region_inference_regions() assert isinstance(regions, list) + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestGetBedrockBaseModel: """Tests for get_bedrock_base_model function.""" @@ -1702,6 +1660,7 @@ class TestGetBedrockBaseModel: """Test that throughput tier suffixes like :51k are stripped. Issue #19113.""" assert get_bedrock_base_model(input_model) == expected + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestBedrockModelInfoWrappers: """Tests that BedrockModelInfo methods correctly wrap standalone functions.""" @@ -1723,6 +1682,7 @@ class TestBedrockModelInfoWrappers: model = "bedrock/converse/claude-3" assert BedrockModelInfo.get_non_litellm_routing_model_name(model) == strip_bedrock_routing_prefix(model) + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestBedrockTokenCounter: """Tests for BedrockTokenCounter class.""" @@ -1759,12 +1719,14 @@ class TestBedrockTokenCounter: ) assert result is None + @pytest.fixture def _pr4_bedrock_env(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("AWS_ACCESS_KEY_ID", "pr4-test-aws-access-key") monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "pr4-test-aws-secret-key") monkeypatch.setenv("AWS_REGION_NAME", "us-east-1") + @pytest.fixture(scope="module") def _use_local_model_cost_map() -> Iterator[None]: with pytest.MonkeyPatch.context() as monkeypatch: @@ -1772,6 +1734,7 @@ def _use_local_model_cost_map() -> Iterator[None]: importlib.reload(litellm) yield + @pytest.mark.usefixtures( "_pr4_bedrock_env", "_use_local_model_cost_map", @@ -2348,8 +2311,12 @@ def test_bedrock_capability_model_reads_base_model_only_for_application_profile_ assert bedrock_capability_model(model, base_model) == expected -_FORMAT_REJECTION: Final = "The model returned the following errors: output_config.format: Extra inputs are not permitted" -_COMPILE_REJECTION: Final = "The model returned the following errors: compilation exceeded limits: Schema is too complex." +_FORMAT_REJECTION: Final = ( + "The model returned the following errors: output_config.format: Extra inputs are not permitted" +) +_COMPILE_REJECTION: Final = ( + "The model returned the following errors: compilation exceeded limits: Schema is too complex." +) _GRAMMAR_REJECTION: Final = ( "The model returned the following errors: The compiled grammar is too large, which would cause performance issues. " "Simplify your tool schemas or reduce the number of strict tools." @@ -2394,10 +2361,10 @@ def test_is_bedrock_native_output_rejection_matches_only_the_400s_that_reject_na ("model", "model_info", "expected"), [ ("us.anthropic.claude-opus-4-8", None, False), - ("global.anthropic.claude-sonnet-5", None, True), + ("global.anthropic.claude-sonnet-5", None, False), ("us-gov.anthropic.claude-opus-4-8", None, False), ("us.anthropic.claude-opus-4-8", {"supports_bedrock_native_structured_output": False}, False), - ("us.anthropic.claude-opus-5-5", {"supports_bedrock_native_structured_output": True}, True), + ("us.anthropic.claude-opus-5-5", {"supports_bedrock_native_structured_output": True}, False), ("us.anthropic.claude-sonnet-4-6", {"supports_bedrock_native_structured_output": True}, False), ], ids=["opus_4_8", "sonnet_5", "govcloud_opus_4_8", "deployment_opt_out", "deployment_opt_in", "native_model"], @@ -2490,7 +2457,10 @@ def test_bedrock_strict_tools_fallback_drops_only_strict_and_keeps_everything_el assert fallback == NativeOutputFallback( optional_params={ "tools": [ - {**strict_tool, "function": {"name": "lookup", "description": "Look up", "parameters": {"type": "object"}}}, + { + **strict_tool, + "function": {"name": "lookup", "description": "Look up", "parameters": {"type": "object"}}, + }, plain_tool, anthropic_tool, {"toolSpec": {"name": "spec", "inputSchema": {"json": {"type": "object"}}}}, diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index dd0e5762845..a758f4d4250 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -4890,6 +4890,92 @@ async def test_json_mode_stream_reaches_the_iterator_of_a_config_without_a_reque ] +class _PlainStreamRouteConfig(_TransformRecordingConfig): + def __init__(self): + super().__init__(transform_async=False) + + def get_complete_url(self, api_base, api_key, model, optional_params, litellm_params, stream=None): + return f"{api_base}/{'stream' if stream else 'answer'}" + + +class _StreamRouteConfig(_PlainStreamRouteConfig): + @property + def has_custom_stream_wrapper(self) -> bool: + return True + + +def _answers_only_the_non_streaming_route(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/answer"): + return httpx.Response(200, json=json.loads(request.content)) + return httpx.Response(403, json={"message": "not authorized to stream"}) + + +def _fake_streamed_completion( + acompletion: bool, + client: HTTPHandler | AsyncHTTPHandler, + fake_stream: object = True, + provider_config: BaseConfig | None = None, +): + return BaseLLMHTTPHandler().completion( + model="stub-model", + messages=[{"role": "user", "content": "hi"}], + api_base="https://llm.example/v1/chat", + custom_llm_provider="openai", + model_response=ModelResponse(), + encoding=None, + logging_obj=_streaming_logging_obj(), + optional_params={"fake_stream": fake_stream}, + timeout=10.0, + litellm_params={}, + acompletion=acompletion, + stream=True, + client=client, + provider_config=provider_config or _StreamRouteConfig(), + ) + + +async def test_a_fake_streamed_completion_is_sent_to_the_non_streaming_route(): + sync_client: Final = HTTPHandler( + client=httpx.Client(transport=httpx.MockTransport(_answers_only_the_non_streaming_route)) + ) + async_client: Final = AsyncHTTPHandler() + async_client.client = httpx.AsyncClient(transport=httpx.MockTransport(_answers_only_the_non_streaming_route)) + + sync_chunks: Final = list(_fake_streamed_completion(acompletion=False, client=sync_client)) + async_chunks: Final = [ + chunk async for chunk in await _fake_streamed_completion(acompletion=True, client=async_client) + ] + + assert (_streamed_text(sync_chunks), _streamed_text(async_chunks)) == ("sync", "sync") + + +def _answers_only_the_streaming_route(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/stream"): + return _sse_echoing_transformed_by(request) + return httpx.Response(403, json={"message": "only streaming is allowed"}) + + +@pytest.mark.parametrize("fake_stream", ["true", 1], ids=["string", "int"]) +async def test_a_fake_stream_value_other_than_true_still_streams_from_the_streaming_route(fake_stream: object): + sync_client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_answers_only_the_streaming_route))) + async_client: Final = AsyncHTTPHandler() + async_client.client = httpx.AsyncClient(transport=httpx.MockTransport(_answers_only_the_streaming_route)) + + sync_chunks: Final = list( + _fake_streamed_completion( + acompletion=False, client=sync_client, fake_stream=fake_stream, provider_config=_PlainStreamRouteConfig() + ) + ) + async_chunks: Final = [ + chunk + async for chunk in await _fake_streamed_completion( + acompletion=True, client=async_client, fake_stream=fake_stream, provider_config=_PlainStreamRouteConfig() + ) + ] + + assert (_streamed_text(sync_chunks), _streamed_text(async_chunks)) == ("sync", "sync") + + class _ImageEditRecordingConfig(BaseImageEditConfig): def __init__(self): self.transform_calls = [] diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py index 6ad5ef1a4eb..79f3e697a4f 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_info.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py @@ -490,6 +490,51 @@ def test_model_info_reports_null_cost_for_unpriced_deployment_and_zero_for_decla assert input_cost > 0 and output_cost > 0 +@pytest.mark.parametrize( + ("deployment_model_info", "native"), + [ + ({}, True), + ({"supports_bedrock_native_structured_output": False}, False), + ({"supports_native_structured_output": True, "supports_bedrock_native_structured_output": False}, True), + ], + ids=["native-by-default", "bedrock-opt-out", "explicit-generic-flag"], +) +def test_model_info_reports_a_deployment_bedrock_native_output_opt_out( + local_model_cost_map: None, deployment_model_info: dict[str, object], native: bool +) -> None: + info: Final = proxy_server._enrich_model_info_with_litellm_data( + { + "model_name": "sonnet", + "litellm_params": {"model": "bedrock/us.anthropic.claude-sonnet-5"}, + "model_info": {"id": "sonnet-deployment", **deployment_model_info}, + } + )["model_info"] + + assert info["supports_native_structured_output"] is native + + +@pytest.mark.parametrize( + ("model", "by_default"), + [("bedrock/us.anthropic.claude-opus-4-7", True), ("bedrock/us.anthropic.claude-sonnet-5", False)], + ids=["set-on-a-model-the-cost-map-leaves-on-the-json-tool", "unset-on-a-model-the-cost-map-makes-native"], +) +def test_model_info_ignores_a_deployment_by_default_key_the_router_reads_only_from_the_cost_map( + local_model_cost_map: None, model: str, by_default: bool +) -> None: + def reported_native(deployment_model_info: dict[str, object]) -> object: + return proxy_server._enrich_model_info_with_litellm_data( + { + "model_name": "claude", + "litellm_params": {"model": model}, + "model_info": {"id": "claude-deployment", **deployment_model_info}, + } + )["model_info"]["supports_native_structured_output"] + + assert reported_native({"supports_bedrock_native_structured_output_by_default": by_default}) is reported_native( + {} + ) + + def test_model_info_id_lookup_reports_the_same_cost_as_the_list( client: TestClient, auth_as: Callable[[], AbstractContextManager[object]], diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index c0c8a0655fe..cd71c3f9bee 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -542,7 +542,13 @@ def test_should_not_pollute_shared_key_with_custom_nonzero_pricing(): @pytest.mark.parametrize( "flag", - ["supports_regex_lookaround", "supports_assistant_prefill", "supports_sampling_params", "supports_continuation"], + [ + "supports_regex_lookaround", + "supports_assistant_prefill", + "supports_sampling_params", + "supports_continuation", + "supports_bedrock_native_structured_output_by_default", + ], ) def test_capability_flag_stays_on_the_deployment_that_set_it(flag: str) -> None: """A deployment's capability override must not land on the shared diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index f8ec0e56e4a..b0c55d3f0f2 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1354,6 +1354,7 @@ _COST_MAP_ROW_SCHEMA: Final = { "supports_image_size": {"type": "boolean"}, "supports_native_structured_output": {"type": "boolean"}, "supports_bedrock_native_structured_output": {"type": "boolean"}, + "supports_bedrock_native_structured_output_by_default": {"type": "boolean"}, "use_openai_responses_path": {"type": "boolean"}, "off_peak_pricing": { "type": "object", diff --git a/tests/unit/test_utils_get_model_info.py b/tests/unit/test_utils_get_model_info.py index d08742be4af..7548001ce4c 100644 --- a/tests/unit/test_utils_get_model_info.py +++ b/tests/unit/test_utils_get_model_info.py @@ -13,7 +13,7 @@ from litellm import get_model_info from litellm.llms.bedrock.common_utils import BedrockModelInfo from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.types.utils import ModelInfoBase -from litellm.utils import _invalidate_model_cost_lowercase_map +from litellm.utils import _invalidate_model_cost_lowercase_map, supports_native_structured_output import os from unittest.mock import MagicMock, patch @@ -388,6 +388,29 @@ def test_get_model_info_bedrock_cross_region_capability_parity(): assert checked > 0, "no cross-region bedrock profiles found - the filter is inert" +def test_bedrock_native_output_default_reaches_only_versions_that_read_its_key(local_model_cost_map): + flagged: Final = tuple( + key for key, row in litellm.model_cost.items() if row.get("supports_bedrock_native_structured_output_by_default") + ) + + assert flagged + assert [key for key in flagged if litellm.model_cost[key].get("supports_native_structured_output")] == [] + assert [key for key in flagged if not get_model_info(key)["supports_native_structured_output"]] == [] + + +@pytest.mark.parametrize( + ("registered", "native"), + [({"supports_bedrock_native_structured_output": False}, False), ({"max_output_tokens": 1024}, True)], + ids=["bedrock-opt-out", "unrelated-override"], +) +def test_a_registered_bedrock_opt_out_beats_the_native_output_default(local_model_cost_map, registered, native): + model: Final = "us.anthropic.claude-sonnet-5" + litellm.register_model({model: registered}, persist_across_reloads=False) + + assert get_model_info(f"bedrock/{model}")["supports_native_structured_output"] is native + assert supports_native_structured_output(f"bedrock/{model}") is native + + def _is_positive_cost(value: object) -> bool: return isinstance(value, (int, float)) and value > 0 From 75d4e15e64395518d04e445b393caded87a4b341 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 6 Nov 2026 17:53:25 +0000 Subject: [PATCH 05/39] test(spend): exercise TTL-preserving increments through rate limit store (#50418) Co-authored-by: mateo <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...test_redis_ttl_preserving_token_increment.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/integration/spend/test_redis_ttl_preserving_token_increment.py b/tests/integration/spend/test_redis_ttl_preserving_token_increment.py index 8b7a21f73d5..daf76bf4740 100644 --- a/tests/integration/spend/test_redis_ttl_preserving_token_increment.py +++ b/tests/integration/spend/test_redis_ttl_preserving_token_increment.py @@ -11,7 +11,7 @@ from litellm.proxy.hooks.parallel_request_limiter_v3 import ( PROXY_MaxParallelRequestsHandler_v3 as _PROXY_MaxParallelRequestsHandler, ) from litellm.proxy.utils import InternalUsageCache -from litellm.types.caching import RedisPipelineIncrementOperation +from litellm.redis.stores.rate_limits import CounterIncrement, RateLimitScope @pytest.mark.asyncio @@ -22,11 +22,12 @@ async def test_async_increment_tokens_with_ttl_preservation() -> None: handler: Final = _PROXY_MaxParallelRequestsHandler( internal_usage_cache=InternalUsageCache(DualCache(redis_cache=redis_cache)) ) - assert handler.token_increment_script is not None suffix: Final = uuid.uuid4().hex[:8] - key_with_ttl: Final = f"{{test_ttl}}:with_ttl:{suffix}" - key_without_ttl: Final = f"{{test_ttl}}:without_ttl:{suffix}" + scope_with_ttl: Final = RateLimitScope("test_ttl", f"with_ttl:{suffix}") + scope_without_ttl: Final = RateLimitScope("test_ttl", f"without_ttl:{suffix}") + key_with_ttl: Final = f"{{test_ttl:with_ttl:{suffix}}}:tokens" + key_without_ttl: Final = f"{{test_ttl:without_ttl:{suffix}}}:tokens" try: await redis_cache.async_delete_cache(key_with_ttl) @@ -34,8 +35,8 @@ async def test_async_increment_tokens_with_ttl_preservation() -> None: await handler.async_increment_tokens_with_ttl_preservation( pipeline_operations=[ - RedisPipelineIncrementOperation(key=key_with_ttl, increment_value=10.0, ttl=60), - RedisPipelineIncrementOperation(key=key_without_ttl, increment_value=5.0, ttl=None), + CounterIncrement(scope_with_ttl.counter("tokens"), 10.0, 60), + CounterIncrement(scope_without_ttl.counter("tokens"), 5.0, None), ] ) @@ -50,8 +51,8 @@ async def test_async_increment_tokens_with_ttl_preservation() -> None: await handler.async_increment_tokens_with_ttl_preservation( pipeline_operations=[ - RedisPipelineIncrementOperation(key=key_with_ttl, increment_value=15.0, ttl=60), - RedisPipelineIncrementOperation(key=key_without_ttl, increment_value=7.0, ttl=None), + CounterIncrement(scope_with_ttl.counter("tokens"), 15.0, 60), + CounterIncrement(scope_without_ttl.counter("tokens"), 7.0, None), ] ) From bc1309f18eb2c440782500e4c5c902f5e4e0e1de Mon Sep 17 00:00:00 2001 From: mrinal-berri Date: Fri, 6 Nov 2026 09:55:30 -0800 Subject: [PATCH 06/39] fix(responses): map streamed flex_unavailable errors to 429 not 500 (#50336) * fix(responses): map streamed flex_unavailable errors to 429 not 500 When OpenAI has no flex capacity, a service_tier flex request fails with type resource_unavailable / code flex_unavailable. Non-streamed, this arrives as HTTP 429 and litellm raises RateLimitError. Streamed /v1/responses delivers the same error as an in-stream event, and neither field was in _ERROR_CODE_HTTP_STATUS, so it defaulted to 500 InternalServerError. The HTTP status then depended only on stream: one provider outage looked like a 429 or a 500. The 500 reads as a server error in metrics and alerting, and callers that retry on 429 do not retry it. Map both resource_unavailable and flex_unavailable to 429 so the streamed path matches the non-streamed path. Fixes #49736 * test(responses): cover streamed flex outage end to end for error and response.failed events Streams the flex_unavailable payload from #49736 through litellm.aresponses with an injected client and asserts a 429 RateLimitError for both in-stream event kinds, matching the non-streamed path * test(responses): drop explanatory comment from flex status mapping test * test(integration): cover streamed flex outage status on responses and chat bridge Drives the flex_unavailable payload from #49736 through a real proxy as an in-stream error event and as response.failed. The native /v1/responses stream fails with rate_limit_exceeded and logs 429, and the chat bridge answers a 429 throttling error, where both reported a 500 before the mapping fix --------- Co-authored-by: BinYi LIU --- litellm/responses/streaming_iterator.py | 2 + .../test_responses_top_level_error_events.py | 34 ++++++++ .../test_streaming_iterator_error_events.py | 78 +++++++++++++++++++ 3 files changed, 114 insertions(+) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 288e4a7119d..0c002a770e8 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -250,6 +250,8 @@ _ERROR_CODE_HTTP_STATUS: Final[Mapping[str, int]] = MappingProxyType( "server_error": 500, "rate_limit_exceeded": 429, "insufficient_quota": 429, + "resource_unavailable": 429, + "flex_unavailable": 429, "vector_store_timeout": 504, "invalid_prompt": 400, "invalid_image": 400, diff --git a/tests/integration/streaming/test_responses_top_level_error_events.py b/tests/integration/streaming/test_responses_top_level_error_events.py index 63bdadceebc..2cbc4570dec 100644 --- a/tests/integration/streaming/test_responses_top_level_error_events.py +++ b/tests/integration/streaming/test_responses_top_level_error_events.py @@ -22,12 +22,14 @@ from integration._support.responses_stream import ( data_frames, delta, error_body, + error_event, frame, function_tools, healthy_stream, messages_body, openai_error_event, openai_error_stream, + response_object, serve, spend_row, sse_events, @@ -277,6 +279,38 @@ def test_native_responses_stream_fails_with_the_provider_code_and_message(gatewa _assert_failure_logged(response.headers["x-litellm-call-id"], "429", RATE_LIMIT_MESSAGE) +_FLEX_MESSAGE: Final = "Flex processing is temporarily unavailable. Please try again later or use standard processing." +_FLEX_ERROR: Final = {"type": "resource_unavailable", "code": "flex_unavailable", "message": _FLEX_MESSAGE} + + +def _flex_outage(identity: str, kind: str) -> tuple[bytes, ...]: + failure: Final = ( + error_event(_FLEX_ERROR) + if kind == "error" + else { + "type": "response.failed", + "sequence_number": 1, + "response": response_object(identity, "failed", error=_FLEX_ERROR), + } + ) + return (frame(created(identity)), frame(failure)) + + +@pytest.mark.parametrize("kind", ["error", "response.failed"]) +def test_native_responses_stream_flex_outage_fails_as_a_rate_limit(gateway: Gateway, kind: str) -> None: + response, error = _responses_frames(gateway, _flex_outage(_identity(), kind)) + assert error["code"] == "rate_limit_exceeded" and _FLEX_MESSAGE in string_value(error["message"]), error + _assert_failure_logged(response.headers["x-litellm-call-id"], "429", _FLEX_MESSAGE) + + +@pytest.mark.parametrize("kind", ["error", "response.failed"]) +def test_chat_stream_flex_outage_is_a_throttling_error_like_the_non_streamed_429(gateway: Gateway, kind: str) -> None: + response, _ = _chat(gateway, _flex_outage(_identity(), kind)) + assert response.status_code == 429, response.text + assert_provider_typed_rate_limit(error_body(response), _FLEX_MESSAGE) + _assert_failure_logged(response.headers["x-litellm-call-id"], "429", _FLEX_MESSAGE) + + def test_native_responses_stream_error_after_text_fails_with_the_provider_code(gateway: Gateway) -> None: identity: Final = _identity() stream: Final = ( diff --git a/tests/unit/responses/test_streaming_iterator_error_events.py b/tests/unit/responses/test_streaming_iterator_error_events.py index 14ecb185f06..0acf5d9ce62 100644 --- a/tests/unit/responses/test_streaming_iterator_error_events.py +++ b/tests/unit/responses/test_streaming_iterator_error_events.py @@ -37,6 +37,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.get_litellm_params import get_litellm_params from litellm.litellm_core_utils.litellm_logging import _get_provider_request_id from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.responses.streaming_iterator import ( _ERROR_CODE_HTTP_STATUS, BaseResponsesAPIStreamingIterator, @@ -75,6 +76,83 @@ def _make_error_chunk(error_type: str, code: str, message: str = "err") -> Error return ErrorEvent(type=ResponsesAPIStreamEvents.ERROR, sequence_number=0, error=error_obj) +@pytest.mark.parametrize( + "error_type,error_code", + ( + ("resource_unavailable", "flex_unavailable"), + ("resource_unavailable", None), + (None, "flex_unavailable"), + ), +) +def test_streamed_flex_unavailable_maps_to_429(error_type: str | None, error_code: str | None) -> None: + assert _status_code_for_error_fields(error_type, error_code) == 429 + + +def test_streamed_unrelated_server_error_still_maps_to_500() -> None: + assert _status_code_for_error_fields("server_error", "server_error") == 500 + + +FLEX_UNAVAILABLE_ERROR: Final = { + "type": "resource_unavailable", + "code": "flex_unavailable", + "message": "Flex processing is temporarily unavailable. Please try again later or use standard processing.", +} + + +def _flex_outage_sse(kind: str) -> str: + pending: Final = { + "id": "resp_flex", + "object": "response", + "created_at": 1700000000, + "model": "gpt-5", + "status": "in_progress", + "output": [], + } + failure: Final = ( + {"type": "error", "sequence_number": 1, "error": FLEX_UNAVAILABLE_ERROR} + if kind == "error" + else { + "type": "response.failed", + "sequence_number": 1, + "response": {**pending, "status": "failed", "error": FLEX_UNAVAILABLE_ERROR}, + } + ) + events: Final = ({"type": "response.created", "sequence_number": 0, "response": pending}, failure) + return "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("kind", ("error", "response.failed")) +async def test_streamed_flex_outage_raises_rate_limit_like_the_non_streamed_path( + kind: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + async_client: Final = AsyncHTTPHandler( + transport=httpx.MockTransport( + lambda request: httpx.Response( + 200, headers={"content-type": "text/event-stream"}, text=_flex_outage_sse(kind) + ) + ) + ) + try: + stream: Final = await litellm.aresponses( + model="openai/gpt-5", + input="hi", + service_tier="flex", + stream=True, + api_key="sk-test", + client=async_client, + ) + with pytest.raises(MidStreamFallbackError) as raised: + _ = [event async for event in stream] + finally: + await async_client.close() + assert raised.value.status_code == 429, raised.value + assert isinstance(raised.value.original_exception, litellm.RateLimitError), raised.value.original_exception + assert "Flex processing is temporarily unavailable" in str(raised.value.original_exception) + + @pytest.mark.parametrize("provider", ("openai", "azure", "bedrock")) @pytest.mark.parametrize("header", ("x-amzn-RequestId", "x-request-id", None)) @pytest.mark.parametrize( From 4fe1cfe09751a409f6732655c4628df74ed78bd7 Mon Sep 17 00:00:00 2001 From: Nate Armstrong Date: Fri, 6 Nov 2026 10:08:04 -0800 Subject: [PATCH 07/39] feat(router-core): derive serialize on router events (#50416) RouterEvent and the types it carries (FailureKind, Blocked, MockFailure, RetryCause, PickError) now serialize, so a host can log or forward events as JSON instead of parsing their Debug text. Events are tagged by "event", durations are seconds, failure kinds use litellm's exception class names --- litellm-rust/crates/router-core/src/error.rs | 12 ++ litellm-rust/crates/router-core/src/host.rs | 104 +++++++++++++++++- litellm-rust/crates/router-core/src/mock.rs | 10 +- .../crates/router-core/tests/call_loop.rs | 54 ++++++++- 4 files changed, 174 insertions(+), 6 deletions(-) diff --git a/litellm-rust/crates/router-core/src/error.rs b/litellm-rust/crates/router-core/src/error.rs index 926772f435c..1b718ed2c12 100644 --- a/litellm-rust/crates/router-core/src/error.rs +++ b/litellm-rust/crates/router-core/src/error.rs @@ -284,6 +284,7 @@ impl Error { strum::EnumString, strum::IntoStaticStr, strum::VariantArray, + serde_with::SerializeDisplay, )] pub enum FailureKind { #[strum(serialize = "RateLimitError")] @@ -471,6 +472,17 @@ mod tests { assert_eq!(<&'static str>::from(kind), wire); } + #[rstest] + #[case::rate_limit(FailureKind::RateLimit, "RateLimitError")] + #[case::timeout(FailureKind::Timeout, "Timeout")] + #[case::api_connection(FailureKind::ApiConnection, "APIConnectionError")] + fn failure_kind_serializes_as_its_exception_name( + #[case] kind: FailureKind, + #[case] wire: &str, + ) { + assert_eq!(serde_json::to_value(kind).unwrap(), serde_json::json!(wire)); + } + #[rstest] #[case::empty("")] #[case::lowercase("ratelimiterror")] diff --git a/litellm-rust/crates/router-core/src/host.rs b/litellm-rust/crates/router-core/src/host.rs index cfc97480bf1..0ef37873e6e 100644 --- a/litellm-rust/crates/router-core/src/host.rs +++ b/litellm-rust/crates/router-core/src/host.rs @@ -342,7 +342,10 @@ pub struct Output { } /// How a provider blocked a response the call otherwise served. -#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::IntoStaticStr, strum::EnumString)] +#[derive( + Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, strum::IntoStaticStr, strum::EnumString, +)] +#[serde(rename_all = "snake_case")] #[strum(serialize_all = "snake_case")] pub enum Blocked { /// A chat response whose first choice was content-filtered (or that has no choices). @@ -402,8 +405,10 @@ pub enum SelectStage { OverrideCheck(RoutingStrategy), } -/// Routing events, delivered through [`Host::observe`]. -#[derive(Clone, Debug, PartialEq)] +/// Routing events, delivered through [`Host::observe`]. They serialize as JSON objects tagged by +/// `event` (the variant name), durations in seconds. +#[derive(Clone, Debug, PartialEq, serde::Serialize)] +#[serde(tag = "event")] pub enum RouterEvent { /// A pick for `model` (the level's group) begins: its route phase opens. PickStarted { @@ -413,6 +418,7 @@ pub enum RouterEvent { model_group: String, deployment_id: String, /// How long the pick took (`_acompletion`'s `async_get_available_deployment` service event). + #[serde(serialize_with = "seconds")] pick: Duration, }, Succeeded { @@ -425,11 +431,13 @@ pub enum RouterEvent { }, CooldownSet { deployment_id: String, + #[serde(serialize_with = "seconds")] cooldown: Duration, }, Retry { model_group: String, retry: u32, + #[serde(serialize_with = "seconds")] delay: Duration, }, Fallback { @@ -584,6 +592,13 @@ impl PickError { } } +/// A pick's error serializes as its message. +impl serde::Serialize for PickError { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_str(&self.0) + } +} + impl PartialEq for PickError { fn eq(&self, other: &Self) -> bool { format!("{:?}", self.0) == format!("{:?}", other.0) @@ -591,7 +606,8 @@ impl PartialEq for PickError { } /// Which error a [`RouterEvent::RetryLogged`] records. -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq, serde::Serialize)] +#[serde(tag = "cause", content = "error", rename_all = "snake_case")] pub enum RetryCause { /// The last attempt's failure. Attempt, @@ -745,3 +761,83 @@ pub trait Host: Send + Sync { Ok(Value::Null) } } + +fn seconds(duration: &Duration, serializer: S) -> Result { + serializer.serialize_f64(duration.as_secs_f64()) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::{Value, json}; + + use super::*; + + #[rstest] + #[case::selected( + RouterEvent::Selected { + model_group: "gpt-4o".to_owned(), + deployment_id: "azure-eastus".to_owned(), + pick: Duration::from_millis(1500), + }, + json!({"event": "Selected", "model_group": "gpt-4o", "deployment_id": "azure-eastus", "pick": 1.5}) + )] + #[case::attempt_failed( + RouterEvent::AttemptFailed { + deployment_id: "openai".to_owned(), + kind: FailureKind::RateLimit, + status: Some(429), + }, + json!({"event": "AttemptFailed", "deployment_id": "openai", "kind": "RateLimitError", "status": 429}) + )] + #[case::cooldown_set( + RouterEvent::CooldownSet { deployment_id: "openai".to_owned(), cooldown: Duration::from_secs(30) }, + json!({"event": "CooldownSet", "deployment_id": "openai", "cooldown": 30.0}) + )] + #[case::retry( + RouterEvent::Retry { model_group: "gpt-4o".to_owned(), retry: 1, delay: Duration::ZERO }, + json!({"event": "Retry", "model_group": "gpt-4o", "retry": 1, "delay": 0.0}) + )] + #[case::blocked( + RouterEvent::Blocked { level: 0, model: "claude".to_owned(), blocked: Blocked::ContentFilter }, + json!({"event": "Blocked", "level": 0, "model": "claude", "blocked": "content_filter"}) + )] + #[case::retry_logged_after_an_attempt( + RouterEvent::RetryLogged { + scope: 0, + level: 0, + model: "gpt-4o".to_owned(), + attempted_retries: None, + cause: RetryCause::Attempt, + }, + json!({ + "event": "RetryLogged", "scope": 0, "level": 0, "model": "gpt-4o", + "attempted_retries": null, "cause": {"cause": "attempt"}, + }) + )] + #[case::retry_logged_after_a_pick( + RouterEvent::RetryLogged { + scope: 1, + level: 1, + model: "backup".to_owned(), + attempted_retries: Some(2), + cause: RetryCause::Pick(PickError(Arc::new(Error::UnknownModel { model: "backup".to_owned() }))), + }, + json!({ + "event": "RetryLogged", "scope": 1, "level": 1, "model": "backup", "attempted_retries": 2, + "cause": {"cause": "pick", "error": "no deployment serves model `backup`"}, + }) + )] + #[case::mock_failed( + RouterEvent::MockFailed { + mock: MockFailure::RateLimit, + level: 0, + model: "gpt-4o".to_owned(), + num_retries: None, + }, + json!({"event": "MockFailed", "mock": "rate_limit", "level": 0, "model": "gpt-4o", "num_retries": null}) + )] + fn an_event_serializes_tagged_by_its_name(#[case] event: RouterEvent, #[case] expected: Value) { + assert_eq!(serde_json::to_value(&event).unwrap(), expected); + } +} diff --git a/litellm-rust/crates/router-core/src/mock.rs b/litellm-rust/crates/router-core/src/mock.rs index 898782e6203..1db6c0d8019 100644 --- a/litellm-rust/crates/router-core/src/mock.rs +++ b/litellm-rust/crates/router-core/src/mock.rs @@ -7,7 +7,15 @@ use crate::error::{AttemptError, FailureKind}; /// Which mock failure a request asks for. #[derive( - Clone, Copy, Debug, PartialEq, Eq, Deserialize, strum::IntoStaticStr, strum::EnumString, + Clone, + Copy, + Debug, + PartialEq, + Eq, + Deserialize, + serde::Serialize, + strum::IntoStaticStr, + strum::EnumString, )] #[serde(rename_all = "snake_case")] #[strum(serialize_all = "snake_case")] diff --git a/litellm-rust/crates/router-core/tests/call_loop.rs b/litellm-rust/crates/router-core/tests/call_loop.rs index a2a741d6135..cd51e53ec6b 100644 --- a/litellm-rust/crates/router-core/tests/call_loop.rs +++ b/litellm-rust/crates/router-core/tests/call_loop.rs @@ -17,7 +17,7 @@ use litellm_router_core::{ }; use parking_lot::Mutex; use rstest::{fixture, rstest}; -use serde_json::json; +use serde_json::{Value, json}; #[derive(Default)] struct ScriptedHost { @@ -1809,3 +1809,55 @@ fn request_for(model: &str) -> RouterRequest { ..RouterRequest::default() } } + +#[rstest] +#[tokio::test] +async fn every_event_of_a_retried_and_fallen_back_call_serializes_tagged_by_its_name( + config: RouterConfig, +) { + let host = Arc::new(ScriptedHost::failing( + "primary", + vec![failure(FailureKind::RateLimit, 429); 3], + )); + let router = router( + config, + json!({"num_retries": 1, "fallbacks": [{"primary": ["backup"]}]}), + host.clone(), + ); + router + .call(Op::from("acompletion"), request()) + .await + .unwrap(); + let tags: Value = host + .events + .lock() + .iter() + .map(|event| serde_json::to_value(event).unwrap()["event"].clone()) + .collect(); + assert_eq!( + tags, + json!([ + "RetriesOpened", + "PickStarted", + "Selected", + "AttemptFailed", + "CooldownSet", + "RetryBudget", + "RetryLogged", + "Retry", + "PickStarted", + "Selected", + "AttemptFailed", + "CooldownSet", + "RetryLogged", + "FallingBack", + "RetryLogged", + "Fallback", + "RetriesOpened", + "PickStarted", + "Selected", + "Succeeded", + "FallbackDone", + ]) + ); +} From fc1e47e580d9e3d2e8b46ad05a293a58ed0b8f1f Mon Sep 17 00:00:00 2001 From: Nate Armstrong Date: Fri, 6 Nov 2026 10:08:54 -0800 Subject: [PATCH 08/39] chore: remove paragraph-ending period restriction (#50419) Co-authored-by: nate Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- AGENTS.md | 1 - 1 file changed, 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 01127303da3..d8de447be90 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -52,7 +52,6 @@ If you ever write any human-facing text (pull requests, issues, commit messages, - don't use "—". Don't use ";" or ":" unless it would be nonsensical not to. Instead, reach for ",", ".", conjunction words, etc. in descending order of preference: vary among them, weighted toward the front of the list, and skip "," where it would cause a comma splice or the sentence is getting long. Overusing any one of them also feels AI-y. A word cap does not penalize you for adding more sentences - don't use the pattern "It's not X, it's Y", "You're not X, you're Y", etc. - unless explicitly asked, don't use bulleted or numbered lists unless it would be nonsensical not to. Instead, prefer prose -- don't add a trailing "." at the end of paragraphs (just like this file). That means every paragraph, not just the last one (of the markdown file, PR description, GitHub comment, etc.). Rule of thumb: if you're adding new line(s) before the next sentence, don't add a ".". Exception is that docs (i.e., litellm-docs) should have trailing "." - don't use →. Instead, prefer not to use arrows, and if need be, use -> instead - use plain, simple, everyday engineering language: the common phrase engineers actually say over rare compact phrasing, in grammatically complete sentences. When explicitly asked to use bullets or ordered lists and structure legitimately helps the reader, prefer nested bullets (any depth is fine) over dense lines in a flat structure From f9f04d38788afbf15122ce3fc46dc7256970c608 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 6 Nov 2026 18:29:35 +0000 Subject: [PATCH 09/39] fix(gemini): correct Embedding 2 Preview shutdown date (#50429) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 2 +- model_prices_and_context_window.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index d9d996208d8..a897f1ab5e0 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -31653,7 +31653,7 @@ "input_cost_per_audio_token_batches": 2.5e-07 }, "gemini/gemini-embedding-2-preview": { - "deprecation_date": "2026-08-10", + "deprecation_date": "2026-11-06", "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", "input_cost_per_audio_token": 6.5e-06, "input_cost_per_audio_token_batches": 3.25e-06, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index d9d996208d8..a897f1ab5e0 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -31653,7 +31653,7 @@ "input_cost_per_audio_token_batches": 2.5e-07 }, "gemini/gemini-embedding-2-preview": { - "deprecation_date": "2026-08-10", + "deprecation_date": "2026-11-06", "deprecation_source": "https://ai.google.dev/gemini-api/docs/deprecations", "input_cost_per_audio_token": 6.5e-06, "input_cost_per_audio_token_batches": 3.25e-06, From d4f338ef1e25ad5c68f3244e3b53a1be6c2bf60d Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Fri, 6 Nov 2026 10:41:22 -0800 Subject: [PATCH 10/39] feat(tags): reset a tag's spend through /tag/update (#50323) * feat(tags): reset a tag's spend through /tag/update Co-authored-by: Arnold Galovics * test(tags): assert spend update response contract * fix(tags): seed spend counter after evicting tag cache * refactor(spend): expose shared counter reset helper * fix(spend): sanitize counter key in spend counter warning log --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Arnold Galovics --- .../key_management_endpoints.py | 41 +----- .../tag_management_endpoints.py | 15 +- litellm/proxy/proxy_server.py | 32 +++++ litellm/types/tag_management.py | 1 + .../integration/spend/test_tag_spend_reset.py | 75 ++++++++++ .../test_tag_management_endpoints.py | 133 ++++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 3 + 7 files changed, 259 insertions(+), 41 deletions(-) create mode 100644 tests/integration/spend/test_tag_spend_reset.py diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index d56250a5504..f990f043dea 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -65,7 +65,6 @@ from litellm.proxy.auth.auth_utils import ( from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import ( evict_and_broadcast, - publish_auth_cache_invalidation, ) from litellm.proxy.common_utils.callback_config_validation import logging_metadata_config_error from litellm.proxy.common_utils.callback_utils import ( @@ -171,7 +170,6 @@ from litellm.proxy.utils import ( require_enterprise_license, ) from litellm.redis.stores.spend_counters import ( - SPEND_COUNTERS_TARGET, key_counter, key_window_counter, ) @@ -6809,38 +6807,6 @@ def _validate_reset_spend_value(reset_to: object, key_in_db: LiteLLM_Verificatio return reset_to -@with_service_target(SPEND_COUNTERS_TARGET) -async def _set_spend_counter_with_floor_and_broadcast(counter_key: str, value: float) -> None: - """ - Set a Redis-backed spend counter to `value`, mirror it into the short-lived - spend_db_floor marker `_authoritative_floor_spend` reads, and broadcast both - to every worker (LIT-3803 pattern: setting, not deleting, means a worker's - own self-delivered broadcast still carries the reset value forward). - - Without the floor marker, `_authoritative_floor_spend` can re-derive a - stale, pre-reset value from a marker another worker cached moments earlier - and raise the just-reset counter right back up via `_repair_stale_spend_counter`. - Without the broadcast, a worker that already cached the pre-reset key object - or floor marker keeps enforcing against it until its own TTL expires. - """ - from litellm.proxy.proxy_server import SPEND_DB_FLOOR_CACHE_TTL_SECONDS, spend_counter_cache, spend_counter_store - - try: - await spend_counter_store.set(counter_key, value, ttl=60) - except Exception as redis_err: - verbose_proxy_logger.warning( - "Failed to update spend counter %s in Redis: %s. Budget checks may use stale value until counter expires.", - counter_key, - redis_err, - ) - - floor_key: Final = f"spend_db_floor:{counter_key}" - spend_counter_cache.in_memory_cache.set_cache(key=floor_key, value=value, ttl=SPEND_DB_FLOOR_CACHE_TTL_SECONDS) - - await publish_auth_cache_invalidation(cache_key=counter_key, new_value=value, ttl=60) - await publish_auth_cache_invalidation(cache_key=floor_key, new_value=value, ttl=SPEND_DB_FLOOR_CACHE_TTL_SECONDS) - - def _budget_limit_windows(budget_limits: Sequence[object] | str | None) -> tuple[Mapping[str, object], ...]: """Coerce a key's stored `budget_limits` into a tuple of plain window dicts. @@ -6925,6 +6891,8 @@ async def _reset_key_budget_windows( pre-reset boundary, re-sum the unchanged historical spend, and put the counter right back where it was before the write ever landed. """ + from litellm.proxy.proxy_server import set_spend_counter_with_floor_and_broadcast + windows: Final = _budget_limit_windows(budget_limits) if not windows: return @@ -6943,7 +6911,7 @@ async def _reset_key_budget_windows( duration = window.get("budget_duration") if isinstance(duration, str) and duration: counter_key = key_window_counter(hashed_api_key, duration) - await _set_spend_counter_with_floor_and_broadcast(counter_key=counter_key, value=0.0) + await set_spend_counter_with_floor_and_broadcast(counter_key=counter_key, value=0.0) @router.post( @@ -6966,6 +6934,7 @@ async def reset_key_spend_fn( hash_token, prisma_client, proxy_logging_obj, + set_spend_counter_with_floor_and_broadcast, user_api_key_cache, ) @@ -7017,7 +6986,7 @@ async def reset_key_spend_fn( # amount for every enforcement check immediately instead of the stale # pre-reset value. _counter_key: Final = key_counter(hashed_api_key) - await _set_spend_counter_with_floor_and_broadcast(counter_key=_counter_key, value=reset_to) + await set_spend_counter_with_floor_and_broadcast(counter_key=_counter_key, value=reset_to) await _reset_key_budget_windows( prisma_client=prisma_client, hashed_api_key=hashed_api_key, diff --git a/litellm/proxy/management_endpoints/tag_management_endpoints.py b/litellm/proxy/management_endpoints/tag_management_endpoints.py index 3ec92165a84..0b7a7ca43cd 100644 --- a/litellm/proxy/management_endpoints/tag_management_endpoints.py +++ b/litellm/proxy/management_endpoints/tag_management_endpoints.py @@ -30,6 +30,7 @@ from litellm.proxy.management_endpoints.common_daily_activity import ( SpendAnalyticsPaginatedResponse, get_daily_activity, ) +from litellm.proxy.management_endpoints.common_utils import validate_finite_spend from litellm.proxy.management_helpers.utils import handle_budget_for_entity from litellm.redis.stores.spend_counters import ( tag_counter, @@ -415,6 +416,7 @@ async def update_tag( Parameters: - name: str - The name of the tag to update + - spend: Optional[float] - Updated spend for the tag - description: Optional[str] - Updated description - models: List[str] - Updated list of allowed LLM models - budget_id: Optional[str] - The id for a budget to associate with the tag @@ -428,11 +430,13 @@ async def update_tag( - model_max_budget: Optional[dict] - Max budget for a specific model - budget_duration: Optional[str] - Frequency of resetting tag budget """ - from litellm.proxy.proxy_server import prisma_client + from litellm.proxy.proxy_server import prisma_client, set_spend_counter_with_floor_and_broadcast if prisma_client is None: raise HTTPException(status_code=500, detail="Database not connected") + validate_finite_spend(tag.spend) + try: # Check if tag exists existing_tag: Final = await _table(TagRepository(prisma_client)).find_unique(where={"tag_name": tag.name}) @@ -459,12 +463,10 @@ async def update_tag( "description": tag.description, "models": tag.models or [], "model_info": json.dumps(model_info), + **({"spend": tag.spend} if tag.spend is not None else {}), + **({"budget_id": budget_id} if budget_id != existing_tag.budget_id else {}), } - # Add budget_id if it changed - if budget_id != existing_tag.budget_id: - update_data["budget_id"] = budget_id - # Update tag in database updated_tag_record: Final = await _table(TagRepository(prisma_client)).update( where={"tag_name": tag.name}, @@ -473,6 +475,9 @@ async def update_tag( await _evict_tag_cache_keys((tag_cache_key(tag.name),)) + if tag.spend is not None: + await set_spend_counter_with_floor_and_broadcast(counter_key=f"spend:tag:{tag.name}", value=tag.spend) + # Build response tag_config: Final = TagConfig( name=updated_tag_record.tag_name, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 7f48d77ff09..781455c6815 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -517,6 +517,7 @@ from litellm.proxy.common_utils.anthropic_error_payload import ( ) from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import ( AuthCacheInvalidationSubscriber, + publish_auth_cache_invalidation, ) from litellm.proxy.common_utils.callback_utils import initialize_callbacks_on_proxy from litellm.proxy.common_utils.codex_model_catalog import codex_model_list_body @@ -727,6 +728,7 @@ from litellm.proxy.litellm_pre_call_utils import ( SCHEDULER_PRIORITY_HEADER, LiteLLMProxyRequestSetup, add_litellm_data_to_request, + sanitize_for_log, ) from litellm.proxy.logging_endpoints.callback_logs_endpoints import ( rust_control_plane_router, @@ -4425,6 +4427,36 @@ async def invalidate_spend_counter(counter_key: str) -> None: _invalidate_spend_counter: Final = invalidate_spend_counter +@with_service_target(SPEND_COUNTERS_TARGET) +async def set_spend_counter_with_floor_and_broadcast(counter_key: str, value: float) -> None: + """ + Set a Redis-backed spend counter to `value`, mirror it into the short-lived + spend_db_floor marker `_authoritative_floor_spend` reads, and broadcast both + to every worker (LIT-3803 pattern: setting, not deleting, means a worker's + own self-delivered broadcast still carries the reset value forward). + + Without the floor marker, `_authoritative_floor_spend` can re-derive a + stale, pre-reset value from a marker another worker cached moments earlier + and raise the just-reset counter right back up via `_repair_stale_spend_counter`. + Without the broadcast, a worker that already cached the pre-reset key object + or floor marker keeps enforcing against it until its own TTL expires. + """ + try: + await spend_counter_store.set(counter_key, value, ttl=60) + except Exception as redis_err: + verbose_proxy_logger.warning( + "Failed to update spend counter %s in Redis: %s. Budget checks may use stale value until counter expires.", + sanitize_for_log(counter_key), + redis_err, + ) + + floor_key: Final = f"spend_db_floor:{counter_key}" + spend_counter_cache.in_memory_cache.set_cache(key=floor_key, value=value, ttl=SPEND_DB_FLOOR_CACHE_TTL_SECONDS) + + await publish_auth_cache_invalidation(cache_key=counter_key, new_value=value, ttl=60) + await publish_auth_cache_invalidation(cache_key=floor_key, new_value=value, ttl=SPEND_DB_FLOOR_CACHE_TTL_SECONDS) + + async def _apply_spend_counter_increments(pending: Sequence[PendingSpendIncrement]) -> None: if _defer_spend_counter_increments(pending): return diff --git a/litellm/types/tag_management.py b/litellm/types/tag_management.py index e59b02af71a..2958c07ce95 100644 --- a/litellm/types/tag_management.py +++ b/litellm/types/tag_management.py @@ -27,6 +27,7 @@ class TagNewRequest(TagBase): class TagUpdateRequest(TagBase): + spend: float | None = None budget_id: str | None = None # Budget fields - if provided, will update the budget max_budget: float | None = None diff --git a/tests/integration/spend/test_tag_spend_reset.py b/tests/integration/spend/test_tag_spend_reset.py new file mode 100644 index 00000000000..2104539f5d7 --- /dev/null +++ b/tests/integration/spend/test_tag_spend_reset.py @@ -0,0 +1,75 @@ +import json +import uuid +from typing import Final + +import httpx + +from tests.integration._support.client import Gateway, eventually +from tests.integration._support.database import read_rows +from tests.integration._support.provider import PROVIDER_URL, SharedProvider +from tests.integration._support.wire import Reply + + +def test_admin_can_reset_tag_spend_and_resume_budget_blocked_requests( + gateway: Gateway, provider: SharedProvider +) -> None: + tag: Final = f"tag-reset-{uuid.uuid4().hex}" + completion: Final = Reply( + body=json.dumps( + { + "id": f"chatcmpl-{uuid.uuid4().hex}", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + } + ).encode() + ) + with gateway.scenario() as scenario: + model: Final = scenario.model( + api_base=f"{PROVIDER_URL}/v1", input_cost_per_token=0.001, output_cost_per_token=0.002 + ) + gateway.post("/tag/new", {"name": tag, "max_budget": 0.0001, "budget_duration": "30d"}) + + def delete_tag() -> None: + gateway.post("/tag/delete", {"name": tag}) + + scenario.cleanups.callback(delete_tag) + + def tagged_request() -> httpx.Response: + return gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [{"role": "user", "content": f"tag reset {uuid.uuid4().hex}"}], + "metadata": {"tags": [tag]}, + }, + ) + + provider.expect(completion) + first: Final = tagged_request() + assert first.status_code == 200, first.text + assert len(provider.received()) == 1 + eventually( + lambda: read_rows('SELECT spend FROM "LiteLLM_TagTable" WHERE tag_name=%s', (tag,)), + lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) > 0.0001, + seconds=70, + ) + blocked: Final = tagged_request() + assert blocked.status_code == 422, blocked.text + assert blocked.json()["error"]["type"] == "budget_exceeded", blocked.text + assert f"Tag={tag}" in blocked.text, blocked.text + assert provider.received() == () + reset: Final = gateway.request("POST", "/tag/update", {"name": tag, "spend": 0}) + assert reset.status_code == 200, reset.text + provider.expect(completion) + try: + resumed: Final = tagged_request() + assert resumed.status_code == 200, resumed.text + assert len(provider.received()) == 1 + finally: + provider.replies.clear() diff --git a/tests/unit/proxy/management_endpoints/test_tag_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_tag_management_endpoints.py index 531bf98b60e..813875bce7c 100644 --- a/tests/unit/proxy/management_endpoints/test_tag_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_tag_management_endpoints.py @@ -16,6 +16,7 @@ from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.proxy_server import app from litellm.proxy.utils import PrismaClient from litellm.types.tag_management import TagNewRequest +from tests.unit.redis.stores.spend_counters_double import FakeSpendCounters client = TestClient(app) @@ -230,6 +231,138 @@ async def test_update_tag(): app.dependency_overrides.clear() +@pytest.mark.asyncio +@pytest.mark.parametrize("spend_fields", [{"spend": 0}, {}, {"spend": None}]) +async def test_update_tag_changes_spend_and_counter_only_when_provided( + spend_fields: Mapping[str, float | None], +) -> None: + from litellm.proxy.management_endpoints.tag_management_endpoints import update_tag + from litellm.types.tag_management import TagUpdateRequest + + updated_tag: Final = SimpleNamespace( + tag_name="reset-tag", + description=None, + models=[], + budget_id=None, + created_at=datetime(2024, 1, 1), + updated_at=datetime(2024, 1, 1), + created_by="admin", + ) + update_row: Final = AsyncMock(return_value=updated_tag) + spend: Final = FakeSpendCounters(redis=None) + mock_prisma: Final = SimpleNamespace( + db=SimpleNamespace( + litellm_tagtable=SimpleNamespace( + find_unique=AsyncMock(return_value=updated_tag), update=update_row + ), + litellm_proxymodeltable=SimpleNamespace(find_many=AsyncMock(return_value=[])), + ) + ) + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + _tag_cache_doubles(), + patch("litellm.proxy.proxy_server.invalidate_spend_counter", new_callable=AsyncMock) as invalidate, + patch("litellm.proxy.proxy_server.spend_counter_cache") as counter_cache, + patch("litellm.proxy.proxy_server.spend_counter_store", spend.store()), + ): + counter_cache.redis_cache = None + response: Final = await update_tag( + tag=TagUpdateRequest.model_validate({"name": "reset-tag", **spend_fields}), + user_api_key_dict=UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + + assert response["message"] == "Tag reset-tag updated successfully" + update_row.assert_awaited_once_with( + where={"tag_name": "reset-tag"}, + data={ + "description": None, + "models": [], + "model_info": "{}", + **({"spend": spend_fields["spend"]} if spend_fields.get("spend") is not None else {}), + }, + ) + if spend_fields.get("spend") is not None: + assert spend.memory["spend:tag:reset-tag"] == 0 + else: + assert "spend:tag:reset-tag" not in spend.memory + counter_cache.in_memory_cache.set_cache.assert_not_called() + invalidate.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("spend", [float("nan"), float("inf"), float("-inf")]) +async def test_update_tag_rejects_non_finite_spend_before_writing(spend: float) -> None: + from litellm.proxy.management_endpoints.tag_management_endpoints import update_tag + from litellm.types.tag_management import TagUpdateRequest + + update_row: Final = AsyncMock() + mock_prisma: Final = SimpleNamespace( + db=SimpleNamespace(litellm_tagtable=SimpleNamespace(update=update_row)) + ) + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch("litellm.proxy.proxy_server.invalidate_spend_counter", new_callable=AsyncMock) as invalidate, + pytest.raises(HTTPException) as error, + ): + await update_tag( + tag=TagUpdateRequest(name="reset-tag", spend=spend), + user_api_key_dict=UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + assert error.value.status_code == 400 + assert "spend must be a finite number" in str(error.value.detail) + update_row.assert_not_awaited() + invalidate.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("redis_state", ["available", "absent", "error"]) +async def test_update_tag_overwrites_stale_counter_repopulated_during_tag_cache_eviction(redis_state: str) -> None: + from litellm.proxy.management_endpoints.tag_management_endpoints import update_tag + from litellm.types.tag_management import TagUpdateRequest + + counter_key: Final = "spend:tag:overlap-reset-tag" + spend: Final = FakeSpendCounters(redis=None if redis_state == "absent" else {}, fail=redis_state == "error") + updated_tag: Final = SimpleNamespace( + tag_name="overlap-reset-tag", + description=None, + models=[], + budget_id=None, + created_at=datetime(2024, 1, 1), + updated_at=datetime(2024, 1, 1), + created_by="admin", + ) + mock_prisma: Final = SimpleNamespace( + db=SimpleNamespace( + litellm_tagtable=SimpleNamespace( + find_unique=AsyncMock(return_value=updated_tag), update=AsyncMock(return_value=updated_tag) + ), + litellm_proxymodeltable=SimpleNamespace(find_many=AsyncMock(return_value=[])), + ) + ) + + async def overlapping_completion(cache_keys: Sequence[str]) -> None: + spend.set_local(counter_key, 1.5, ttl_seconds=60) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch( + "litellm.proxy.proxy_server.spend_counter_store", spend.store(), + ), + patch( + "litellm.proxy.management_endpoints.tag_management_endpoints._evict_tag_cache_keys", + overlapping_completion, + ), + ): + await update_tag( + tag=TagUpdateRequest(name="overlap-reset-tag", spend=0), + user_api_key_dict=UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + assert spend.memory[counter_key] == 0 + if redis_state == "available": + assert spend.redis is not None + assert spend.redis[counter_key] == 0 + + @pytest.mark.asyncio async def test_new_tag_persists_a_budget(): from datetime import datetime diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 2272b4bea43..c351e3b1b90 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -17159,6 +17159,7 @@ export interface paths { * * Parameters: * - name: str - The name of the tag to update + * - spend: Optional[float] - Updated spend for the tag * - description: Optional[str] - Updated description * - models: List[str] - Updated list of allowed LLM models * - budget_id: Optional[str] - The id for a budget to associate with the tag @@ -47982,6 +47983,8 @@ export interface components { rpm_limit?: number | null; /** Soft Budget */ soft_budget?: number | null; + /** Spend */ + spend?: number | null; /** Tpm Limit */ tpm_limit?: number | null; }; From ec46a981fc14aba24d700b7b97df88e20b98151e Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Fri, 6 Nov 2026 10:45:44 -0800 Subject: [PATCH 11/39] feat(model-hub): show each deployment's full prices in the model hub (#50145) * feat(model-hub): show each deployment's full prices in the model hub The model hub only showed one input and one output price per model, the highest across its deployments. Each model group now lists every deployment's prices, read from the same entry billing charges, including cached tokens, long-context tiers, flex, priority, batch and tiered ranges. Both the logged-in and public hub detail views render them in one table per deployment, labeled by provider only * refactor(ui): move model hub pricing breakdown text onto i18n Route every string the deployment pricing breakdown writes through t() under a new deploymentPricing group in en.json, plus the three hub headings that host it. The English on screen is unchanged * fix(model-hub): show Vertex regional prices and explicit cached reads as billed A Vertex deployment outside the global endpoint now shows its token prices with the regional uplift billing applies at the deployment's location. The explicit cached read price shows per 1M tokens on its own row instead of a raw per-token value. The breakdown also says that adjustments made per request, like off-peak discounts and regional or data residency surcharges, aren't included * fix(model-hub): show prices written as text the way billing reads them A price written in config.yaml as 3e-7 arrives as text. Billing turns it into a number, but the hub dropped it, so a deployment showed no price or the cost map's price instead of its own. The hub now reads numeric text for regular prices and size-based price ranges, and skips text that isn't a number * style(ui): format public model hub test * test(model-hub): name the deployment pricing fixture to stay within the inline object lint budget --- .../litellm_core_utils/llm_cost_calc/utils.py | 2 +- litellm/router_rust/surface/model_info.py | 21 ++ litellm/router_utils/model_group_pricing.py | 69 ++++ litellm/types/router.py | 16 + .../public_endpoints/test_public_endpoints.py | 51 +++ tests/unit/test_router/test_router.py | 275 ++++++++++++++++ .../components/AIHub/ModelHubTable.test.tsx | 31 +- .../src/components/AIHub/ModelHubTable.tsx | 6 + .../components/AIHub/ModelHubTableColumns.tsx | 2 + .../components/PublicModelHubTableColumns.tsx | 2 + .../DeploymentPricingBreakdown.test.tsx | 97 ++++++ .../models/DeploymentPricingBreakdown.tsx | 139 ++++++++ .../models/deploymentPricing.test.ts | 186 +++++++++++ .../molecules/models/deploymentPricing.ts | 305 ++++++++++++++++++ .../publicModelHub/ModelInspector.tsx | 6 + .../PublicAiHub.integration.test.tsx | 25 ++ .../src/components/public_model_hub.test.tsx | 27 ++ .../src/components/public_model_hub.tsx | 8 + ui/litellm-dashboard/src/i18n/locales/en.json | 73 ++++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 31 ++ 20 files changed, 1367 insertions(+), 5 deletions(-) create mode 100644 litellm/router_utils/model_group_pricing.py create mode 100644 ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.test.tsx create mode 100644 ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.tsx create mode 100644 ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.test.ts create mode 100644 ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.ts diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 9ac4f9d5ee8..5eb9cdb11ce 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -1464,7 +1464,7 @@ def get_fedramp_uplift_multiplier(model_info: ModelInfo, data_residency: str | N return 1.0 -def get_vertex_regional_endpoint_uplift(model_info: ModelInfo, vertex_location: str | None) -> float: +def get_vertex_regional_endpoint_uplift(model_info: Mapping[str, object], vertex_location: str | None) -> float: """ Resolve the per-model uplift multiplier for Vertex AI non-global (regional and multi-region) endpoints. diff --git a/litellm/router_rust/surface/model_info.py b/litellm/router_rust/surface/model_info.py index b1649617557..7fd4c22df6d 100644 --- a/litellm/router_rust/surface/model_info.py +++ b/litellm/router_rust/surface/model_info.py @@ -46,6 +46,7 @@ from litellm.litellm_core_utils.add_retry_fallback_headers import ( from litellm.litellm_core_utils.batch_utils import deployment_supported_endpoints from litellm.litellm_core_utils.core_helpers import as_str_mapping, coerce_token_limit from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap +from litellm.litellm_core_utils.litellm_logging import resolve_vertex_location_for_cost from litellm.litellm_core_utils.reasoning_effort_capability import ( deployment_is_catalog_mapped, intersect_supported_reasoning_efforts, @@ -56,11 +57,13 @@ from litellm.llms.anthropic.common_utils import AnthropicModelInfo from litellm.llms.bedrock.common_utils import get_bedrock_base_model from litellm.redis.stores.router_usage import ROUTER_USAGE_TARGET, RouterUsage, UsageCounter from litellm.router_python_host import python_host +from litellm.router_utils.model_group_pricing import deployment_pricing from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import build_io_token_rate_limit_headers from litellm.router_utils.router_callbacks.track_deployment_metrics import find_deployment_metadata from litellm.types.router import ( Deployment, DeploymentModelListingInfo, + DeploymentPricing, DeploymentTypedDict, DiscoveredDeploymentModelInfo, LiteLLM_Params, @@ -140,6 +143,11 @@ def _configured_model_info(deployment: Mapping[str, object]) -> Mapping[str, obj return _MODEL_INFO_ADAPTER.validate_python(deployment.get("model_info") or {}) +def _configured_deployment_id(deployment: Mapping[str, object]) -> str | None: + deployment_id: Final = _configured_model_info(deployment).get("id") + return deployment_id if isinstance(deployment_id, str) else None + + def _operator_max_input_tokens(deployment: Mapping[str, object]) -> int | None: """``_operator_max_input_tokens``: the deployment's own limit, skipping one a saved ``/model/info`` response copied from the cost map.""" @@ -187,6 +195,7 @@ class _GroupBuild: # Filled as the lookups arrive. params: dict[int, LiteLLM_Params] = dataclasses.field(default_factory=dict) # mutable-ok: see above info: ModelGroupInfo | None = None + pricing: tuple[DeploymentPricing, ...] = () def _usage_total(values: Sequence[object]) -> int | None: @@ -358,6 +367,7 @@ class HostModelInfo: return None for field, value in (as_str_mapping(answer["assignments"]) or {}).items(): setattr(group.info, field, _group_field(field, wire.from_wire(value))) + group.info.deployment_pricing = group.pricing return group.info def _answer_group_lookup(self, step: Mapping[str, object], group: "_GroupBuild", wire: "_Wire") -> object: @@ -384,6 +394,17 @@ class HostModelInfo: efforts: Final = resolve_supported_reasoning_efforts( model_info, deployment_is_mapped=step["mapped"] is True ) + pricing: Final = deployment_pricing( + llm_provider, + _configured_deployment_id(deployment), + group.params[int(str(step["index"]))].model, + model_info, + resolve_vertex_location_for_cost( + llm_provider, deployment.get("litellm_params"), None, litellm_model + ), + ) + if pricing is not None and pricing not in group.pricing: + group.pricing = (*group.pricing, pricing) # rebind-ok: the merge's state return { "supported_endpoints": wire.to_wire( list( diff --git a/litellm/router_utils/model_group_pricing.py b/litellm/router_utils/model_group_pricing.py new file mode 100644 index 00000000000..afadbc3eb38 --- /dev/null +++ b/litellm/router_utils/model_group_pricing.py @@ -0,0 +1,69 @@ +import math +from collections.abc import Mapping +from typing import Final + +from pydantic import TypeAdapter, ValidationError + +from litellm.litellm_core_utils.litellm_logging import deployment_pricing_model_info +from litellm.litellm_core_utils.llm_cost_calc.utils import get_vertex_regional_endpoint_uplift +from litellm.types.router import DeploymentPricing, PriceTier + +_RAW_PRICE_TIERS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) + + +def _numeric_text(value: str) -> float | None: + try: + parsed: Final = float(value) + except ValueError: + return None + return parsed if math.isfinite(parsed) else None + + +def _cost_value(value: object) -> float | None: + match value: + case bool(): + return None + case int() | float(): + return float(value) + case str(): + return _numeric_text(value) + case _: + return None + + +def _cost_fields(fields: Mapping[str, object], token_uplift: float) -> Mapping[str, float]: + return { + name: cost * token_uplift if "token" in name else cost + for name, value in fields.items() + if "cost" in name and (cost := _cost_value(value)) is not None + } + + +def _price_tiers(tiered_pricing: object, token_uplift: float) -> tuple[PriceTier, ...]: + try: + return tuple( + PriceTier.model_validate({"range": tier.get("range"), "costs": _cost_fields(tier, token_uplift)}) + for tier in _RAW_PRICE_TIERS.validate_python(tiered_pricing) + ) + except ValidationError: + return () + + +def deployment_pricing( + provider: str, + deployment_id: str | None, + deployment_model: str, + resolved_model_info: Mapping[str, object], + vertex_location: str | None, +) -> DeploymentPricing | None: + """Prices from the entry billing charges this deployment at, so a cost-map variant its own pricing shadows + is never shown. A deployment that declares no pricing is billed at its resolved cost-map entry. Token prices + carry the Vertex regional-endpoint uplift billing applies at the deployment's location.""" + billed_model_info: Final = deployment_pricing_model_info(model_id=deployment_id, deployment_model=deployment_model) + model_info: Final = resolved_model_info if billed_model_info is None else billed_model_info + token_uplift: Final = get_vertex_regional_endpoint_uplift(model_info, vertex_location) + costs: Final = _cost_fields(model_info, token_uplift) + tiers: Final = _price_tiers(model_info.get("tiered_pricing"), token_uplift) + if not costs and not tiers: + return None + return DeploymentPricing(provider=provider, costs=costs, tiered_pricing=tiers) diff --git a/litellm/types/router.py b/litellm/types/router.py index a9f28fbc673..3dcfc694003 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -1041,6 +1041,21 @@ def _resolved_annotations(model_class: type[object]) -> Mapping[str, object]: return MappingProxyType(get_type_hints(model_class)) +class PriceTier(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + range: tuple[float, float] + costs: Mapping[str, float] + + +class DeploymentPricing(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + provider: str + costs: Mapping[str, float] + tiered_pricing: tuple[PriceTier, ...] = () + + class ModelGroupInfo(LiteLLMBaseModel): model_group: str providers: list[str] @@ -1071,6 +1086,7 @@ class ModelGroupInfo(LiteLLMBaseModel): configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = None description: str | None = None supported_endpoints: tuple[str, ...] | None = None + deployment_pricing: tuple[DeploymentPricing, ...] = () def __init__(self, **data) -> None: for field_name, field_type in _resolved_annotations(self.__class__).items(): diff --git a/tests/unit/proxy/public_endpoints/test_public_endpoints.py b/tests/unit/proxy/public_endpoints/test_public_endpoints.py index d1afcf80ec9..27ba8968200 100644 --- a/tests/unit/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/unit/proxy/public_endpoints/test_public_endpoints.py @@ -987,6 +987,57 @@ def test_public_model_hub_reports_the_newest_check_of_a_group_not_the_last_row_r app.dependency_overrides.clear() +def test_public_model_hub_lists_each_deployments_prices_without_exposing_the_deployment() -> None: + llm_router: Final = litellm.Router( + model_list=[ + { + "model_name": "hub-public", + "litellm_params": { + "model": "openai/hub-secret-backend-a", + "api_base": "https://secret-a.internal", + "api_key": "sk-secret-a", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + }, + "model_info": {"id": "hub-secret-id-a"}, + }, + { + "model_name": "hub-public", + "litellm_params": { + "model": "openai/hub-secret-backend-b", + "api_base": "https://secret-b.internal", + "api_key": "sk-secret-b", + "input_cost_per_token": 3e-06, + "output_cost_per_token": 4e-06, + }, + "model_info": {"id": "hub-secret-id-b"}, + }, + ] + ) + app: Final = FastAPI() + app.include_router(router) + + with ( + patch("litellm.public_model_groups", ["hub-public"]), + patch("litellm.proxy.proxy_server.llm_router", llm_router), + patch("litellm.proxy.proxy_server.prisma_client", None), + ): + response: Final = TestClient(app).get("/public/model_hub") + + assert response.status_code == 200, response.text + groups: Final = TypeAdapter(list[ModelGroupInfoProxy]).validate_json(response.text) + assert [dict(p.costs) for p in groups[0].deployment_pricing] == [ + {"input_cost_per_token": 1e-06, "output_cost_per_token": 2e-06}, + {"input_cost_per_token": 3e-06, "output_cost_per_token": 4e-06}, + ] + leaked: Final = [ + secret + for secret in ("secret-a.internal", "secret-b.internal", "sk-secret", "hub-secret-backend", "hub-secret-id") + if secret in response.text + ] + assert leaked == [], response.text + + # --------------------------------------------------------------------------- # /public/agent_hub # --------------------------------------------------------------------------- diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 271615b18cf..8f8676103b2 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -1889,6 +1889,281 @@ def test_model_group_info_cost_none_for_unpriced_deployment_but_zero_when_declar assert priced.output_cost_per_token is not None and priced.output_cost_per_token > 0 +_HUB_PRICES: Final = MappingProxyType( + { + "input_cost_per_token": 1e-06, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": 1e-07, + "cache_creation_input_token_cost": 1.25e-06, + "input_cost_per_token_above_200k_tokens": 2e-06, + "output_cost_per_token_above_200k_tokens": 8e-06, + "input_cost_per_token_priority": 3e-06, + "input_cost_per_token_flex": 5e-07, + "input_cost_per_token_batches": 5e-07, + } +) + + +def _cost_map_model(key: str, entry: Mapping[str, object]) -> str: + litellm.register_model( + {key: {"litellm_provider": "openai", "mode": "chat", **entry}}, + persist_across_reloads=False, + add_to_provider_model_sets=False, + ) + return f"openai/{key}" + + +def test_model_group_info_lists_every_cost_map_price_of_a_deployment(local_model_cost_map: None) -> None: + model: Final = _cost_map_model("hub-pricing-model", {**_HUB_PRICES, "max_input_tokens": 400000}) + router: Final = litellm.Router(model_list=[{"model_name": "hub", "litellm_params": {"model": model}}]) + + group: Final = router.registry().group("hub") + + assert group is not None + assert [(p.provider, dict(p.costs), p.tiered_pricing) for p in group.deployment_pricing] == [ + ("openai", dict(_HUB_PRICES), ()) + ] + + +def test_model_group_info_lists_each_deployments_price_once_next_to_the_unchanged_max( + local_model_cost_map: None, +) -> None: + model: Final = _cost_map_model("hub-pricing-model", _HUB_PRICES) + override: Final = 3 * _HUB_PRICES["input_cost_per_token"] + router: Final = litellm.Router( + model_list=[ + {"model_name": "hub", "litellm_params": {"model": model, "api_base": "https://a.example"}}, + {"model_name": "hub", "litellm_params": {"model": model, "api_base": "https://b.example"}}, + {"model_name": "hub", "litellm_params": {"model": model, "input_cost_per_token": override}}, + ] + ) + + group: Final = router.registry().group("hub") + + assert group is not None + assert [p.costs["input_cost_per_token"] for p in group.deployment_pricing] == [ + _HUB_PRICES["input_cost_per_token"], + override, + ] + assert group.input_cost_per_token == override + assert group.output_cost_per_token == _HUB_PRICES["output_cost_per_token"] + + +def _usage_billed_at(cost_key: str, prompt_tokens: int = 1000) -> tuple[Usage, str | None]: + match cost_key: + case "input_cost_per_token": + return Usage(prompt_tokens=prompt_tokens, completion_tokens=0, total_tokens=prompt_tokens), None + case "output_cost_per_token": + return Usage(prompt_tokens=0, completion_tokens=1000, total_tokens=1000), None + case "cache_read_input_token_cost": + return Usage( + prompt_tokens=1000, + completion_tokens=0, + total_tokens=1000, + prompt_tokens_details={"cached_tokens": 1000}, + ), None + case "cache_creation_input_token_cost": + return Usage( + prompt_tokens=1000, completion_tokens=0, total_tokens=1000, cache_creation_input_tokens=1000 + ), None + case "input_cost_per_token_priority": + return Usage(prompt_tokens=1000, completion_tokens=0, total_tokens=1000), "priority" + case "input_cost_per_token_flex": + return Usage(prompt_tokens=1000, completion_tokens=0, total_tokens=1000), "flex" + case _: + raise AssertionError(f"no billing scenario for {cost_key}") + + +def _billed_rate(model: str, deployment_id: str, usage: Usage, service_tier: str | None) -> float: + cost: Final = litellm.completion_cost( + completion_response=litellm.ModelResponse(usage=usage), + model=model, + custom_llm_provider="openai", + custom_pricing=True, + router_model_id=deployment_id, + service_tier=service_tier, + ) + return cost / max(usage.prompt_tokens, usage.completion_tokens) + + +def test_model_group_info_shows_an_overridden_deployment_only_the_prices_billing_charges( + local_model_cost_map: None, +) -> None: + model: Final = _cost_map_model("hub-pricing-model", _HUB_PRICES) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "hub", + "litellm_params": { + "model": model, + "input_cost_per_token": 3 * _HUB_PRICES["input_cost_per_token"], + "output_cost_per_token": 3 * _HUB_PRICES["output_cost_per_token"], + }, + "model_info": {"id": "hub-override"}, + } + ] + ) + + group: Final = router.registry().group("hub") + + assert group is not None + costs: Final = group.deployment_pricing[0].costs + assert {key: _billed_rate(model, "hub-override", *_usage_billed_at(key)) for key in costs} == pytest.approx( + dict(costs) + ) + long_prompt: Final = _usage_billed_at("input_cost_per_token", prompt_tokens=250_000) + assert _billed_rate(model, "hub-override", *long_prompt) == pytest.approx(costs["input_cost_per_token"]) + shadowed: Final = { + "input_cost_per_token_above_200k_tokens", + "output_cost_per_token_above_200k_tokens", + "input_cost_per_token_batches", + } + assert shadowed.isdisjoint(costs), dict(costs) + + +@pytest.mark.parametrize("vertex_location", ["us-east5", "global"]) +def test_model_group_info_shows_a_vertex_deployment_the_token_prices_billing_charges_at_its_location( + local_model_cost_map: None, vertex_location: str +) -> None: + model: Final = "vertex_ai/hub-vertex-model" + litellm.register_model( + { + model: { + "litellm_provider": "vertex_ai-language-models", + "mode": "chat", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": 1e-07, + "regional_endpoint_uplift_multiplier": 1.1, + } + }, + persist_across_reloads=False, + add_to_provider_model_sets=False, + ) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "hub", + "litellm_params": {"model": model, "vertex_project": "p", "vertex_location": vertex_location}, + "model_info": {"id": "hub-vertex"}, + } + ] + ) + + group: Final = router.registry().group("hub") + + assert group is not None + costs: Final = group.deployment_pricing[0].costs + + def billed(cost_key: str) -> float: + usage, _ = _usage_billed_at(cost_key) + cost: Final = litellm.completion_cost( + completion_response=litellm.ModelResponse(usage=usage), + model=model, + custom_llm_provider="vertex_ai", + router_model_id="hub-vertex", + vertex_location=vertex_location, + ) + return cost / max(usage.prompt_tokens, usage.completion_tokens) + + shown: Final = ("input_cost_per_token", "output_cost_per_token", "cache_read_input_token_cost") + assert {key: costs[key] for key in shown} == pytest.approx({key: billed(key) for key in shown}) + + +def test_model_group_info_lists_tiered_pricing_ranges(local_model_cost_map: None) -> None: + tiers: Final = ( + {"range": [0, 256000], "input_cost_per_token": 5e-08, "output_cost_per_token": 4e-07}, + {"range": [256000, 1000000], "input_cost_per_token": 2.5e-07, "output_cost_per_token": 2e-06}, + ) + model: Final = _cost_map_model("hub-tiered-model", {"tiered_pricing": list(tiers)}) + router: Final = litellm.Router( + model_list=[ + {"model_name": "tiered", "litellm_params": {"model": model}}, + { + "model_name": "tiered", + "litellm_params": {"model": model, "input_cost_per_token": 1e-06, "output_cost_per_token": 4e-06}, + }, + ] + ) + + group: Final = router.registry().group("tiered") + + assert group is not None + assert [ + [(tier.range, dict(tier.costs)) for tier in pricing.tiered_pricing] for pricing in group.deployment_pricing + ] == [ + [ + ((0.0, 256000.0), {"input_cost_per_token": 5e-08, "output_cost_per_token": 4e-07}), + ((256000.0, 1000000.0), {"input_cost_per_token": 2.5e-07, "output_cost_per_token": 2e-06}), + ], + [], + ] + + +def test_model_group_info_reads_prices_written_as_text_the_way_billing_does(local_model_cost_map: None) -> None: + tier: Final = {"range": [0, 256000], "input_cost_per_token": "5e-08", "output_cost_per_token": "4e-07"} + model: Final = _cost_map_model( + "hub-text-priced-model", {"tiered_pricing": [tier], "cache_read_input_token_cost": "N/A"} + ) + router: Final = litellm.Router( + model_list=[ + {"model_name": "tiered", "litellm_params": {"model": model}}, + { + "model_name": "flat", + "litellm_params": {"model": model, "input_cost_per_token": "3e-7", "output_cost_per_token": "1.2e-6"}, + "model_info": {"id": "hub-text-override"}, + }, + ] + ) + + tiered: Final = router.registry().group("tiered") + flat: Final = router.registry().group("flat") + + assert tiered is not None + assert flat is not None + assert [dict(t.costs) for t in tiered.deployment_pricing[0].tiered_pricing] == [ + {"input_cost_per_token": 5e-08, "output_cost_per_token": 4e-07} + ] + assert "cache_read_input_token_cost" not in tiered.deployment_pricing[0].costs + flat_costs: Final = flat.deployment_pricing[0].costs + shown: Final = ("input_cost_per_token", "output_cost_per_token") + assert {key: flat_costs[key] for key in shown} == pytest.approx( + {key: _billed_rate(model, "hub-text-override", *_usage_billed_at(key)) for key in shown} + ) + + +def test_model_group_info_lists_custom_prices_of_an_unmapped_deployment_and_skips_an_unpriced_one() -> None: + router: Final = litellm.Router( + model_list=[ + { + "model_name": "self-hosted", + "litellm_params": {"model": "openai/hub-unpriced", "api_base": "http://localhost:8000/v1"}, + "model_info": {"cost_tracking_exempt": True}, + }, + { + "model_name": "self-hosted", + "litellm_params": { + "model": "openai/hub-priced", + "api_base": "http://localhost:8001/v1", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + }, + "model_info": {"cache_read_input_token_cost": 1e-07}, + }, + ] + ) + + group: Final = router.registry().group("self-hosted") + + assert group is not None + assert [(p.provider, dict(p.costs)) for p in group.deployment_pricing] == [ + ( + "openai", + {"input_cost_per_token": 1e-06, "output_cost_per_token": 2e-06, "cache_read_input_token_cost": 1e-07}, + ) + ] + + def test_model_group_info_description_from_model_info(): """ model_info.description set in the config should surface on ModelGroupInfo, diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.test.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.test.tsx index 123c7431732..1b30b060ab0 100644 --- a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.test.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.test.tsx @@ -2,6 +2,7 @@ import * as networking from "@/components/networking"; import userEvent from "@testing-library/user-event"; import { act } from "@testing-library/react"; import type { MCPServerData } from "./MCPHubTableColumns"; +import type { DeploymentPricing } from "@/components/molecules/models/deploymentPricing"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { renderWithProviders, screen, testQueryClient, waitFor, within } from "../../../tests/test-utils"; import ModelHubTable from "./ModelHubTable"; @@ -374,7 +375,7 @@ describe("ModelHubTable", () => { describe("model details usage example", () => { const openDetails = async ( - model: { model_group: string; providers: string[]; mode: string }, + model: { model_group: string; providers: string[]; mode: string; deployment_pricing?: DeploymentPricing[] }, canOpenPlayground = true, ) => { vi.mocked(networking.modelHubCall).mockResolvedValue({ data: [model] }); @@ -425,6 +426,34 @@ describe("ModelHubTable", () => { expect(screen.queryByRole("link", { name: "Try it in the Playground" })).not.toBeInTheDocument(); }); + it("shows each deployment's own prices, by service tier, in the model details", async () => { + const gpt5OnOpenAIAndAzure = { + model_group: "gpt-5", + providers: ["openai", "azure"], + mode: "chat", + deployment_pricing: [ + { + provider: "openai", + costs: { input_cost_per_token: 2.5e-6, input_cost_per_token_priority: 5e-6 }, + tiered_pricing: [], + }, + { provider: "azure", costs: { input_cost_per_token: 2.75e-6 }, tiered_pricing: [] }, + ], + }; + const dialog = await openDetails(gpt5OnOpenAIAndAzure); + + const openai = within(dialog).getByRole("region", { name: "OpenAI pricing" }); + expect(within(openai).getByRole("row", { name: /Input \/ 1M tokens/ })).toHaveTextContent("$2.50$5.00"); + expect(within(dialog).getByRole("region", { name: "Azure pricing" })).toHaveTextContent("$2.75"); + }); + + it("leaves the pricing breakdown out when the proxy sends no deployment prices", async () => { + const dialog = await openDetails({ model_group: "gpt-5", providers: ["openai"], mode: "chat" }); + + expect(within(dialog).getByText("Input Cost per 1M Tokens:")).toBeInTheDocument(); + expect(within(dialog).queryByText("Pricing by Deployment:")).not.toBeInTheDocument(); + }); + it("keeps the chat completions example for a chat model", async () => { const dialog = await openDetails({ model_group: "claude-opus-4-8", providers: ["anthropic"], mode: "chat" }); diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx index 4d44d0bb809..60259b1f7f0 100644 --- a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx @@ -57,6 +57,7 @@ import { isDecisionMode, } from "@/lib/decisionModels"; import DecisionModelGarden from "@/components/AIHub/DecisionModelGarden"; +import { DeploymentPricingBreakdown } from "@/components/molecules/models/DeploymentPricingBreakdown"; import { ModelCustomMetadata } from "./ModelCustomMetadata"; interface ModelHubTableProps { @@ -819,6 +820,11 @@ const ModelHubTable: React.FC = ({

+ {t("aiHub.modelHubTable.modelDialog.pricingByDeployment")}

} + /> {/* Capabilities */} diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.tsx index 2e4af18455f..7f2f3a49d44 100644 --- a/ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.tsx @@ -22,6 +22,7 @@ import { copyToClipboard } from "@/utils/dataUtils"; import { ModelCustomTags, type ModelCustomMetadataValue } from "./ModelCustomMetadata"; import { formatModelModeLabel, ProviderLogo } from "./ModelHubProviders"; import { getProviderLogoAndName } from "@/components/provider_info_helpers"; +import type { DeploymentPricing } from "@/components/molecules/models/deploymentPricing"; export interface ModelHubData { model_group: string; @@ -41,6 +42,7 @@ export interface ModelHubData { description?: string; custom_tags?: string[] | null; custom_metadata?: Record | null; + deployment_pricing?: DeploymentPricing[] | null; [key: string]: any; } diff --git a/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx b/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx index 08d6b973b87..eb89af22f17 100644 --- a/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx @@ -11,6 +11,7 @@ import i18n from "@/i18n/i18n"; import { displayMode } from "@/lib/decisionModels"; import type { ModelCustomMetadataValue } from "@/components/AIHub/ModelCustomMetadata"; import { formatModelModeLabel, ProviderLogo } from "@/components/AIHub/ModelHubProviders"; +import type { DeploymentPricing } from "@/components/molecules/models/deploymentPricing"; export interface ModelGroupInfo { model_group: string; @@ -32,6 +33,7 @@ export interface ModelGroupInfo { description?: string; custom_tags?: string[] | null; custom_metadata?: Record | null; + deployment_pricing?: DeploymentPricing[] | null; [key: string]: any; } diff --git a/ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.test.tsx b/ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.test.tsx new file mode 100644 index 00000000000..08ee05aeb04 --- /dev/null +++ b/ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.test.tsx @@ -0,0 +1,97 @@ +import { render, screen, within } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; + +import { DeploymentPricingBreakdown } from "./DeploymentPricingBreakdown"; +import type { DeploymentPricing } from "./deploymentPricing"; + +const OPENAI: DeploymentPricing = { + provider: "openai", + costs: { + input_cost_per_token: 2.5e-6, + output_cost_per_token: 1e-5, + input_cost_per_token_priority: 5e-6, + cache_read_input_token_cost_above_272k_tokens_priority: 1e-6, + output_cost_per_second_720p: 0.05, + }, + tiered_pricing: [{ range: [0, 256_000], costs: { input_cost_per_token: 5e-8, output_cost_per_token: 4e-7 } }], +}; + +const AZURE: DeploymentPricing = { provider: "azure", costs: { input_cost_per_token: 2.75e-6 }, tiered_pricing: [] }; + +const pricesOf = (region: HTMLElement, group: string, charge: string) => { + const rows = within(within(region).getByRole("rowgroup", { name: group })).getAllByRole("row"); + const row = rows.find((candidate) => within(candidate).queryByRole("rowheader", { name: charge })); + expect(row).toBeDefined(); + return within(row as HTMLElement) + .getAllByRole("cell") + .map((cell) => cell.textContent); +}; + +describe("DeploymentPricingBreakdown", () => { + it("renders one block per deployment, headed by its provider, with that deployment's own prices", () => { + render(Pricing} />); + + const openai = screen.getByRole("region", { name: "OpenAI pricing" }); + const azure = screen.getByRole("region", { name: "Azure pricing" }); + expect(within(openai).getByRole("heading", { name: /OpenAI/ })).toBeInTheDocument(); + expect(pricesOf(openai, "Base prices", "Input / 1M tokens")).toEqual(["$2.50", "$5.00"]); + expect(pricesOf(azure, "Base prices", "Input / 1M tokens")).toEqual(["$2.75"]); + expect(within(azure).queryByText(/Blank cells/)).not.toBeInTheDocument(); + expect(screen.getByText(/This model has 2 deployments/)).toBeInTheDocument(); + expect(screen.getAllByText(/Adjustments made per request/)).toHaveLength(1); + }); + + it("lays prices out by service tier column and context threshold group", () => { + render(); + + const openai = screen.getByRole("region", { name: "OpenAI pricing" }); + const header = within(openai).getAllByRole("rowgroup")[0]; + expect( + within(header) + .getAllByRole("columnheader") + .map((cell) => cell.textContent), + ).toEqual(["Charge", "Standard", "Priority"]); + expect(pricesOf(openai, "Base prices", "Output / 1M tokens")).toEqual(["$10.00", ""]); + expect(pricesOf(openai, "Above 272K tokens", "Cache read / 1M tokens")).toEqual(["", "$1.00"]); + expect(within(openai).getByText("Blank cells aren't priced separately for this deployment.")).toBeInTheDocument(); + expect(screen.queryByText(/This model has/)).not.toBeInTheDocument(); + }); + + it("shows tiered pricing ranges and keys it could not read", () => { + render(); + + expect(screen.getByText("0–256K tokens: $0.05 input / $0.40 output")).toBeInTheDocument(); + expect(screen.getByText("Output / second 720p")).toBeInTheDocument(); + expect(screen.queryByText("output_cost_per_second_720p")).not.toBeInTheDocument(); + expect(screen.getByText("$0.05")).toBeInTheDocument(); + }); + + it("shows only the request-size ranges for a deployment priced purely by tiers", () => { + render( + , + ); + + expect(screen.getByText("0–256K tokens: $0.05 input")).toBeInTheDocument(); + expect(screen.queryByRole("table")).not.toBeInTheDocument(); + }); + + it.each([ + ["missing", undefined], + ["null", null], + ["empty", []], + ])("renders nothing, heading included, when deployment pricing is %s", (_, deployments) => { + const { container } = render( + Pricing by deployment} />, + ); + + expect(container).toBeEmptyDOMElement(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.tsx b/ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.tsx new file mode 100644 index 00000000000..3d967e1d43e --- /dev/null +++ b/ui/litellm-dashboard/src/components/molecules/models/DeploymentPricingBreakdown.tsx @@ -0,0 +1,139 @@ +import type * as React from "react"; +import { useTranslation } from "react-i18next"; + +import { ProviderLogo } from "@/components/molecules/models/ProviderLogo"; +import { getProviderLogoAndName } from "@/components/provider_info_helpers"; +import { cn } from "@/lib/cva.config"; +import { + NUMERIC_CELL_CLASS, + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "@/components/ui/table"; + +import { + buildPricingTable, + type DeploymentPricing, + describePricingTier, + formatTokenCount, + type PriceGroup, + SERVICE_TIER_LABELS, + type ServiceTier, +} from "./deploymentPricing"; + +function PriceGroupRows({ group, tiers }: { group: PriceGroup; tiers: readonly ServiceTier[] }) { + const { t } = useTranslation(); + const name = + group.threshold === null + ? t("deploymentPricing.basePrices") + : t("deploymentPricing.aboveThreshold", { threshold: formatTokenCount(group.threshold) }); + return ( + + {group.threshold !== null && ( + + + {name} + + + )} + {group.rows.map((row) => ( + + + {row.label} + + {tiers.map((tier) => ( + + {row.prices[tier]} + + ))} + + ))} + + ); +} + +function DeploymentPrices({ deployment }: { deployment: DeploymentPricing }) { + const { t } = useTranslation(); + const table = buildPricingTable(deployment.costs); + const tiers = deployment.tiered_pricing; + const providerName = getProviderLogoAndName(deployment.provider).displayName; + return ( +
+

+ + {providerName} +

+ {table.groups.length > 0 && ( + + + + {t("deploymentPricing.chargeColumn")} + {table.tiers.map((tier) => ( + + {t(SERVICE_TIER_LABELS[tier])} + + ))} + + + {table.groups.map((group) => ( + + ))} +
+ )} + {table.hasBlankCells &&

{t("deploymentPricing.blankCells")}

} + {tiers.length > 0 && ( +
+

{t("deploymentPricing.tieredPricing")}

+
    + {tiers.map((tier) => ( +
  • {describePricingTier(tier)}
    • + ))} +
+
+ )} + {table.unknown.length > 0 && ( +
+ {table.unknown.map(({ key, label, price }) => ( +
+
{label}
+
{price}
+
+ ))} +
+ )} +
+ ); +} + +export function DeploymentPricingBreakdown({ + deployments, + heading, + className, +}: { + deployments: readonly DeploymentPricing[] | null | undefined; + heading?: React.ReactNode; + className?: string; +}) { + const { t } = useTranslation(); + if (!deployments || deployments.length === 0) return null; + return ( +
+ {heading} + {deployments.length > 1 && ( +

+ {t("deploymentPricing.deploymentCount", { count: deployments.length })} +

+ )} + {deployments.map((deployment, index) => ( + + ))} +

{t("deploymentPricing.requestAdjustments")}

+
+ ); +} diff --git a/ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.test.ts b/ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.test.ts new file mode 100644 index 00000000000..4e38f55d1b3 --- /dev/null +++ b/ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.test.ts @@ -0,0 +1,186 @@ +import { describe, expect, it } from "vitest"; + +import { buildPricingTable, describePricingTier, fallbackCostLabel, parseCostKey } from "./deploymentPricing"; + +describe("parseCostKey", () => { + it.each([ + ["input_cost_per_token", { charge: "input", modality: "", unit: "token", threshold: null, tier: "standard" }], + ["output_cost_per_token_flex", { charge: "output", unit: "token", threshold: null, tier: "flex" }], + ["input_cost_per_token_batches", { charge: "input", tier: "batches" }], + ["cache_read_input_token_cost", { charge: "cache_read", unit: "token", tier: "standard" }], + ["explicit_cache_read_input_token_cost", { charge: "explicit_cache_read", unit: "token", tier: "standard" }], + [ + "cache_read_input_token_cost_above_272k_tokens_priority", + { charge: "cache_read", threshold: 272_000, tier: "priority" }, + ], + [ + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex", + { charge: "cache_write", cacheDuration: "12hr", threshold: 272_000, tier: "flex" }, + ], + ["cache_creation_input_token_cost_above_1hr", { charge: "cache_write", cacheDuration: "1hr", threshold: null }], + ["cache_read_input_audio_token_cost_priority", { charge: "cache_read", modality: "audio", tier: "priority" }], + ["input_cost_per_audio_token", { charge: "input", modality: "audio", unit: "token" }], + ["output_cost_per_reasoning_token", { charge: "output", modality: "reasoning", unit: "token" }], + [ + "input_cost_per_audio_per_second_above_128k_tokens", + { charge: "input", modality: "audio", unit: "second", threshold: 128_000 }, + ], + ["output_cost_per_second", { charge: "output", modality: "", unit: "second" }], + ["input_cost_per_image", { charge: "input", modality: "", unit: "image" }], + ["input_cost_per_image_token", { charge: "input", modality: "image", unit: "token" }], + ["output_cost_per_pixel", { unit: "pixel" }], + ["input_cost_per_character", { unit: "character" }], + ["input_cost_per_query", { unit: "query" }], + ["input_cost_per_token_above_200k_tokens_ultrafast", { threshold: 200_000, tier: "ultrafast" }], + ["output_cost_per_token_balanced", { tier: "balanced" }], + ])("reads %s", (key, expected) => { + expect(parseCostKey(key)).toMatchObject({ kind: "price", ...expected }); + }); + + it.each([ + "output_cost_per_second_720p", + "output_cost_per_image_1024", + "input_cost_per_token_cache_hit", + "input_cost_per_token_above_272k_tokens_decisions", + "input_cost_per_video_per_second_above_8s_interval", + "ocr_cost_per_page", + "input_dbu_cost_per_token", + "input_cost_per_text_message", + ])("keeps %s as an unknown key rather than guessing", (key) => { + expect(parseCostKey(key)).toEqual({ kind: "unknown", key }); + }); +}); + +const CONTRACT_COSTS = { + input_cost_per_token: 2.5e-6, + output_cost_per_token: 1e-5, + cache_read_input_token_cost: 1.25e-6, + cache_creation_input_token_cost: 3e-6, + input_cost_per_token_priority: 5e-6, + input_cost_per_token_flex: 1.25e-6, + input_cost_per_token_above_200k_tokens: 5e-6, + cache_read_input_token_cost_above_272k_tokens_priority: 1e-6, + input_cost_per_token_batches: 1.25e-6, + output_cost_per_second: 0.001, + input_cost_per_image: 0.002, +}; + +describe("buildPricingTable", () => { + const table = buildPricingTable(CONTRACT_COSTS); + + it("offers a column per service tier the deployment prices, in a fixed order", () => { + expect(table.tiers).toEqual(["standard", "flex", "priority", "batches"]); + }); + + it("groups prices by context threshold, base prices first", () => { + expect(table.groups.map((group) => group.threshold)).toEqual([null, 200_000, 272_000]); + }); + + it("puts every tier of one charge on one row, token prices scaled to 1M tokens", () => { + expect(table.groups[0].rows).toEqual([ + { label: "Input / 1M tokens", prices: { standard: "$2.50", flex: "$1.25", priority: "$5.00", batches: "$1.25" } }, + { label: "Input / image", prices: { standard: "$0.002" } }, + { label: "Cache read / 1M tokens", prices: { standard: "$1.25" } }, + { label: "Cache write / 1M tokens", prices: { standard: "$3.00" } }, + { label: "Output / 1M tokens", prices: { standard: "$10.00" } }, + { label: "Output / second", prices: { standard: "$0.001" } }, + ]); + }); + + it("shows an explicit cached read per 1M tokens on its own row", () => { + const { groups, unknown } = buildPricingTable({ + cache_read_input_token_cost: 1e-7, + explicit_cache_read_input_token_cost: 5e-8, + }); + expect(unknown).toEqual([]); + expect(groups[0].rows).toEqual([ + { label: "Cache read / 1M tokens", prices: { standard: "$0.10" } }, + { label: "Explicit cache read / 1M tokens", prices: { standard: "$0.05" } }, + ]); + }); + + it("answers a cached read on priority above a threshold from its own group", () => { + expect(table.groups[2].rows).toEqual([{ label: "Cache read / 1M tokens", prices: { priority: "$1.00" } }]); + }); + + it("lists keys it cannot read under their raw name with the unscaled price", () => { + const { groups, unknown } = buildPricingTable({ input_cost_per_token: 1e-6, output_cost_per_second_720p: 0.05 }); + expect(unknown).toEqual([{ key: "output_cost_per_second_720p", label: "Output / second 720p", price: "$0.05" }]); + expect(groups.flatMap((group) => group.rows.map((row) => row.label))).toEqual(["Input / 1M tokens"]); + }); + + it("shows a zero price as free and keeps precision for tiny per-unit prices", () => { + const { groups } = buildPricingTable({ input_cost_per_token: 0, output_cost_per_pixel: 1e-8 }); + expect(groups[0].rows).toEqual([ + { label: "Input / 1M tokens", prices: { standard: "Free" } }, + { label: "Output / pixel", prices: { standard: "$0.00000001" } }, + ]); + }); + + it("labels modality and cache duration in the row", () => { + const { groups } = buildPricingTable({ + cache_creation_input_token_cost_above_1hr: 6e-6, + input_cost_per_audio_token: 4e-5, + output_cost_per_reasoning_token: 2e-5, + }); + expect(groups[0].rows.map((row) => row.label)).toEqual([ + "Audio input / 1M tokens", + "Cache write (1hr) / 1M tokens", + "Reasoning output / 1M tokens", + ]); + }); + + it("returns an empty table for a deployment with no prices", () => { + const empty = { tiers: [], groups: [], hasBlankCells: false, unknown: [] }; + expect(buildPricingTable({})).toEqual(empty); + }); + + it("flags blank cells only when some charge lacks a price in a tier the deployment offers", () => { + const everyTierPriced = { + input_cost_per_token: 1e-6, + input_cost_per_token_priority: 2e-6, + output_cost_per_token: 3e-6, + output_cost_per_token_priority: 6e-6, + }; + expect(buildPricingTable(CONTRACT_COSTS).hasBlankCells).toBe(true); + expect(buildPricingTable(everyTierPriced).hasBlankCells).toBe(false); + }); +}); + +describe("fallbackCostLabel", () => { + it.each([ + ["google_maps_grounding_cost_per_query", "Google maps grounding / query"], + ["file_search_cost_per_gb_per_day", "File search / gb per day"], + ["cost_per_second", "Cost / second"], + ["output_cost_per_second_720p", "Output / second 720p"], + ["explicit_cache_read_input_token_cost", "Explicit cache read input token"], + ["some_new_price", "Some new price"], + ])("names %s as %s", (key, label) => { + expect(fallbackCostLabel(key)).toBe(label); + }); +}); + +describe("describePricingTier", () => { + it("states the token range and each price per 1M tokens, input before output", () => { + expect( + describePricingTier({ + range: [0, 256_000], + costs: { output_cost_per_token: 4e-7, input_cost_per_token: 5e-8 }, + }), + ).toBe("0–256K tokens: $0.05 input / $0.40 output"); + }); + + it("names extra charges and non-token units", () => { + expect( + describePricingTier({ + range: [256_000, 1_000_000], + costs: { + input_cost_per_token: 2.5e-7, + output_cost_per_reasoning_token: 1.2e-5, + input_cost_per_query: 0.01, + web_search_cost_per_call: 0.02, + }, + }), + ).toBe("256K–1M tokens: $0.25 input / $0.01 input per query / $12.00 reasoning output / $0.02 web search / call"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.ts b/ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.ts new file mode 100644 index 00000000000..337a8900b7d --- /dev/null +++ b/ui/litellm-dashboard/src/components/molecules/models/deploymentPricing.ts @@ -0,0 +1,305 @@ +import type { ParseKeys } from "i18next"; + +import i18n from "@/i18n/i18n"; +import type { components } from "@/lib/http/schema"; + +export type PricingTier = components["schemas"]["PriceTier"]; +export type DeploymentPricing = components["schemas"]["DeploymentPricing"]; + +export const SERVICE_TIERS = ["standard", "flex", "balanced", "priority", "ultrafast", "batches"] as const; +export type ServiceTier = (typeof SERVICE_TIERS)[number]; + +export const SERVICE_TIER_LABELS: Readonly> = { + standard: "deploymentPricing.tiers.standard", + flex: "deploymentPricing.tiers.flex", + balanced: "deploymentPricing.tiers.balanced", + priority: "deploymentPricing.tiers.priority", + ultrafast: "deploymentPricing.tiers.ultrafast", + batches: "deploymentPricing.tiers.batches", +}; + +const UNITS = ["token", "second", "image", "pixel", "character", "query", "request"] as const; +type Unit = (typeof UNITS)[number]; + +const CHARGES = ["input", "cache_read", "explicit_cache_read", "cache_write", "output"] as const; +type Charge = (typeof CHARGES)[number]; + +const MODALITIES = ["", "text", "image", "audio", "video", "reasoning"] as const; +type Modality = (typeof MODALITIES)[number]; + +export type CostKey = + | { + readonly kind: "price"; + readonly charge: Charge; + readonly modality: Modality; + readonly cacheDuration: string; + readonly unit: Unit; + readonly threshold: number | null; + readonly tier: ServiceTier; + } + | { readonly kind: "unknown"; readonly key: string }; + +const TIER_SUFFIX = /^(.+)_(flex|balanced|priority|ultrafast|batches)$/; +const THRESHOLD_SUFFIX = /^(.+)_above_(\d+)k_tokens$/; +const CACHE_DURATION_SUFFIX = /^(.+)_above_(\d+hr)$/; +const DIRECT_PRICE = /^(input|output)_cost_per_(?:(text|image|audio|video|reasoning)_(?:per_)?)?([a-z]+)$/; +const CACHE_PRICE = /^(explicit_)?cache_(read|creation)_input_(?:(image|audio)_)?token_cost$/; + +const splitSuffix = (key: string, pattern: RegExp): readonly [string, string | null] => { + const match = pattern.exec(key); + return match ? [match[1], match[2]] : [key, null]; +}; + +const isUnit = (value: string): value is Unit => (UNITS as readonly string[]).includes(value); + +const parseBase = (base: string): Pick, "charge" | "modality" | "unit"> | null => { + const direct = DIRECT_PRICE.exec(base); + if (direct) { + const [, charge, modality = "", unit] = direct; + return isUnit(unit) ? { charge: charge as Charge, modality: modality as Modality, unit } : null; + } + const cache = CACHE_PRICE.exec(base); + if (cache) { + const [, explicit, kind, modality = ""] = cache; + if (kind === "creation") + return explicit ? null : { charge: "cache_write", modality: modality as Modality, unit: "token" }; + return { charge: explicit ? "explicit_cache_read" : "cache_read", modality: modality as Modality, unit: "token" }; + } + return null; +}; + +export const parseCostKey = (key: string): CostKey => { + const [withoutTier, tier] = splitSuffix(key, TIER_SUFFIX); + const [withoutThreshold, thousands] = splitSuffix(withoutTier, THRESHOLD_SUFFIX); + const [base, cacheDuration] = splitSuffix(withoutThreshold, CACHE_DURATION_SUFFIX); + const parsed = parseBase(base); + if (!parsed) return { kind: "unknown", key }; + return { + kind: "price", + ...parsed, + cacheDuration: cacheDuration ?? "", + threshold: thousands === null ? null : Number(thousands) * 1000, + tier: (tier ?? "standard") as ServiceTier, + }; +}; + +type Price = Extract; + +type LabelForm = "start" | "inline"; + +const CHARGE_LABELS: Readonly>>> = { + start: { + input: "deploymentPricing.charges.input", + cache_read: "deploymentPricing.charges.cacheRead", + explicit_cache_read: "deploymentPricing.charges.explicitCacheRead", + cache_write: "deploymentPricing.charges.cacheWrite", + output: "deploymentPricing.charges.output", + }, + inline: { + input: "deploymentPricing.chargesInline.input", + cache_read: "deploymentPricing.chargesInline.cacheRead", + explicit_cache_read: "deploymentPricing.chargesInline.explicitCacheRead", + cache_write: "deploymentPricing.chargesInline.cacheWrite", + output: "deploymentPricing.chargesInline.output", + }, +}; + +const MODALITY_LABELS: Readonly, ParseKeys>>>> = { + start: { + text: "deploymentPricing.modalities.text", + image: "deploymentPricing.modalities.image", + audio: "deploymentPricing.modalities.audio", + video: "deploymentPricing.modalities.video", + reasoning: "deploymentPricing.modalities.reasoning", + }, + inline: { + text: "deploymentPricing.modalitiesInline.text", + image: "deploymentPricing.modalitiesInline.image", + audio: "deploymentPricing.modalitiesInline.audio", + video: "deploymentPricing.modalitiesInline.video", + reasoning: "deploymentPricing.modalitiesInline.reasoning", + }, +}; + +const UNIT_LABELS: Readonly> = { + token: "deploymentPricing.units.token", + second: "deploymentPricing.units.second", + image: "deploymentPricing.units.image", + pixel: "deploymentPricing.units.pixel", + character: "deploymentPricing.units.character", + query: "deploymentPricing.units.query", + request: "deploymentPricing.units.request", +}; + +const chargeName = ({ charge, modality }: Price, form: LabelForm): string => + modality + ? i18n.t("deploymentPricing.withModality", { + modality: i18n.t(MODALITY_LABELS[form][modality]), + charge: i18n.t(CHARGE_LABELS.inline[charge]), + }) + : i18n.t(CHARGE_LABELS[form][charge]); + +const chargeLabel = (price: Price, form: LabelForm): string => + price.cacheDuration + ? i18n.t("deploymentPricing.withCacheDuration", { charge: chargeName(price, form), duration: price.cacheDuration }) + : chargeName(price, form); + +const unitLabel = (unit: Unit): string => i18n.t(UNIT_LABELS[unit]); + +const rowLabel = (price: Price): string => + i18n.t("deploymentPricing.rowLabel", { charge: chargeLabel(price, "start"), unit: unitLabel(price.unit) }); + +const rowRank = (price: Price): readonly number[] => [ + CHARGES.indexOf(price.charge), + UNITS.indexOf(price.unit), + MODALITIES.indexOf(price.modality), + parseInt(price.cacheDuration || "0", 10), +]; + +const compareRanks = (a: readonly number[], b: readonly number[]): number => + a.reduce((result, value, index) => result || value - b[index], 0); + +const PRICE_FORMAT_OPTIONS: Intl.NumberFormatOptions = { + style: "currency", + currency: "USD", + minimumFractionDigits: 2, + maximumFractionDigits: 10, + useGrouping: false, +}; + +const PRICE_FORMAT = new Intl.NumberFormat("en-US", PRICE_FORMAT_OPTIONS); + +export const formatRawPrice = (value: number): string => + value === 0 ? i18n.t("deploymentPricing.free") : PRICE_FORMAT.format(value); + +const formatUnitPrice = (value: number, unit: Unit): string => + formatRawPrice(unit === "token" ? value * 1_000_000 : value); + +export interface PriceRow { + readonly label: string; + readonly prices: Readonly>>; +} + +export interface PriceGroup { + readonly threshold: number | null; + readonly rows: readonly PriceRow[]; +} + +export interface UnknownPrice { + readonly key: string; + readonly label: string; + readonly price: string; +} + +export interface PricingTable { + readonly tiers: readonly ServiceTier[]; + readonly groups: readonly PriceGroup[]; + readonly hasBlankCells: boolean; + readonly unknown: readonly UnknownPrice[]; +} + +const PER_UNIT_KEY = /^(.*?)_?cost_per_(.+)$/; +const COST_SUFFIX = /^(.+?)_cost(?:_(.+))?$/; + +const sentenceCase = (text: string): string => { + const words = text.replaceAll("_", " ").trim(); + return words.charAt(0).toUpperCase() + words.slice(1); +}; + +export const fallbackCostLabel = (key: string): string => { + const perUnit = PER_UNIT_KEY.exec(key); + if (perUnit) { + return i18n.t("deploymentPricing.rowLabel", { + charge: perUnit[1] ? sentenceCase(perUnit[1]) : i18n.t("deploymentPricing.fallbackCost"), + unit: perUnit[2].replaceAll("_", " "), + }); + } + const suffixed = COST_SUFFIX.exec(key); + if (suffixed) return sentenceCase([suffixed[1], suffixed[2]].filter(Boolean).join(" ")); + return sentenceCase(key); +}; + +interface ParsedEntry { + readonly key: string; + readonly value: number; + readonly cost: CostKey; +} + +interface PriceEntry extends ParsedEntry { + readonly cost: Price; +} + +const isPriceEntry = (entry: ParsedEntry): entry is PriceEntry => entry.cost.kind === "price"; + +const parseCosts = (costs: Readonly>): readonly ParsedEntry[] => + Object.entries(costs).map(([key, value]) => ({ key, value, cost: parseCostKey(key) })); + +const uniqueBy = (items: readonly T[], keyOf: (item: T) => K): readonly T[] => + items.filter((item, index) => items.findIndex((other) => keyOf(other) === keyOf(item)) === index); + +const byRowRank = (a: PriceEntry, b: PriceEntry): number => compareRanks(rowRank(a.cost), rowRank(b.cost)); + +const groupRows = (entries: readonly PriceEntry[]): readonly PriceRow[] => + [...uniqueBy(entries, ({ cost }) => rowLabel(cost))].sort(byRowRank).map(({ cost }) => ({ + label: rowLabel(cost), + prices: Object.fromEntries( + entries + .filter((other) => rowLabel(other.cost) === rowLabel(cost)) + .map((other) => [other.cost.tier, formatUnitPrice(other.value, other.cost.unit)]), + ), + })); + +export const buildPricingTable = (costs: Readonly>): PricingTable => { + const parsed = parseCosts(costs); + const prices = parsed.filter(isPriceEntry); + const thresholds = [...uniqueBy(prices, ({ cost }) => cost.threshold)] + .map(({ cost }) => cost.threshold) + .sort((a, b) => (a ?? 0) - (b ?? 0)); + const tiers = SERVICE_TIERS.filter((tier) => prices.some(({ cost }) => cost.tier === tier)); + const groups = thresholds.map((threshold) => ({ + threshold, + rows: groupRows(prices.filter(({ cost }) => cost.threshold === threshold)), + })); + return { + tiers, + groups, + hasBlankCells: groups.some(({ rows }) => rows.some(({ prices: row }) => tiers.some((tier) => !(tier in row)))), + unknown: parsed + .filter((entry) => !isPriceEntry(entry)) + .map(({ key, value }) => ({ key, label: fallbackCostLabel(key), price: formatRawPrice(value) })), + }; +}; + +export const formatTokenCount = (tokens: number): string => { + if (tokens >= 1_000_000) return `${+(tokens / 1_000_000).toFixed(1)}M`; + if (tokens >= 1_000) return `${+(tokens / 1_000).toFixed(1)}K`; + return String(tokens); +}; + +const tierPart = ({ cost, value }: PriceEntry): string => { + const charge = chargeLabel(cost, "inline"); + const price = formatUnitPrice(value, cost.unit); + return cost.unit === "token" + ? i18n.t("deploymentPricing.tierPrice", { price, charge }) + : i18n.t("deploymentPricing.tierPricePerUnit", { price, charge, unit: unitLabel(cost.unit) }); +}; + +export const describePricingTier = ({ range: [low, high], costs }: PricingTier): string => { + const parsed = parseCosts(costs); + const parts = [ + ...[...parsed.filter(isPriceEntry)].sort(byRowRank).map(tierPart), + ...parsed + .filter((entry) => !isPriceEntry(entry)) + .map(({ key, value }) => + i18n.t("deploymentPricing.tierPrice", { + price: formatRawPrice(value), + charge: fallbackCostLabel(key).toLowerCase(), + }), + ), + ]; + return i18n.t("deploymentPricing.tierRange", { + low: formatTokenCount(low), + high: formatTokenCount(high), + prices: parts.join(" / "), + }); +}; diff --git a/ui/litellm-dashboard/src/components/publicModelHub/ModelInspector.tsx b/ui/litellm-dashboard/src/components/publicModelHub/ModelInspector.tsx index 95901f2bf1d..39495baeea3 100644 --- a/ui/litellm-dashboard/src/components/publicModelHub/ModelInspector.tsx +++ b/ui/litellm-dashboard/src/components/publicModelHub/ModelInspector.tsx @@ -4,6 +4,7 @@ import { ChevronDown } from "lucide-react"; import * as React from "react"; import { Trans, useTranslation } from "react-i18next"; +import { DeploymentPricingBreakdown } from "@/components/molecules/models/DeploymentPricingBreakdown"; import type { ModelGroupInfo } from "@/components/PublicModelHubTableColumns"; import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; import { Button } from "@/components/ui/button"; @@ -17,6 +18,7 @@ import { modeLabel, modelCapabilities, providerName, + SECTION_TITLE, wildcardExample, } from "./hubFormat"; import { Badges, CodeBlock, type Fact, Facts, PaneSection, ProviderAvatar, ProviderName } from "./HubParts"; @@ -175,6 +177,10 @@ export function ModelInspector({ model }: { model: ModelGroupInfo }) { )} + {t("publicModelHub.modelInspector.pricingByDeployment")}} + /> {capabilities.length > 0 && ( diff --git a/ui/litellm-dashboard/src/components/publicModelHub/PublicAiHub.integration.test.tsx b/ui/litellm-dashboard/src/components/publicModelHub/PublicAiHub.integration.test.tsx index e966836fa3a..cf40a685d76 100644 --- a/ui/litellm-dashboard/src/components/publicModelHub/PublicAiHub.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/publicModelHub/PublicAiHub.integration.test.tsx @@ -77,6 +77,13 @@ const MODELS = [ output_cost_per_token: 0, max_input_tokens: 0, description: "General purpose", + deployment_pricing: [ + { + provider: "azure", + costs: { output_cost_per_token: 1e-5, output_cost_per_token_priority: 2e-5 }, + tiered_pricing: [], + }, + ], }, ]; @@ -362,6 +369,24 @@ describe("PublicAiHub", () => { expect(within(pricedRow).queryByRole("cell", { name: "Free" })).not.toBeInTheDocument(); }); + it("shows the open model's prices for each of its deployments", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + await user.click(await screen.findByRole("link", { name: /gpt-5/ })); + + expect(await screen.findByRole("heading", { name: "Pricing by deployment" })).toBeInTheDocument(); + const azure = screen.getByRole("region", { name: "Azure pricing" }); + expect(within(azure).getByRole("row", { name: /Output \/ 1M tokens/ })).toHaveTextContent("$10.00$20.00"); + }); + + it("leaves the pricing breakdown out for a model without deployment prices", async () => { + renderWithProviders(, { searchParams: { open: "claude-sonnet-4-5" } }); + + expect(await screen.findByRole("heading", { level: 2, name: "claude-sonnet-4-5" })).toBeInTheDocument(); + expect(screen.queryByRole("heading", { name: "Pricing by deployment" })).not.toBeInTheDocument(); + }); + it("treats an empty open param as nothing opened", async () => { renderWithProviders(, { searchParams: { open: "" } }); diff --git a/ui/litellm-dashboard/src/components/public_model_hub.test.tsx b/ui/litellm-dashboard/src/components/public_model_hub.test.tsx index 515b8230d5b..4081751ca73 100644 --- a/ui/litellm-dashboard/src/components/public_model_hub.test.tsx +++ b/ui/litellm-dashboard/src/components/public_model_hub.test.tsx @@ -327,6 +327,33 @@ describe("PublicModelHub", () => { expect(example).toHaveTextContent('model="gpt-4"'); }); + it("shows each deployment's full pricing in the model details", async () => { + respondWith([ + model({ + model_group: "gpt-4", + input_cost_per_token: 2.5e-6, + deployment_pricing: [ + { + provider: "openai", + costs: { input_cost_per_token: 2.5e-6, cache_read_input_token_cost_above_272k_tokens_priority: 1e-6 }, + tiered_pricing: [{ range: [0, 256_000], costs: { input_cost_per_token: 5e-8 } }], + }, + ], + }), + ]); + renderHub(); + + fireEvent.click(await screen.findByRole("button", { name: "gpt-4" })); + + const openai = await screen.findByRole("region", { name: "OpenAI pricing" }); + const aboveThreshold = within(openai).getByRole("rowgroup", { name: "Above 272K tokens" }); + expect(within(aboveThreshold).getByRole("row", { name: /Cache read/ })).toHaveTextContent( + "Cache read / 1M tokens$1.00", + ); + expect(within(openai).getByText("0–256K tokens: $0.05 input")).toBeInTheDocument(); + expect(within(screen.getByRole("dialog")).getByText("$2.5000")).toBeInTheDocument(); + }); + it("filters by several providers at once, and returns to the first page", async () => { const user = userEvent.setup(); respondWith(DEFAULT_MODELS, 300); diff --git a/ui/litellm-dashboard/src/components/public_model_hub.tsx b/ui/litellm-dashboard/src/components/public_model_hub.tsx index a80bf939165..552ed6fc490 100644 --- a/ui/litellm-dashboard/src/components/public_model_hub.tsx +++ b/ui/litellm-dashboard/src/components/public_model_hub.tsx @@ -56,6 +56,7 @@ import { displayMode, isDecisionMode } from "@/lib/decisionModels"; import { modelUsageExample } from "@/components/AIHub/modelUsageExample"; import { filterBySearchTerm, rankBySearchRelevance } from "@/utils/searchUtils"; import { ModelCustomMetadata } from "@/components/AIHub/ModelCustomMetadata"; +import { DeploymentPricingBreakdown } from "@/components/molecules/models/DeploymentPricingBreakdown"; import ModelHubProviders, { formatModelModeLabel } from "@/components/AIHub/ModelHubProviders"; interface PublicModelHubProps { @@ -954,6 +955,13 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded

+ {t("aiHub.publicModelHub.modelDialog.pricingByDeployment")}

+ } + /> {/* Capabilities */} diff --git a/ui/litellm-dashboard/src/i18n/locales/en.json b/ui/litellm-dashboard/src/i18n/locales/en.json index 919d6cd25cc..3c9dbc99a6b 100644 --- a/ui/litellm-dashboard/src/i18n/locales/en.json +++ b/ui/litellm-dashboard/src/i18n/locales/en.json @@ -7217,7 +7217,8 @@ "rpm": "Requests per Minute:", "openaiParams": "Supported OpenAI Parameters", "usageExample": "Usage Example", - "copyToClipboard": "Copy to clipboard" + "copyToClipboard": "Copy to clipboard", + "pricingByDeployment": "Pricing by Deployment:" }, "agentDialog": { "fallbackTitle": "Agent Details", @@ -7370,7 +7371,8 @@ "usageExample": "Usage Example", "decisionNote": "Decision models answer on /v1/decisions and /v1/systemone, not /chat/completions.", "howToCall": "How to call decision models", - "tryPlayground": "Try it in the Playground" + "tryPlayground": "Try it in the Playground", + "pricingByDeployment": "Pricing by Deployment:" }, "agentDialog": { "fallbackTitle": "Agent Details", @@ -7905,7 +7907,8 @@ "title": "Wildcard model", "description": "Call it as , with any model name this provider offers" }, - "capabilities": "Capabilities" + "capabilities": "Capabilities", + "pricingByDeployment": "Pricing by deployment" }, "agentInspector": { "copyName": "Copy agent name", @@ -20300,5 +20303,69 @@ "save": "Save Changes" } } + }, + "deploymentPricing": { + "deploymentCount_one": "This model has {{count}} deployment. Each request is billed at the prices of the deployment that served it.", + "deploymentCount_other": "This model has {{count}} deployments. Each request is billed at the prices of the deployment that served it.", + "requestAdjustments": "These are list prices. Adjustments made per request, like off-peak discounts, regional or data residency surcharges, and some providers' own pricing rules, aren't included.", + "providerPricing": "{{provider}} pricing", + "chargeColumn": "Charge", + "basePrices": "Base prices", + "aboveThreshold": "Above {{threshold}} tokens", + "blankCells": "Blank cells aren't priced separately for this deployment.", + "tieredPricing": "Prices by request size, per 1M tokens", + "tierRange": "{{low}}–{{high}} tokens: {{prices}}", + "tierPrice": "{{price}} {{charge}}", + "tierPricePerUnit": "{{price}} {{charge}} per {{unit}}", + "free": "Free", + "rowLabel": "{{charge}} / {{unit}}", + "withModality": "{{modality}} {{charge}}", + "withCacheDuration": "{{charge}} ({{duration}})", + "fallbackCost": "Cost", + "tiers": { + "standard": "Standard", + "flex": "Flex", + "balanced": "Balanced", + "priority": "Priority", + "ultrafast": "Ultrafast", + "batches": "Batch" + }, + "charges": { + "input": "Input", + "cacheRead": "Cache read", + "explicitCacheRead": "Explicit cache read", + "cacheWrite": "Cache write", + "output": "Output" + }, + "chargesInline": { + "input": "input", + "cacheRead": "cache read", + "explicitCacheRead": "explicit cache read", + "cacheWrite": "cache write", + "output": "output" + }, + "modalities": { + "text": "Text", + "image": "Image", + "audio": "Audio", + "video": "Video", + "reasoning": "Reasoning" + }, + "modalitiesInline": { + "text": "text", + "image": "image", + "audio": "audio", + "video": "video", + "reasoning": "reasoning" + }, + "units": { + "token": "1M tokens", + "second": "second", + "image": "image", + "pixel": "pixel", + "character": "character", + "query": "query", + "request": "request" + } } } diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index c351e3b1b90..ab1f3c253ea 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -32837,6 +32837,20 @@ export interface components { } & { [key: string]: unknown; }; + /** DeploymentPricing */ + DeploymentPricing: { + /** Costs */ + costs: { + [key: string]: number; + }; + /** Provider */ + provider: string; + /** + * Tiered Pricing + * @default [] + */ + tiered_pricing: components["schemas"]["PriceTier"][]; + }; /** DeploymentWarning */ DeploymentWarning: { /** Id */ @@ -39687,6 +39701,11 @@ export interface components { } | null; /** Custom Tags */ custom_tags?: string[] | null; + /** + * Deployment Pricing + * @default [] + */ + deployment_pricing: components["schemas"]["DeploymentPricing"][]; /** Description */ description?: string | null; /** Health Checked At */ @@ -42891,6 +42910,18 @@ export interface components { */ version_status: string; }; + /** PriceTier */ + PriceTier: { + /** Costs */ + costs: { + [key: string]: number; + }; + /** Range */ + range: [ + number, + number + ]; + }; /** Prompt */ Prompt: { litellm_params: components["schemas"]["PromptLiteLLMParams"]; From fbe040a6dc4f0f9f8e741c83a287d5312dc35f5b Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 6 Nov 2026 10:56:27 -0800 Subject: [PATCH 12/39] fix(azure): sync speech retirement dates and sources (#50430) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 19 ++++++++++++------- model_prices_and_context_window.json | 19 ++++++++++++------- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index a897f1ab5e0..4796ed3ac9c 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -12120,16 +12120,19 @@ }, "azure/tts-1": { "deprecation_date": "2026-12-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_character": 1.5e-05, "litellm_provider": "azure", - "mode": "audio_speech" + "mode": "audio_speech", + "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule" }, "azure/tts-1-hd": { "deprecation_date": "2026-12-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_character": 3.3e-05, "litellm_provider": "azure", "mode": "audio_speech", - "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/" + "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule" }, "azure/us/gpt-4.1-2025-04-14": { "deprecation_date": "2027-04-14", @@ -12882,11 +12885,13 @@ "supports_vision": true }, "azure/whisper-1": { - "deprecation_date": "2026-12-15", + "deprecation_date": "2027-02-26", + "deprecation_source": "https://management.azure.com/subscriptions/a88257a5-3323-4ef9-95bf-5681a46c9624/providers/Microsoft.CognitiveServices/locations/eastus/models?api-version=2024-10-01", "input_cost_per_second": 0.0001, "litellm_provider": "azure", "mode": "audio_transcription", - "output_cost_per_second": 0.0001 + "output_cost_per_second": 0.0001, + "source": "https://management.azure.com/subscriptions/a88257a5-3323-4ef9-95bf-5681a46c9624/providers/Microsoft.CognitiveServices/locations/eastus/models?api-version=2024-10-01" }, "azure_ai/Cohere-embed-v3-english": { "deprecation_date": "2027-02-15", @@ -77027,13 +77032,13 @@ "supports_vision": true }, "azure/whisper": { - "deprecation_date": "2026-12-15", - "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "deprecation_date": "2027-02-26", + "deprecation_source": "https://management.azure.com/subscriptions/a88257a5-3323-4ef9-95bf-5681a46c9624/providers/Microsoft.CognitiveServices/locations/eastus/models?api-version=2024-10-01", "input_cost_per_second": 0.0001, "litellm_provider": "azure", "mode": "audio_transcription", "output_cost_per_second": 0.0001, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://management.azure.com/subscriptions/a88257a5-3323-4ef9-95bf-5681a46c9624/providers/Microsoft.CognitiveServices/locations/eastus/models?api-version=2024-10-01" }, "gemini/gemini-nano-banana-2.1": { "input_cost_per_token": 1.5e-06, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index a897f1ab5e0..4796ed3ac9c 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -12120,16 +12120,19 @@ }, "azure/tts-1": { "deprecation_date": "2026-12-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_character": 1.5e-05, "litellm_provider": "azure", - "mode": "audio_speech" + "mode": "audio_speech", + "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule" }, "azure/tts-1-hd": { "deprecation_date": "2026-12-15", + "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", "input_cost_per_character": 3.3e-05, "litellm_provider": "azure", "mode": "audio_speech", - "source": "https://azure.microsoft.com/en-us/pricing/details/azure-openai/" + "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule" }, "azure/us/gpt-4.1-2025-04-14": { "deprecation_date": "2027-04-14", @@ -12882,11 +12885,13 @@ "supports_vision": true }, "azure/whisper-1": { - "deprecation_date": "2026-12-15", + "deprecation_date": "2027-02-26", + "deprecation_source": "https://management.azure.com/subscriptions/a88257a5-3323-4ef9-95bf-5681a46c9624/providers/Microsoft.CognitiveServices/locations/eastus/models?api-version=2024-10-01", "input_cost_per_second": 0.0001, "litellm_provider": "azure", "mode": "audio_transcription", - "output_cost_per_second": 0.0001 + "output_cost_per_second": 0.0001, + "source": "https://management.azure.com/subscriptions/a88257a5-3323-4ef9-95bf-5681a46c9624/providers/Microsoft.CognitiveServices/locations/eastus/models?api-version=2024-10-01" }, "azure_ai/Cohere-embed-v3-english": { "deprecation_date": "2027-02-15", @@ -77027,13 +77032,13 @@ "supports_vision": true }, "azure/whisper": { - "deprecation_date": "2026-12-15", - "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", + "deprecation_date": "2027-02-26", + "deprecation_source": "https://management.azure.com/subscriptions/a88257a5-3323-4ef9-95bf-5681a46c9624/providers/Microsoft.CognitiveServices/locations/eastus/models?api-version=2024-10-01", "input_cost_per_second": 0.0001, "litellm_provider": "azure", "mode": "audio_transcription", "output_cost_per_second": 0.0001, - "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'" + "source": "https://management.azure.com/subscriptions/a88257a5-3323-4ef9-95bf-5681a46c9624/providers/Microsoft.CognitiveServices/locations/eastus/models?api-version=2024-10-01" }, "gemini/gemini-nano-banana-2.1": { "input_cost_per_token": 1.5e-06, From a94118c6cda593e9d2ea69d9b9fabdd61f11bf89 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 6 Nov 2026 11:04:32 -0800 Subject: [PATCH 13/39] test(bedrock): expect native output on Opus 4.8 profiles after #50222 (#50343) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../test_bedrock_batch_files_wire.py | 14 ++++- ...t_bedrock_native_structured_output_wire.py | 56 +++++++++++++------ ...vertex_anthropic_structured_output_wire.py | 2 +- 3 files changed, 50 insertions(+), 22 deletions(-) diff --git a/tests/integration/providers/test_bedrock_batch_files_wire.py b/tests/integration/providers/test_bedrock_batch_files_wire.py index 1657895109d..162fc5aac27 100644 --- a/tests/integration/providers/test_bedrock_batch_files_wire.py +++ b/tests/integration/providers/test_bedrock_batch_files_wire.py @@ -89,8 +89,13 @@ _SCHEMA: Final = { @pytest.mark.parametrize( ("backend", "native_output"), - [("us.anthropic.claude-sonnet-5-5", False), ("us.anthropic.claude-haiku-5-5", True)], - ids=["opt_in_native_output_model", "native_output_model"], + [ + ("us.anthropic.claude-sonnet-5-5", False), + ("us.anthropic.claude-haiku-5-5", True), + ("us.anthropic.claude-opus-4-7", False), + ("us.anthropic.claude-opus-4-8", True), + ], + ids=["opt_in_native_output_model", "native_output_model", "opus_4_7", "opus_4_8"], ) def test_json_schema_batch_record_with_thinking_uploads_native_output_only_on_models_that_always_take_it( gateway: Gateway, backend: str, native_output: bool @@ -130,7 +135,10 @@ def test_json_schema_batch_record_with_thinking_uploads_native_output_only_on_mo output_format: Final = object_value(model_input.get("output_config", {})).get("format") tools: Final = tuple(object_value(entry) for entry in list_value(model_input.get("tools", []))) if native_output: - assert object_value(object_value(output_format)["schema"])["properties"] == _SCHEMA["properties"], model_input + assert model_input["output_config"] == {"format": {"type": "json_schema", "schema": _SCHEMA}}, model_input + assert model_input["anthropic_beta"] == ["structured-outputs-2025-11-13"], model_input + assert "tools" not in model_input, model_input + assert model_input["thinking"] == {"type": "adaptive"}, model_input assert not tools, model_input return assert output_format is None, model_input diff --git a/tests/integration/providers/test_bedrock_native_structured_output_wire.py b/tests/integration/providers/test_bedrock_native_structured_output_wire.py index 68942ed295b..ceb40879b9f 100644 --- a/tests/integration/providers/test_bedrock_native_structured_output_wire.py +++ b/tests/integration/providers/test_bedrock_native_structured_output_wire.py @@ -623,11 +623,9 @@ def test_messages_thinking_on_a_bedrock_converse_native_model_gets_native_output @pytest.mark.parametrize("route", ["converse", "invoke"]) -@pytest.mark.parametrize( - ("backend", "thinks_by_default"), [(_OPUS_4_8, False), (_HAIKU, True)], ids=["opus_4_8", "haiku_5_5"] -) -def test_bedrock_native_model_with_no_thinking_param_gets_native_output_only_when_it_thinks_by_default( - gateway: Gateway, route: str, backend: str, thinks_by_default: bool +@pytest.mark.parametrize("backend", [_OPUS_4_8, _HAIKU], ids=["opus_4_8", "haiku_5_5"]) +def test_bedrock_native_model_with_no_thinking_param_gets_native_output( + gateway: Gateway, route: str, backend: str ) -> None: marker, marker_hex = _marker() with wire_server(_bedrock) as wire, gateway.scenario() as scenario: @@ -640,11 +638,9 @@ def test_bedrock_native_model_with_no_thinking_param_gets_native_output_only_whe assert not message.get("tool_calls"), response.text _, body = _one_call(wire, marker_hex) invoke: Final = route == "invoke" - if thinks_by_default: - _assert_native_without_tools(body, invoke, _SCHEMA) - return - assert _native_schema(body, invoke) is None, body - assert _tools(body, invoke)[_JSON_TOOL]["properties"] == _SCHEMA["properties"], body + _assert_native_without_tools(body, invoke, _SCHEMA) + if invoke: + assert body["anthropic_beta"] == ["structured-outputs-2025-11-13"], body @pytest.mark.parametrize("route", ["converse", "invoke"]) @@ -687,8 +683,13 @@ def test_a_model_that_cannot_force_the_json_tool_retries_a_rejected_native_outpu (_NEITHER_KEY, None, False), (_OPT_IN, {"supports_bedrock_native_structured_output": True}, True), (_OPUS_4_8, {"supports_bedrock_native_structured_output": False}, False), + ( + _OPUS_4_8, + {"supports_native_structured_output": True, "supports_bedrock_native_structured_output": False}, + True, + ), ], - ids=["neither_key", "deployment_opt_in", "deployment_opt_out"], + ids=["neither_key", "deployment_opt_in", "deployment_opt_out", "explicit_native_with_bedrock_key_off"], ) def test_thinking_request_follows_the_bedrock_native_key_the_model_resolves_to( gateway: Gateway, backend: str, model_info: Mapping[str, JsonValue] | None, native: bool @@ -1047,7 +1048,7 @@ def test_bedrock_rejecting_opt_in_native_output_gets_one_retry_without_it( ids=["other_400", "native_model"], ) @pytest.mark.parametrize("route", ["converse", "invoke"]) -def test_bedrock_400_is_not_retried_unless_it_rejects_opt_in_native_output( +def test_bedrock_400_is_not_retried_unless_it_rejects_native_output( gateway: Gateway, route: str, backend: str, rejection: str ) -> None: marker, marker_hex = _marker() @@ -1058,10 +1059,19 @@ def test_bedrock_400_is_not_retried_unless_it_rejects_opt_in_native_output( "/v1/chat/completions", _chat_body(model, marker, thinking=_ADAPTIVE, response_format=_response_format(_SCHEMA)), ) - assert response.status_code == 400, response.text - assert rejection.split(": ", 1)[1] in response.text, response.text - _, body = _one_call(wire, marker_hex) - assert _native_schema(body, route == "invoke") == _SCHEMA, body + invoke: Final = route == "invoke" + if rejection == _MAX_TOKENS_400: + assert response.status_code == 400, response.text + assert rejection.split(": ", 1)[1] in response.text, response.text + _, body = _one_call(wire, marker_hex) + _assert_native_without_tools(body, invoke, _SCHEMA) + return + assert _content_json(_chat_message(response)) == _payload(marker_hex), response.text + rejected, retried = (_JSON.validate_json(request.body) for request in wire.drain()) + _assert_native_without_tools(rejected, invoke, _SCHEMA) + assert _native_schema(retried, invoke) is None, retried + assert _tools(retried, invoke)[_JSON_TOOL]["properties"] == _SCHEMA["properties"], retried + assert marker_hex in json.dumps(retried), retried def _always_rejecting(message: str) -> Callable[[Request], Reply]: @@ -1076,7 +1086,7 @@ def _always_rejecting(message: str) -> Callable[[Request], Reply]: [(_OPUS_4_8, _MAX_TOKENS_400), (_SONNET, _FABLE_REJECTION)], ids=["other_400", "native_model"], ) -def test_messages_400_on_bedrock_invoke_reaches_the_caller_after_one_call_unless_it_rejects_opt_in_native_output( +def test_messages_400_on_bedrock_invoke_reaches_the_caller_after_one_call_unless_it_rejects_native_output( gateway: Gateway, backend: str, rejection: str ) -> None: marker, marker_hex = _marker() @@ -1089,7 +1099,17 @@ def test_messages_400_on_bedrock_invoke_reaches_the_caller_after_one_call_unless ) assert response.status_code == 400, response.text assert rejection.split(": ", 1)[1] in response.text, response.text - _one_call(wire, marker_hex) + if rejection == _MAX_TOKENS_400: + _, body = _one_call(wire, marker_hex) + _assert_native_without_tools(body, True, _SCHEMA) + return + rejected, retried = (_JSON.validate_json(request.body) for request in wire.drain()) + _assert_native_without_tools(rejected, True, _SCHEMA) + assert _native_schema(retried, True) is None, retried + assert not _tools(retried, True), retried + prompt: Final = json.dumps(retried["messages"]) + assert marker_hex in prompt, retried + assert all(name in prompt for name in object_value(_SCHEMA["properties"])), retried def test_a_messages_retry_without_native_output_still_omits_the_deployments_dropped_params(gateway: Gateway) -> None: diff --git a/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py b/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py index 19928ecd6e0..95e07593b2a 100644 --- a/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py +++ b/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py @@ -48,7 +48,7 @@ _THINKING_OFF_TOOL_MODEL: Final = "claude-opus-4-8" _UNMAPPED_MODEL: Final = "claude-scripted-9" _BEDROCK_INVOKE_MODEL: Final = "anthropic.claude-sonnet-5" _BEDROCK_TOOL_PATH_MODEL: Final = "anthropic.claude-opus-4-8" -_BEDROCK_JSON_TOOL_MODEL: Final = "us.anthropic.claude-opus-4-8" +_BEDROCK_JSON_TOOL_MODEL: Final = "us.anthropic.claude-opus-4-7" _THINKING_SIGNATURE: Final = "c2NyaXB0ZWQtc2lnbmF0dXJl" _PROJECT: Final = "scripted-project" _LOCATION: Final = "us-east5" From 1f15088fc91da484531d445679fa4de1396cd33c Mon Sep 17 00:00:00 2001 From: simonp9 <349053457+simonp9@users.noreply.github.com> Date: Fri, 6 Nov 2026 11:14:47 -0800 Subject: [PATCH 14/39] feat(proxy): mixed Codex model picker and LiteLLM routing on native codex routes (#49408) * feat(proxy): mixed Codex model picker and LiteLLM routing on native codex routes The native Codex catalog now lists the key's other chat and responses models after ChatGPT's, within Codex's catalog size limit, and drops the ETag so the merged list refreshes on Codex's timer. A request for a model configured by exact name on another provider goes through /v1/responses without the ChatGPT bearer or account header, while ChatGPT names and wildcard-only matches keep the subscription passthrough. Reasoning items another provider produced are dropped before forwarding to ChatGPT. * fix(proxy): keep Codex's picker from turning reasoning off for LiteLLM models Codex fills a missing default_reasoning_level with effort "none" when a model is picked in /model, while `codex -m ` sends no effort at all. A thinking-only provider model then answers 400. On the native /backend-api/codex/responses route, drop effort "none" for a LiteLLM model whose catalog entry names no default, so the provider applies its own default exactly as `codex -m` does * fix(proxy): harden Codex mixed-model routing on the native ChatGPT routes Keep the ChatGPT credentials out of hook raw headers on the inner /v1/responses request, route to LiteLLM and merge LiteLLM catalog rows only for keys that may call /v1/responses, drop unpersisted reasoning ids on the passthrough when store is not true, keep any slug the caller's ChatGPT account lists on the subscription via a shared account catalog cache, and keep X-Models-Etag for keys whose catalog added no LiteLLM rows. The guide covers the new behavior and the /v1 wildcard precedence over a catch-all deployment. * fix(proxy): keep Codex's native route on the subscription when the account list is unknown A name OpenAI lists for the caller's ChatGPT account now goes to LiteLLM only when the account's list is known and leaves it out. A failed catalog read is remembered for 60 s and the Codex version comes only from a User-Agent whose product matches the originator header. The merged catalog serves a stable ETag digest of OpenAI's ETag and the key's LiteLLM rows, and /responses replaces X-Models-Etag with the same value, memoized per key for 300 s, so Codex stops refetching or dropping its catalog cache. LiteLLM-routed native replies wrap their reasoning with the model id, so an OpenAI or Azure blob is dropped before chatgpt.com and unwrapped for LiteLLM, and the chat-completions bridge decodes wrapped thinking blocks with their signature. The access-group recipe uses chatgpt/ model names so /v1 routing does not change, and the native route checks the chatgpt/ form of a name the key holds only through that group. A none reasoning effort is dropped only under drop_params. The guide covers app tools, every routing case, and the ETag, and the catalog logs how many models the byte limit left out. * fix(fireworks_ai): move Codex additional_tools items into tools on the Responses API * fix(proxy): keep Codex native turns working when the key's model listing fails A listing callback that raises no longer turns chatgpt.com's catalog or responses answer into a 500. The catalog keeps upstream's body and ETag, and responses keep upstream's X-Models-Etag. Under drop_params, a `none` reasoning effort is now dropped only when a candidate deployment does not explicitly support it, so an alias to a model like openai/gpt-5.4 keeps it. The account slug list and its unavailable marker are keyed on the account id plus the caller's ChatGPT Authorization header, so a failed fetch with one bearer cannot turn off LiteLLM routing for another caller on the same account. The auth step checks the Codex route before importing the Codex module. * fix(proxy): keep Codex model switches working across LiteLLM providers On the Codex native route, a LiteLLM model now gets only the encrypted reasoning every deployment it can route to is able to decrypt, so ChatGPT or another deployment's blobs no longer make xAI answer 400 Codex additional_tools items are hoisted into tools for every provider except the OpenAI API, and namespace tools become flat tools named mcp__docs__search, the shape xAI accepts A JWT caller's models ETag is cached on a hash of its LiteLLM credential, so ChatGPT turns stop rebuilding the catalog rows * fix(proxy): keep Codex namespace tool calls intact across LiteLLM providers Fireworks and Bedrock Mantle accept Codex namespace tools natively, so the native route now hoists them without flattening, like OpenAI and Azure and the chat completions bridge. Other native Responses providers still get flat names, now built with the bridge's namespace_tool_name_map, and each function_call they return gets its tool name and namespace back in the non-streaming body and in streamed output_item.added, output_item.done, and completed/incomplete/failed events. Every other event passes through unchanged * fix(proxy): bridge Codex turns for providers whose Responses API rejects Codex tools Codex sends its freeform exec tool as a custom tool on every turn, and xAI's Responses API answers 422 for custom, namespace and additional_tools shapes. On the Codex native route, a model whose deployments include a native Responses provider other than OpenAI, Azure, Fireworks or Bedrock Mantle now asks for the chat completions bridge with use_chat_completions_api, which already turns custom and namespace tools into functions, hoists additional_tools, and puts the namespace back on each returned call OpenAI and Azure still get the request unchanged, and Fireworks and Bedrock Mantle still get it on their own Responses API with additional_tools hoisted. The route's namespace flattening and the response-side namespace restore are removed, since no path sends flattened names anymore * fix(proxy): drop reasoning ids a Codex turn's LiteLLM target cannot resolve A reasoning item another deployment produced, replayed with store false and no blob the target can decrypt, now reaches the target as its readable summary without id or status, and is dropped when nothing readable is left. OpenAI answered 400 on the status field and 404 on the id for a glm-5p3 item replayed whole to an openai deployment. Same-deployment replays and stored conversations keep the item whole * refactor(codex): reach encrypted reasoning stripping through the router registry The Codex route read Router.optional_callbacks and imported EncryptedContentAffinityCheck from litellm.router_utils, which the router surface fence forbids outside the router. RouterRegistry gains decryptable_reasoning(model_group, request_input, targets), which keeps the same rule: with encrypted content affinity on for the group, reasoning one target can decrypt stays, otherwise every target must decrypt it * fix(proxy): keep Codex's priority tier off LiteLLM rows in the native model catalog * fix(proxy): drop Codex's none effort for an alias whose native entry names no default * fix(proxy): start a native Codex alias at its pinned reasoning effort and build the catalog off the event loop * test(proxy): map the pinned reasoning efforts field in the router registry stub * fix(proxy): honor admin-only /v1/responses and keep Codex's default tier off native aliases Codex native routes now send a key to LiteLLM only when admin_only_routes allows it, a native alias with configured service tiers starts with no tier picked, and the account slug cache declares its Redis key family. * fix(proxy): check the key's tool allowlist on native Codex LiteLLM turns and additional_tools items * fix(proxy): check vector store grants on file_search tools inside additional_tools items A Responses request can define file_search inside an additional_tools input item, which the native Codex route hoists for LiteLLM-served models and OpenAI runs as is on /v1/responses. The vector store check now reads those tools in both strict and legacy modes. The input item scan uses a Mapping guard instead of a pydantic error per item, since auth now runs it on every list input. * fix(proxy): apply the proxy-wide allowed_routes to LiteLLM models on native Codex routes * fix(proxy): cap native Codex account catalog reads per LiteLLM caller before rate limits apply * fix(guardrails): check additional_tools items next to top-level function tools in tool_policy ToolPolicyGuardrail read tools from the request body only when no top-level function tool existed, so a blocked tool defined in an additional_tools item reached Fireworks once the Responses transform moved the item into tools after the pre-call hooks. Check every list on each request, including namespace members inside items. * fix(router): serve decryptable_reasoning from the Rust core's optional checks The merge with main removed the Python router core, so the registry's host has no optional_callbacks and no routing context, and native Codex requests with encrypted reasoning raised RustCoreUnsupported. The filter now lives with the optional checks, which own both. The ported pinned_reasoning_efforts field gets a test on the Rust core Also give the static litellm.Router a declared constructor. Since the Python core was deleted a bare __init__ assignment reads as RustCore's, so every Router(model_list=...) is a type error and make lint fails on main. TYPE_CHECKING only, no runtime change * fix(proxy): check the budget of a saved alias target on native Codex LiteLLM turns Native /backend-api/codex/responses binds {endpoint}, so auth skipped the key's or team's model_group_alias and read budgets on the requested name, while the later /v1/responses handoff served the target unchecked. Auth now resolves the alias on a turn that routes to LiteLLM, as it does on /v1, and leaves ChatGPT-only names on the subscription. Also makes the tool_policy tests build a real ToolPolicyRegistry and type their fixtures. * fix(proxy): keep a name the ChatGPT account lists on the subscription despite a saved alias * fix(codex): keep LiteLLM keys off ChatGPT, enforce tool policy on namespaces and pin reasoning on aliases Auth no longer asks ChatGPT for the account's model list when the bearer is missing or is the LiteLLM key, so a key sent as the bearer never reaches it. tool_policy now reads the members of a top-level namespace, which is where the native route's hoist puts them. A global alias now reads encrypted content affinity from its target group. A non-list top-level tools value and a non-string function name now reach the provider, which refuses them, instead of raising a 500. The catalog ETag no longer depends on the order a process iterates priced service tiers in, so workers and pods agree on it. * fix(proxy): list a team's models in the order the team set them get_team_models built the names in a set, so each process listed a team key's models in its own hash order and the native Codex catalog sent a different row order and ETag from each worker * fix(proxy): apply JWT route policy before native Codex routes serve LiteLLM models A standard JWT caller now gets a LiteLLM model on /backend-api/codex/models and /backend-api/codex/responses only when the JWT auth flow would admit a direct /v1/responses call for that model, so team_allowed_routes, admin_allowed_routes and RBAC route limits hold on the handoff too. The Fireworks Responses transform also leaves additional_tools input items in place when the tools param is not a list, instead of raising while hoisting * fix(fireworks): hoist additional_tools when tools is a tuple litellm.responses passes a tuple tools argument through to the Fireworks transform unchanged, so the non-list guard skipped hoisting and left the additional_tools item in the input. Treat tuples like lists and cover both shapes in the drop_params merge test --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../chatgpt_subscription_passthrough.md | 40 +- .../transformation.py | 12 + .../fireworks_ai/responses/transformation.py | 34 +- .../guardrail_translation/handler.py | 10 +- litellm/proxy/_lazy_openapi_snapshot.json | 6 +- litellm/proxy/auth/auth_checks.py | 25 +- litellm/proxy/auth/auth_utils.py | 21 +- litellm/proxy/auth/model_checks.py | 35 +- litellm/proxy/auth/route_checks.py | 13 + litellm/proxy/auth/user_api_key_auth.py | 46 +- .../proxy/common_utils/codex_model_catalog.py | 159 +- .../common_utils/codex_model_translation.py | 6 +- .../tool_policy/tool_policy_guardrail.py | 34 +- .../codex_passthrough_endpoints.py | 751 ++++++- litellm/proxy/proxy_server.py | 205 +- litellm/responses/additional_tools.py | 52 +- .../count_request.py | 22 +- .../reasoning_items.py | 28 +- .../transformation.py | 8 +- litellm/responses/utils.py | 7 +- litellm/router_api.py | 18 +- litellm/router_rust/steps/callbacks.py | 22 + litellm/router_rust/surface/model_info.py | 15 + litellm/router_rust/surface/registry.py | 8 + .../encrypted_content_affinity_check.py | 4 +- ...t_fireworks_ai_responses_transformation.py | 78 + ...st_auth_checks_object_access_and_lookup.py | 31 + tests/unit/proxy/auth/test_model_checks.py | 9 + .../test_user_api_key_auth_request_flow.py | 13 + .../common_utils/test_codex_model_catalog.py | 140 ++ .../test_tool_policy_guardrail.py | 106 +- .../test_codex_passthrough_endpoints.py | 1882 ++++++++++++++++- tests/unit/proxy/router_entry_point_stubs.py | 2 + .../proxy/test_tools_allowlist_enforcement.py | 36 + .../test_reasoning_items.py | 85 +- tests/unit/responses/test_additional_tools.py | 10 +- ...st_registry_characterization_model_info.py | 21 + .../test_router/test_router_entry_points.py | 75 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 8 +- 39 files changed, 3834 insertions(+), 243 deletions(-) diff --git a/cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md b/cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md index e1f606a2eb7..538a5f8a404 100644 --- a/cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md +++ b/cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md @@ -46,7 +46,43 @@ For a managed workspace rollout, use OpenAI's [gateway configuration guide](http ## Native compatibility and Pi -For a native Codex model catalog and request format, change Codex's `base_url` to `https://litellm.example.com/backend-api/codex`. This route uses the same credentials but does not need a model deployment or resolve model aliases. Restricted keys must allow the upstream model name. +For a native Codex model catalog and request format, change Codex's `base_url` to `https://litellm.example.com/backend-api/codex`. This route uses the same credentials. A ChatGPT model keeps its upstream name and needs no deployment of its own. + +### Mix ChatGPT and LiteLLM models + +On the native route, Codex's model picker shows OpenAI's catalog first and then the other chat and responses models the virtual key can use. Pick one of those and Codex's requests go through LiteLLM's `/v1/responses` with the key's usual routing, budgets, guardrails, and logging. The proxy removes the ChatGPT bearer and account header from that request, so the other provider and your callbacks never receive them. A name is routed this way when another provider serves it as an exact `model_name`, through a router `model_group_alias`, or as a team's public model name. A ChatGPT model, a name that only a wildcard deployment matches, or a name OpenAI's catalog lists for the caller's ChatGPT account keeps the passthrough, so a model OpenAI adds later still uses the subscription even when a deployment of another provider has the same name. The proxy learns the account's list from the catalog Codex fetches, or asks OpenAI with the caller's login. When it cannot get the list, for example because OpenAI is down, the request times out, or the client is not Codex, every request keeps the passthrough for the next minute, as it did before LiteLLM models were added to the picker. When the key can use many models, the picker lists the ones that fit in Codex's catalog size limit, and the proxy logs a warning with how many it left out. + +Codex's picker sends a `none` reasoning effort for a LiteLLM model whose catalog entry lists no effort levels. With `litellm_settings: drop_params: true`, the proxy drops it for a model that does not support `none`, and the provider applies its own default. A model whose cost map entry or `model_info` sets `supports_none_reasoning_effort: true` keeps it. Without `drop_params`, the effort reaches the provider unchanged. + +Codex also sends its ChatGPT app tools with every turn, which on a LiteLLM model adds a few hundred thousand prompt tokens to each turn. Set `apps = false` under `[features]` in Codex's `config.toml` to turn them off. + +Only a key that may call `/v1/responses` gets these models. A key whose `allowed_routes` covers only the native routes sees OpenAI's catalog unchanged, and every request it sends uses the subscription. + +Codex sends earlier reasoning items back with each turn. The proxy tags the reasoning that a LiteLLM model returns on this route, OpenAI and Azure deployments included. When a session switches to a ChatGPT model, the proxy drops those items before forwarding the request to OpenAI, which would reject them. Messages, tool calls, and ChatGPT's own reasoning are kept, and a LiteLLM model still gets its own reasoning back. + +When the catalog adds LiteLLM models for a key, its `ETag` is a digest of OpenAI's ETag and those models, and that key's responses carry the same value in `X-Models-Etag`. Codex keeps its cached catalog while the two match and fetches it again when OpenAI's catalog or the key's models change. A key whose catalog adds no LiteLLM models keeps OpenAI's ETags. + +### Restrict native models with an access group + +A key with a model allowlist needs a rule for ChatGPT names that OpenAI adds over time. Put the subscription wildcards in an access group under `chatgpt/` names: + +```yaml +model_list: + - model_name: chatgpt/gpt-* + litellm_params: + model: chatgpt/gpt-* + model_info: + access_groups: ["chatgpt-native"] + - model_name: chatgpt/codex-* + litellm_params: + model: chatgpt/codex-* + model_info: + access_groups: ["chatgpt-native"] +``` + +Then add `chatgpt-native` to the key's models, next to any other model names it should reach. On the native route, a key that holds only the `chatgpt/` form of a name is checked as that form, so a request for `gpt-5.6-sol` is allowed when the key may call `chatgpt/gpt-5.6-sol`. The key can call every ChatGPT model that matches the wildcards, including ones the proxy configuration does not name. + +On `/v1` these wildcards match only a request that asks for a `chatgpt/` name, so a request for a plain `gpt-` or `codex-` name still goes where it went before, such as a `*` catch-all deployment. A `/v1` request for a `chatgpt/` name without a ChatGPT login falls back to the proxy's own saved login if it has one, so keep that login unset as described above. ### Configure Pi @@ -71,7 +107,7 @@ Keep Pi's subscription login as the provider credential. The header value names Send a prompt and check the completed request in LiteLLM's Logs page. Its gateway user/key and token counts should match the caller. Give each person their own LiteLLM virtual key for separate attribution. The gateway credential and ChatGPT login are independently authenticated; LiteLLM does not establish that both identities belong to the same person. -The native routes are `GET /backend-api/codex/models` and `POST /backend-api/codex/responses`. Model requests use the upstream model name, so a restricted LiteLLM key must allow that name. OpenAI's model catalog is returned in its native format; seeing a model there does not override gateway permissions. +The native routes are `GET /backend-api/codex/models` and `POST /backend-api/codex/responses`. Model requests use the upstream model name, so a restricted LiteLLM key needs the `chatgpt-native` access group above or that exact name. OpenAI's model catalog is returned in its native format; seeing a model there does not override gateway permissions. The proxy forwards the ChatGPT bearer and account header to the fixed OpenAI Codex endpoint. It removes gateway credentials and cookies from the upstream request and redacts credential headers in logging callbacks. The client remains responsible for refresh and reauthentication. Native passthrough returns authentication and quota errors without switching subscriptions. Standard model routes retain the caller login across ChatGPT retries; explicitly configured cross-provider fallbacks use their own credentials. An explicit workspace header is preserved for OpenAI to validate. diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 9902fe1d8fc..b24c75e69bc 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -54,6 +54,7 @@ from litellm.llms.base_llm.bridges.completion_transformation import ( CompletionTransformationBridge, ) from litellm.llms.custom_httpx.sse_lines import split_sse_lines +from litellm.responses.additional_tools import additional_tools_in_input from litellm.responses.litellm_completion_transformation.tool_search import is_anthropic_tool_search_chat_tool from litellm.responses.sse_output_recovery import ( parse_sse_json_chunk, @@ -426,6 +427,17 @@ def replayed_tool_search_tool_names(request: object) -> tuple[str, ...]: return tuple(chain.from_iterable(_tool_definition_name(tool) for tool in replayed_tool_search_tools(request))) +def _top_level_namespaces(request: object) -> Iterator[object]: + tools: Final = _json_list(_json_field(request, "tools")) or () + return (tool for tool in tools if _json_field(tool, "type") == "namespace") + + +def nested_tool_member_names(request: object) -> tuple[str, ...]: + tools: Final = (*_top_level_namespaces(request), *additional_tools_in_input(_json_field(request, "input"))) + members: Final = chain.from_iterable(_namespace_members(tool) for tool in tools) + return tuple(chain.from_iterable(_tool_definition_name(member) for member in members)) + + def _loads_any_tool(message: object, names: frozenset[str]) -> bool: loaded_names: Final = chain.from_iterable(_tool_definition_name(tool) for tool in _message_loaded_tools(message)) return any(name in names for name in loaded_names) diff --git a/litellm/llms/fireworks_ai/responses/transformation.py b/litellm/llms/fireworks_ai/responses/transformation.py index d599f0b65f7..a45c25b27f3 100644 --- a/litellm/llms/fireworks_ai/responses/transformation.py +++ b/litellm/llms/fireworks_ai/responses/transformation.py @@ -1,10 +1,11 @@ -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from types import MappingProxyType -from typing import TYPE_CHECKING, Final +from typing import TYPE_CHECKING, Final, cast # noqa: TID251 # the base signature leaves the params bag untyped from urllib.parse import unquote import httpx +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.responses.transformation import ( STANDARD_MODE_WITHOUT_CONTEXT_REASONING_FIELD_VALUES, @@ -20,8 +21,9 @@ from litellm.llms.fireworks_ai.common_utils import ( ) from litellm.llms.openai.responses.instruction_items import developer_items_as_system, instruction_items_folded from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.responses.additional_tools import HoistedAdditionalTools, hoist_additional_tools from litellm.secret_managers.main import get_secret_str -from litellm.types.llms.openai import ResponseInputParam +from litellm.types.llms.openai import ALL_RESPONSES_API_TOOL_PARAMS, ResponseInputParam from litellm.types.responses.main import DeleteResponseResult from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders @@ -98,6 +100,23 @@ class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig): headers: dict[str, str], # mutable-ok: overrides the base class signature ) -> dict[str, object]: # mutable-ok: overrides the base class signature validated_input: Final = self._validate_input_param(input) + tools_param: Final = response_api_optional_request_params.get("tools") + hoisted: Final = ( + hoist_additional_tools( + validated_input, + cast( # cast-ok: the base signature leaves the params bag untyped + "Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None", tools_param + ), + drop_params=drops_unsupported_params(litellm_params.drop_params), + ) + if tools_param is None or isinstance(tools_param, (list, tuple)) + else HoistedAdditionalTools(input=validated_input, tools=(), hoisted=()) + ) + hoisted_params: Final = ( + {**response_api_optional_request_params, "tools": list(hoisted.tools)} + if hoisted.hoisted + else response_api_optional_request_params + ) model_info: Final = deployment_model_info(litellm_params) declared: Final = declared_mid_conversation_system_support(model, self.custom_llm_provider.value, model_info) keeps_developer_items_in_place: Final = declared is True and ( @@ -107,14 +126,9 @@ class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig): folded: Final = ( None if keeps_developer_items_in_place - else instruction_items_folded( - validated_input, response_api_optional_request_params, "user" if declared is False else "system" - ) - ) - placed_input, placed_params = folded or ( - developer_items_as_system(validated_input), - response_api_optional_request_params, + else instruction_items_folded(hoisted.input, hoisted_params, "user" if declared is False else "system") ) + placed_input, placed_params = folded or (developer_items_as_system(hoisted.input), hoisted_params) request: Final = super().transform_responses_api_request( model=resolve_fireworks_resource_name(model), input=placed_input, diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index 0ec8497b0c6..aa4dea196ff 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -78,6 +78,7 @@ from litellm.llms.openai.responses.guardrail_translation.tool_merge import ( append_returned_tools, merge_guardrailed_tools, ) +from litellm.responses.additional_tools import additional_tools_in_input from litellm.responses.litellm_completion_transformation.compaction_items import ( compaction_summary, decode_compaction_item, @@ -989,9 +990,14 @@ class OpenAIResponsesHandler(BaseTranslation): def extract_request_tool_names(self, data: Mapping[str, object]) -> list[str]: """Extract tool names from Responses API request (tools[].name for function - and custom, tools[].server_label for mcp).""" + and custom, tools[].server_label for mcp), including tools carried by + ``additional_tools`` input items.""" names: Final[list[str]] = [] - for tool in list_field(data, "tools"): + request_tools: Final[tuple[object, ...]] = ( + *list_field(data, "tools"), + *additional_tools_in_input(data.get("input")), + ) + for tool in request_tools: if not isinstance(tool, dict): continue if tool.get("type") in ("function", "custom") and tool.get("name"): diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 5cb63c98d56..137d6016402 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -9633,7 +9633,7 @@ "paths": { "/backend-api/codex/{endpoint}": { "get": { - "description": "Forward a client's ChatGPT subscription with separate LiteLLM authentication.", + "description": "Forward a client's ChatGPT subscription with separate LiteLLM authentication, listing and serving the\nkey's other chat models through LiteLLM.", "operationId": "codex_passthrough_route_backend_api_codex__endpoint__get", "parameters": [ { @@ -9681,7 +9681,7 @@ ] }, "post": { - "description": "Forward a client's ChatGPT subscription with separate LiteLLM authentication.", + "description": "Forward a client's ChatGPT subscription with separate LiteLLM authentication, listing and serving the\nkey's other chat models through LiteLLM.", "operationId": "codex_passthrough_route_backend_api_codex__endpoint__post", "parameters": [ { @@ -9724,7 +9724,7 @@ } } }, - "description": "Unmodified Codex response or server-sent events" + "description": "Codex response, the model catalog with the key's LiteLLM models, or server-sent events" }, "422": { "content": { diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 11ca3d9083c..b40eb451979 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -180,6 +180,7 @@ from litellm.repositories.table_repositories import ( ) from litellm.repositories.team_repository import TeamRepository from litellm.repositories.user_repository import UserRepository +from litellm.responses.additional_tools import additional_tools_in_input from litellm.router import Router from litellm.router_api import RegistryReader from litellm.types.proxy.auth.auth_checks import UserNotFoundError @@ -1037,9 +1038,6 @@ async def check_tools_allowlist( call_types: Final = get_call_types_for_route(route) if not call_types or not any(ct.value in TOOL_CAPABLE_CALL_TYPES for ct in call_types): return - tool_names: Final = extract_request_tool_names(route, request_body) - if not tool_names: - return key_meta: Final = (valid_token.metadata or {}) if isinstance(valid_token.metadata, dict) else {} team_meta: Final = (valid_token.team_metadata or {}) if isinstance(valid_token.team_metadata, dict) else {} key_allowed: Final = key_meta.get("allowed_tools") @@ -1047,6 +1045,9 @@ async def check_tools_allowlist( effective: Final = key_allowed if (isinstance(key_allowed, list) and len(key_allowed) > 0) else team_allowed if not isinstance(effective, list) or len(effective) == 0: return + tool_names: Final = extract_request_tool_names(route, request_body) + if not tool_names: + return allowed_set: Final = {str(t) for t in effective} disallowed: Final = [n for n in tool_names if n not in allowed_set] if disallowed: @@ -7988,6 +7989,10 @@ def _tools(tools: object) -> tuple[object, ...]: return () +def _request_tools(request_body: Mapping[str, object]) -> tuple[object, ...]: + return (*_tools(request_body.get("tools")), *additional_tools_in_input(request_body.get("input"))) + + def _tool_vector_store_ids(tool: object) -> tuple[str, ...]: try: tool_fields: Final = _TOOL_ADAPTER.validate_python(tool, strict=True) @@ -7998,14 +8003,14 @@ def _tool_vector_store_ids(tool: object) -> tuple[str, ...]: def _strict_requested_vector_store_ids(request_body: Mapping[str, object]) -> tuple[str, ...]: """ - Same fields VectorStoreRegistry.get_vector_store_ids_to_run reads, but a vector_store_ids that is - not a list of strings is a 400 instead of being skipped or iterated, and tools that are not - objects name no store. + Same fields VectorStoreRegistry.get_vector_store_ids_to_run reads, plus the tools of `additional_tools` + input items, but a vector_store_ids that is not a list of strings is a 400 instead of being skipped or + iterated, and tools that are not objects name no store. """ return tuple( chain( _validated_vector_store_ids(request_body.get("vector_store_ids")), - chain.from_iterable(_tool_vector_store_ids(tool) for tool in _tools(request_body.get("tools"))), + chain.from_iterable(_tool_vector_store_ids(tool) for tool in _request_tools(request_body)), ) ) @@ -8044,7 +8049,7 @@ def _legacy_requested_vector_store_ids( return tuple( chain( top_level, - chain.from_iterable(vector_store_ids_in_tool(tool) for tool in _tools(request_body.get("tools"))), + chain.from_iterable(vector_store_ids_in_tool(tool) for tool in _request_tools(request_body)), ) ) @@ -8113,8 +8118,8 @@ async def vector_store_access_check( """ Checks whether the caller may use every vector store the request names. - Requested stores come from `vector_store_ids`, `tools[].vector_store_ids` and the RAG - `retrieval_config.vector_store_id`. Grants come from each identity's + Requested stores come from `vector_store_ids`, `tools[].vector_store_ids`, the tools of + `additional_tools` input items and the RAG `retrieval_config.vector_store_id`. Grants come from each identity's `object_permission.vector_stores`. Both modes read the same request fields, with or without a vector store registry, and answer 400 diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 2b5fc6d877e..24602dec677 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -863,7 +863,7 @@ async def pre_db_read_auth_checks( Raises: - HTTPException if request fails initial auth checks """ - from litellm.proxy.proxy_server import general_settings, llm_router, premium_user + from litellm.proxy.proxy_server import general_settings, llm_router # Check 1. request size await check_if_request_size_is_safe(request=request) @@ -891,19 +891,12 @@ async def pre_db_read_auth_checks( ) # Check 4. Check if request route is an allowed route on the proxy - if "allowed_routes" in general_settings: - _allowed_routes: Final = general_settings["allowed_routes"] - if premium_user is not True: - verbose_proxy_logger.error( - "Trying to set allowed_routes. This is an Enterprise feature. %s", - CommonProxyErrors.not_premium_user.value, - ) - if route not in _allowed_routes: - verbose_proxy_logger.error("Route %s not in allowed_routes=%s", route, _allowed_routes) - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail=f"Access forbidden: Route {route} not allowed", - ) + if not RouteChecks.is_allowed_on_proxy(route): + verbose_proxy_logger.error("Route %s not in allowed_routes=%s", route, general_settings["allowed_routes"]) + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail=f"Access forbidden: Route {route} not allowed", + ) def route_in_additonal_public_routes(current_route: str): diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index d57aa5c9df1..c1db8b7b4c7 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -192,34 +192,25 @@ def get_team_models( - Empty list if no models set - If model_access_groups is provided, only return models that are in the access groups """ - all_models_set: Final[set[str]] = set() - if len(team_models) > 0: - all_models_set.update(team_models) - if SpecialModelNames.all_team_models.value in all_models_set: - all_models_set.update(team_models) - # GH#30619: expand all-team-models sentinel - # to the actual proxy model list - all_models_set.discard(SpecialModelNames.all_team_models.value) - all_models_set.update(proxy_model_list) - if include_model_access_groups: - all_models_set.update(model_access_groups.keys()) - if SpecialModelNames.all_proxy_models.value in all_models_set: - all_models_set.update(proxy_model_list) - if include_model_access_groups: - all_models_set.update(model_access_groups.keys()) - - all_models = _get_models_from_access_groups( + expands_to_proxy_models: Final = ( + SpecialModelNames.all_team_models.value in team_models + or SpecialModelNames.all_proxy_models.value in team_models + ) + expansion: Final = ( + (*proxy_model_list, *(model_access_groups if include_model_access_groups else ())) + if expands_to_proxy_models + else () + ) + named_models: Final = (model for model in team_models if model != SpecialModelNames.all_team_models.value) + all_models: Final = _get_models_from_access_groups( model_access_groups=model_access_groups, - all_models=list(all_models_set), + all_models=list(dict.fromkeys((*named_models, *expansion))), include_model_access_groups=include_model_access_groups, proxy_model_list=proxy_model_list, ) - # deduplicate while preserving order - all_models = list(dict.fromkeys(all_models)) - verbose_proxy_logger.debug("ALL TEAM MODELS - %s", len(all_models)) - return all_models + return list(dict.fromkeys(all_models)) def get_complete_model_list( diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py index 125bfab4edf..0aa2528d46b 100644 --- a/litellm/proxy/auth/route_checks.py +++ b/litellm/proxy/auth/route_checks.py @@ -439,6 +439,19 @@ class RouteChecks: detail=f"user not allowed to access this route. Route={route} is an admin only route", ) + @staticmethod + def is_allowed_on_proxy(route: str) -> bool: + from litellm.proxy.proxy_server import general_settings, premium_user + + if "allowed_routes" not in general_settings: + return True + if premium_user is not True: + verbose_proxy_logger.error( + "Trying to set allowed_routes. This is an Enterprise feature. %s", + CommonProxyErrors.not_premium_user.value, + ) + return route in general_settings["allowed_routes"] + @staticmethod def is_unbilled_route(route: str) -> bool: resource, _, action = route.rsplit("/", 1)[-1].partition(":") diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index ba253b44ea3..a52ad14e293 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -309,6 +309,17 @@ async def _normalize_claude_model( rewrite_request_model(request_data, request, source) +_CODEX_NATIVE_RESPONSES_ROUTE: Final = "/backend-api/codex/responses" + + +def _is_codex_native_litellm_turn(route: str, model: object, llm_router: litellm.Router, team_id: str | None) -> bool: + if route != _CODEX_NATIVE_RESPONSES_ROUTE: + return False + from litellm.proxy.pass_through_endpoints.codex_passthrough_endpoints import routes_to_litellm + + return routes_to_litellm(model, llm_router, team_id) + + async def _resolve_router_settings_model_group_alias( request_data: dict[str, object], # mutable-ok: the request body is rewritten in place for every downstream reader valid_token: UserAPIKeyAuth, @@ -325,9 +336,12 @@ async def _resolve_router_settings_model_group_alias( if request.scope.get(MODEL_GROUP_ALIAS_RESOLVED_SCOPE_KEY) is True: return request.scope[MODEL_GROUP_ALIAS_RESOLVED_SCOPE_KEY] = True - if request_dispatched_to_pass_through_endpoint(request) or request_dispatched_to_provider_pass_through(request): - return requested: Final = request_data.get("model") + if request_dispatched_to_pass_through_endpoint(request) or ( + request_dispatched_to_provider_pass_through(request) + and not _is_codex_native_litellm_turn(route, requested, llm_router, valid_token.team_id) + ): + return if not isinstance(requested, str) or await read_raw_json_body(request=request) is None: return settings: Final = await proxy_config.get_hierarchical_router_settings( @@ -338,6 +352,11 @@ async def _resolve_router_settings_model_group_alias( target: Final = resolve_model_group_alias(settings.get("model_group_alias"), requested) if target is None or target == requested: return + if request_dispatched_to_provider_pass_through(request): + from litellm.proxy.pass_through_endpoints.codex_passthrough_endpoints import native_turn_serves_through_litellm + + if not await native_turn_serves_through_litellm(requested, llm_router, request, valid_token): + return verbose_proxy_logger.debug( "router_settings.model_group_alias resolved %s -> %s before auth", requested.replace("\r", "").replace("\n", ""), @@ -347,6 +366,27 @@ async def _resolve_router_settings_model_group_alias( rewrite_request_model(request_data, request, target) +def _resolve_codex_native_chatgpt_model( + request_data: dict[str, object], # mutable-ok: the request body is rewritten in place for every downstream reader + valid_token: UserAPIKeyAuth, + request: Request | None, + route: str, +) -> None: + requested: Final = request_data.get("model") + if route != _CODEX_NATIVE_RESPONSES_ROUTE or request is None or not isinstance(requested, str): + return + from litellm.proxy.pass_through_endpoints.codex_passthrough_endpoints import native_chatgpt_model + from litellm.proxy.proxy_server import llm_router + + if llm_router is None: + return + target: Final = native_chatgpt_model(requested, llm_router, valid_token) + if target is None: + return + request.scope.setdefault(CLIENT_REQUESTED_MODEL_SCOPE_KEY, requested) + rewrite_request_model(request_data, request, target) + + def _get_model_names_for_budget_checks( model: str | list[str] | None, ) -> list[str]: @@ -4172,6 +4212,7 @@ async def _authorize_authenticated_request( RouteChecks.should_call_route(route=route, valid_token=user_api_key_auth_obj, request=request) await _normalize_claude_model(request_data, user_api_key_auth_obj, request, route) await _resolve_router_settings_model_group_alias(request_data, user_api_key_auth_obj, request, route) + _resolve_codex_native_chatgpt_model(request_data, user_api_key_auth_obj, request, route) # Single authorization point. Builder paths MUST NOT call common_checks. # Route through the same exception handler the builder uses so @@ -4650,6 +4691,7 @@ async def enforce_key_and_fallback_model_access( """ await _normalize_claude_model(request_data, valid_token, request, route) await _resolve_router_settings_model_group_alias(request_data, valid_token, request, route) + _resolve_codex_native_chatgpt_model(request_data, valid_token, request, route) config: Final = valid_token.config if config != {}: diff --git a/litellm/proxy/common_utils/codex_model_catalog.py b/litellm/proxy/common_utils/codex_model_catalog.py index 462e372f3d8..01cce9a6ce6 100644 --- a/litellm/proxy/common_utils/codex_model_catalog.py +++ b/litellm/proxy/common_utils/codex_model_catalog.py @@ -24,8 +24,10 @@ a newer tag. from __future__ import annotations +import hashlib +import json from collections.abc import Callable, Mapping, Sequence -from dataclasses import dataclass +from dataclasses import asdict, dataclass from functools import cache from itertools import accumulate, pairwise from pathlib import Path @@ -33,7 +35,7 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Final, Literal, TypeAlias from packaging.version import Version -from pydantic import ConfigDict, TypeAdapter, ValidationError +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from litellm._logging import verbose_proxy_logger from litellm.proxy.common_utils.codex_model_translation import ( @@ -61,6 +63,8 @@ CODEX_CHAT_MODES: Final = frozenset({"chat", "responses"}) _BODY_PREFIX: Final = '{"models":[' _BODY_SUFFIX: Final = "]}" +_JSON_OBJECT: Final = TypeAdapter(dict[str, object]) +_JSON_OBJECTS: Final = TypeAdapter(tuple[dict[str, object], ...]) _NO_UPDATE: Final[Mapping[str, object]] = MappingProxyType({}) @@ -119,6 +123,7 @@ class CodexStockCatalog(LiteLLMBaseModel): CodexCatalogEntry = CodexFallbackModel | CodexStockModel CodexAliasDefaults: TypeAlias = Literal["codex", "deployment"] +_NATIVE_ALIAS_DEFAULTS: Final[CodexAliasDefaults] = "deployment" @dataclass(frozen=True, slots=True) @@ -408,6 +413,7 @@ def _entry( stock_model.upgrade if stock_model.upgrade is not None and stock_model.upgrade.model in served else None ) is_alias: Final = row.id != stock_model.slug + deployment_decides: Final = is_alias and alias_defaults == "deployment" stock_name: Final = row.id if is_alias else stock_model.display_name listing: Final = { "slug": row.id, @@ -417,12 +423,18 @@ def _entry( "supported_in_api": True, "upgrade": upgrade, **({} if search_tool is None else {"supports_search_tool": search_tool}), - **(_deployment_defaults(row, stock_model) if is_alias and alias_defaults == "deployment" else {}), + **(_deployment_defaults(row, stock_model) if deployment_decides else {}), **_configured_window(row, stock_model), } if configured is None: return stock_model.model_copy(update={**_priced_tiers(row, stock_model), **listing}) - return stock_model.model_copy(update={**listing, **_tier_fields(stock_model, configured)}) + return stock_model.model_copy( + update={ + **listing, + **_tier_fields(stock_model, configured), + **({"default_service_tier": None} if deployment_decides else {}), + } + ) def _serialized(entry: CodexCatalogEntry) -> str: @@ -435,13 +447,12 @@ def _bytes_left_after(left: int, size: int) -> int: return left - size if size <= left else left -def _fits(serialized: Sequence[str], byte_limit: int | None) -> tuple[bool, ...]: - """Whether each entry, taken in order, fits in what is left of a body of at most `byte_limit` - bytes: an entry larger than the bytes left is passed over and a smaller one after it is still - taken, each entry counted with its separating comma and the last one's comma given back.""" - if byte_limit is None: +def _fits(serialized: Sequence[str], budget: int | None) -> tuple[bool, ...]: + """Whether each entry, taken in order, fits in what is left of `budget` bytes: an entry larger + than the bytes left is passed over and a smaller one after it is still taken, each entry + counted with its separating comma.""" + if budget is None: return (True,) * len(serialized) - budget: Final = byte_limit - len(_BODY_PREFIX) - len(_BODY_SUFFIX) + 1 left: Final = tuple( accumulate((len(entry.encode()) + 1 for entry in serialized), _bytes_left_after, initial=budget) ) @@ -449,17 +460,34 @@ def _fits(serialized: Sequence[str], byte_limit: int | None) -> tuple[bool, ...] def _kept_positions( - entries: Sequence[CodexCatalogEntry], serialized: Sequence[str], byte_limit: int | None + entries: Sequence[CodexCatalogEntry], serialized: Sequence[str], budget: int | None ) -> frozenset[int]: """The listing positions that survive the byte limit: an entry offering a service tier is taken ahead of one offering none, each group in listing order, since a model without a tier is driven by name just as well off Codex's bundled catalog; an entry too large for the bytes left never costs the smaller ones after it their place.""" survival: Final = sorted(range(len(entries)), key=lambda position: (not entries[position].service_tiers, position)) - fits: Final = _fits(tuple(serialized[position] for position in survival), byte_limit) + fits: Final = _fits(tuple(serialized[position] for position in survival), budget) return frozenset(position for position, fit in zip(survival, fits, strict=True) if fit) +def _entries( + drivable: Sequence[CodexCatalogRow], + stock: Mapping[str, CodexStockModel] | None, + instructions: str | None, + first_priority: int, + alias_defaults: CodexAliasDefaults, + client_version: str | None, +) -> tuple[CodexCatalogEntry, ...]: + known: Final = bundled_codex_models() if stock is None else stock + prompt: Final = codex_base_instructions() if instructions is None else instructions + served: Final = frozenset(row.id for row in drivable) + return tuple( + _entry(first_priority + index, row, known, served, prompt, alias_defaults, client_version) + for index, row in enumerate(drivable) + ) + + @dataclass(frozen=True, slots=True) class CodexCatalogBody: """A `ModelsResponse` body, the ids it lists, and the chat-capable ids the byte limit left out.""" @@ -495,15 +523,12 @@ def codex_models_response_json( requesting Codex's version, which decides the reasoning levels a fallback entry may offer; None counts as a version too old for custom levels. """ - known: Final = bundled_codex_models() if stock is None else stock - prompt: Final = codex_base_instructions() if instructions is None else instructions drivable: Final = tuple(row for row in rows if _codex_can_drive(row.id, row.mode)) - served: Final = frozenset(row.id for row in drivable) - entries: Final = tuple( - _entry(index, row, known, served, prompt, alias_defaults, client_version) for index, row in enumerate(drivable) - ) + entries: Final = _entries(drivable, stock, instructions, 0, alias_defaults, client_version) serialized: Final = tuple(_serialized(entry) for entry in entries) - kept: Final = _kept_positions(entries, serialized, byte_limit) + kept: Final = _kept_positions( + entries, serialized, None if byte_limit is None else byte_limit - len(_BODY_PREFIX) - len(_BODY_SUFFIX) + 1 + ) return CodexCatalogBody( json=f"{_BODY_PREFIX}{','.join(serialized[position] for position in sorted(kept))}{_BODY_SUFFIX}", listed=tuple(row.id for position, row in enumerate(drivable) if position in kept), @@ -511,6 +536,74 @@ def codex_models_response_json( ) +def _priority(model: Mapping[str, object]) -> int: + priority: Final = model.get("priority") + return priority if isinstance(priority, int) else -1 + + +def _compact_json(value: object) -> str: + return json.dumps(value, separators=(",", ":")) + + +def _entry_json(entry: CodexCatalogEntry) -> Mapping[str, object]: + return _JSON_OBJECT.validate_python(entry.model_dump(mode="json", exclude_unset=isinstance(entry, CodexStockModel))) + + +def merged_codex_catalog( + upstream_body: bytes, + rows: Sequence[CodexCatalogRow], + *, + stock: Mapping[str, CodexStockModel] | None = None, + instructions: str | None = None, + byte_limit: int = CODEX_CATALOG_BYTE_LIMIT, + client_version: str | None = None, +) -> str | None: + try: + top: Final = _JSON_OBJECT.validate_json(upstream_body) + upstream: Final = _JSON_OBJECTS.validate_python(top.get("models")) + except ValidationError: + return None + slugs: Final = frozenset(model.get("slug") for model in upstream) + added: Final = tuple(row for row in rows if _codex_can_drive(row.id, row.mode) and row.id not in slugs) + first_priority: Final = max((_priority(model) for model in upstream), default=-1) + 1 + entries: Final = _entries(added, stock, instructions, first_priority, _NATIVE_ALIAS_DEFAULTS, client_version) + added_models: Final = tuple(_entry_json(entry) for entry in entries) + serialized: Final = tuple(_compact_json(model) for model in added_models) + budget: Final = byte_limit - len(_compact_json({**top, "models": upstream})) + kept: Final = tuple(sorted(_kept_positions(entries, serialized, budget))) + if len(kept) < len(entries): + verbose_proxy_logger.warning( + "Codex native model catalog left out %d of %d LiteLLM models to stay within Codex's %d byte limit", + len(entries) - len(kept), + len(entries), + byte_limit, + ) + if not kept: + return None + return _compact_json({**top, "models": [*upstream, *(added_models[position] for position in kept)]}) + + +def _canonical_value(value: object) -> object: + return value.model_dump(mode="json") if isinstance(value, BaseModel) else repr(value) + + +def codex_catalog_etag(upstream_etag: str, rows: Sequence[CodexCatalogRow]) -> str: + canonical: Final = json.dumps( + [asdict(row) for row in rows], sort_keys=True, separators=(",", ":"), default=_canonical_value + ) + digest: Final = hashlib.sha256(json.dumps([upstream_etag, canonical]).encode()).hexdigest() + return f'"{digest}"' + + +def upstream_codex_slugs(upstream_body: bytes) -> frozenset[str] | None: + try: + upstream: Final = _JSON_OBJECTS.validate_python(_JSON_OBJECT.validate_json(upstream_body).get("models")) + except ValidationError: + return None + slugs: Final = (model.get("slug") for model in upstream) + return frozenset(slug for slug in slugs if isinstance(slug, str)) + + def _group_reasoning_efforts( row: ModelInfoResponse, lookup_id: str, registry: RouterRegistry | None, team_id: str | None ) -> tuple[str, ...] | None: @@ -520,6 +613,11 @@ def _group_reasoning_efforts( return group.supported_reasoning_efforts if group is not None else None +def _shared_pinned_effort(pinned: tuple[object, ...]) -> str | None: + first: Final = next(iter(pinned), None) + return first if isinstance(first, str) and all(effort == first for effort in pinned) else None + + def deployment_capabilities(model_info: Mapping[str, object] | None, model_id: str) -> CodexDeploymentCapabilities: """One deployment's model info, merged with the cost map, as the capabilities a Codex entry describes. Model info that does not validate declares nothing, logged, so one bad value never fails the listing.""" @@ -562,6 +660,11 @@ def _catalog_row( registry.resolve(lookup_id, kind="upstream_provider", team_id=team_id) if registry is not None else None ), display_name=registry.group(label_id, field="display_name") if registry is not None else None, + reasoning_effort=( + _shared_pinned_effort(registry.group(lookup_id, field="pinned_reasoning_efforts", team_id=team_id)) + if registry is not None + else None + ), service_tiers=registry.group(lookup_id, field="service_tiers", team_id=team_id) if registry is not None else (), supported_reasoning_efforts=_group_reasoning_efforts(row, lookup_id, registry, team_id), search_tool_support=( @@ -619,3 +722,21 @@ def codex_model_list_body( ", ".join(body.left_out), ) return body.json + + +def codex_entry_names_default_reasoning_level( + model_id: str, llm_router: Router, team_id: str | None, client_version: str | None +) -> bool: + """Whether the native catalog entry for `model_id` names a `default_reasoning_level`. Codex's model picker + sends reasoning effort `none` for a model whose entry names none, while showing that choice as + "default".""" + listed: Final[ModelInfoResponse] = {"id": model_id, "object": "model", "created": 0, "owned_by": ""} + (entry,) = _entries( + (_catalog_row(listed, model_id, model_id, llm_router, team_id),), + None, + "", + 0, + _NATIVE_ALIAS_DEFAULTS, + client_version, + ) + return _entry_json(entry).get("default_reasoning_level") is not None diff --git a/litellm/proxy/common_utils/codex_model_translation.py b/litellm/proxy/common_utils/codex_model_translation.py index 2c4ea8896e3..694dc422994 100644 --- a/litellm/proxy/common_utils/codex_model_translation.py +++ b/litellm/proxy/common_utils/codex_model_translation.py @@ -16,7 +16,7 @@ from types import NoneType from typing import Annotated, Final, Literal, TypeAlias from packaging.version import Version -from pydantic import ConfigDict, Field, StringConstraints +from pydantic import ConfigDict, Field, StringConstraints, field_serializer from litellm.litellm_core_utils.reasoning_effort_strength import nearest_declared_reasoning_effort from litellm.types.llms.base_model import LiteLLMBaseModel @@ -117,6 +117,10 @@ class CodexDeploymentCapabilities(LiteLLMBaseModel): prices_tokens: bool = False priced_service_tier_keys: frozenset[str] = frozenset() + @field_serializer("priced_service_tier_keys") + def _sorted_priced_service_tier_keys(self, keys: frozenset[str]) -> tuple[str, ...]: + return tuple(sorted(keys)) + @cache def tier_price_keys() -> tuple[str, ...]: diff --git a/litellm/proxy/guardrails/guardrail_hooks/tool_policy/tool_policy_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/tool_policy/tool_policy_guardrail.py index fbf1f2576ff..1feb6d533f4 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/tool_policy/tool_policy_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/tool_policy/tool_policy_guardrail.py @@ -27,7 +27,10 @@ from pydantic import TypeAdapter from typing_extensions import TypedDict, Unpack from litellm._logging import verbose_proxy_logger -from litellm.completion_extras.litellm_responses_transformation.transformation import replayed_tool_search_tool_names +from litellm.completion_extras.litellm_responses_transformation.transformation import ( + nested_tool_member_names, + replayed_tool_search_tool_names, +) from litellm.integrations.custom_guardrail import ( CustomGuardrail, log_guardrail_information, @@ -47,6 +50,14 @@ class _CustomGuardrailOptions(TypedDict, total=False, extra_items=object): """Base-class constructor options this guardrail forwards untouched to CustomGuardrail.""" +def _function_tool_name(tool: object) -> str | None: + match tool: + case {"function": {"name": str(name)}}: + return name + case _: + return None + + def _attribute(source: object, name: str) -> object: return getattr(source, name, None) @@ -140,17 +151,18 @@ class ToolPolicyGuardrail(CustomGuardrail): """ if input_type == "request": tools: Final = inputs.get("tools") or [] - function_tool_names: Final = [ - t["function"]["name"] - for t in tools - if isinstance(t, dict) and isinstance(t.get("function"), dict) and t["function"].get("name") - ] + function_tool_names: Final = [name for name in map(_function_tool_name, tools) if name] route: Final = _get_request_route_from_data(request_data) - replayed_tool_names: Final = replayed_tool_search_tool_names(_REQUEST_DATA.validate_python(request_data)) - tool_names = ( - extract_request_tool_names(route, request_data) - if route and not function_tool_names - else [*function_tool_names, *replayed_tool_names] + request: Final = _REQUEST_DATA.validate_python(request_data) + tool_names = list( + dict.fromkeys( + ( + *function_tool_names, + *(extract_request_tool_names(route, request_data) if route else ()), + *replayed_tool_search_tool_names(request), + *nested_tool_member_names(request), + ) + ) ) else: tool_calls: Final = inputs.get("tool_calls") or [] diff --git a/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py index 3087faa9f67..d321f5b5861 100644 --- a/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/codex_passthrough_endpoints.py @@ -1,15 +1,94 @@ +import asyncio +import hashlib import hmac +import json +import re +from collections.abc import Iterator, Mapping +from dataclasses import dataclass from types import MappingProxyType -from typing import Annotated, Final, Literal +from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol +import httpx from fastapi import APIRouter, Depends, HTTPException, Request, Response +from fastapi.responses import StreamingResponse +from pydantic import TypeAdapter, ValidationError +from starlette.types import Message +import litellm +from litellm._internal_context import with_service_target +from litellm._logging import verbose_proxy_logger +from litellm.exceptions import BadRequestError +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.llms.chatgpt.common_utils import CHATGPT_API_BASE -from litellm.proxy._types import UserAPIKeyAuth +from litellm.llms.custom_httpx.http_handler import ( + get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # shared client factory has untyped params +) +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.auth.auth_checks import check_model_access_helper, check_tools_allowlist +from litellm.proxy.auth.handle_jwt import JWTAuthManager +from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.codex_model_catalog import ( + CodexCatalogRow, + bundled_codex_models, + codex_catalog_etag, + codex_entry_names_default_reasoning_level, + merged_codex_catalog, + upstream_codex_slugs, +) +from litellm.proxy.common_utils.http_parsing_utils import ( + get_client_requested_model, + read_request_body, # pyright: ignore[reportUnknownVariableType] # returns a bare dict, validated at the call +) +from litellm.proxy.litellm_pre_call_utils import should_auto_drop_params_for_agentic_cli from litellm.proxy.pass_through_endpoints.pass_through_endpoints import pass_through_request +from litellm.proxy.response_api_endpoints.endpoints import responses_api +from litellm.responses.additional_tools import with_additional_tools_hoisted +from litellm.responses.litellm_completion_transformation.reasoning_items import is_other_provider_reasoning_item +from litellm.responses.utils import ResponsesAPIRequestUtils +from litellm.types.llms.custom_http import httpxSpecialProvider +from litellm.types.router import Deployment +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager, supports_none_reasoning_effort + +if TYPE_CHECKING: + from litellm.router import Router router: Final = APIRouter() +_CHATGPT_PROVIDER: Final = "chatgpt" +_CHATGPT_CREDENTIAL_HEADERS: Final = frozenset({b"authorization", b"chatgpt-account-id"}) +_MERGED_CATALOG_DROPPED_HEADERS: Final = frozenset({"etag", "content-length", "content-type"}) +_ETAG_HEADER: Final = "etag" +_MODELS_ETAG_HEADER: Final = "x-models-etag" +_ENCRYPTED_CONTENT_AFFINITY_FLAG: Final = "encrypted_content_affinity_enabled" +_LITELLM_RESPONSES_PATH: Final = "/v1/responses" +_CODEX_NO_DEFAULT_EFFORT: Final = "none" +_NONE_EFFORT_SUPPORT_FLAG: Final = "supports_none_reasoning_effort" +_JSON_OBJECT: Final = TypeAdapter(dict[str, object]) +_INPUT_ITEMS: Final = TypeAdapter(tuple[object, ...]) +_CACHED_SLUGS: Final = TypeAdapter(frozenset[str]) +_CACHED_MODELS_ETAG: Final = TypeAdapter(dict[Literal["etag"], str]) +_MODEL_NAMES: Final = TypeAdapter(list[str]) +_HEADER_DERIVED_STATE: Final = frozenset({"_cached_headers"}) +_ACCOUNT_CATALOG_HEADERS: Final = frozenset({"authorization", "chatgpt-account-id", "originator", "user-agent"}) +_CODEX_USER_AGENT_PRODUCT: Final = re.compile(r"([^/]+)/(\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.+-]+)?)") +_ACCOUNT_CATALOG_TIMEOUT_SECONDS: Final = 5.0 +_SHARED_CACHE_TTL_SECONDS: Final = 3600 +_ACCOUNT_SLUGS_UNAVAILABLE: Final = "unavailable" +_ACCOUNT_SLUGS_UNAVAILABLE_TTL_SECONDS: Final = 60 +_ACCOUNT_SLUGS_CACHE_PREFIX: Final = "codex_account_slugs:" +_ACCOUNT_SLUGS_TARGET: Final = "codex_account_slugs" +_ACCOUNT_SLUGS_FETCHES_CACHE_PREFIX: Final = "codex_account_slugs_fetches:" +_ACCOUNT_SLUGS_FETCHES_PER_CALLER: Final = 5 +_ACCOUNT_SLUGS_FETCHES_WINDOW_SECONDS: Final = 60 +_MODELS_ETAG_CACHE_PREFIX: Final = "codex_models_etag:" +_MODELS_ETAG_TTL_SECONDS: Final = 300 +_OPENAI_API_PROVIDERS: Final = frozenset({LlmProviders.OPENAI.value, LlmProviders.AZURE.value}) +_CODEX_TOOL_PROVIDERS: Final = frozenset( + {*_OPENAI_API_PROVIDERS, LlmProviders.FIREWORKS_AI.value, LlmProviders.BEDROCK_MANTLE.value} +) +_CHAT_COMPLETIONS_BRIDGE: Final = "use_chat_completions_api" _HEADERS: Final = frozenset( { "authorization", @@ -29,10 +108,637 @@ _HEADERS: Final = frozenset( ) -@router.get("/backend-api/codex/{endpoint}", tags=["ChatGPT Pass-through", "pass-through"]) +def _is_chatgpt(deployment: Deployment) -> bool: + return ( + deployment.litellm_params.custom_llm_provider == _CHATGPT_PROVIDER + or deployment.litellm_params.model.partition("/")[0] == _CHATGPT_PROVIDER + ) + + +def _serves_by_name(deployment: Deployment, names: frozenset[str]) -> bool: + return not _is_chatgpt(deployment) and ( + deployment.model_name in names or deployment.model_info.team_public_model_name in names + ) + + +def _candidate_deployments(model: str, llm_router: "Router", team_id: str | None) -> tuple[Deployment, ...]: + registry: Final = llm_router.registry() + candidates: Final = tuple( + registry.deployment(model_id) for model_id in registry.names(model, kind="route_candidates", team_id=team_id) + ) + return tuple(deployment for deployment in candidates if deployment is not None) + + +def routes_to_litellm(model: object, llm_router: "Router | None", team_id: str | None) -> bool: + if ( + not isinstance(model, str) + or llm_router is None + or model in bundled_codex_models() + or f"{_CHATGPT_PROVIDER}/{model}" in litellm.model_cost # pyright: ignore[reportUnknownMemberType] # registry is dynamically typed + ): + return False + deployments: Final = _candidate_deployments(model, llm_router, team_id) + names: Final = frozenset({model, llm_router.registry().resolve(model, kind="alias") or model}) + return bool(deployments) and all(_serves_by_name(deployment, names) for deployment in deployments) + + +def _key_and_team_allow(model: str, llm_router: "Router", user_api_key_dict: UserAPIKeyAuth) -> bool: + allowed_lists: Final = ( + _MODEL_NAMES.validate_python(user_api_key_dict.models), + _MODEL_NAMES.validate_python(user_api_key_dict.team_models), + ) + return all( + check_model_access_helper(model, llm_router, allowed, team_id=user_api_key_dict.team_id) + for allowed in allowed_lists + ) + + +def native_chatgpt_model(model: str, llm_router: "Router", user_api_key_dict: UserAPIKeyAuth) -> str | None: + if model.startswith(f"{_CHATGPT_PROVIDER}/") or _key_and_team_allow(model, llm_router, user_api_key_dict): + return None + chatgpt_model: Final = f"{_CHATGPT_PROVIDER}/{model}" + deployments: Final = _candidate_deployments(chatgpt_model, llm_router, user_api_key_dict.team_id) + if not deployments or not all(_is_chatgpt(deployment) for deployment in deployments): + return None + return chatgpt_model if _key_and_team_allow(chatgpt_model, llm_router, user_api_key_dict) else None + + +class _SharedCache(Protocol): + async def async_get_cache(self, key: str) -> object: ... + + async def async_set_cache(self, key: str, value: object, *, ttl: int) -> None: ... + + async def async_increment_cache(self, key: str, value: float, *, ttl: int) -> float | None: ... + + +def codex_shared_cache() -> _SharedCache: + from litellm.proxy.proxy_server import proxy_logging_obj + + return proxy_logging_obj.internal_usage_cache.dual_cache + + +def _key_may_call_litellm_responses(user_api_key_dict: UserAPIKeyAuth) -> bool: + if not RouteChecks.is_allowed_on_proxy(_LITELLM_RESPONSES_PATH): + return False + try: + _ = RouteChecks.should_call_route(route=_LITELLM_RESPONSES_PATH, valid_token=user_api_key_dict) + if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: + RouteChecks.custom_admin_only_route_check(route=_LITELLM_RESPONSES_PATH) + except HTTPException: + return False + return True + + +async def _jwt_policy_admits_litellm_responses( + gateway_token: str, model: str, request_headers: Mapping[str, str] +) -> bool: + from litellm.proxy.proxy_server import ( + general_settings_view, + jwt_handler, + prisma_client, + proxy_logging_obj, + user_api_key_cache, + ) + + try: + _ = await JWTAuthManager.auth_builder( + api_key=gateway_token, + jwt_handler=jwt_handler, + request_data={"model": model}, + general_settings=dict(general_settings_view()), + route=_LITELLM_RESPONSES_PATH, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=None, + proxy_logging_obj=proxy_logging_obj, + request_headers=request_headers, + request_method="POST", + ) + except Exception: # noqa: BLE001 # any JWT policy refusal keeps the caller on ChatGPT's models + return False + return True + + +@dataclass(frozen=True, slots=True) +class _LitellmResponsesAccess: + key_admits: bool + jwt_token: str | None + request_headers: Mapping[str, str] + + async def admits(self, model: str) -> bool: + if not self.key_admits: + return False + if self.jwt_token is None: + return True + return await _jwt_policy_admits_litellm_responses(self.jwt_token, model, self.request_headers) + + +def _litellm_responses_access(user_api_key_dict: UserAPIKeyAuth, request: Request) -> _LitellmResponsesAccess: + standard_jwt: Final = bool(user_api_key_dict.jwt_claims) and not user_api_key_dict.token + return _LitellmResponsesAccess( + key_admits=_key_may_call_litellm_responses(user_api_key_dict), + jwt_token=_gateway_token(request) if standard_jwt else None, + request_headers=request.headers, + ) + + +def _account_slugs_cache_key(headers: Mapping[str, str]) -> str | None: + account_id: Final = headers.get("chatgpt-account-id") + if not account_id: + return None + caller: Final = json.dumps([account_id, headers.get("authorization", "")]) + return f"{_ACCOUNT_SLUGS_CACHE_PREFIX}{hashlib.sha256(caller.encode()).hexdigest()}" + + +def _cached_slugs(cached: object) -> frozenset[str] | None: + if cached is None: + return None + try: + return _CACHED_SLUGS.validate_python(cached) + except ValidationError: + return None + + +async def _remember_account_slugs(cache_key: str, slugs: frozenset[str] | None, cache: _SharedCache) -> None: + if slugs is None: + await cache.async_set_cache(cache_key, _ACCOUNT_SLUGS_UNAVAILABLE, ttl=_ACCOUNT_SLUGS_UNAVAILABLE_TTL_SECONDS) + return + await cache.async_set_cache(cache_key, tuple(sorted(slugs)), ttl=_SHARED_CACHE_TTL_SECONDS) + + +def _codex_client_version(headers: Mapping[str, str]) -> str | None: + product: Final = _CODEX_USER_AGENT_PRODUCT.fullmatch(headers.get("user-agent", "").partition(" ")[0]) + if product is None or product.group(1) != headers.get("originator"): + return None + return product.group(2) + + +async def _fetched_account_slugs(headers: Mapping[str, str], client_version: str) -> frozenset[str] | None: + client: Final = get_async_httpx_client( + llm_provider=httpxSpecialProvider.PassThroughEndpoint, + params={"timeout": _ACCOUNT_CATALOG_TIMEOUT_SECONDS}, + ) + try: + response: Final = await client.client.get( + f"{CHATGPT_API_BASE}/models", + params={"client_version": client_version}, + headers={name: value for name, value in headers.items() if name in _ACCOUNT_CATALOG_HEADERS}, + ) + except httpx.HTTPError: + return None + return upstream_codex_slugs(response.content) if response.status_code == 200 else None + + +async def _admits_account_slugs_fetch(caller: str, cache: _SharedCache) -> bool: + fetches: Final = await cache.async_increment_cache( + f"{_ACCOUNT_SLUGS_FETCHES_CACHE_PREFIX}{caller}", 1, ttl=_ACCOUNT_SLUGS_FETCHES_WINDOW_SECONDS + ) + return fetches is not None and fetches <= _ACCOUNT_SLUGS_FETCHES_PER_CALLER + + +@with_service_target(_ACCOUNT_SLUGS_TARGET) +async def _account_slugs(headers: Mapping[str, str], caller: str, cache: _SharedCache) -> frozenset[str] | None: + cache_key: Final = _account_slugs_cache_key(headers) + if cache_key is None: + return None + cached_value: Final = await cache.async_get_cache(cache_key) + if cached_value == _ACCOUNT_SLUGS_UNAVAILABLE: + return None + cached: Final = _cached_slugs(cached_value) + if cached is not None: + return cached + client_version: Final = _codex_client_version(headers) + if client_version is None or not await _admits_account_slugs_fetch(caller, cache): + return None + fetched: Final = await _fetched_account_slugs(headers, client_version) + await _remember_account_slugs(cache_key, fetched, cache) + return fetched + + +async def _serves_through_litellm( + model: object, + llm_router: "Router | None", + user_api_key_dict: UserAPIKeyAuth, + access: _LitellmResponsesAccess, + caller: str, + headers: Mapping[str, str], + cache: _SharedCache, +) -> bool: + if not isinstance(model, str) or not routes_to_litellm(model, llm_router, user_api_key_dict.team_id): + return False + if not await access.admits(model): + return False + account_slugs: Final = await _account_slugs(headers, caller, cache) + return account_slugs is not None and model not in account_slugs + + +def _gateway_token(request: Request) -> str: + gateway_parts: Final = request.headers.get("x-litellm-api-key", "").split() + return ( + gateway_parts[-1] + if len(gateway_parts) == 1 or (len(gateway_parts) == 2 and gateway_parts[0].lower() == "bearer") + else "" + ) + + +def _chatgpt_bearer(request: Request) -> str: + scheme, _, token = request.headers.get("authorization", "").partition(" ") + return token.strip() if scheme.lower() == "bearer" else "" + + +def _forwarded_headers(request: Request) -> Mapping[str, str]: + return MappingProxyType({name: value for name, value in request.headers.items() if name in _HEADERS}) + + +async def native_turn_serves_through_litellm( + model: str, llm_router: "Router", request: Request, user_api_key_dict: UserAPIKeyAuth +) -> bool: + chatgpt_bearer: Final = _chatgpt_bearer(request) + gateway_token: Final = _gateway_token(request) + if not chatgpt_bearer or not gateway_token or hmac.compare_digest(chatgpt_bearer.encode(), gateway_token.encode()): + return False + return await _serves_through_litellm( + model, + llm_router, + user_api_key_dict, + _litellm_responses_access(user_api_key_dict, request), + _models_etag_caller(user_api_key_dict, gateway_token), + _forwarded_headers(request), + codex_shared_cache(), + ) + + +def _json_object(value: object) -> Mapping[str, object]: + try: + return _JSON_OBJECT.validate_python(value) + except ValidationError: + return MappingProxyType({}) + + +def _scope_state(request: Request) -> Mapping[str, object]: + return _json_object(request.scope.get("state")) + + +def _as_litellm_responses_request(request: Request, body: bytes, parsed_body: Mapping[str, object]) -> Request: + replayed: Final[Iterator[Message]] = iter(({"type": "http.request", "body": body, "more_body": False},)) + + async def receive() -> Message: + return next(replayed, None) or await request.receive() + + return Request( + { + **request.scope, + "path": _LITELLM_RESPONSES_PATH, + "raw_path": _LITELLM_RESPONSES_PATH.encode(), + "headers": [ + (name, value) for name, value in request.headers.raw if name not in _CHATGPT_CREDENTIAL_HEADERS + ], + "parsed_body": (tuple(parsed_body), parsed_body), + "state": { + name: value for name, value in _scope_state(request).items() if name not in _HEADER_DERIVED_STATE + }, + }, + receive, + ) + + +def _drops_params(body: Mapping[str, object], model: str, llm_router: "Router", team_id: str | None) -> bool: + deployments: Final = _candidate_deployments(model, llm_router, team_id) + return drops_unsupported_params(body.get("drop_params")) or ( + bool(deployments) + and all(normalize_drop_params(deployment.litellm_params.drop_params) is True for deployment in deployments) + ) + + +def _deployment_supports_no_effort(deployment: Deployment) -> bool: + configured: Final = _json_object(deployment.model_info.model_extra).get(_NONE_EFFORT_SUPPORT_FLAG) + if isinstance(configured, bool): + return configured + return supports_none_reasoning_effort( + model=deployment.litellm_params.model, custom_llm_provider=deployment.litellm_params.custom_llm_provider + ) + + +def _supports_no_effort(model: str, llm_router: "Router", team_id: str | None) -> bool: + deployments: Final = _candidate_deployments(model, llm_router, team_id) + return bool(deployments) and all(_deployment_supports_no_effort(deployment) for deployment in deployments) + + +def _with_provider_default_reasoning( + body: Mapping[str, object], llm_router: "Router | None", team_id: str | None, client_version: str | None +) -> Mapping[str, object]: + model: Final = body.get("model") + try: + reasoning: Final = _JSON_OBJECT.validate_python(body.get("reasoning")) + except ValidationError: + return body + if ( + reasoning.get("effort") != _CODEX_NO_DEFAULT_EFFORT + or not isinstance(model, str) + or llm_router is None + or not _drops_params(body, model, llm_router, team_id) + or codex_entry_names_default_reasoning_level(model, llm_router, team_id, client_version) + or _supports_no_effort(model, llm_router, team_id) + ): + return body + return {**body, "reasoning": {key: value for key, value in reasoning.items() if key != "effort"}} + + +def _with_wrapped_reasoning(body: Mapping[str, object]) -> Mapping[str, object]: + return { + **body, + "litellm_metadata": {**_json_object(body.get("litellm_metadata")), _ENCRYPTED_CONTENT_AFFINITY_FLAG: True}, + } + + +def _with_reasoning_the_targets_can_decrypt( + body: Mapping[str, object], model: str, llm_router: "Router", deployments: tuple[Deployment, ...] +) -> Mapping[str, object]: + try: + items: Final = _INPUT_ITEMS.validate_python(body.get("input")) + except ValidationError: + return body + model_group: Final = llm_router.registry().resolve(model, kind="alias") or model + kept: Final = llm_router.registry().decryptable_reasoning(model_group, items, deployments) + if body.get("store") is True: + return {**body, "input": list(kept)} + resolvable: Final = (_without_unpersisted_reasoning_reference(item) for item in kept) + return {**body, "input": [item for item in resolvable if item is not None]} + + +def _without_unpersisted_reasoning_reference(item: object) -> object | None: + if not _is_unpersisted_reasoning_reference(item): + return item + return ResponsesAPIRequestUtils.readable_reasoning(_JSON_OBJECT.validate_python(item)) + + +def _model_and_provider(deployment: Deployment) -> tuple[str, str] | None: + try: + model, provider, _, _ = get_llm_provider( + model=deployment.litellm_params.model, custom_llm_provider=deployment.litellm_params.custom_llm_provider + ) + except BadRequestError: + return None + return model, provider + + +def _is_openai_api(deployment: Deployment) -> bool: + resolved: Final = _model_and_provider(deployment) + return resolved is not None and resolved[1] in _OPENAI_API_PROVIDERS + + +def _takes_codex_tools(deployment: Deployment) -> bool: + resolved: Final = _model_and_provider(deployment) + if resolved is None: + return False + model, provider = resolved + return ( + provider in _CODEX_TOOL_PROVIDERS + or ProviderConfigManager.get_provider_responses_api_config( + provider=provider, model=model, api_base=deployment.litellm_params.api_base + ) + is None + ) + + +def _with_tools_the_targets_accept( + body: Mapping[str, object], deployments: tuple[Deployment, ...], drop_params: bool +) -> Mapping[str, object]: + if all(_is_openai_api(deployment) for deployment in deployments): + return body + if all(_takes_codex_tools(deployment) for deployment in deployments): + return with_additional_tools_hoisted(body, drop_params=drop_params) + return {**body, _CHAT_COMPLETIONS_BRIDGE: True} + + +def _drops_params_for_client( + body: Mapping[str, object], model: str, llm_router: "Router", team_id: str | None, user_agent: str +) -> bool: + from litellm.proxy.proxy_server import proxy_config + + return _drops_params(body, model, llm_router, team_id) or should_auto_drop_params_for_agentic_cli( + user_agent, "drop_params" in body, proxy_config + ) + + +def _litellm_body( + body: Mapping[str, object], llm_router: "Router | None", team_id: str | None, headers: Mapping[str, str] +) -> Mapping[str, object]: + wrapped: Final = _with_wrapped_reasoning( + _with_provider_default_reasoning(body, llm_router, team_id, _codex_client_version(headers)) + ) + model: Final = body.get("model") + if not isinstance(model, str) or llm_router is None: + return wrapped + deployments: Final = _candidate_deployments(model, llm_router, team_id) + return _with_tools_the_targets_accept( + _with_reasoning_the_targets_can_decrypt(wrapped, model, llm_router, deployments), + deployments, + _drops_params_for_client(body, model, llm_router, team_id, headers.get("user-agent", "")), + ) + + +def _is_unpersisted_reasoning_reference(item: object) -> bool: + try: + fields: Final = _JSON_OBJECT.validate_python(item) + except ValidationError: + return False + item_id: Final = fields.get("id") + encrypted_content: Final = fields.get("encrypted_content") + return ( + fields.get("type") == "reasoning" + and isinstance(item_id, str) + and bool(item_id) + and not (isinstance(encrypted_content, str) and bool(encrypted_content)) + ) + + +def _rejected_by_chatgpt(item: object, stored: bool) -> bool: + return is_other_provider_reasoning_item(item) or (not stored and _is_unpersisted_reasoning_reference(item)) + + +def _input_chatgpt_accepts(body: Mapping[str, object]) -> tuple[object, ...] | None: + try: + items: Final = _INPUT_ITEMS.validate_python(body.get("input")) + except ValidationError: + return None + stored: Final = body.get("store") is True + kept: Final = tuple(item for item in items if not _rejected_by_chatgpt(item, stored)) + return kept if len(kept) < len(items) else None + + +def _chatgpt_body(body: Mapping[str, object], client_model: str | None) -> Mapping[str, object] | None: + kept_input: Final = _input_chatgpt_accepts(body) + restored_model: Final = client_model is not None and body.get("model") == f"{_CHATGPT_PROVIDER}/{client_model}" + if kept_input is None and not restored_model: + return None + return { + **body, + **({"input": list(kept_input)} if kept_input is not None else {}), + **({"model": client_model} if restored_model else {}), + } + + +async def _litellm_catalog_rows( + user_api_key_dict: UserAPIKeyAuth, access: _LitellmResponsesAccess, llm_router: "Router | None" +) -> tuple[CodexCatalogRow, ...]: + from litellm.proxy.proxy_server import codex_catalog_rows_for_key + + if llm_router is None or not access.key_admits: + return () + try: + rows: Final = await codex_catalog_rows_for_key(user_api_key_dict) + except Exception as error: # noqa: BLE001 # a failed listing must never fail the ChatGPT answer it extends + verbose_proxy_logger.warning( + "Codex catalog keeps only ChatGPT's models: listing the key's LiteLLM models raised %s", + type(error).__name__, + ) + return () + routed: Final = tuple(row for row in rows if routes_to_litellm(row.id, llm_router, user_api_key_dict.team_id)) + admitted: Final = await asyncio.gather(*(access.admits(row.id) for row in routed)) + return tuple(row for row, row_admitted in zip(routed, admitted, strict=True) if row_admitted) + + +def _models_etag_caller(user_api_key_dict: UserAPIKeyAuth, gateway_token: str) -> str: + return user_api_key_dict.token or hashlib.sha256(gateway_token.encode()).hexdigest() + + +def _models_etag_cache_key(caller: str, upstream_etag: str) -> str: + return f"{_MODELS_ETAG_CACHE_PREFIX}{hashlib.sha256(json.dumps([caller, upstream_etag]).encode()).hexdigest()}" + + +async def _remembered_models_etag( + caller: str, upstream_etag: str, rows: tuple[CodexCatalogRow, ...], cache: _SharedCache +) -> str: + etag: Final = codex_catalog_etag(upstream_etag, rows) if rows else upstream_etag + await cache.async_set_cache( + _models_etag_cache_key(caller, upstream_etag), {"etag": etag}, ttl=_MODELS_ETAG_TTL_SECONDS + ) + return etag + + +async def _cached_models_etag(caller: str, upstream_etag: str, cache: _SharedCache) -> str | None: + try: + return _CACHED_MODELS_ETAG.validate_python( + await cache.async_get_cache(_models_etag_cache_key(caller, upstream_etag)) + )["etag"] + except ValidationError: + return None + + +async def _key_models_etag( + upstream_etag: str, + user_api_key_dict: UserAPIKeyAuth, + access: _LitellmResponsesAccess, + caller: str, + llm_router: "Router | None", + cache: _SharedCache, +) -> str: + cached: Final = await _cached_models_etag(caller, upstream_etag, cache) + if cached is not None: + return cached + rows: Final = await _litellm_catalog_rows(user_api_key_dict, access, llm_router) + return await _remembered_models_etag(caller, upstream_etag, rows, cache) + + +def _with_etag(response: Response, content: bytes | str, etag: str | None) -> Response: + kept_headers: Final = { + name: value for name, value in response.headers.items() if name not in _MERGED_CATALOG_DROPPED_HEADERS + } + return Response( + content=content, + status_code=response.status_code, + headers={**kept_headers, **({_ETAG_HEADER: etag} if etag is not None else {})}, + media_type="application/json", + ) + + +async def _catalog( + request: Request, + user_api_key_dict: UserAPIKeyAuth, + access: _LitellmResponsesAccess, + caller: str, + headers: Mapping[str, str], + cache: _SharedCache, +) -> Response: + from litellm.proxy.proxy_server import llm_router + + upstream: Final = await pass_through_request( + request=request, + target=f"{CHATGPT_API_BASE}/models", + custom_headers=dict(headers), + user_api_key_dict=user_api_key_dict, + custom_llm_provider=_CHATGPT_PROVIDER, + upstream_headers_filter=lambda _: headers, + ) + if upstream.status_code != 200 or isinstance(upstream, StreamingResponse): + return upstream + upstream_body: Final = bytes(upstream.body) + slugs: Final = upstream_codex_slugs(upstream_body) + if slugs is None: + return upstream + account_slugs_key: Final = _account_slugs_cache_key(headers) + if account_slugs_key is not None: + await _remember_account_slugs(account_slugs_key, slugs, cache) + rows: Final = await _litellm_catalog_rows(user_api_key_dict, access, llm_router) + upstream_etag: Final = upstream.headers.get(_ETAG_HEADER) + etag: Final = None if upstream_etag is None else await _remembered_models_etag(caller, upstream_etag, rows, cache) + if not rows: + return upstream + merged: Final = await asyncio.to_thread( + merged_codex_catalog, upstream_body, rows, client_version=request.query_params.get("client_version") + ) + return _with_etag(upstream, merged if merged is not None else upstream_body, etag) + + +async def _responses( + request: Request, + fastapi_response: Response, + user_api_key_dict: UserAPIKeyAuth, + access: _LitellmResponsesAccess, + caller: str, + headers: Mapping[str, str], + cache: _SharedCache, +) -> object: + from litellm.proxy.proxy_server import llm_router + + body: Final = _JSON_OBJECT.validate_python(await read_request_body(request)) + if await _serves_through_litellm(body.get("model"), llm_router, user_api_key_dict, access, caller, headers, cache): + litellm_body: Final = _litellm_body(body, llm_router, user_api_key_dict.team_id, headers) + await check_tools_allowlist( + request_body=dict(litellm_body), + valid_token=user_api_key_dict, + team_object=None, + route=_LITELLM_RESPONSES_PATH, + ) + return await responses_api( + _as_litellm_responses_request(request, json.dumps(litellm_body).encode(), litellm_body), + fastapi_response, + user_api_key_dict.model_copy(update={"request_route": _LITELLM_RESPONSES_PATH}), + ) + chatgpt_body: Final = _chatgpt_body(body, get_client_requested_model(request)) + upstream: Final = await pass_through_request( + request=request, + target=f"{CHATGPT_API_BASE}/responses", + custom_headers=dict(headers), + user_api_key_dict=user_api_key_dict, + custom_body=dict(chatgpt_body) if chatgpt_body is not None else None, + custom_llm_provider=_CHATGPT_PROVIDER, + upstream_headers_filter=lambda _: headers, + ) + upstream_models_etag: Final = upstream.headers.get(_MODELS_ETAG_HEADER) + if upstream_models_etag is not None: + upstream.headers[_MODELS_ETAG_HEADER] = await _key_models_etag( + upstream_models_etag, user_api_key_dict, access, caller, llm_router, cache + ) + return upstream + + +@router.get("/backend-api/codex/{endpoint}", tags=["ChatGPT Pass-through", "pass-through"], response_model=None) @router.post( "/backend-api/codex/{endpoint}", tags=["ChatGPT Pass-through", "pass-through"], + response_model=None, openapi_extra={ "requestBody": { "required": True, @@ -41,7 +747,7 @@ _HEADERS: Final = frozenset( }, responses={ 200: { - "description": "Unmodified Codex response or server-sent events", + "description": "Codex response, the model catalog with the key's LiteLLM models, or server-sent events", "content": { "application/json": {"schema": {"type": "object", "additionalProperties": True}}, "text/event-stream": {"schema": {"type": "string"}}, @@ -52,36 +758,29 @@ _HEADERS: Final = frozenset( async def codex_passthrough_route( endpoint: Literal["models", "responses"], request: Request, + fastapi_response: Response, user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], -) -> Response: - """Forward a client's ChatGPT subscription with separate LiteLLM authentication.""" + cache: Annotated[_SharedCache, Depends(codex_shared_cache)], +) -> object: + """Forward a client's ChatGPT subscription with separate LiteLLM authentication, listing and serving the + key's other chat models through LiteLLM.""" expected_method: Final = "GET" if endpoint == "models" else "POST" if request.method != expected_method: raise HTTPException(status_code=405, detail="Method not allowed", headers={"Allow": expected_method}) - authorization: Final = request.headers.get("authorization", "") - scheme, _, token = authorization.partition(" ") - gateway_key: Final = request.headers.get("x-litellm-api-key", "") - gateway_parts: Final = gateway_key.split() - gateway_token: Final = ( - gateway_parts[-1] - if len(gateway_parts) == 1 or (len(gateway_parts) == 2 and gateway_parts[0].lower() == "bearer") - else "" - ) - if scheme.lower() != "bearer" or not token.strip() or not gateway_token: + chatgpt_bearer: Final = _chatgpt_bearer(request) + gateway_token: Final = _gateway_token(request) + if not chatgpt_bearer or not gateway_token: raise HTTPException( status_code=401, detail="Send the ChatGPT bearer in Authorization and the LiteLLM key in x-litellm-api-key", ) - if hmac.compare_digest(token.strip().encode(), gateway_token.encode()): + if hmac.compare_digest(chatgpt_bearer.encode(), gateway_token.encode()): raise HTTPException(status_code=401, detail="ChatGPT and LiteLLM credentials must be separate") - headers: Final = MappingProxyType({name: value for name, value in request.headers.items() if name in _HEADERS}) - return await pass_through_request( - request=request, - target=f"{CHATGPT_API_BASE}/{endpoint}", - custom_headers=dict(headers), - user_api_key_dict=user_api_key_dict, - custom_llm_provider="chatgpt", - upstream_headers_filter=lambda _: headers, - ) + headers: Final = _forwarded_headers(request) + caller: Final = _models_etag_caller(user_api_key_dict, gateway_token) + access: Final = _litellm_responses_access(user_api_key_dict, request) + if endpoint == "models": + return await _catalog(request, user_api_key_dict, access, caller, headers, cache) + return await _responses(request, fastapi_response, user_api_key_dict, access, caller, headers, cache) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 781455c6815..d21f5952ba2 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -236,6 +236,7 @@ if TYPE_CHECKING: from litellm.integrations.opentelemetry import OpenTelemetry from litellm.proxy.health_check_utils.shared_health_check_manager import SharedHealthCheckManager + from litellm.types.proxy.model_listing import ModelInfoResponse Span = _Span | Any else: @@ -520,7 +521,7 @@ from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import ( publish_auth_cache_invalidation, ) from litellm.proxy.common_utils.callback_utils import initialize_callbacks_on_proxy -from litellm.proxy.common_utils.codex_model_catalog import codex_model_list_body +from litellm.proxy.common_utils.codex_model_catalog import CodexCatalogRow, codex_catalog_rows, codex_model_list_body from litellm.proxy.common_utils.config_includes import resolve_include_file_path, resolve_includes from litellm.proxy.common_utils.config_sync_pubsub import ConfigSyncSubscriber from litellm.proxy.common_utils.credential_hydration import decrypted_or_stored @@ -559,6 +560,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( from litellm.proxy.common_utils.load_config_utils import get_config_from_bucket from litellm.proxy.common_utils.model_deprecation import collect_model_deprecations from litellm.proxy.common_utils.model_listing_utils import ( + CallerAliases, ClaudeCodeRoutingNames, DeployedModelGroups, TeamModelNameTranslator, @@ -12616,6 +12618,116 @@ async def _deployment_hidden_by_listing_callbacks(deployment: Deployment, user_a return listed_name in await _names_hidden_by_listing_callbacks(user_api_key_dict, (listed_name,)) +async def _caller_aliases(user_api_key_dict: UserAPIKeyAuth, team_id: str | None) -> CallerAliases: + caller_router_settings: Final = await proxy_config.get_hierarchical_router_settings( + user_api_key_dict, prisma_client, proxy_logging_obj + ) + return caller_alias_maps( + user_api_key_dict.aliases, + user_api_key_dict.team_model_aliases, + user_api_key_dict.team_id, + team_id, + caller_router_settings.get("model_group_alias") if isinstance(caller_router_settings, Mapping) else None, + ) + + +async def _listing_hidden_names(healthy_only: bool | None) -> frozenset[str]: + blocked_names: Final = llm_router.health().fully_blocked_model_names() if llm_router is not None else set[str]() + unhealthy_names: Final = await get_hidden_unhealthy_model_names( + healthy_only=healthy_only, + general_settings=general_settings_view(), + llm_router=llm_router, + ) + return frozenset(blocked_names | unhealthy_names) + + +@dataclass(frozen=True, slots=True) +class _KeyModelListing: + rows: "tuple[ModelInfoResponse, ...]" + entries: tuple[tuple[str, str], ...] + labels: tuple[tuple[str, str], ...] + + +def _listed_model_info( + response_id: str, lookup_id: str, include_metadata: bool, fallback_type: str | None +) -> "ModelInfoResponse": + from litellm.proxy.utils import create_model_info_response + + model_info: Final = create_model_info_response( + model_id=lookup_id, + provider="openai", + include_metadata=include_metadata, + fallback_type=fallback_type, + llm_router=llm_router, + ) + model_info["id"] = response_id + return model_info + + +async def _key_model_listing( + user_api_key_dict: UserAPIKeyAuth, + caller_aliases: CallerAliases, + hidden_names: frozenset[str], + *, + team_id: str | None = None, + include_model_access_groups: bool = False, + only_model_access_groups: bool = False, + include_wildcard_routes: bool = False, + include_metadata: bool = False, + fallback_type: str | None = None, + offered_model_names: Sequence[str] = (), +) -> _KeyModelListing: + from litellm.proxy.utils import get_available_models_for_user + + available: Final = await get_available_models_for_user( + user_api_key_dict=user_api_key_dict, + llm_router=llm_router, + general_settings=general_settings, + user_model=user_model, + prisma_client=prisma_client, + proxy_logging_obj=proxy_logging_obj, + team_id=team_id, + include_model_access_groups=include_model_access_groups, + only_model_access_groups=only_model_access_groups, + return_wildcard_routes=include_wildcard_routes, + user_api_key_cache=user_api_key_cache, + ) + names: Final = (*available, *(name for name in offered_model_names if name not in available)) + excluded: Final = hidden_names | undiscoverable_model_names( + names, llm_router, user_api_key_dict, team_id or user_api_key_dict.team_id + ) + listed_entries: Final = await _entries_kept_by_listing_callbacks( + TeamModelNameTranslator.listing_entries( + [name for name in names if name not in excluded], llm_router, general_settings_view() + ), + user_api_key_dict, + ) + entries: Final = alias_listing_entries( + listed_entries, + caller_aliases, + _describable_model_groups(listed_entries, hidden_names, user_api_key_dict, team_id), + ) + return _KeyModelListing( + rows=tuple( + _listed_model_info(response_id, lookup_id, include_metadata, fallback_type) + for response_id, lookup_id in entries + ), + entries=entries, + labels=label_entries(listed_entries, entries), + ) + + +async def codex_catalog_rows_for_key(user_api_key_dict: UserAPIKeyAuth) -> tuple[CodexCatalogRow, ...]: + listing: Final = await _key_model_listing( + user_api_key_dict, + await _caller_aliases(user_api_key_dict, None), + await _listing_hidden_names(None), + ) + return await asyncio.to_thread( + codex_catalog_rows, listing.rows, listing.entries, llm_router, user_api_key_dict.team_id, listing.labels + ) + + @router.get("/v1/models", dependencies=[Depends(user_api_key_auth)], tags=["model management"]) @router.get( "/models", dependencies=[Depends(user_api_key_auth)], tags=["model management"] @@ -12683,10 +12795,7 @@ async def model_list( from litellm.proxy.management_endpoints.common_utils import ( user_has_admin_privileges, ) - from litellm.proxy.utils import ( - create_model_info_response, - get_available_models_for_user, - ) + from litellm.proxy.utils import create_model_info_response from litellm.types.proxy.model_listing import ModelInfoResponse http_request: Final = cast(Request | None, request) # cast-ok: in-process callers pass no request @@ -12694,16 +12803,7 @@ async def model_list( http_request is not None and http_request.headers.get("anthropic-version") is not None ) client_headers: Final[Mapping[str, str]] = http_request.headers if http_request is not None else _EMPTY_HEADERS - caller_router_settings: Final = await proxy_config.get_hierarchical_router_settings( - user_api_key_dict, prisma_client, proxy_logging_obj - ) - caller_aliases: Final = caller_alias_maps( - user_api_key_dict.aliases, - user_api_key_dict.team_model_aliases, - user_api_key_dict.team_id, - team_id, - caller_router_settings.get("model_group_alias") if isinstance(caller_router_settings, Mapping) else None, - ) + caller_aliases: Final = await _caller_aliases(user_api_key_dict, team_id) routing_names: Final = ClaudeCodeRoutingNames( llm_router, team_id or user_api_key_dict.team_id, @@ -12729,18 +12829,7 @@ async def model_list( proxy_logging_obj=proxy_logging_obj, ) - # Compute once — used in both branches below to hide paused models from the listing. - blocked_names: Final = llm_router.health().fully_blocked_model_names() if llm_router is not None else set[str]() - - # Opt-in: also hide models whose deployments are all unhealthy per background - # health checks. Empty when health state is unavailable or stale (fail open). - unhealthy_names: Final = await get_hidden_unhealthy_model_names( - healthy_only=healthy_only, - general_settings=settings, - llm_router=llm_router, - ) - - hidden_names: Final = blocked_names | unhealthy_names + hidden_names: Final = await _listing_hidden_names(healthy_only) from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import typesafe_allowlist_model_names @@ -12838,58 +12927,24 @@ async def model_list( object="list", ) - # Otherwise, use the normal behavior (current implementation) - # Get available models for the user - available_models: Final = await get_available_models_for_user( - user_api_key_dict=user_api_key_dict, - llm_router=llm_router, - general_settings=general_settings, - user_model=user_model, - prisma_client=prisma_client, - proxy_logging_obj=proxy_logging_obj, - team_id=team_id, - include_model_access_groups=include_model_access_groups or False, - only_model_access_groups=only_model_access_groups or False, - return_wildcard_routes=include_wildcard_routes, - user_api_key_cache=user_api_key_cache, - ) offers_typesafe_allowlist_names: Final = ( asks_for_allowlist_names and team_id is None and user_api_key_has_admin_view(user_api_key_dict) ) - typesafe_names_to_offer: Final = typesafe_allowlist_model_names() if offers_typesafe_allowlist_names else () - all_models = [*available_models, *(name for name in typesafe_names_to_offer if name not in available_models)] - - undiscoverable_names: Final = undiscoverable_model_names( - all_models, llm_router, user_api_key_dict, team_id or user_api_key_dict.team_id - ) - if hidden_names or undiscoverable_names: - all_models = [m for m in all_models if m not in hidden_names and m not in undiscoverable_names] - - # Surface the public team name by default; legacy internal keys via flag. - # The internal routing key drives the metadata/fallback lookup, while the - # public name is what the client sees as the model id. - model_data = [] - listed_entries: Final = await _entries_kept_by_listing_callbacks( - TeamModelNameTranslator.listing_entries(all_models, llm_router, settings), user_api_key_dict - ) - entries: Final = alias_listing_entries( - listed_entries, + key_listing: Final = await _key_model_listing( + user_api_key_dict, caller_aliases, - _describable_model_groups(listed_entries, hidden_names, user_api_key_dict, team_id), + hidden_names, + team_id=team_id, + include_model_access_groups=include_model_access_groups or False, + only_model_access_groups=only_model_access_groups or False, + include_wildcard_routes=include_wildcard_routes, + include_metadata=include_metadata or False, + fallback_type=fallback_type, + offered_model_names=typesafe_allowlist_model_names() if offers_typesafe_allowlist_names else (), ) - labels: Final = label_entries(listed_entries, entries) - for response_id, lookup_id in entries: - model_info = create_model_info_response( - model_id=lookup_id, - provider="openai", - include_metadata=include_metadata or False, - fallback_type=fallback_type, - llm_router=llm_router, - ) - model_info["id"] = response_id - model_data.append(model_info) - - listing: Final = cast(Sequence[ModelInfoResponse], model_data) # cast-ok: rows built above + listing: Final = key_listing.rows + entries: Final = key_listing.entries + labels: Final = key_listing.labels if client_version is not None: return Response( content=await asyncio.to_thread( @@ -12916,7 +12971,7 @@ async def model_list( ) return dict( - data=model_data, + data=list(listing), object="list", ) diff --git a/litellm/responses/additional_tools.py b/litellm/responses/additional_tools.py index 9c7a3a5f72e..879bc8ba41c 100644 --- a/litellm/responses/additional_tools.py +++ b/litellm/responses/additional_tools.py @@ -1,21 +1,20 @@ import json from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass -from itertools import groupby +from itertools import chain, groupby from types import MappingProxyType from typing import Final, cast # noqa: TID251 # validating the openai tool union strips vendor keys from raw tools -from pydantic import ValidationError +from pydantic import TypeAdapter, ValidationError +from typing_extensions import TypeIs # noqa: TID251 # narrows a client input item before its type is read from litellm._logging import verbose_logger from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.openai import ALL_RESPONSES_API_TOOL_PARAMS, ResponseInputParam ADDITIONAL_TOOLS_INPUT_ITEM_TYPE: Final = "additional_tools" - - -class _InputItemType(LiteLLMBaseModel): - type: str = "" +_NAMESPACE_TOOL_TYPE: Final = "namespace" +_ITEMS: Final = TypeAdapter(tuple[object, ...]) class _ToolList(LiteLLMBaseModel): @@ -29,11 +28,14 @@ class HoistedAdditionalTools: hoisted: tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...] +def _is_an_input_item( + item: object, +) -> TypeIs[Mapping[str, object]]: # guard-ok: auth scans every list input, and a pydantic error per item is slow + return isinstance(item, Mapping) + + def _is_additional_tools_item(item: object) -> bool: - try: - return _InputItemType.model_validate(item).type == ADDITIONAL_TOOLS_INPUT_ITEM_TYPE - except ValidationError: - return False + return _is_an_input_item(item) and item.get("type") == ADDITIONAL_TOOLS_INPUT_ITEM_TYPE def _tool_list_of(value: object) -> tuple[Mapping[str, object], ...]: @@ -61,7 +63,7 @@ def _members_of_copies(copies: Sequence[Mapping[str, object]]) -> Iterator[Mappi def _merged(copies: Sequence[Mapping[str, object]]) -> Mapping[str, object]: newest: Final = copies[-1] - if newest.get("type") != "namespace" or len(copies) == 1: + if newest.get("type") != _NAMESPACE_TOOL_TYPE or len(copies) == 1: return newest members: Final = tuple(_members_of_copies(copies)) return {**newest, "tools": list(_unique_tools(members))} if members else newest @@ -86,6 +88,11 @@ def _unique_tools(tools: Sequence[Mapping[str, object]]) -> tuple[Mapping[str, o return tuple(_merged(copies_by_identity[identity]) for identity in first_seen_order) +def additional_tools_in_input(input: object) -> tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]: + items: Final = _ITEMS.validate_python(input) if isinstance(input, list) else () + return tuple(chain.from_iterable(_tools_of_item(item) for item in items if _is_additional_tools_item(item))) + + def hoist_additional_tools( input: str | ResponseInputParam, tools: Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None, @@ -112,3 +119,26 @@ def hoist_additional_tools( for tool in _unique_tools((*existing, *hoisted)) ) return HoistedAdditionalTools(input=remaining_input, tools=unique, hoisted=hoisted) + + +def _top_level_tools(request: Mapping[str, object]) -> tuple[object, ...] | None: + try: + return _ITEMS.validate_python(request.get("tools") or ()) + except ValidationError: + return None + + +def with_additional_tools_hoisted(request: Mapping[str, object], *, drop_params: bool) -> Mapping[str, object]: + request_input: Final = request.get("input") + top_level_tools: Final = _top_level_tools(request) + if not isinstance(request_input, list) or top_level_tools is None: + return request + input_items: Final = cast(ResponseInputParam, request_input) # cast-ok: forwarded as sent, as inference does + hoisted: Final = hoist_additional_tools( + input_items, + cast("tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]", top_level_tools), # cast-ok: forwarded as sent + drop_params=drop_params, + ) + if not hoisted.hoisted: + return request + return {**request, "input": hoisted.input, "tools": list(hoisted.tools)} diff --git a/litellm/responses/litellm_completion_transformation/count_request.py b/litellm/responses/litellm_completion_transformation/count_request.py index 3deceac07aa..2141b55850b 100644 --- a/litellm/responses/litellm_completion_transformation/count_request.py +++ b/litellm/responses/litellm_completion_transformation/count_request.py @@ -12,7 +12,7 @@ from litellm.litellm_core_utils.prompt_templates.file_id_utils import ( ) from litellm.litellm_core_utils.tool_fields import DropPolicy from litellm.llms.base_llm.files.batch_records import responses_batch_body_to_chat_body -from litellm.responses.additional_tools import hoist_additional_tools +from litellm.responses.additional_tools import with_additional_tools_hoisted from litellm.responses.file_search.emulated_handler import replace_file_search_tools from litellm.responses.litellm_completion_transformation.compaction_items import holds_compaction_summary from litellm.responses.litellm_completion_transformation.transformation import ( @@ -20,7 +20,7 @@ from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) from litellm.responses.main import previous_response_client_ids -from litellm.types.llms.openai import ALL_RESPONSES_API_TOOL_PARAMS, ResponseInputParam +from litellm.types.llms.openai import ResponseInputParam from litellm.utils import ProviderConfigManager _COUNTED_CHAT_PARAMS: Final = ( @@ -114,20 +114,6 @@ def with_provider_file_ids( } -def _with_hoisted_additional_tools(responses_request: Mapping[str, object], drop_params: bool) -> Mapping[str, object]: - responses_input: Final = responses_request.get("input") - if not isinstance(responses_input, list): - return responses_request - input_items: Final = cast(ResponseInputParam, responses_input) # cast-ok: forwarded as sent, as inference does - top_level_tools: Final = cast( # cast-ok: tools are forwarded as sent, as inference does - "tuple[ALL_RESPONSES_API_TOOL_PARAMS, ...]", _ITEMS.validate_python(responses_request.get("tools") or ()) - ) - hoisted: Final = hoist_additional_tools(input_items, top_level_tools, drop_params=drop_params) - if not hoisted.hoisted: - return responses_request - return {**responses_request, "input": hoisted.input, "tools": list(hoisted.tools)} - - def _caller_input(responses_request: Mapping[str, object]) -> ResponseInputParam | None: responses_input: Final = responses_request.get("input") return ( @@ -205,7 +191,9 @@ async def chat_count_request( chat_body: Final = await _with_previous_turns( responses_batch_body_to_chat_body( _with_compaction_as_summary_text( - _with_emulated_file_search(_with_hoisted_additional_tools(responses_request, policy.drop_params)) + _with_emulated_file_search( + with_additional_tools_hoisted(responses_request, drop_params=policy.drop_params) + ) ), custom_llm_provider, policy.drop_params, diff --git a/litellm/responses/litellm_completion_transformation/reasoning_items.py b/litellm/responses/litellm_completion_transformation/reasoning_items.py index ab982222902..dda313bfc1b 100644 --- a/litellm/responses/litellm_completion_transformation/reasoning_items.py +++ b/litellm/responses/litellm_completion_transformation/reasoning_items.py @@ -6,6 +6,7 @@ from typing import Final from pydantic import BaseModel, TypeAdapter, ValidationError REASONING_ITEM_ID_PREFIX: Final = "rs_" +WRAPPED_ENCRYPTED_CONTENT_PREFIX: Final = "litellm_enc:" _JSON_LIST: Final = TypeAdapter(list[object]) _JSON_OBJECT: Final = TypeAdapter(dict[str, object]) @@ -58,16 +59,35 @@ def is_minted_reasoning_item_id(item_id: object) -> bool: return parsed.version == 4 and str(parsed) == suffix -def is_litellm_minted_reasoning_item(item: object) -> bool: +def _reasoning_fields(item: object) -> Mapping[str, object] | None: try: fields: Final = _JSON_OBJECT.validate_python( item.model_dump(exclude_none=True) if isinstance(item, BaseModel) else item ) except ValidationError: - return False - if fields.get("type") != "reasoning": - return False + return None + return fields if fields.get("type") == "reasoning" else None + + +def _is_minted(fields: Mapping[str, object]) -> bool: return ( is_minted_reasoning_item_id(fields.get("id")) or decode_thinking_blocks(fields.get("encrypted_content")) is not None ) + + +def is_litellm_minted_reasoning_item(item: object) -> bool: + fields: Final = _reasoning_fields(item) + return fields is not None and _is_minted(fields) + + +def is_other_provider_reasoning_item(item: object) -> bool: + fields: Final = _reasoning_fields(item) + if fields is None: + return False + encrypted_content: Final = fields.get("encrypted_content") + return ( + _is_minted(fields) + or bool(fields.get("content")) + or (isinstance(encrypted_content, str) and encrypted_content.startswith(WRAPPED_ENCRYPTED_CONTENT_PREFIX)) + ) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 5b1964a4b5e..33ea85d7d53 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -89,6 +89,7 @@ from litellm.responses.litellm_completion_transformation.session_handler import ResponsesSessionHandler, StoredTurns, ) +from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.anthropic import AnthropicToolSearchToolRegex, claude_hosted_web_tool_call from litellm.types.llms.base import CachedTokensDetails from litellm.types.llms.openai import ( @@ -2556,7 +2557,12 @@ class LiteLLMCompletionResponsesConfig: Returns None for anything this deployment did not write, so a genuinely opaque blob is still skipped rather than forwarded as garbage. """ - decoded: Final = decode_thinking_blocks(input_item.get("encrypted_content")) + encrypted_content: Final = input_item.get("encrypted_content") + decoded: Final = decode_thinking_blocks( + ResponsesAPIRequestUtils.unwrap_encrypted_content_with_model_id(encrypted_content)[1] + if isinstance(encrypted_content, str) + else encrypted_content + ) if decoded is None: return None return tuple( diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 43ed9a3b96f..7fc8d5aed32 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -50,6 +50,7 @@ from litellm.types.utils import ( _PROVIDER_SPECIFIC_HEADER_ADAPTER: Final[ TypeAdapter[ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None] ] = TypeAdapter(ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None) +_REASONING_FIELDS_ONLY_ITS_ORIGIN_RESOLVES: Final = frozenset({"encrypted_content", "id", "status"}) class _UsageExtraFields(ExtraItemsTypedDict, total=False, extra_items=object): @@ -678,7 +679,7 @@ class ResponsesAPIRequestUtils: return items: Final = cast(list[object], request_input) # cast-ok: untyped client json stripped: Final = tuple( - ResponsesAPIRequestUtils._without_encrypted_reasoning(reasoning) + ResponsesAPIRequestUtils.readable_reasoning(reasoning) if (reasoning := ResponsesAPIRequestUtils._encrypted_reasoning_item(item)) is not None and (should_strip is None or should_strip(reasoning)) else item @@ -696,14 +697,14 @@ class ResponsesAPIRequestUtils: return reasoning @staticmethod - def _without_encrypted_reasoning(reasoning: Mapping[str, object]) -> object | None: + def readable_reasoning(reasoning: Mapping[str, object]) -> Mapping[str, object] | None: readable: Final = any( ResponsesAPIRequestUtils._has_readable_text(reasoning.get(key)) for key in ("summary", "content") ) if not readable: return None kept: Final[dict[str, object]] = { # mutable-ok: request item rebuilt without the undecryptable keys - key: value for key, value in reasoning.items() if key not in ("encrypted_content", "id") + key: value for key, value in reasoning.items() if key not in _REASONING_FIELDS_ONLY_ITS_ORIGIN_RESOLVES } return kept diff --git a/litellm/router_api.py b/litellm/router_api.py index fa6ad69fab4..1ade356f494 100644 --- a/litellm/router_api.py +++ b/litellm/router_api.py @@ -158,6 +158,7 @@ GroupField: TypeAlias = Literal[ "display_name", "mode", "service_tiers", + "pinned_reasoning_efforts", "max_input_tokens", "deployment_model_info", "token_limits", @@ -939,7 +940,7 @@ class RouterRegistry: self, name: str, *, - field: Literal["service_tiers"], + field: Literal["service_tiers", "pinned_reasoning_efforts"], include_hidden: bool = False, team_id: str | None = None, ) -> tuple[object, ...]: ... @@ -1001,6 +1002,7 @@ class RouterRegistry: (were ``get_configured_display_name``, ``get_configured_mode``, ``get_configured_token_limits``, ``get_model_listing_info``) - ``service_tiers``: (was ``get_configured_service_tiers``) + - ``pinned_reasoning_efforts``: the ``reasoning_effort`` each routable deployment pins - ``max_input_tokens``: each routable deployment's own ``model_info.max_input_tokens`` - ``deployment_model_info``: each routable deployment's model info, merged with the pricing map """ @@ -1015,6 +1017,8 @@ class RouterRegistry: return r._core_get_configured_mode(name) if field == "service_tiers": return r._core_get_configured_service_tiers(name, team_id=team_id) + if field == "pinned_reasoning_efforts": + return r._core_get_pinned_reasoning_efforts(name, team_id=team_id) if field == "max_input_tokens": return r._core_get_configured_max_input_tokens(name, team_id=team_id) if field == "deployment_model_info": @@ -1136,6 +1140,18 @@ class RouterRegistry: return r._core_prompt_caching_pin_cache raise ValueError(f"unknown plugin kind {kind!r}") + def decryptable_reasoning( + self, model_group: str, request_input: Sequence[object], targets: "Sequence[Deployment]" + ) -> tuple[object, ...]: + """ + ``request_input`` without the encrypted reasoning that ``targets``, the deployments a + ``model_group`` request may go to, cannot decrypt, keeping each item's readable summary. + When encrypted content affinity is on for the group, the router sends the request to the + reasoning's origin, so reasoning one target can decrypt stays; otherwise every target must + decrypt it (was reading ``optional_callbacks`` for ``EncryptedContentAffinityCheck``). + """ + return self._router._core_decryptable_reasoning(model_group, request_input, targets) + # Settings a caller may read. Assigning one from outside the router goes through apply(ChangeSet(assign=...)). SETTING_FIELDS: Final[tuple[str, ...]] = ( diff --git a/litellm/router_rust/steps/callbacks.py b/litellm/router_rust/steps/callbacks.py index 75afe38548e..c96a9c1bbae 100644 --- a/litellm/router_rust/steps/callbacks.py +++ b/litellm/router_rust/steps/callbacks.py @@ -407,6 +407,28 @@ class OptionalChecks: bound: Final = request.get("litellm_logging_obj") in self._bound_requests return {"success": bound and not sync, "failure": bound, "make_call": make_call} + def decryptable_reasoning( + self, model_group: str, request_input: Sequence[object], targets: "Sequence[Deployment]" + ) -> tuple[object, ...]: + """``registry().decryptable_reasoning``: one target decrypting is enough only when encrypted + content affinity pins ``model_group`` to the reasoning's origin.""" + target_dicts: Final = tuple(target.model_dump(exclude_none=True) for target in targets) + pinned: Final = any( + isinstance(callback, EncryptedContentAffinityCheck) and callback.is_enabled_for_model_group(model_group) + for callback in self.optional_callbacks or () + ) + target_groups: Final = (target_dicts,) if pinned else tuple((target,) for target in target_dicts) + kept: Final = list(request_input) + for group in target_groups: + EncryptedContentAffinityCheck.strip_reasoning_the_targets_cannot_decrypt( + self._ctx, # pyright: ignore[reportArgumentType] # the core's RoutingContext serves registry() + kept, + None, + group, + unmarked_origin=None, + ) + return tuple(kept) + def routing_callbacks(self) -> Iterable[object]: """``_routing_callbacks``: the host's callbacks, or after ``observe(request_scoped=True)`` the host's callbacks with the router's own selectors and policies in place of theirs.""" diff --git a/litellm/router_rust/surface/model_info.py b/litellm/router_rust/surface/model_info.py index 7fd4c22df6d..508d2bb1145 100644 --- a/litellm/router_rust/surface/model_info.py +++ b/litellm/router_rust/surface/model_info.py @@ -81,6 +81,7 @@ if TYPE_CHECKING: ) _MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) # The key router-core's merge reads as an opaque value (``router-core/src/model_info.rs``). _OPAQUE_KEY: Final = "__litellm_python_value__" @@ -761,6 +762,20 @@ class HostModelInfo: ) return tuple(wire.from_wire(tier) for tier in tiers) + def _core_get_pinned_reasoning_efforts(self, model_name: str, team_id: str | None = None) -> tuple[object, ...]: + """ + Return the litellm_params.reasoning_effort each deployment a request + from team_id to model_name can route to (see _routable_deployments) + pins, unvalidated and in model_list order, None for a deployment that + pins none; the caller decides how the deployments combine. + + Returns an empty tuple for wildcard-expanded or unknown names. + """ + return tuple( + _LITELLM_PARAMS_ADAPTER.validate_python(deployment["litellm_params"]).get("reasoning_effort") + for deployment in self._routable_deployments(model_name, team_id) + ) + def _core_get_configured_max_input_tokens( self, model_name: str, team_id: str | None = None ) -> tuple[int | None, ...]: diff --git a/litellm/router_rust/surface/registry.py b/litellm/router_rust/surface/registry.py index eaeac10cb5f..7c3abdcd45e 100644 --- a/litellm/router_rust/surface/registry.py +++ b/litellm/router_rust/surface/registry.py @@ -1510,6 +1510,14 @@ class HostRegistry(HostModelInfo): return self.__getattr__("_core_prompt_caching_pin_cache") # pyright: ignore[reportReturnType] # raises RustCoreUnsupported return self.checks.prompt_caching_pin_cache + def _core_decryptable_reasoning( + self, model_group: str, request_input: Sequence[object], targets: Sequence[Deployment] + ) -> tuple[object, ...]: + """The input ``targets`` can decrypt (``registry().decryptable_reasoning``).""" + if self.checks is None: + return self.__getattr__("_core_decryptable_reasoning") # pyright: ignore[reportReturnType] # raises RustCoreUnsupported + return self.checks.decryptable_reasoning(model_group, request_input, targets) + def replay_model_cost_registrations(self) -> None: """Re-register every served deployment's cost-map entries (``replay_model_cost_registrations``).""" provider_unresolved: Final = self._provider_unresolved_deployments diff --git a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py index 062886b44fd..9badee854fb 100644 --- a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py @@ -148,7 +148,7 @@ class EncryptedContentAffinityCheck(CustomLogger): return any("encrypted_content_affinity" in checks for checks in model_group_affinity_config.values()) - def _is_enabled_for_model_group(self, model_group: str) -> bool: + def is_enabled_for_model_group(self, model_group: str) -> bool: group_checks: Final = self.model_group_affinity_config.get(model_group) return self.enable_global_affinity or ( group_checks is not None and "encrypted_content_affinity" in group_checks @@ -530,7 +530,7 @@ class EncryptedContentAffinityCheck(CustomLogger): """ request_kwargs = request_kwargs or {} typed_healthy_deployments: Final = cast(list[dict[str, object]], healthy_deployments) - if not self._is_enabled_for_model_group(model): + if not self.is_enabled_for_model_group(model): return typed_healthy_deployments typed_request_kwargs: Final = _REQUEST_KWARGS_ADAPTER.validate_python(request_kwargs) target_order: Final = typed_request_kwargs.get("_target_order") diff --git a/tests/unit/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py b/tests/unit/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py index 6538ccf1cac..fe33a26dfd7 100644 --- a/tests/unit/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py +++ b/tests/unit/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py @@ -477,6 +477,84 @@ def test_transform_request_forwards_non_string_instructions_and_input_untouched( ) +CODEX_CLOCK_NAMESPACE_TOOL: Final = { + "type": "namespace", + "name": "clock", + "description": "Clock tools.", + "tools": [ + { + "type": "function", + "name": "sleep", + "description": "Sleep for some seconds.", + "parameters": {"type": "object", "properties": {"seconds": {"type": "number"}}, "required": ["seconds"]}, + } + ], +} +CODEX_ADDITIONAL_TOOLS_ITEM: Final = { + "type": "additional_tools", + "role": "developer", + "tools": [CODEX_CLOCK_NAMESPACE_TOOL], +} + + +@pytest.mark.parametrize("instructions", [None, "Answer with exactly one word."], ids=["no-instructions", "instructions"]) +def test_transform_request_moves_codex_additional_tools_item_into_tools(instructions: str | None) -> None: + user_item: Final = {"role": "user", "content": "Sleep for two seconds."} + request: Final = FireworksAIResponsesAPIConfig().transform_responses_api_request( + model="accounts/fireworks/models/glm-5p3", + input=cast(ResponseInputParam, [CODEX_ADDITIONAL_TOOLS_ITEM, user_item]), + response_api_optional_request_params={} if instructions is None else {"instructions": instructions}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert tuple(request["tools"]) == (CODEX_CLOCK_NAMESPACE_TOOL,) + assert tuple(request["input"]) == (user_item,) + assert request.get("instructions") == instructions + + +@pytest.mark.parametrize( + ("drop_params", "expected_clock_copies"), + [ + pytest.param(True, 1, id="merged once under drop_params"), + pytest.param(None, 2, id="forwarded as sent without drop_params"), + ], +) +@pytest.mark.parametrize("tools_as_tuple", [False, True], ids=["tools-list", "tools-tuple"]) +def test_transform_request_merges_codex_additional_tools_into_existing_tools_only_under_drop_params( + drop_params: bool | None, expected_clock_copies: int, tools_as_tuple: bool +) -> None: + weather_tool: Final = { + "type": "function", + "name": "get_weather", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + } + user_item: Final = {"role": "user", "content": "Sleep for two seconds, then check the weather in Paris."} + request: Final = FireworksAIResponsesAPIConfig().transform_responses_api_request( + model="accounts/fireworks/models/glm-5p3", + input=cast(ResponseInputParam, [CODEX_ADDITIONAL_TOOLS_ITEM, user_item, CODEX_ADDITIONAL_TOOLS_ITEM]), + response_api_optional_request_params={"tools": (weather_tool,) if tools_as_tuple else [weather_tool]}, + litellm_params=GenericLiteLLMParams(drop_params=drop_params), + headers={}, + ) + assert tuple(request["tools"]) == (weather_tool, *(CODEX_CLOCK_NAMESPACE_TOOL,) * expected_clock_copies) + assert tuple(request["input"]) == (user_item,) + + +@pytest.mark.parametrize("drop_params", [True, None], ids=["drop_params", "no-drop_params"]) +def test_transform_request_leaves_codex_additional_tools_in_input_when_tools_is_malformed( + drop_params: bool | None, +) -> None: + user_item: Final = {"role": "user", "content": "Sleep for two seconds."} + request: Final = FireworksAIResponsesAPIConfig().transform_responses_api_request( + model="accounts/fireworks/models/glm-5p3", + input=cast(ResponseInputParam, [CODEX_ADDITIONAL_TOOLS_ITEM, user_item]), + response_api_optional_request_params={"tools": {"type": "function", "name": "get_weather"}}, + litellm_params=GenericLiteLLMParams(drop_params=drop_params), + headers={}, + ) + assert tuple(request["input"]) == (CODEX_ADDITIONAL_TOOLS_ITEM, user_item) + + def test_responses_call_maps_pydantic_developer_items_and_replays_pydantic_output_items() -> None: client: Final = _mock_http_client(_fireworks_response("accounts/fireworks/models/qwen3p8-2p4t-a95b")) pydantic_input: Final = cast( diff --git a/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py b/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py index dd532c0ccf5..78ce9728f3b 100644 --- a/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py +++ b/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py @@ -2401,6 +2401,37 @@ async def test_deny_by_default_ignores_tools_that_name_no_vector_store( ) +@pytest.mark.asyncio +@pytest.mark.parametrize("deny_by_default", [False, True], ids=["legacy", "deny-by-default"]) +@pytest.mark.parametrize( + ("key_vector_stores", "denied_by"), + [(["KBSTOREA"], _KEY_DENIED), (["KBSTOREB"], None)], + ids=["ungranted", "granted"], +) +async def test_file_search_inside_additional_tools_input_item_needs_a_vector_store_grant( + deny_by_default: bool, key_vector_stores: list[str], denied_by: ProxyErrorTypes | None +): + await _assert_rag_query_outcome( + _common_checks_for_rag_query( + {"vector_store_deny_by_default": deny_by_default}, + _virtual_key(object_permission_id="key-permission"), + SimpleNamespace(vector_stores=key_vector_stores), + request_body={ + "model": "gpt-4o-mini", + "input": [ + { + "type": "additional_tools", + "role": "developer", + "tools": [{"type": "file_search", "vector_store_ids": ["KBSTOREB"]}], + }, + {"type": "message", "role": "user", "content": "what is in this KB?"}, + ], + }, + ), + denied_by, + ) + + @pytest.mark.asyncio @pytest.mark.parametrize( "request_body", diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index 250c83fa61a..cd5410ffd18 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -35,6 +35,15 @@ def test_get_team_models_for_all_models_and_team_only_models(): assert set(result) == set(combined_models) +def test_get_team_models_lists_models_in_the_order_the_team_set_them() -> None: + from litellm.proxy.auth.model_checks import get_team_models + + team_models: Final = ["zeta", "alpha", "mu", "beta", "omega", "kappa", "delta", "all-proxy-models"] + result: Final = get_team_models(team_models, ["proxy-b", "proxy-a"], {"grp": ["g1"]}, True) + + assert result == [*team_models, "proxy-b", "proxy-a", "grp", "g1"] + + def test_get_team_models_all_proxy_models_includes_access_groups(): """ When a team has 'all-proxy-models' and include_model_access_groups=True, diff --git a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py index 741669337cd..8440d3d2ab2 100644 --- a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py +++ b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py @@ -12752,6 +12752,19 @@ def _user_defined_pass_through_endpoint(): return endpoint +@pytest.mark.asyncio +async def test_router_settings_model_group_alias_leaves_a_deployed_name_alone_on_provider_pass_through(monkeypatch): + router = _alias_router() + monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router) + data = {"model": "claude-sonnet", "messages": [{"role": "user", "content": "hi"}]} + route = "/anthropic/v1/messages" + request = _alias_request(route, data, path_params={"endpoint": "v1/messages"}) + token = _alias_token(monkeypatch, "key", {"claude-sonnet": "claude-haiku"}, ["claude-haiku", "claude-sonnet"]) + await _enforce_alias_access(token, data, route, request, router) + assert data["model"] == "claude-sonnet" + assert (await request.json())["model"] == "claude-sonnet" + + @pytest.mark.asyncio @pytest.mark.parametrize("user_defined", [False, True]) async def test_router_settings_model_group_alias_leaves_pass_through_bodies_alone(monkeypatch, user_defined): diff --git a/tests/unit/proxy/common_utils/test_codex_model_catalog.py b/tests/unit/proxy/common_utils/test_codex_model_catalog.py index 760f79f4ec2..f0b3a8fabc7 100644 --- a/tests/unit/proxy/common_utils/test_codex_model_catalog.py +++ b/tests/unit/proxy/common_utils/test_codex_model_catalog.py @@ -1,4 +1,5 @@ import json +import logging from collections.abc import Mapping from dataclasses import replace from types import MappingProxyType @@ -7,6 +8,7 @@ from typing import Final, Literal import pytest import litellm +from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.llm_cost_calc.utils import get_service_tier_cost_key from litellm.proxy.common_utils.codex_model_catalog import ( @@ -15,11 +17,14 @@ from litellm.proxy.common_utils.codex_model_catalog import ( CodexStockModel, CodexStockUpgrade, bundled_codex_models, + codex_catalog_etag, codex_catalog_rows, + codex_entry_names_default_reasoning_level, codex_model_list_body, codex_models_response_json, configured_service_tiers, deployment_capabilities, + merged_codex_catalog, ) from litellm.proxy.common_utils.codex_model_translation import CodexDeploymentCapabilities, CodexServiceTier @@ -628,6 +633,15 @@ def test_default_service_tier_survives_only_while_still_offered(configured, expe assert entry["default_service_tier"] == expected + +@pytest.mark.parametrize(("alias_defaults", "expected"), [("codex", "priority"), ("deployment", None)]) +def test_alias_with_offered_tiers_takes_codexs_default_tier_only_under_codex_defaults(alias_defaults, expected): + (entry,) = _models( + _row("team-sol", upstream_model="gpt-6-sol", service_tiers=["priority"]), alias_defaults=alias_defaults + ) + + assert (_tiers(entry), entry["default_service_tier"]) == ([("priority", "Fast", "2x speed, increased usage")], expected) + def test_non_chat_and_wildcard_rows_are_left_out_of_the_catalog(): body = _body( _row("gpt-6-astra"), @@ -705,6 +719,55 @@ def test_an_entry_too_large_for_the_bytes_left_is_passed_over_and_the_smaller_on assert len(body.json.encode()) <= limit +def test_merged_catalog_keeps_upstream_whole_and_adds_only_the_entries_that_fit_the_byte_limit(): + upstream = {"models": [{"slug": "upstream-model", "priority": 4, "base_instructions": "u" * 500}]} + upstream_only = len(json.dumps(upstream, separators=(",", ":"))) + rows = tuple(_row(f"model-{index}") for index in range(4)) + + def merged(*merged_rows, byte_limit): + return merged_codex_catalog( + json.dumps(upstream).encode(), merged_rows, stock=_STOCK, instructions=_PROMPT, byte_limit=byte_limit + ) + + unlimited = json.loads(merged(*rows, byte_limit=upstream_only * 100))["models"] + limit = len(merged(*rows[:2], byte_limit=upstream_only * 100).encode()) + + body = merged(*rows, byte_limit=limit) + + assert len(body.encode()) <= limit + assert json.loads(body)["models"] == unlimited[:3] + assert merged(*rows, byte_limit=upstream_only) is None + + +def test_merged_catalog_gives_an_alias_no_service_tier_or_upgrade_of_codexs_own(): + upstream = json.dumps({"models": [{"slug": "upstream-model", "priority": 0}]}).encode() + rows = (_row("team-sol", upstream_model="openai/gpt-6-sol"), _row("team-55", upstream_model="openai/gpt-5.5")) + + added = json.loads(merged_codex_catalog(upstream, rows, stock=_STOCK, instructions=_PROMPT))["models"][1:] + + assert [ + (entry["slug"], entry["service_tiers"], entry["default_service_tier"], entry["upgrade"]) for entry in added + ] == [ + ("team-sol", [], None, None), + ("team-55", [], None, None), + ] + + +def test_merged_catalog_warns_once_with_how_many_models_the_byte_limit_left_out(caplog, monkeypatch): + upstream = json.dumps({"models": [{"slug": "upstream-model", "priority": 0}]}).encode() + rows = tuple(_row(f"model-{index}") for index in range(4)) + limit = len(merged_codex_catalog(upstream, rows[:1], stock=_STOCK, instructions=_PROMPT, byte_limit=10**6).encode()) + monkeypatch.setattr(verbose_proxy_logger, "propagate", True) + + with caplog.at_level(logging.WARNING, logger=verbose_proxy_logger.name): + merged_codex_catalog(upstream, rows, stock=_STOCK, instructions=_PROMPT, byte_limit=limit) + merged_codex_catalog(upstream, rows[:1], stock=_STOCK, instructions=_PROMPT, byte_limit=limit) + + assert [record.getMessage() for record in caplog.records] == [ + f"Codex native model catalog left out 3 of 4 LiteLLM models to stay within Codex's {limit} byte limit" + ] + + def test_unlimited_body_keeps_every_entry(): rows = tuple(_row(f"model-{index}") for index in range(3)) @@ -947,6 +1010,38 @@ def test_catalog_rows_without_a_router_carry_only_the_listing(): assert codex_catalog_rows(listing, (("plain", "plain"),), None) == (CodexCatalogRow(id="plain", mode="chat"),) +@pytest.mark.parametrize("client_version", [None, "0.165.0"]) +def test_entry_names_a_default_reasoning_level_exactly_when_the_native_entry_carries_one(client_version): + router = litellm.Router( + model_list=[ + {"model_name": "open-weights", "litellm_params": {"model": "fireworks_ai/open-weights"}}, + {"model_name": "coding-model", "litellm_params": {"model": "openai/gpt-5.5"}}, + {"model_name": "pinned-model", "litellm_params": {"model": "openai/gpt-5.5", "reasoning_effort": "high"}}, + {"model_name": "split-pin", "litellm_params": {"model": "openai/gpt-5.5", "reasoning_effort": "high"}}, + {"model_name": "split-pin", "litellm_params": {"model": "openai/gpt-5.5", "reasoning_effort": "low"}}, + { + "model_name": "max-only", + "litellm_params": {"model": "hosted_vllm/qa-local-coder", "api_base": "https://vllm.example/v1"}, + "model_info": {"supports_reasoning": True, "reasoning_effort_levels": ["max"]}, + }, + ] + ) + names = ("open-weights", "coding-model", "pinned-model", "split-pin", "max-only") + listing = tuple( + {"id": name, "object": "model", "created": 0, "owned_by": "openai", "mode": "chat"} for name in names + ) + upstream = json.dumps({"models": [{"slug": "upstream-model", "priority": 0}]}).encode() + rows = codex_catalog_rows(listing, tuple((name, name) for name in names), router) + served = json.loads(merged_codex_catalog(upstream, rows, client_version=client_version))["models"][1:] + served_names_one = [entry.get("default_reasoning_level") is not None for entry in served] + + assert [ + codex_entry_names_default_reasoning_level(entry["slug"], router, None, client_version) for entry in served + ] == served_names_one + assert served_names_one == [False, False, True, False, client_version is not None] + assert served[2]["default_reasoning_level"] == "high" + + def test_codex_tool_search_is_offered_off_the_route_and_model_info_and_only_when_every_deployment_answers_it(): claude = "anthropic/claude-sonnet-5-5" converse = "bedrock/us.anthropic.claude-sonnet-5-5" @@ -995,6 +1090,34 @@ def test_codex_tool_search_is_offered_off_the_route_and_model_info_and_only_when assert models[0]["context_window"] == 1000000 +def test_native_merged_catalog_offers_codex_tool_search_as_the_proxy_listing_does(): + claude = "anthropic/claude-sonnet-5-5" + router = litellm.Router( + model_list=[ + {"model_name": "claude", "litellm_params": {"model": claude}}, + { + "model_name": "claude-off", + "litellm_params": {"model": claude}, + "model_info": {"supports_search_tool": False}, + }, + {"model_name": "coding-model", "litellm_params": {"model": "openai/gpt-5.5"}}, + ] + ) + names = ("claude", "claude-off", "coding-model") + listing = tuple({"id": name, "object": "model", "created": 0, "owned_by": "x", "mode": "chat"} for name in names) + entries = tuple((name, name) for name in names) + upstream = json.dumps({"models": [{"slug": "upstream-model", "priority": 0}]}).encode() + + listed = json.loads(codex_model_list_body(listing, entries, router))["models"] + merged = json.loads(merged_codex_catalog(upstream, codex_catalog_rows(listing, entries, router)))["models"][1:] + + assert [entry["slug"] for entry in merged] == list(names) + assert [entry.get("supports_search_tool") for entry in merged] == [ + entry.get("supports_search_tool") for entry in listed + ] + assert [entry["supports_search_tool"] for entry in merged[:2]] == [True, False] + + _SOL_SLUG: Final = next( slug for slug, model in bundled_codex_models().items() if model.default_service_tier is not None ) @@ -1282,3 +1405,20 @@ def test_model_info_that_does_not_validate_declares_no_capability_but_keeps_its_ ) assert capabilities == CodexDeploymentCapabilities(priced_service_tier_keys=frozenset({priced})) + + +class _CollidingName(str): + def __hash__(self) -> int: + return 1 + + +def test_catalog_etag_ignores_the_order_a_process_iterates_priced_service_tiers_in() -> None: + def row_with_tiers(*tiers: str) -> CodexCatalogRow: + capabilities: Final = CodexDeploymentCapabilities.model_construct( + priced_service_tier_keys=frozenset(_CollidingName(tier) for tier in tiers) + ) + return replace(_row("gpt-6-astra"), deployment_capabilities=(capabilities,)) + + assert codex_catalog_etag("upstream", [row_with_tiers("flex", "priority")]) == codex_catalog_etag( + "upstream", [row_with_tiers("priority", "flex")] + ) diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_tool_policy_guardrail.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_tool_policy_guardrail.py index f40465e8d7b..bac470b1365 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_tool_policy_guardrail.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_tool_policy_guardrail.py @@ -2,13 +2,16 @@ Unit tests for ToolPolicyGuardrail. """ -from typing import Any +from collections.abc import Mapping, Sequence +from types import SimpleNamespace +from typing import Any, Final from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi import HTTPException +from litellm.proxy.db import tool_registry_writer from litellm.proxy.guardrails.guardrail_hooks.tool_policy.tool_policy_guardrail import ( ToolPolicyGuardrail, ) @@ -222,3 +225,104 @@ async def test_blocked_tool_loaded_by_a_replayed_tool_search_raises_next_to_allo ) assert exc_info.value.status_code == 400 assert exc_info.value.detail["blocked_tools"] == ["run_shell"] + + +async def _install_tool_policies(monkeypatch: pytest.MonkeyPatch, input_policies: Mapping[str, str]) -> None: + prisma_client = MagicMock() + prisma_client.db.litellm_tooltable.find_many = AsyncMock( + return_value=[ + SimpleNamespace(tool_name=name, input_policy=policy, output_policy="untrusted") + for name, policy in input_policies.items() + ] + ) + prisma_client.db.litellm_objectpermissiontable.find_many = AsyncMock(return_value=[]) + registry = tool_registry_writer.ToolPolicyRegistry() + await registry.sync_tool_policy_from_db(prisma_client) + monkeypatch.setattr(tool_registry_writer, "_tool_policy_registry", registry) + + +def _responses_request( + tools: Sequence[Mapping[str, object]], item_tools: Sequence[Mapping[str, object]] +) -> dict[str, object]: + return { + "user_api_key_request_route": "/v1/responses", + "tools": tools, + "input": [ + {"type": "additional_tools", "role": "developer", "tools": item_tools}, + {"role": "user", "content": "clean up the disk"}, + ], + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "request_data", + [ + _responses_request( + [{"type": "function", "name": "get_weather"}], + [{"type": "function", "name": "drop_database"}], + ), + _responses_request( + [{"type": "function", "name": "get_weather"}], + [ + { + "type": "namespace", + "name": "mcp__db", + "tools": [{"type": "function", "name": "drop_database"}], + } + ], + ), + _responses_request( + [ + {"type": "function", "name": "get_weather"}, + {"type": "mcp", "server_label": "drop_database"}, + ], + [], + ), + _responses_request( + [ + {"type": "function", "name": "get_weather"}, + {"type": "namespace", "name": "mcp__db", "tools": [{"type": "function", "name": "drop_database"}]}, + ], + [], + ), + ], + ids=["additional_tools_item", "namespace_in_item", "top_level_mcp", "top_level_namespace"], +) +async def test_blocked_tool_raises_next_to_a_top_level_function_tool( + guardrail: ToolPolicyGuardrail, request_data: dict[str, object], monkeypatch: pytest.MonkeyPatch +) -> None: + await _install_tool_policies(monkeypatch, {"get_weather": "untrusted", "drop_database": "blocked"}) + with pytest.raises(HTTPException) as exc_info: + await guardrail.apply_guardrail( + inputs=_tool_request_inputs(["get_weather"]), + request_data=request_data, + input_type="request", + ) + assert exc_info.value.status_code == 400 + assert exc_info.value.detail["blocked_tools"] == ["drop_database"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("name", [["drop_database"], {"name": "drop_database"}], ids=["list", "dict"]) +async def test_function_tool_name_that_is_not_a_string_is_left_for_the_provider_to_refuse( + guardrail: ToolPolicyGuardrail, name: object, monkeypatch: pytest.MonkeyPatch +) -> None: + await _install_tool_policies(monkeypatch, {"get_weather": "untrusted"}) + inputs: Final = {"tools": [{"type": "function", "function": {"name": name}}]} + assert await guardrail.apply_guardrail(inputs=inputs, request_data={}, input_type="request") == inputs + + +@pytest.mark.asyncio +async def test_blocked_tool_defined_at_the_top_level_and_in_an_item_is_named_once( + guardrail: ToolPolicyGuardrail, monkeypatch: pytest.MonkeyPatch +) -> None: + tool = {"type": "function", "name": "drop_database"} + await _install_tool_policies(monkeypatch, {"drop_database": "blocked"}) + with pytest.raises(HTTPException) as exc_info: + await guardrail.apply_guardrail( + inputs=_tool_request_inputs(["drop_database"]), + request_data=_responses_request([tool], [tool]), + input_type="request", + ) + assert exc_info.value.detail["blocked_tools"] == ["drop_database"] diff --git a/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py index 354c65776dc..1a4751b4598 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_codex_passthrough_endpoints.py @@ -1,17 +1,39 @@ import asyncio +import base64 import json -from collections.abc import Iterator +import time +from collections.abc import Callable, Iterator, Sequence from datetime import datetime from typing import Final import httpx +import jwt import pytest import respx +from cryptography.hazmat.primitives.asymmetric import rsa from fastapi.testclient import TestClient +from jwt.algorithms import RSAAlgorithm import litellm +from litellm import Router +from litellm.caching.dual_cache import DualCache from litellm.integrations.custom_logger import CustomLogger +from litellm.llms.chatgpt.common_utils import OPENAI_AUTH_CLAIM_KEY from litellm.proxy import proxy_server +from litellm.proxy._types import LiteLLM_JWTAuth, LiteLLM_TeamTableCachedObj, LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.auth.handle_jwt import JWTHandler +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.codex_model_catalog import bundled_codex_models +from litellm.proxy.pass_through_endpoints.codex_passthrough_endpoints import ( + _ACCOUNT_SLUGS_FETCHES_PER_CALLER, + codex_shared_cache, +) +from litellm.proxy.utils import hash_token +from litellm.responses.litellm_completion_transformation.reasoning_items import ( + encode_thinking_blocks, + mint_reasoning_item_id, +) +from litellm.responses.utils import ResponsesAPIRequestUtils from tests._master_key import MASTER_KEY _BASE: Final = "https://chatgpt.com/backend-api/codex" @@ -22,6 +44,7 @@ _HEADERS: Final = {"authorization": f"Bearer {_TOKEN}", "x-litellm-api-key": MAS @pytest.fixture def client(monkeypatch: pytest.MonkeyPatch, httpx_transport: None) -> Iterator[TestClient]: monkeypatch.setattr(proxy_server, "master_key", MASTER_KEY) + monkeypatch.setattr(proxy_server.proxy_logging_obj.internal_usage_cache, "dual_cache", DualCache()) yield TestClient(proxy_server.app) @@ -183,3 +206,1860 @@ async def test_usage_logged_without_credentials( serialized: Final = json.dumps(logged, default=str) for secret in (_TOKEN, MASTER_KEY, "test-hidden-token", "test-hidden-cookie"): assert secret not in serialized + + +_PROVIDER_BASE: Final = "https://open-weights.test/v1" +_CHATGPT_JWT: Final = ( + ".".join( + base64.urlsafe_b64encode(json.dumps(part).encode()).decode().rstrip("=") + for part in ({"alg": "none"}, {OPENAI_AUTH_CLAIM_KEY: {"chatgpt_account_id": "account-a"}}) + ) + + "." +) +_PROVIDER_KEY: Final = "test-provider-key" +_CODEX_USER_AGENT: Final = "codex_cli_rs/0.165.0 (Ubuntu 24.4.0; x86_64) xterm-256color" +_ACCOUNT_HEADERS: Final = { + **_HEADERS, + "authorization": f"Bearer {_CHATGPT_JWT}", + "chatgpt-account-id": "account-a", + "originator": "codex_cli_rs", + "user-agent": _CODEX_USER_AGENT, +} +_BUNDLED_SLUG: Final = next(iter(bundled_codex_models())) +_PRICED_CHATGPT_SLUG: Final = "priced-subscription-model" +_UPSTREAM_CATALOG: Final = { + "models": [ + {"slug": "subscription-model", "priority": 0, "supported_reasoning_levels": ["high"]}, + {"slug": "shared-name", "priority": 7, "visibility": "list"}, + {"slug": "subscription-hidden", "priority": 3, "visibility": "hide"}, + ] +} + + +def _open_weights(model_name: str, mode: str | None = None, upstream: str | None = None) -> dict[str, object]: + return { + "model_name": model_name, + "litellm_params": { + "model": f"fireworks_ai/{upstream or model_name}", + "api_base": _PROVIDER_BASE, + "api_key": _PROVIDER_KEY, + }, + "model_info": {"mode": mode} if mode else {}, + } + + +@pytest.fixture +def mixed_router(monkeypatch: pytest.MonkeyPatch) -> Router: + router: Final = Router( + model_list=[ + { + "model_name": "chatgpt/gpt-*", + "litellm_params": {"model": "chatgpt/gpt-*"}, + "model_info": {"access_groups": ["chatgpt-native"]}, + }, + { + "model_name": "chatgpt/codex-*", + "litellm_params": {"model": "chatgpt/codex-*"}, + "model_info": {"access_groups": ["chatgpt-native"]}, + }, + {"model_name": "chatgpt-subscription", "litellm_params": {"model": "chatgpt/subscription-model"}}, + _open_weights("open-weights"), + _open_weights("shared-name"), + _open_weights(_BUNDLED_SLUG), + _open_weights(_PRICED_CHATGPT_SLUG), + _open_weights("open-embedder", mode="embedding"), + _open_weights("*"), + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + unpriced_bundled: Final = { + name: price for name, price in litellm.model_cost.items() if name != f"chatgpt/{_BUNDLED_SLUG}" + } + monkeypatch.setattr( + litellm, + "model_cost", + {**unpriced_bundled, f"chatgpt/{_PRICED_CHATGPT_SLUG}": {"litellm_provider": "chatgpt", "mode": "responses"}}, + ) + return router + + +def test_native_catalog_lists_the_keys_other_chat_models_after_chatgpts( + client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router +) -> None: + respx_mock.get(f"{_BASE}/models").respond(json=_UPSTREAM_CATALOG, headers={"etag": '"upstream-v1"'}) + response: Final = client.get("/backend-api/codex/models?client_version=1.0.0", headers=_HEADERS) + assert response.status_code == 200 + models: Final = response.json()["models"] + assert models[: len(_UPSTREAM_CATALOG["models"])] == _UPSTREAM_CATALOG["models"] + added: Final = models[len(_UPSTREAM_CATALOG["models"]) :] + assert [model["slug"] for model in added] == ["open-weights"] + assert added[0]["visibility"] == "list" + assert added[0]["priority"] > max(model["priority"] for model in _UPSTREAM_CATALOG["models"]) + + +@pytest.mark.parametrize( + ("client_version", "expected_efforts"), + [ + pytest.param("0.165.0", ["low", "max"], id="a codex that parses custom efforts gets them all"), + pytest.param("0.137.0", ["low"], id="an older codex gets only the efforts it parses"), + ], +) +def test_native_catalog_offers_the_declared_reasoning_efforts_the_requesting_codex_parses( + client: TestClient, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + client_version: str, + expected_efforts: list[str], +) -> None: + declared: Final = { + **_open_weights("open-weights"), + "model_info": {"supports_reasoning": True, "reasoning_effort_levels": ["low", "max"]}, + } + monkeypatch.setattr(proxy_server, "llm_router", Router(model_list=[declared])) + respx_mock.get(f"{_BASE}/models").respond(json=_UPSTREAM_CATALOG, headers={"etag": '"upstream-v1"'}) + response: Final = client.get(f"/backend-api/codex/models?client_version={client_version}", headers=_HEADERS) + assert response.status_code == 200 + (added,) = response.json()["models"][len(_UPSTREAM_CATALOG["models"]) :] + assert [level["effort"] for level in added["supported_reasoning_levels"]] == expected_efforts + + +@pytest.mark.parametrize( + "upstream", + [ + pytest.param(httpx.Response(503, json={"detail": "busy"}, headers={"etag": '"e"'}), id="non-200"), + pytest.param(httpx.Response(200, content=b"login", headers={"etag": '"e"'}), id="not json"), + pytest.param(httpx.Response(200, json={"models": "none"}, headers={"etag": '"e"'}), id="no model list"), + ], +) +def test_native_catalog_relays_an_unmergeable_upstream_answer_unchanged( + client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router, upstream: httpx.Response +) -> None: + respx_mock.get(f"{_BASE}/models").mock(return_value=upstream) + response: Final = client.get("/backend-api/codex/models?client_version=1.0.0", headers=_HEADERS) + assert response.status_code == upstream.status_code + assert response.content == upstream.content + assert response.headers["etag"] == '"e"' + + +def _provider_completed(text: str) -> dict[str, object]: + return { + "id": "resp_open_weights", + "object": "response", + "created_at": 1700000000, + "status": "completed", + "model": "open-weights", + "output": [ + { + "type": "message", + "id": "msg_open_weights", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": text, "annotations": []}], + } + ], + "usage": {"input_tokens": 3, "output_tokens": 1, "total_tokens": 4}, + } + + +def _provider_stream(text: str) -> bytes: + completed: Final = _provider_completed(text) + events: Final = ( + { + "type": "response.created", + "sequence_number": 0, + "response": {**completed, "status": "in_progress", "output": []}, + }, + { + "type": "response.output_text.delta", + "sequence_number": 1, + "item_id": "msg_open_weights", + "output_index": 0, + "content_index": 0, + "delta": text, + }, + {"type": "response.completed", "sequence_number": 2, "response": completed}, + ) + return "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events).encode() + + +def _event_types(stream: bytes) -> list[str]: + payloads: Final = [line.removeprefix(b"data: ") for line in stream.splitlines() if line.startswith(b"data: {")] + return [json.loads(payload)["type"] for payload in payloads] + + +class PreCallRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.requests: Final[list[tuple[str | None, str]]] = [] + self.raw_header_names: Final[list[frozenset[str]]] = [] + + async def async_pre_call_hook( + self, user_api_key_dict: UserAPIKeyAuth, cache: object, data: dict[str, object], call_type: str + ) -> dict[str, object]: + self.requests.append((user_api_key_dict.request_route, repr(data))) + secret_fields: Final = data.get("secret_fields") + raw_headers: Final = secret_fields.get("raw_headers") if isinstance(secret_fields, dict) else None + self.raw_header_names.append(frozenset(raw_headers) if isinstance(raw_headers, dict) else frozenset()) + return data + + +def _provider_reply(request: httpx.Request) -> httpx.Response: + if json.loads(request.content).get("stream") is not True: + return httpx.Response(200, json=_provider_completed("Hi")) + return httpx.Response(200, content=_provider_stream("Hi"), headers={"content-type": "text/event-stream"}) + + +def _open_weights_provider(respx_mock: respx.MockRouter) -> respx.Route: + return respx_mock.post(f"{_PROVIDER_BASE}/responses").mock(side_effect=_provider_reply) + + +_UPSTREAM_MODELS_ETAG: Final = '"upstream-v2"' + + +def _chatgpt_responses(respx_mock: respx.MockRouter) -> respx.Route: + return respx_mock.post(f"{_BASE}/responses").respond( + content=b"event: response.completed\ndata: {}\n\n", + headers={"content-type": "text/event-stream", "x-models-etag": _UPSTREAM_MODELS_ETAG}, + ) + + +def _account_catalog(respx_mock: respx.MockRouter) -> respx.Route: + return respx_mock.get(f"{_BASE}/models").respond(json=_UPSTREAM_CATALOG) + + +def test_litellm_model_on_native_route_streams_like_v1_without_chatgpt_credentials( + client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router, monkeypatch: pytest.MonkeyPatch +) -> None: + recorder: Final = PreCallRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + _account_catalog(respx_mock) + chatgpt: Final = respx_mock.post(f"{_BASE}/responses").respond(json={}) + provider: Final = respx_mock.post(f"{_PROVIDER_BASE}/responses").mock( + side_effect=lambda _: httpx.Response( + 200, content=_provider_stream("Hi"), headers={"content-type": "text/event-stream"} + ) + ) + body: Final = {"model": "open-weights", "input": "Say hi", "stream": True, "store": False} + native: Final = client.post("/backend-api/codex/responses", json=body, headers=_ACCOUNT_HEADERS) + v1: Final = client.post("/v1/responses", json=body, headers={"authorization": f"Bearer {MASTER_KEY}"}) + assert native.status_code == v1.status_code == 200 + assert native.headers["content-type"].startswith("text/event-stream") + assert _event_types(native.content) == _event_types(v1.content) + assert "response.completed" in _event_types(native.content) + assert not chatgpt.called + provider_headers: Final = provider.calls[0].request.headers + assert provider_headers["authorization"] == f"Bearer {_PROVIDER_KEY}" + assert "chatgpt-account-id" not in provider_headers + assert all(_CHATGPT_JWT not in value for value in provider_headers.values()) + (native_route, native_call), (v1_route, _) = recorder.requests + assert native_route == v1_route == "/v1/responses" + assert _CHATGPT_JWT not in native_call + assert "account-a" not in native_call + + +def test_litellm_model_on_native_route_keeps_chatgpt_credentials_out_of_hook_raw_headers( + client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router, monkeypatch: pytest.MonkeyPatch +) -> None: + recorder: Final = PreCallRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + _account_catalog(respx_mock) + provider: Final = _open_weights_provider(respx_mock) + response: Final = client.post( + "/backend-api/codex/responses", + json={"model": "open-weights", "input": "Say hi", "stream": True, "store": False}, + headers=_ACCOUNT_HEADERS, + ) + assert response.status_code == 200 + assert provider.called + (raw_header_names,) = recorder.raw_header_names + assert "x-litellm-api-key" in raw_header_names + assert not raw_header_names & {"authorization", "chatgpt-account-id"}, sorted(raw_header_names) + + +@pytest.mark.parametrize( + ("model", "sent", "drop_params", "expected"), + [ + pytest.param("open-weights", "none", None, "none", id="none without drop_params"), + pytest.param("open-weights", "none", "global", None, id="none under litellm_settings drop_params"), + pytest.param("open-weights", "none", "deployment", None, id="none under the deployment's drop_params"), + pytest.param("open-weights", "none", "request", None, id="none under the request's drop_params"), + pytest.param("open-weights", "high", "global", "high", id="effort set for a model without a default effort"), + pytest.param("coding-model", "none", "global", None, id="none on an alias whose native entry names no effort"), + pytest.param("api-model", "none", "global", "none", id="none on a model that supports it"), + pytest.param("glm-model", "none", "global", None, id="none on a model that does not support it"), + pytest.param("declared-model", "none", "global", "none", id="none on a deployment whose model_info allows it"), + ], +) +def test_litellm_model_on_native_route_gets_the_reasoning_effort_codex_m_would_send( + client: TestClient, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + model: str, + sent: str, + drop_params: str | None, + expected: str | None, +) -> None: + monkeypatch.setattr(litellm, "drop_params", drop_params == "global") + open_weights: Final = _open_weights("open-weights") + monkeypatch.setattr( + proxy_server, + "llm_router", + Router( + model_list=[ + { + **open_weights, + "litellm_params": {**open_weights["litellm_params"], "drop_params": drop_params == "deployment"}, + }, + { + **_open_weights("coding-model", upstream="gpt-5.5"), + "model_info": {"supports_none_reasoning_effort": False}, + }, + _open_weights("glm-model", upstream="glm-5p3"), + {**_open_weights("declared-model"), "model_info": {"supports_none_reasoning_effort": True}}, + { + "model_name": "api-model", + "litellm_params": {"model": "openai/gpt-5.4", "api_base": _PROVIDER_BASE, "api_key": _PROVIDER_KEY}, + }, + ] + ), + ) + _account_catalog(respx_mock) + provider: Final = respx_mock.post(f"{_PROVIDER_BASE}/responses").mock( + side_effect=lambda _: httpx.Response( + 200, content=_provider_stream("Hi"), headers={"content-type": "text/event-stream"} + ) + ) + body: Final = { + "model": model, + "input": "Say hi", + "stream": True, + "store": False, + **({"drop_params": True} if drop_params == "request" else {}), + } + expected_reasoning: Final = {"summary": "auto"} if expected is None else {"effort": expected, "summary": "auto"} + native: Final = client.post( + "/backend-api/codex/responses", + json={**body, "reasoning": {"effort": sent, "summary": "auto"}}, + headers=_ACCOUNT_HEADERS, + ) + v1: Final = client.post( + "/v1/responses", + json={**body, "reasoning": expected_reasoning}, + headers={"authorization": f"Bearer {MASTER_KEY}"}, + ) + assert native.status_code == v1.status_code == 200 + native_request, v1_request = (json.loads(call.request.content) for call in provider.calls) + assert native_request == v1_request + assert native_request["reasoning"].get("effort") == expected + + +@pytest.mark.parametrize( + "model", ["gpt-unlisted", "chatgpt-subscription", _BUNDLED_SLUG, _PRICED_CHATGPT_SLUG, "newly-released"] +) +def test_chatgpt_model_keeps_passthrough_without_other_providers_reasoning( + client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router, model: str +) -> None: + provider: Final = respx_mock.post(f"{_PROVIDER_BASE}/responses").respond(json={}) + chatgpt: Final = respx_mock.post(f"{_BASE}/responses").respond( + content=b"event: response.completed\ndata: {}\n\n", + headers={"content-type": "text/event-stream", "x-models-etag": '"upstream-v2"'}, + ) + user_turn: Final = {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Hello"}]} + chatgpt_reasoning: Final = { + "type": "reasoning", + "id": "rs_upstream", + "summary": [], + "encrypted_content": "opaque-upstream-blob", + } + assistant_turn: Final = {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "Hi"}]} + other_providers_reasoning: Final = [ + {"type": "reasoning", "id": mint_reasoning_item_id(), "summary": []}, + { + "type": "reasoning", + "summary": [], + "encrypted_content": encode_thinking_blocks([{"type": "thinking", "thinking": "t", "signature": "s"}]), + }, + {"type": "reasoning", "summary": [], "content": [{"type": "reasoning_text", "text": "t"}]}, + { + "type": "reasoning", + "summary": [], + "encrypted_content": ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id("blob", "deployment"), + }, + ] + body: Final = { + "model": model, + "input": [user_turn, *other_providers_reasoning, chatgpt_reasoning, assistant_turn], + "stream": True, + } + response: Final = client.post("/backend-api/codex/responses", json=body, headers=_HEADERS) + assert response.status_code == 200 + assert not provider.called + assert json.loads(chatgpt.calls.last.request.content) == { + **body, + "input": [user_turn, chatgpt_reasoning, assistant_turn], + } + + +@pytest.fixture +def restricted_key(monkeypatch: pytest.MonkeyPatch) -> Callable[..., str]: + monkeypatch.setattr(proxy_server, "prisma_client", "test-database") + + def mint( + models: list[str], + allowed_routes: tuple[str, ...] = (), + user_role: LitellmUserRoles | None = None, + allowed_tools: tuple[str, ...] = (), + ) -> str: + key: Final = f"sk-test-{'-'.join((*models, *allowed_routes, str(user_role), *allowed_tools))}" + proxy_server.user_api_key_cache.set_cache( + key=hash_token(key), + value=UserAPIKeyAuth( + token=hash_token(key), + models=models, + allowed_routes=list(allowed_routes), + user_role=user_role, + metadata={"allowed_tools": list(allowed_tools)} if allowed_tools else {}, + ), + ) + return key + + return mint + + +@pytest.mark.parametrize( + ("models", "added"), + [ + pytest.param(["chatgpt-native", "open-weights"], ["open-weights"], id="key with a LiteLLM model"), + pytest.param(["chatgpt-native"], [], id="key with ChatGPT models only"), + ], +) +def test_native_catalog_adds_only_the_litellm_models_the_key_can_call( + client: TestClient, + respx_mock: respx.MockRouter, + mixed_router: Router, + restricted_key: Callable[..., str], + models: list[str], + added: list[str], +) -> None: + respx_mock.get(f"{_BASE}/models").respond(json=_UPSTREAM_CATALOG, headers={"etag": '"upstream-v1"'}) + response: Final = client.get( + "/backend-api/codex/models?client_version=1.0.0", + headers={**_HEADERS, "x-litellm-api-key": restricted_key(models)}, + ) + assert response.status_code == 200 + assert [model["slug"] for model in response.json()["models"]] == [ + *(model["slug"] for model in _UPSTREAM_CATALOG["models"]), + *added, + ] + assert (response.headers["etag"] == '"upstream-v1"') is not added + + +@pytest.mark.parametrize( + ("models", "model", "allowed"), + [ + pytest.param(["chatgpt-native", "open-weights"], "gpt-unlisted", True, id="chatgpt access group"), + pytest.param(["chatgpt-subscription", "open-weights"], "gpt-unlisted", False, id="single subscription model"), + pytest.param(["chatgpt-native"], "open-weights", False, id="litellm model outside the key"), + ], +) +def test_native_route_serves_only_the_models_the_key_allows( + client: TestClient, + respx_mock: respx.MockRouter, + mixed_router: Router, + restricted_key: Callable[..., str], + models: list[str], + model: str, + allowed: bool, +) -> None: + provider: Final = respx_mock.post(f"{_PROVIDER_BASE}/responses").respond(json={}) + chatgpt: Final = respx_mock.post(f"{_BASE}/responses").respond( + content=b"event: response.completed\ndata: {}\n\n", headers={"content-type": "text/event-stream"} + ) + response: Final = client.post( + "/backend-api/codex/responses", + json={"model": model, "input": "Hello", "stream": True}, + headers={**_HEADERS, "x-litellm-api-key": restricted_key(models)}, + ) + assert (response.status_code == 200) is allowed + assert chatgpt.called is allowed + assert not provider.called + assert [json.loads(call.request.content)["model"] for call in chatgpt.calls] == ([model] if allowed else []) + + +def _key_with_alias_to_budgeted_target(monkeypatch: pytest.MonkeyPatch, target_budget: float) -> str: + monkeypatch.setattr(proxy_server, "prisma_client", "test-database") + monkeypatch.setattr( + proxy_server, "llm_router", Router(model_list=[_open_weights("group-a"), _open_weights("group-b")]) + ) + key: Final = f"sk-test-alias-budget-{target_budget}" + proxy_server.user_api_key_cache.set_cache( + key=hash_token(key), + value=UserAPIKeyAuth( + token=hash_token(key), + models=["group-a", "group-b"], + router_settings={"model_group_alias": {"group-a": "group-b"}}, + model_max_budget={"group-b": {"max_budget": target_budget, "budget_duration": "1d"}}, + ), + ) + return key + + +def test_native_route_refuses_a_saved_alias_when_its_target_is_over_budget( + client: TestClient, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + key: Final = _key_with_alias_to_budgeted_target(monkeypatch, target_budget=0.0) + _account_catalog(respx_mock) + provider: Final = _open_weights_provider(respx_mock) + response: Final = client.post( + "/backend-api/codex/responses", + json={"model": "group-a", "input": "Hello"}, + headers={**_ACCOUNT_HEADERS, "x-litellm-api-key": key}, + ) + assert response.json()["error"]["type"] == "budget_exceeded", response.text + assert "model=group-b" in response.json()["error"]["message"] + assert not provider.called + + +def test_native_route_serves_a_saved_alias_target_within_budget_under_the_requested_name( + client: TestClient, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + key: Final = _key_with_alias_to_budgeted_target(monkeypatch, target_budget=5.0) + _account_catalog(respx_mock) + provider: Final = _open_weights_provider(respx_mock) + response: Final = client.post( + "/backend-api/codex/responses", + json={"model": "group-a", "input": "Hello"}, + headers={**_ACCOUNT_HEADERS, "x-litellm-api-key": key}, + ) + assert response.status_code == 200, response.text + assert response.json()["model"] == "group-a" + assert json.loads(provider.calls.last.request.content)["model"].endswith("/group-b") + + +def test_native_route_keeps_a_chatgpt_turn_on_the_subscription_despite_a_saved_alias( + client: TestClient, + respx_mock: respx.MockRouter, + mixed_router: Router, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(proxy_server, "prisma_client", "test-database") + key: Final = "sk-test-alias-on-a-chatgpt-name" + proxy_server.user_api_key_cache.set_cache( + key=hash_token(key), + value=UserAPIKeyAuth( + token=hash_token(key), + models=[_BUNDLED_SLUG, "open-weights"], + router_settings={"model_group_alias": {_BUNDLED_SLUG: "open-weights"}}, + ), + ) + provider: Final = _open_weights_provider(respx_mock) + chatgpt: Final = _chatgpt_responses(respx_mock) + response: Final = client.post( + "/backend-api/codex/responses", + json={"model": _BUNDLED_SLUG, "input": "Hello", "stream": True}, + headers={**_HEADERS, "x-litellm-api-key": key}, + ) + assert response.status_code == 200, response.text + assert not provider.called + assert [json.loads(call.request.content)["model"] for call in chatgpt.calls] == [_BUNDLED_SLUG] + + +def test_native_route_keeps_a_name_the_account_lists_on_the_subscription_despite_a_saved_alias( + client: TestClient, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(proxy_server, "prisma_client", "test-database") + monkeypatch.setattr( + proxy_server, "llm_router", Router(model_list=[_open_weights("gpt-reserve"), _open_weights("group-b")]) + ) + key: Final = "sk-test-alias-on-a-name-the-account-lists" + proxy_server.user_api_key_cache.set_cache( + key=hash_token(key), + value=UserAPIKeyAuth( + token=hash_token(key), + models=["gpt-reserve", "group-b"], + router_settings={"model_group_alias": {"gpt-reserve": "group-b"}}, + model_max_budget={"group-b": {"max_budget": 0.0, "budget_duration": "1d"}}, + ), + ) + catalog: Final = respx_mock.get(f"{_BASE}/models").respond(json={"models": [{"slug": "gpt-reserve"}]}) + provider: Final = _open_weights_provider(respx_mock) + chatgpt: Final = _chatgpt_responses(respx_mock) + response: Final = client.post( + "/backend-api/codex/responses", + json={"model": "gpt-reserve", "input": "Hello", "stream": True}, + headers={**_ACCOUNT_HEADERS, "x-litellm-api-key": key}, + ) + assert response.status_code == 200, response.text + assert not provider.called + assert [json.loads(call.request.content)["model"] for call in chatgpt.calls] == ["gpt-reserve"] + assert catalog.call_count == 1 + + +_BEARER_KEY: Final = "sk-test-alias-key-sent-as-the-bearer" + + +@pytest.mark.parametrize( + "gateway_header", + [{}, {"x-litellm-api-key": _BEARER_KEY}], + ids=["key only in the bearer", "same key in both headers"], +) +def test_native_route_never_sends_a_litellm_key_sent_as_the_bearer_to_chatgpt( + client: TestClient, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, gateway_header: dict[str, str] +) -> None: + monkeypatch.setattr(proxy_server, "prisma_client", "test-database") + monkeypatch.setattr( + proxy_server, "llm_router", Router(model_list=[_open_weights("gpt-reserve"), _open_weights("group-b")]) + ) + proxy_server.user_api_key_cache.set_cache( + key=hash_token(_BEARER_KEY), + value=UserAPIKeyAuth( + token=hash_token(_BEARER_KEY), + models=["gpt-reserve", "group-b"], + router_settings={"model_group_alias": {"gpt-reserve": "group-b"}}, + ), + ) + catalog: Final = respx_mock.get(f"{_BASE}/models").respond(json={"models": []}) + provider: Final = _open_weights_provider(respx_mock) + chatgpt: Final = _chatgpt_responses(respx_mock) + response: Final = client.post( + "/backend-api/codex/responses", + json={"model": "gpt-reserve", "input": "Hello", "stream": True}, + headers={ + "authorization": f"Bearer {_BEARER_KEY}", + "chatgpt-account-id": "account-a", + "originator": "codex_cli_rs", + "user-agent": _CODEX_USER_AGENT, + **gateway_header, + }, + ) + assert response.status_code == 401, response.text + assert not catalog.called + assert not provider.called + assert not chatgpt.called + + +def test_v1_request_for_a_gpt_name_still_reaches_the_catch_all_with_the_access_group_recipe( + client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router +) -> None: + provider: Final = _open_weights_provider(respx_mock) + chatgpt: Final = _chatgpt_responses(respx_mock) + response: Final = client.post( + "/v1/responses", + json={"model": "gpt-unlisted", "input": "Hello", "stream": True}, + headers={"authorization": f"Bearer {MASTER_KEY}"}, + ) + assert response.status_code == 200, response.text + assert provider.called + assert not chatgpt.called + + +_NATIVE_ROUTES: Final = ("/backend-api/codex/models", "/backend-api/codex/responses") + + +@pytest.mark.parametrize( + ("allowed_routes", "added"), + [ + pytest.param(_NATIVE_ROUTES, [], id="key limited to the native routes"), + pytest.param((*_NATIVE_ROUTES, "/v1/responses"), ["open-weights"], id="key that may call /v1/responses"), + ], +) +def test_native_catalog_adds_litellm_models_only_for_a_key_that_may_call_v1_responses( + client: TestClient, + respx_mock: respx.MockRouter, + mixed_router: Router, + restricted_key: Callable[..., str], + allowed_routes: tuple[str, ...], + added: list[str], +) -> None: + respx_mock.get(f"{_BASE}/models").respond(json=_UPSTREAM_CATALOG) + response: Final = client.get( + "/backend-api/codex/models?client_version=1.0.0", + headers={**_HEADERS, "x-litellm-api-key": restricted_key(["chatgpt-native", "open-weights"], allowed_routes)}, + ) + assert response.status_code == 200, response.text + assert [model["slug"] for model in response.json()["models"]] == [ + *(model["slug"] for model in _UPSTREAM_CATALOG["models"]), + *added, + ] + + +@pytest.mark.parametrize( + ("allowed_routes", "through_litellm"), + [ + pytest.param(_NATIVE_ROUTES, False, id="key limited to the native routes"), + pytest.param((*_NATIVE_ROUTES, "/v1/responses"), True, id="key that may call /v1/responses"), + ], +) +def test_native_responses_reach_litellm_models_only_for_a_key_that_may_call_v1_responses( + client: TestClient, + respx_mock: respx.MockRouter, + mixed_router: Router, + restricted_key: Callable[..., str], + allowed_routes: tuple[str, ...], + through_litellm: bool, +) -> None: + _account_catalog(respx_mock) + provider: Final = _open_weights_provider(respx_mock) + chatgpt: Final = _chatgpt_responses(respx_mock) + response: Final = client.post( + "/backend-api/codex/responses", + json={"model": "open-weights", "input": "Hello", "stream": True}, + headers={ + **_ACCOUNT_HEADERS, + "x-litellm-api-key": restricted_key(["chatgpt-native", "open-weights"], allowed_routes), + }, + ) + assert response.status_code == 200, response.text + assert provider.called is through_litellm + assert chatgpt.called is not through_litellm + + + +@pytest.mark.parametrize( + ("user_role", "through_litellm"), + [ + pytest.param(LitellmUserRoles.INTERNAL_USER, False, id="non-admin key"), + pytest.param(LitellmUserRoles.PROXY_ADMIN, True, id="proxy admin key"), + ], +) +def test_admin_only_v1_responses_keeps_litellm_models_to_proxy_admins_on_native_routes( + client: TestClient, + respx_mock: respx.MockRouter, + mixed_router: Router, + restricted_key: Callable[..., str], + monkeypatch: pytest.MonkeyPatch, + user_role: LitellmUserRoles, + through_litellm: bool, +) -> None: + monkeypatch.setattr(proxy_server, "general_settings", {"admin_only_routes": ["/v1/responses"]}) + monkeypatch.setattr(proxy_server, "premium_user", True) + _account_catalog(respx_mock) + provider: Final = _open_weights_provider(respx_mock) + chatgpt: Final = _chatgpt_responses(respx_mock) + key: Final = restricted_key(["chatgpt-native", "open-weights"], user_role=user_role) + catalog: Final = client.get( + "/backend-api/codex/models?client_version=1.0.0", headers={**_ACCOUNT_HEADERS, "x-litellm-api-key": key} + ) + response: Final = client.post( + "/backend-api/codex/responses", + json={"model": "open-weights", "input": "Hello", "stream": True}, + headers={**_ACCOUNT_HEADERS, "x-litellm-api-key": key}, + ) + assert (catalog.status_code, response.status_code) == (200, 200), response.text + assert ("open-weights" in [model["slug"] for model in catalog.json()["models"]]) is through_litellm + assert provider.called is through_litellm + assert chatgpt.called is not through_litellm + + +@pytest.mark.parametrize( + ("allowed_routes", "through_litellm"), + [ + pytest.param(["/backend-api/codex/models", "/backend-api/codex/responses"], False, id="native routes only"), + pytest.param( + ["/backend-api/codex/models", "/backend-api/codex/responses", "/v1/responses"], + True, + id="v1 responses allowed too", + ), + ], +) +def test_proxy_allowed_routes_keep_litellm_models_off_native_routes_unless_v1_responses_is_allowed( + client: TestClient, + respx_mock: respx.MockRouter, + mixed_router: Router, + restricted_key: Callable[..., str], + monkeypatch: pytest.MonkeyPatch, + allowed_routes: list[str], + through_litellm: bool, +) -> None: + monkeypatch.setattr(proxy_server, "general_settings", {"allowed_routes": allowed_routes}) + monkeypatch.setattr(proxy_server, "premium_user", True) + _account_catalog(respx_mock) + provider: Final = _open_weights_provider(respx_mock) + chatgpt: Final = _chatgpt_responses(respx_mock) + key: Final = restricted_key(["chatgpt-native", "open-weights"]) + catalog: Final = client.get( + "/backend-api/codex/models?client_version=1.0.0", headers={**_ACCOUNT_HEADERS, "x-litellm-api-key": key} + ) + response: Final = client.post( + "/backend-api/codex/responses", + json={"model": "open-weights", "input": "Hello", "stream": True}, + headers={**_ACCOUNT_HEADERS, "x-litellm-api-key": key}, + ) + assert (catalog.status_code, response.status_code) == (200, 200), response.text + assert ("open-weights" in [model["slug"] for model in catalog.json()["models"]]) is through_litellm + assert provider.called is through_litellm + assert chatgpt.called is not through_litellm + + +_JWT_ISSUER: Final = "https://idp.codex-test.example" +_JWT_JWKS_URL: Final = f"{_JWT_ISSUER}/keys" +_JWT_AUDIENCE: Final = "litellm-proxy" +_JWT_KID: Final = "codex-test-key" +_JWT_TEAM: Final = "codex-team" + + +def _team_jwt(monkeypatch: pytest.MonkeyPatch, team_allowed_routes: list[str]) -> str: + private_key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) + public_jwk: Final = { + **json.loads(RSAAlgorithm.to_jwk(private_key.public_key())), + "kid": _JWT_KID, + "alg": "RS256", + "use": "sig", + } + keys_cache: Final = DualCache() + keys_cache.set_cache(key=f"litellm_jwt_auth_keys_{_JWT_JWKS_URL}", value=[public_jwk]) + handler: Final = JWTHandler() + handler.update_environment( + prisma_client=None, + user_api_key_cache=keys_cache, + litellm_jwtauth=LiteLLM_JWTAuth( + issuers=[{"issuer": _JWT_ISSUER, "jwks_url": _JWT_JWKS_URL, "audience": _JWT_AUDIENCE}], + team_ids_jwt_field="groups", + team_allowed_routes=team_allowed_routes, + ), + ) + monkeypatch.setattr(proxy_server, "jwt_handler", handler) + monkeypatch.setattr(proxy_server, "general_settings", {"enable_jwt_auth": True}) + monkeypatch.setattr(proxy_server, "premium_user", True) + monkeypatch.setattr(proxy_server, "prisma_client", "test-database") + proxy_server.user_api_key_cache.set_cache( + key=f"team_id:{_JWT_TEAM}", + value=LiteLLM_TeamTableCachedObj(team_id=_JWT_TEAM, models=["chatgpt-native", "open-weights"]), + ) + issued_at: Final = int(time.time()) + return jwt.encode( + { + "sub": "codex-user", + "iss": _JWT_ISSUER, + "aud": _JWT_AUDIENCE, + "iat": issued_at, + "exp": issued_at + 300, + "groups": [_JWT_TEAM], + }, + private_key, + algorithm="RS256", + headers={"kid": _JWT_KID}, + ) + + +@pytest.mark.parametrize( + ("team_allowed_routes", "through_litellm"), + [ + pytest.param(["/backend-api/codex/*"], False, id="team allowed only the native routes"), + pytest.param(["/backend-api/codex/*", "openai_routes"], True, id="team allowed openai routes too"), + ], +) +def test_jwt_team_routes_keep_litellm_models_off_native_routes_unless_v1_responses_is_allowed( + client: TestClient, + respx_mock: respx.MockRouter, + mixed_router: Router, + monkeypatch: pytest.MonkeyPatch, + team_allowed_routes: list[str], + through_litellm: bool, +) -> None: + token: Final = _team_jwt(monkeypatch, team_allowed_routes) + _account_catalog(respx_mock) + provider: Final = _open_weights_provider(respx_mock) + chatgpt: Final = _chatgpt_responses(respx_mock) + catalog: Final = client.get( + "/backend-api/codex/models?client_version=1.0.0", headers={**_ACCOUNT_HEADERS, "x-litellm-api-key": token} + ) + response: Final = client.post( + "/backend-api/codex/responses", + json={"model": "open-weights", "input": "Hello", "stream": True}, + headers={**_ACCOUNT_HEADERS, "x-litellm-api-key": token}, + ) + assert (catalog.status_code, response.status_code) == (200, 200), response.text + assert ("open-weights" in [model["slug"] for model in catalog.json()["models"]]) is through_litellm + assert provider.called is through_litellm + assert chatgpt.called is not through_litellm + + +_USER_TURN: Final = {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Hello"}]} +_ASSISTANT_TURN: Final = {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "Hi"}]} +_FIREWORKS_REASONING: Final = { + "type": "reasoning", + "id": f"rs_{'0f' * 16}", + "summary": [{"type": "summary_text", "text": "Thinking it over"}], +} + + +@pytest.mark.parametrize( + ("item", "store", "forwarded"), + [ + pytest.param(_FIREWORKS_REASONING, False, False, id="another provider's summary-only item"), + pytest.param({**_FIREWORKS_REASONING, "encrypted_content": None}, None, False, id="null blob, store unset"), + pytest.param( + {"type": "reasoning", "id": "rs_upstream", "summary": [], "encrypted_content": "gAAAAB-test-blob"}, + False, + True, + id="ChatGPT's own item", + ), + pytest.param({**_FIREWORKS_REASONING, "id": None}, False, True, id="summary-only item without an id"), + pytest.param(_FIREWORKS_REASONING, True, True, id="stored conversation"), + ], +) +def test_chatgpt_passthrough_drops_reasoning_ids_chatgpt_never_persisted( + client: TestClient, + respx_mock: respx.MockRouter, + item: dict[str, object], + store: bool | None, + forwarded: bool, +) -> None: + chatgpt: Final = _chatgpt_responses(respx_mock) + body: Final = { + "model": "subscription-model", + "input": [_USER_TURN, item, _ASSISTANT_TURN], + "stream": True, + **({} if store is None else {"store": store}), + } + response: Final = client.post("/backend-api/codex/responses", json=body, headers=_HEADERS) + assert response.status_code == 200 + assert json.loads(chatgpt.calls.last.request.content)["input"] == ( + [_USER_TURN, item, _ASSISTANT_TURN] if forwarded else [_USER_TURN, _ASSISTANT_TURN] + ) + + +def test_exact_deployment_named_like_an_account_slug_stays_on_the_subscription_after_the_catalog_read( + client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router +) -> None: + catalog: Final = respx_mock.get(f"{_BASE}/models").respond(json=_UPSTREAM_CATALOG) + provider: Final = _open_weights_provider(respx_mock) + chatgpt: Final = _chatgpt_responses(respx_mock) + assert client.get("/backend-api/codex/models?client_version=0.165.0", headers=_ACCOUNT_HEADERS).status_code == 200 + response: Final = client.post( + "/backend-api/codex/responses", + json={"model": "shared-name", "input": "Hello", "stream": True}, + headers=_ACCOUNT_HEADERS, + ) + assert response.status_code == 200 + assert chatgpt.called + assert not provider.called + assert catalog.call_count == 1 + + +_LISTED: Final = httpx.Response(200, json=_UPSTREAM_CATALOG) + + +@pytest.mark.parametrize( + ("model", "user_agent", "catalog_answer", "subscription"), + [ + pytest.param("shared-name", _CODEX_USER_AGENT, _LISTED, True, id="listed"), + pytest.param("open-weights", _CODEX_USER_AGENT, _LISTED, False, id="unlisted"), + pytest.param("open-weights", _CODEX_USER_AGENT, httpx.Response(503), True, id="catalog unavailable"), + pytest.param("open-weights", _CODEX_USER_AGENT, httpx.ReadTimeout("slow"), True, id="catalog timeout"), + pytest.param("open-weights", "codex_cli_rs", _LISTED, True, id="no version"), + pytest.param("open-weights", "curl/8.5.0", _LISTED, True, id="user agent of another client"), + ], +) +def test_native_responses_ask_chatgpt_which_models_the_account_lists_on_a_cache_miss( + client: TestClient, + respx_mock: respx.MockRouter, + mixed_router: Router, + model: str, + user_agent: str, + catalog_answer: httpx.Response | httpx.TimeoutException, + subscription: bool, +) -> None: + catalog: Final = respx_mock.get(f"{_BASE}/models").mock(side_effect=[catalog_answer]) + provider: Final = _open_weights_provider(respx_mock) + chatgpt: Final = _chatgpt_responses(respx_mock) + response: Final = client.post( + "/backend-api/codex/responses", + json={"model": model, "input": "Hello", "stream": True}, + headers={**_ACCOUNT_HEADERS, "user-agent": user_agent}, + ) + assert response.status_code == 200 + assert chatgpt.called is subscription + assert provider.called is not subscription + assert catalog.called is (user_agent == _CODEX_USER_AGENT) + + +def test_a_failed_account_catalog_read_keeps_the_next_turns_on_the_subscription_without_asking_again( + client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router +) -> None: + catalog: Final = respx_mock.get(f"{_BASE}/models").respond(503) + provider: Final = _open_weights_provider(respx_mock) + chatgpt: Final = _chatgpt_responses(respx_mock) + responses: Final = tuple( + client.post( + "/backend-api/codex/responses", + json={"model": "open-weights", "input": "Hello", "stream": True}, + headers=_ACCOUNT_HEADERS, + ) + for _ in range(2) + ) + assert [response.status_code for response in responses] == [200, 200] + assert chatgpt.call_count == 2 + assert not provider.called + assert catalog.call_count == 1 + + +def test_a_failed_account_catalog_read_with_another_login_leaves_the_accounts_own_list_reachable( + client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router +) -> None: + own_login: Final = f"Bearer {_CHATGPT_JWT}" + catalog: Final = respx_mock.get(f"{_BASE}/models").mock( + side_effect=lambda request: ( + httpx.Response(200, json=_UPSTREAM_CATALOG) + if request.headers["authorization"] == own_login + else httpx.Response(401) + ) + ) + provider: Final = _open_weights_provider(respx_mock) + chatgpt: Final = _chatgpt_responses(respx_mock) + turn: Final = {"model": "open-weights", "input": "Hello", "stream": True} + other_login: Final = client.post( + "/backend-api/codex/responses", + json=turn, + headers={**_ACCOUNT_HEADERS, "authorization": "Bearer test-someone-elses-login"}, + ) + own: Final = client.post("/backend-api/codex/responses", json=turn, headers=_ACCOUNT_HEADERS) + assert other_login.status_code == own.status_code == 200 + assert chatgpt.call_count == 1 + assert provider.call_count == 1 + assert [call.request.headers["authorization"] for call in catalog.calls] == [ + "Bearer test-someone-elses-login", + own_login, + ] + + +def test_rotating_chatgpt_logins_on_one_key_reads_the_account_catalog_a_bounded_number_of_times_per_key( + client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router, restricted_key: Callable[..., str] +) -> None: + catalog: Final = respx_mock.get(f"{_BASE}/models").respond(401) + provider: Final = _open_weights_provider(respx_mock) + chatgpt: Final = _chatgpt_responses(respx_mock) + turn: Final = {"model": "open-weights", "input": "Hello", "stream": True} + rotated: Final = tuple( + client.post( + "/backend-api/codex/responses", + json=turn, + headers={**_ACCOUNT_HEADERS, "authorization": f"Bearer test-rotated-login-{attempt}"}, + ) + for attempt in range(_ACCOUNT_SLUGS_FETCHES_PER_CALLER + 2) + ) + teammate: Final = client.post( + "/backend-api/codex/responses", + json=turn, + headers={ + **_ACCOUNT_HEADERS, + "authorization": "Bearer test-teammate-login", + "x-litellm-api-key": restricted_key(["open-weights"]), + }, + ) + assert {response.status_code for response in (*rotated, teammate)} == {200} + assert chatgpt.call_count == len(rotated) + 1 + assert not provider.called + assert [call.request.headers["authorization"] for call in catalog.calls][-1] == "Bearer test-teammate-login" + assert catalog.call_count == _ACCOUNT_SLUGS_FETCHES_PER_CALLER + 1 + + +def test_account_catalog_lookup_sends_the_callers_chatgpt_login_and_codex_version( + client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router +) -> None: + catalog: Final = respx_mock.get(f"{_BASE}/models").respond(json=_UPSTREAM_CATALOG) + _chatgpt_responses(respx_mock) + response: Final = client.post( + "/backend-api/codex/responses", + json={"model": "shared-name", "input": "Hello", "stream": True}, + headers=_ACCOUNT_HEADERS, + ) + assert response.status_code == 200 + fetched: Final = catalog.calls.last.request + assert fetched.url.params["client_version"] == "0.165.0" + assert {name: fetched.headers[name] for name in ("authorization", "chatgpt-account-id", "originator")} == { + "authorization": f"Bearer {_CHATGPT_JWT}", + "chatgpt-account-id": "account-a", + "originator": "codex_cli_rs", + } + assert fetched.headers["user-agent"] == _CODEX_USER_AGENT + assert "x-litellm-api-key" not in fetched.headers + + +def _models_etags(client: TestClient, headers: dict[str, str]) -> tuple[str | None, str | None]: + catalog: Final = client.get("/backend-api/codex/models?client_version=0.165.0", headers=headers) + responses: Final = client.post( + "/backend-api/codex/responses", + json={"model": "gpt-unlisted", "input": "Hello", "stream": True}, + headers=headers, + ) + assert catalog.status_code == responses.status_code == 200, (catalog.text, responses.text) + return catalog.headers.get("etag"), responses.headers.get("x-models-etag") + + +@pytest.mark.parametrize( + ("models", "merged"), + [ + pytest.param(["chatgpt-native", "open-weights"], True, id="catalog with LiteLLM models"), + pytest.param(["chatgpt-native"], False, id="catalog of ChatGPT models only"), + ], +) +def test_native_catalog_and_responses_carry_the_same_models_etag( + client: TestClient, + respx_mock: respx.MockRouter, + mixed_router: Router, + restricted_key: Callable[..., str], + models: list[str], + merged: bool, +) -> None: + respx_mock.get(f"{_BASE}/models").respond(json=_UPSTREAM_CATALOG, headers={"etag": _UPSTREAM_MODELS_ETAG}) + _chatgpt_responses(respx_mock) + catalog_etag, responses_etag = _models_etags(client, {**_HEADERS, "x-litellm-api-key": restricted_key(models)}) + assert catalog_etag is not None + assert catalog_etag == responses_etag + assert (catalog_etag != _UPSTREAM_MODELS_ETAG) is merged + + +def test_native_models_etag_is_the_same_on_replicas_with_empty_caches( + client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setitem(proxy_server.app.dependency_overrides, codex_shared_cache, lambda: DualCache()) + respx_mock.get(f"{_BASE}/models").respond(json=_UPSTREAM_CATALOG, headers={"etag": _UPSTREAM_MODELS_ETAG}) + _chatgpt_responses(respx_mock) + first_replica: Final = _models_etags(client, _HEADERS) + second_replica: Final = _models_etags(client, _HEADERS) + assert first_replica[0] not in (None, _UPSTREAM_MODELS_ETAG) + assert len({*first_replica, *second_replica}) == 1 + + +def test_native_models_etag_changes_with_the_keys_litellm_models_and_with_upstreams_etag( + client: TestClient, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + open_weights: Final = _open_weights("open-weights") + wider_window: Final = {**open_weights, "model_info": {"max_input_tokens": 64000}} + + def catalog_etag(deployment: dict[str, object], upstream_etag: str) -> str | None: + monkeypatch.setattr(proxy_server, "llm_router", Router(model_list=[deployment])) + respx_mock.get(f"{_BASE}/models").respond(json=_UPSTREAM_CATALOG, headers={"etag": upstream_etag}) + return client.get("/backend-api/codex/models?client_version=0.165.0", headers=_HEADERS).headers.get("etag") + + baseline: Final = catalog_etag(open_weights, '"upstream-v1"') + assert baseline == catalog_etag(open_weights, '"upstream-v1"') + assert ( + len({baseline, catalog_etag(wider_window, '"upstream-v1"'), catalog_etag(open_weights, '"upstream-v3"')}) == 3 + ) + assert baseline not in (None, '"upstream-v1"') + + +class FailingModelListing(CustomLogger): + async def async_filter_listed_models( + self, user_api_key_dict: UserAPIKeyAuth, model_names: Sequence[str] + ) -> Sequence[str]: + raise RuntimeError("model listing backend unavailable") + + +def test_native_catalog_relays_chatgpts_answer_when_listing_the_keys_models_fails( + client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "callbacks", [FailingModelListing()]) + respx_mock.get(f"{_BASE}/models").respond(json=_UPSTREAM_CATALOG, headers={"etag": '"upstream-v1"'}) + response: Final = client.get("/backend-api/codex/models?client_version=0.165.0", headers=_HEADERS) + assert response.status_code == 200 + assert response.json() == _UPSTREAM_CATALOG + assert response.headers["etag"] == '"upstream-v1"' + + +def test_native_responses_keep_chatgpts_models_etag_when_listing_the_keys_models_fails( + client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "callbacks", [FailingModelListing()]) + chatgpt: Final = _chatgpt_responses(respx_mock) + response: Final = client.post( + "/backend-api/codex/responses", + json={"model": "gpt-unlisted", "input": "Hello", "stream": True}, + headers=_HEADERS, + ) + assert response.status_code == 200 + assert chatgpt.called + assert response.headers["x-models-etag"] == _UPSTREAM_MODELS_ETAG + + +_OPENAI_BASE: Final = "https://openai-api.test/v1" +_OPENAI_BLOB: Final = "gAAAAB-openai-reasoning-blob" +_OPENAI_REASONING: Final = {"type": "reasoning", "id": "rs_openai", "summary": [], "encrypted_content": _OPENAI_BLOB} + + +def _openai_reply(request: httpx.Request) -> httpx.Response: + completed: Final = { + "id": "resp_openai", + "object": "response", + "created_at": 1700000000, + "status": "completed", + "model": "gpt-5.6", + "output": [ + _OPENAI_REASONING, + { + "type": "message", + "id": "msg_openai", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "Hi", "annotations": []}], + }, + ], + "usage": {"input_tokens": 3, "output_tokens": 1, "total_tokens": 4}, + } + if json.loads(request.content).get("stream") is not True: + return httpx.Response(200, json=completed) + events: Final = ( + { + "type": "response.created", + "sequence_number": 0, + "response": {**completed, "status": "in_progress", "output": []}, + }, + {"type": "response.output_item.done", "sequence_number": 1, "output_index": 0, "item": _OPENAI_REASONING}, + {"type": "response.completed", "sequence_number": 2, "response": completed}, + ) + return httpx.Response( + 200, + content="".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events).encode(), + headers={"content-type": "text/event-stream"}, + ) + + +def _reasoning_item_of(response: httpx.Response, stream: bool) -> dict[str, object]: + if not stream: + return response.json()["output"][0] + payloads: Final = ( + json.loads(line.removeprefix(b"data: ")) + for line in response.content.splitlines() + if line.startswith(b"data: {") + ) + return next(event["item"] for event in payloads if event["type"] == "response.output_item.done") + + +@pytest.mark.parametrize("stream", [False, True]) +def test_openai_reasoning_from_the_native_route_is_dropped_for_chatgpt_and_unwrapped_for_litellm( + client: TestClient, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, stream: bool +) -> None: + monkeypatch.setattr( + proxy_server, + "llm_router", + Router( + model_list=[ + { + "model_name": "api-model", + "litellm_params": { + "model": "openai/gpt-5.6", + "api_base": _OPENAI_BASE, + "api_key": "test-openai-key", + }, + } + ] + ), + ) + _account_catalog(respx_mock) + provider: Final = respx_mock.post(f"{_OPENAI_BASE}/responses").mock(side_effect=_openai_reply) + chatgpt: Final = _chatgpt_responses(respx_mock) + turn: Final = {"model": "api-model", "input": [_USER_TURN], "stream": stream, "store": False} + first: Final = client.post("/backend-api/codex/responses", json=turn, headers=_ACCOUNT_HEADERS) + assert first.status_code == 200, first.text + reasoning: Final = _reasoning_item_of(first, stream) + model_id, blob = ResponsesAPIRequestUtils.unwrap_encrypted_content_with_model_id( + str(reasoning["encrypted_content"]) + ) + assert (model_id is not None, blob) == (True, _OPENAI_BLOB) + history: Final = [_USER_TURN, reasoning, _ASSISTANT_TURN, _USER_TURN] + to_chatgpt: Final = client.post( + "/backend-api/codex/responses", + json={**turn, "model": "subscription-model", "input": history, "stream": True}, + headers=_ACCOUNT_HEADERS, + ) + back_to_litellm: Final = client.post( + "/backend-api/codex/responses", json={**turn, "input": history}, headers=_ACCOUNT_HEADERS + ) + v1: Final = client.post( + "/v1/responses", json={**turn, "stream": False}, headers={"authorization": f"Bearer {MASTER_KEY}"} + ) + assert to_chatgpt.status_code == back_to_litellm.status_code == v1.status_code == 200 + assert json.loads(chatgpt.calls.last.request.content)["input"] == [_USER_TURN, _ASSISTANT_TURN, _USER_TURN] + replayed: Final = json.loads(provider.calls[1].request.content)["input"][1] + assert (replayed["id"], replayed["encrypted_content"]) == ("rs_openai", _OPENAI_BLOB) + assert v1.json()["output"][0]["encrypted_content"] == _OPENAI_BLOB + + +_XAI_BASE: Final = "https://xai-api.test/v1" +_LUNA_ID: Final = "oai-luna-deployment" +_LUNA_BLOB: Final = "gAAAAB-luna-reasoning-blob" +_LUNA_REASONING: Final = { + "type": "reasoning", + "summary": [], + "encrypted_content": ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id(_LUNA_BLOB, _LUNA_ID), +} +_CHATGPT_REASONING: Final = {"type": "reasoning", "summary": [], "encrypted_content": "gAAAAB-chatgpt-reasoning-blob"} + + +@pytest.fixture +def xai_fireworks_and_openai_router(monkeypatch: pytest.MonkeyPatch) -> Router: + router: Final = Router( + model_list=[ + { + "model_name": "grok-4.7", + "litellm_params": {"model": "xai/grok-4.7", "api_base": _XAI_BASE, "api_key": "test-xai-key"}, + }, + _open_weights("open-weights"), + { + "model_name": "oai-luna", + "litellm_params": { + "model": "openai/gpt-6-luna", + "api_base": _OPENAI_BASE, + "api_key": "test-openai-key", + }, + "model_info": {"id": _LUNA_ID}, + }, + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + return router + + +@pytest.mark.parametrize( + ("model", "replayed", "expected"), + [ + pytest.param("open-weights", _CHATGPT_REASONING, None, id="chatgpt reasoning to fireworks is dropped"), + pytest.param("open-weights", _LUNA_REASONING, None, id="another deployment's reasoning to fireworks is dropped"), + pytest.param("oai-luna", _LUNA_REASONING, _LUNA_BLOB, id="the minting deployment's reasoning is kept"), + ], +) +def test_litellm_model_on_native_route_gets_only_the_reasoning_it_can_decrypt( + client: TestClient, + respx_mock: respx.MockRouter, + xai_fireworks_and_openai_router: Router, + model: str, + replayed: dict[str, object], + expected: str | None, +) -> None: + _account_catalog(respx_mock) + fireworks: Final = _open_weights_provider(respx_mock) + luna: Final = respx_mock.post(f"{_OPENAI_BASE}/responses").mock(side_effect=_provider_reply) + response: Final = client.post( + "/backend-api/codex/responses", + json={ + "model": model, + "input": [_USER_TURN, replayed, _ASSISTANT_TURN, _USER_TURN], + "stream": False, + "store": False, + }, + headers=_ACCOUNT_HEADERS, + ) + assert response.status_code == 200, response.text + provider: Final = fireworks if model == "open-weights" else luna + sent: Final = json.loads(provider.calls.last.request.content)["input"] + assert [item.get("encrypted_content") for item in sent if item.get("type") == "reasoning"] == ( + [] if expected is None else [expected] + ) + + +_WHOLE_FIREWORKS_REASONING: Final = {**_FIREWORKS_REASONING, "status": "completed"} +_SUMMARY_ONLY_REASONING: Final = {key: value for key, value in _FIREWORKS_REASONING.items() if key != "id"} +_OTHER_DEPLOYMENTS_BLOB: Final = ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id( + "gAAAAB-other-deployment-blob", "other-deployment" +) + + +@pytest.mark.parametrize( + ("model", "replayed", "store", "expected"), + [ + pytest.param( + "oai-luna", _WHOLE_FIREWORKS_REASONING, False, [_SUMMARY_ONLY_REASONING], id="whole fireworks item to openai" + ), + pytest.param( + "oai-luna", + {**_WHOLE_FIREWORKS_REASONING, "encrypted_content": _OTHER_DEPLOYMENTS_BLOB}, + False, + [_SUMMARY_ONLY_REASONING], + id="whole item with another deployment's blob to openai", + ), + pytest.param( + "oai-luna", + {**_SUMMARY_ONLY_REASONING, "content": [], "encrypted_content": _OTHER_DEPLOYMENTS_BLOB}, + False, + [{**_SUMMARY_ONLY_REASONING, "content": []}], + id="codex's replay of another deployment's item to openai", + ), + pytest.param( + "oai-luna", + {**_SUMMARY_ONLY_REASONING, "content": None, "encrypted_content": None}, + False, + [_SUMMARY_ONLY_REASONING], + id="codex's replay of a blobless item to openai", + ), + pytest.param( + "oai-luna", + {"type": "reasoning", "id": "rs_unreadable", "summary": []}, + False, + [], + id="an unresolvable id with nothing readable is dropped", + ), + pytest.param( + "open-weights", + {**_SUMMARY_ONLY_REASONING, "id": "rs_openai"}, + False, + [_SUMMARY_ONLY_REASONING], + id="openai summary-only item to fireworks", + ), + pytest.param( + "oai-luna", + {**_LUNA_REASONING, "id": "rs_luna", "content": []}, + False, + [{**_LUNA_REASONING, "id": "rs_luna", "content": [], "encrypted_content": _LUNA_BLOB}], + id="the minting deployment gets its item whole", + ), + pytest.param( + "oai-luna", _FIREWORKS_REASONING, True, [_FIREWORKS_REASONING], id="a stored conversation keeps the id" + ), + ], +) +def test_reasoning_the_target_cannot_resolve_reaches_it_without_its_id( + client: TestClient, + respx_mock: respx.MockRouter, + xai_fireworks_and_openai_router: Router, + model: str, + replayed: dict[str, object], + store: bool, + expected: list[dict[str, object]], +) -> None: + _account_catalog(respx_mock) + fireworks: Final = _open_weights_provider(respx_mock) + luna: Final = respx_mock.post(f"{_OPENAI_BASE}/responses").mock(side_effect=_provider_reply) + response: Final = client.post( + "/backend-api/codex/responses", + json={ + "model": model, + "input": [_USER_TURN, replayed, _ASSISTANT_TURN, _USER_TURN], + "stream": False, + "store": store, + }, + headers=_ACCOUNT_HEADERS, + ) + assert response.status_code == 200, response.text + sent: Final = json.loads((fireworks if model == "open-weights" else luna).calls.last.request.content)["input"] + assert [item for item in sent if item.get("type") == "reasoning"] == expected + + +def test_reasoning_the_chat_bridge_target_cannot_resolve_reaches_it_as_its_summary( + client: TestClient, respx_mock: respx.MockRouter, xai_fireworks_and_openai_router: Router +) -> None: + _account_catalog(respx_mock) + chat_endpoint: Final = respx_mock.post(f"{_XAI_BASE}/chat/completions").mock(side_effect=_xai_search_call_reply) + response: Final = client.post( + "/backend-api/codex/responses", + json={ + "model": "grok-4.7", + "input": [_USER_TURN, _WHOLE_FIREWORKS_REASONING, _ASSISTANT_TURN, _USER_TURN], + "stream": False, + "store": False, + }, + headers=_ACCOUNT_HEADERS, + ) + assert response.status_code == 200, response.text + messages: Final = json.loads(chat_endpoint.calls.last.request.content)["messages"] + assert [message.get("reasoning_content") for message in messages if message["role"] == "assistant"] == [ + "Thinking it over" + ] + + +def _namespace(name: str, *members: str) -> dict[str, object]: + return { + "type": "namespace", + "name": name, + "description": "", + "tools": [{"type": "function", "name": member, "parameters": {"type": "object"}} for member in members], + } + + +_EXEC_TOOL: Final = {"type": "function", "name": "exec_command", "parameters": {"type": "object"}} +_ADDITIONAL_TOOLS: Final = { + "type": "additional_tools", + "role": "developer", + "tools": [_namespace("collaboration", "followup_task"), _namespace("mcp__docs__", "search")], +} + + +@pytest.mark.parametrize( + ("model", "expected_input", "expected_tool_names"), + [ + pytest.param( + "open-weights", + [_USER_TURN], + ["exec_command", "collaboration", "mcp__docs__"], + id="fireworks gets the item's tools as top-level tools", + ), + pytest.param("oai-luna", [_ADDITIONAL_TOOLS, _USER_TURN], ["exec_command"], id="the openai api keeps the item"), + ], +) +def test_codex_additional_tools_item_reaches_each_provider_in_a_shape_it_accepts( + client: TestClient, + respx_mock: respx.MockRouter, + xai_fireworks_and_openai_router: Router, + model: str, + expected_input: list[dict[str, object]], + expected_tool_names: list[str], +) -> None: + _account_catalog(respx_mock) + fireworks: Final = _open_weights_provider(respx_mock) + luna: Final = respx_mock.post(f"{_OPENAI_BASE}/responses").mock(side_effect=_provider_reply) + response: Final = client.post( + "/backend-api/codex/responses", + json={ + "model": model, + "input": [_ADDITIONAL_TOOLS, _USER_TURN], + "tools": [_EXEC_TOOL], + "stream": False, + "store": False, + }, + headers=_ACCOUNT_HEADERS, + ) + assert response.status_code == 200, response.text + sent: Final = json.loads((fireworks if model == "open-weights" else luna).calls.last.request.content) + assert sent["input"] == expected_input + assert [tool["name"] for tool in sent["tools"]] == expected_tool_names + + +_FORBIDDEN_TOOL: Final = {"type": "function", "name": "delete_user", "parameters": {"type": "object"}} + + +@pytest.mark.parametrize( + ("tools", "input_items", "allowed"), + [ + pytest.param([_EXEC_TOOL, _FORBIDDEN_TOOL], [_USER_TURN], False, id="a forbidden top-level tool"), + pytest.param( + [_EXEC_TOOL], + [{"type": "additional_tools", "role": "developer", "tools": [_FORBIDDEN_TOOL]}, _USER_TURN], + False, + id="a forbidden tool in an additional_tools item", + ), + pytest.param([_EXEC_TOOL], [_USER_TURN], True, id="only allowed tools"), + ], +) +def test_native_litellm_turn_enforces_the_key_tool_allowlist( + client: TestClient, + respx_mock: respx.MockRouter, + xai_fireworks_and_openai_router: Router, + restricted_key: Callable[..., str], + tools: list[dict[str, object]], + input_items: list[dict[str, object]], + allowed: bool, +) -> None: + _account_catalog(respx_mock) + fireworks: Final = _open_weights_provider(respx_mock) + key: Final = restricted_key( + ["open-weights"], (*_NATIVE_ROUTES, "/v1/responses"), allowed_tools=(str(_EXEC_TOOL["name"]),) + ) + response: Final = client.post( + "/backend-api/codex/responses", + json={"model": "open-weights", "input": input_items, "tools": tools, "stream": False, "store": False}, + headers={**_ACCOUNT_HEADERS, "x-litellm-api-key": key}, + ) + assert response.status_code == (200 if allowed else 403), response.text + assert fireworks.called is allowed + + +_ANTHROPIC_BASE: Final = "https://anthropic-api.test" + + +def test_codex_namespace_tool_called_through_the_chat_bridge_comes_back_with_its_namespace( + client: TestClient, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr( + proxy_server, + "llm_router", + Router( + model_list=[ + { + "model_name": "claude-sonnet-5-5", + "litellm_params": { + "model": "anthropic/claude-sonnet-5-5", + "api_base": _ANTHROPIC_BASE, + "api_key": "test-anthropic-key", + }, + } + ] + ), + ) + _account_catalog(respx_mock) + anthropic: Final = respx_mock.post(f"{_ANTHROPIC_BASE}/v1/messages").respond( + json={ + "id": "msg_test", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [ + {"type": "tool_use", "id": "toolu_test", "name": "mcp__docs____search", "input": {"query": "proxy"}} + ], + "stop_reason": "tool_use", + "usage": {"input_tokens": 3, "output_tokens": 1}, + } + ) + response: Final = client.post( + "/backend-api/codex/responses", + json={ + "model": "claude-sonnet-5-5", + "input": [_ADDITIONAL_TOOLS, _USER_TURN], + "tools": [_EXEC_TOOL], + "stream": False, + "store": False, + }, + headers=_ACCOUNT_HEADERS, + ) + assert response.status_code == 200, response.text + assert anthropic.called + (call,) = (item for item in response.json()["output"] if item["type"] == "function_call") + assert (call["namespace"], call["name"]) == ("mcp__docs__", "search") + + +_CLOCK_NAMESPACE: Final = _namespace("clock", "sleep") +_CLOCK_TOOLS: Final = {"type": "additional_tools", "role": "developer", "tools": [_CLOCK_NAMESPACE]} +def test_codex_namespace_tool_reaches_fireworks_as_a_namespace( + client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router +) -> None: + _account_catalog(respx_mock) + provider: Final = _open_weights_provider(respx_mock) + response: Final = client.post( + "/backend-api/codex/responses", + json={"model": "open-weights", "input": [_CLOCK_TOOLS, _USER_TURN], "stream": False, "store": False}, + headers=_ACCOUNT_HEADERS, + ) + assert response.status_code == 200, response.text + assert json.loads(provider.calls.last.request.content)["tools"] == [_CLOCK_NAMESPACE] + + + +@pytest.mark.parametrize( + ("drop_params", "expected_tools"), + [ + pytest.param({}, [_CLOCK_NAMESPACE], id="codex's drop_params default merges the copies"), + pytest.param({"drop_params": False}, [_CLOCK_NAMESPACE, _CLOCK_NAMESPACE], id="the caller's off sends each"), + ], +) +def test_codex_tool_list_repeated_on_every_turn_reaches_fireworks_merged_only_under_drop_params( + client: TestClient, + respx_mock: respx.MockRouter, + mixed_router: Router, + drop_params: dict[str, bool], + expected_tools: list[dict[str, object]], +) -> None: + _account_catalog(respx_mock) + provider: Final = _open_weights_provider(respx_mock) + response: Final = client.post( + "/backend-api/codex/responses", + json={ + "model": "open-weights", + "input": [_CLOCK_TOOLS, _USER_TURN, _CLOCK_TOOLS], + "stream": False, + "store": False, + **drop_params, + }, + headers=_ACCOUNT_HEADERS, + ) + assert response.status_code == 200, response.text + assert json.loads(provider.calls.last.request.content)["tools"] == expected_tools + +_CUSTOM_EXEC_TOOL: Final = {"type": "custom", "name": "exec", "description": "Run a command", "format": {"type": "text"}} +_XAI_SEARCH_CALL: Final = { + "id": "call_search", + "type": "function", + "function": {"name": "mcp__docs____search", "arguments": '{"query":"proxy"}'}, +} +_XAI_USAGE: Final = {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4} + + +def _xai_chat_chunk(choice: dict[str, object], **fields: object) -> str: + chunk: Final = { + "id": "chatcmpl-xai", + "object": "chat.completion.chunk", + "created": 1700000000, + "model": "grok-4.7", + "choices": [{"index": 0, **choice}], + **fields, + } + return f"data: {json.dumps(chunk)}\n\n" + + +def _xai_search_call_reply(request: httpx.Request) -> httpx.Response: + if json.loads(request.content).get("stream") is not True: + return httpx.Response( + 200, + json={ + "id": "chatcmpl-xai", + "object": "chat.completion", + "created": 1700000000, + "model": "grok-4.7", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": None, "tool_calls": [_XAI_SEARCH_CALL]}, + "finish_reason": "tool_calls", + } + ], + "usage": _XAI_USAGE, + }, + ) + stream: Final = ( + _xai_chat_chunk( + {"delta": {"role": "assistant", "tool_calls": [{"index": 0, **_XAI_SEARCH_CALL}]}, "finish_reason": None} + ) + + _xai_chat_chunk({"delta": {}, "finish_reason": "tool_calls"}, usage=_XAI_USAGE) + + "data: [DONE]\n\n" + ) + return httpx.Response(200, content=stream.encode(), headers={"content-type": "text/event-stream"}) + + +def _data_lines(stream: bytes) -> list[bytes]: + return [line.removeprefix(b"data: ") for line in stream.splitlines() if line.startswith(b"data: {")] + + +def _function_calls_of(response: httpx.Response, stream: bool) -> list[dict[str, object]]: + items: Final = ( + response.json()["output"] + if not stream + else [ + event["item"] + for event in map(json.loads, _data_lines(response.content)) + if event["type"] == "response.output_item.done" + ] + ) + return [item for item in items if item["type"] == "function_call"] + + +@pytest.mark.parametrize("stream", [False, True]) +def test_codex_turn_for_xai_goes_through_chat_completions_and_its_namespaced_call_comes_back_whole( + client: TestClient, respx_mock: respx.MockRouter, xai_fireworks_and_openai_router: Router, stream: bool +) -> None: + _account_catalog(respx_mock) + responses_endpoint: Final = respx_mock.post(f"{_XAI_BASE}/responses").respond(json=_provider_completed("Hi")) + chat_endpoint: Final = respx_mock.post(f"{_XAI_BASE}/chat/completions").mock(side_effect=_xai_search_call_reply) + response: Final = client.post( + "/backend-api/codex/responses", + json={ + "model": "grok-4.7", + "input": [_ADDITIONAL_TOOLS, _USER_TURN], + "tools": [_EXEC_TOOL, _CUSTOM_EXEC_TOOL], + "stream": stream, + "store": False, + }, + headers=_ACCOUNT_HEADERS, + ) + assert response.status_code == 200, response.text + assert not responses_endpoint.called + sent_tools: Final = json.loads(chat_endpoint.calls.last.request.content)["tools"] + assert [(tool["type"], tool["function"]["name"]) for tool in sent_tools] == [ + ("function", "exec_command"), + ("function", "exec"), + ("function", "collaboration__followup_task"), + ("function", "mcp__docs____search"), + ] + (call,) = _function_calls_of(response, stream) + assert (call["name"], call["namespace"], call["call_id"]) == ("search", "mcp__docs__", "call_search") + + +def test_encrypted_reasoning_stays_only_when_every_deployment_the_router_may_pick_can_decrypt_it( + client: TestClient, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + second_base: Final = "https://second-openai-org.test/v1" + + def replayed_blobs(optional_pre_call_checks: list[str]) -> list[object]: + monkeypatch.setattr( + proxy_server, + "llm_router", + Router( + model_list=[ + { + "model_name": "oai-pair", + "litellm_params": {"model": "openai/gpt-6-luna", "api_base": base, "api_key": "test-key"}, + "model_info": {"id": model_id}, + } + for base, model_id in ((_OPENAI_BASE, _LUNA_ID), (second_base, "second-org-deployment")) + ], + optional_pre_call_checks=optional_pre_call_checks, + ), + ) + providers: Final = tuple( + respx_mock.post(f"{base}/responses").mock(side_effect=_provider_reply) + for base in (_OPENAI_BASE, second_base) + ) + response: Final = client.post( + "/backend-api/codex/responses", + json={"model": "oai-pair", "input": [_USER_TURN, _LUNA_REASONING, _USER_TURN], "store": False}, + headers=_ACCOUNT_HEADERS, + ) + assert response.status_code == 200, response.text + (sent,) = (json.loads(call.request.content)["input"] for provider in providers for call in provider.calls) + return [item.get("encrypted_content") for item in sent if item.get("type") == "reasoning"] + + _account_catalog(respx_mock) + assert replayed_blobs([]) == [] + respx_mock.reset() + _account_catalog(respx_mock) + assert replayed_blobs(["encrypted_content_affinity"]) == [_LUNA_BLOB] + + +def test_encrypted_reasoning_stays_on_a_global_alias_of_a_group_pinned_to_its_origin( + client: TestClient, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + second_base: Final = "https://second-openai-org.test/v1" + monkeypatch.setattr( + proxy_server, + "llm_router", + Router( + model_list=[ + { + "model_name": "oai-pair", + "litellm_params": {"model": "openai/gpt-6-luna", "api_base": base, "api_key": "test-key"}, + "model_info": {"id": model_id}, + } + for base, model_id in ((_OPENAI_BASE, _LUNA_ID), (second_base, "second-org-deployment")) + ], + model_group_alias={"pair-alias": "oai-pair"}, + model_group_affinity_config={"oai-pair": ["encrypted_content_affinity"]}, + ), + ) + _account_catalog(respx_mock) + providers: Final = tuple( + respx_mock.post(f"{base}/responses").mock(side_effect=_provider_reply) for base in (_OPENAI_BASE, second_base) + ) + response: Final = client.post( + "/backend-api/codex/responses", + json={"model": "pair-alias", "input": [_USER_TURN, _LUNA_REASONING, _USER_TURN], "store": False}, + headers=_ACCOUNT_HEADERS, + ) + assert response.status_code == 200, response.text + (sent,) = (json.loads(call.request.content)["input"] for provider in providers for call in provider.calls) + assert [item.get("encrypted_content") for item in sent if item.get("type") == "reasoning"] == [_LUNA_BLOB] + + +class ModelListingCounter(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.listings: Final[list[tuple[str, ...]]] = [] + + async def async_filter_listed_models( + self, user_api_key_dict: UserAPIKeyAuth, model_names: Sequence[str] + ) -> Sequence[str]: + self.listings.append(tuple(model_names)) + return model_names + + +def test_jwt_caller_builds_the_models_etag_once_across_chatgpt_turns( + client: TestClient, respx_mock: respx.MockRouter, mixed_router: Router, monkeypatch: pytest.MonkeyPatch +) -> None: + counter: Final = ModelListingCounter() + monkeypatch.setattr(litellm, "callbacks", [counter]) + monkeypatch.setitem( + proxy_server.app.dependency_overrides, + user_api_key_auth, + lambda: UserAPIKeyAuth(api_key=None, user_id="jwt-user", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + _chatgpt_responses(respx_mock) + turns: Final = tuple( + client.post( + "/backend-api/codex/responses", + json={"model": "gpt-unlisted", "input": "Hello", "stream": True}, + headers={**_HEADERS, "x-litellm-api-key": "test-jwt-credential"}, + ) + for _ in range(2) + ) + assert [turn.status_code for turn in turns] == [200, 200] + assert len(counter.listings) == 1 + first_etag, second_etag = (turn.headers["x-models-etag"] for turn in turns) + assert first_etag == second_etag != _UPSTREAM_MODELS_ETAG diff --git a/tests/unit/proxy/router_entry_point_stubs.py b/tests/unit/proxy/router_entry_point_stubs.py index 1d30cb99b44..1445fe517c6 100644 --- a/tests/unit/proxy/router_entry_point_stubs.py +++ b/tests/unit/proxy/router_entry_point_stubs.py @@ -125,6 +125,8 @@ def route_entry_points_to_stubs(router: _RouterDouble) -> _RouterDouble: return router.get_configured_display_name(name) if field == "service_tiers": return router.get_configured_service_tiers(name, **_given(team_id=team_id)) + if field == "pinned_reasoning_efforts": + return router.get_pinned_reasoning_efforts(name, **_given(team_id=team_id)) if field == "max_input_tokens": return router.get_configured_max_input_tokens(name, **_given(team_id=team_id)) if field == "deployment_model_info": diff --git a/tests/unit/proxy/test_tools_allowlist_enforcement.py b/tests/unit/proxy/test_tools_allowlist_enforcement.py index e306f7be4f8..efaf25c480c 100644 --- a/tests/unit/proxy/test_tools_allowlist_enforcement.py +++ b/tests/unit/proxy/test_tools_allowlist_enforcement.py @@ -242,6 +242,42 @@ class TestCheckToolsAllowlist: assert exc_info.value.type == ProxyErrorTypes.tool_access_denied assert "restricted_tool" in str(exc_info.value.message) + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("allowed_tools", "denied"), + [ + pytest.param(["get_weather"], True, id="forbidden"), + pytest.param(["get_weather", "delete_user"], False, id="allowed"), + ], + ) + async def test_tool_defined_in_an_additional_tools_input_item_is_checked_on_responses_route( + self, allowed_tools: list[str], denied: bool + ) -> None: + body = { + "model": "fireworks_ai/glm-5p3", + "input": [ + { + "type": "additional_tools", + "role": "developer", + "tools": [{"type": "function", "name": "delete_user", "parameters": {"type": "object"}}], + }, + {"type": "message", "role": "user", "content": "Delete user 42"}, + ], + } + check = check_tools_allowlist( + request_body=body, + valid_token=_token(metadata={"allowed_tools": allowed_tools}), + team_object=None, + route="/v1/responses", + ) + if not denied: + await check + return + with pytest.raises(ProxyException) as exc_info: + await check + assert exc_info.value.type == ProxyErrorTypes.tool_access_denied + assert "delete_user" in str(exc_info.value.message) + @pytest.mark.parametrize( "loaded_tool", [ diff --git a/tests/unit/responses/litellm_completion_transformation/test_reasoning_items.py b/tests/unit/responses/litellm_completion_transformation/test_reasoning_items.py index 093d1744418..580ee672500 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_reasoning_items.py +++ b/tests/unit/responses/litellm_completion_transformation/test_reasoning_items.py @@ -1,12 +1,18 @@ import json +import pytest + from litellm.responses.litellm_completion_transformation.reasoning_items import ( decode_thinking_blocks, encode_thinking_blocks, is_litellm_minted_reasoning_item, is_minted_reasoning_item_id, + is_other_provider_reasoning_item, mint_reasoning_item_id, ) +from litellm.responses.litellm_completion_transformation.transformation import LiteLLMCompletionResponsesConfig +from litellm.responses.utils import ResponsesAPIRequestUtils +from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams A_PROVIDER_OWNED_REASONING_ITEM_ID = "rs_08d3a89dbb92277a006abf04f4266087d0b4eedacd7848f306" A_PROVIDER_OWNED_ENCRYPTED_BLOB = "gAAAAABo-opaque-provider-blob" @@ -58,5 +64,82 @@ def test_a_reasoning_item_is_litellm_minted_by_its_id_or_by_its_encoded_thinking "encrypted_content": A_PROVIDER_OWNED_ENCRYPTED_BLOB, } ) - assert not is_litellm_minted_reasoning_item({"type": "message", "id": mint_reasoning_item_id(), "role": "assistant"}) + assert not is_litellm_minted_reasoning_item( + {"type": "message", "id": mint_reasoning_item_id(), "role": "assistant"} + ) assert not is_litellm_minted_reasoning_item("a bare string input") + + +SUMMARY = [{"type": "summary_text", "text": "Paris first."}] +PROVIDER_OWNED_ITEM = { + "type": "reasoning", + "id": A_PROVIDER_OWNED_REASONING_ITEM_ID, + "summary": SUMMARY, + "encrypted_content": A_PROVIDER_OWNED_ENCRYPTED_BLOB, +} + + +@pytest.mark.parametrize( + "item", + [ + pytest.param({"type": "reasoning", "id": mint_reasoning_item_id(), "summary": SUMMARY}, id="minted id"), + pytest.param( + {"type": "reasoning", "summary": SUMMARY, "encrypted_content": encode_thinking_blocks([SIGNED_BLOCK])}, + id="encoded thinking blocks", + ), + pytest.param( + {"type": "reasoning", "summary": [], "content": [{"type": "reasoning_text", "text": "Paris first."}]}, + id="plain-text reasoning content", + ), + pytest.param( + { + **PROVIDER_OWNED_ITEM, + "encrypted_content": ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id( + A_PROVIDER_OWNED_ENCRYPTED_BLOB, "another-deployment" + ), + }, + id="blob wrapped with a litellm deployment id", + ), + ], +) +def test_reasoning_items_another_provider_produced_are_recognized(item): + assert is_other_provider_reasoning_item(item) + + +@pytest.mark.parametrize( + "item", + [ + pytest.param(PROVIDER_OWNED_ITEM, id="provider-owned blob"), + pytest.param({"type": "reasoning", "summary": SUMMARY}, id="summary only, no id"), + pytest.param( + {"type": "reasoning", "summary": SUMMARY, "content": None, "encrypted_content": None}, id="nulled replay" + ), + pytest.param({"type": "reasoning", "summary": SUMMARY, "content": []}, id="empty content"), + pytest.param( + {"type": "message", "id": mint_reasoning_item_id(), "role": "user", "content": "hi"}, id="not reasoning" + ), + pytest.param("a stray string", id="not an object"), + ], +) +def test_reasoning_items_the_provider_can_replay_are_kept(item): + assert not is_other_provider_reasoning_item(item) + + +def test_signed_thinking_survives_the_model_id_wrap_on_its_way_back_to_the_chat_completions_bridge(): + wrapped = ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id( + encode_thinking_blocks([SIGNED_BLOCK]), "anthropic-deployment" + ) + messages = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model="anthropic/claude-sonnet-5-5", + input=[ + {"role": "user", "content": "Capital of France?"}, + {"type": "reasoning", "id": mint_reasoning_item_id(), "summary": [], "encrypted_content": wrapped}, + {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "Paris."}]}, + {"role": "user", "content": "And of Italy?"}, + ], + responses_api_request=ResponsesAPIOptionalRequestParams(), + )["messages"] + + assert [message.get("thinking_blocks") for message in messages if message["role"] == "assistant"] == [ + [SIGNED_BLOCK] + ] diff --git a/tests/unit/responses/test_additional_tools.py b/tests/unit/responses/test_additional_tools.py index 290670a7c0a..d2c8cbacf78 100644 --- a/tests/unit/responses/test_additional_tools.py +++ b/tests/unit/responses/test_additional_tools.py @@ -1,8 +1,9 @@ from typing import Final +import pytest from pydantic import JsonValue -from litellm.responses.additional_tools import hoist_additional_tools +from litellm.responses.additional_tools import hoist_additional_tools, with_additional_tools_hoisted _EXEC_TOOL = { "type": "custom", @@ -134,3 +135,10 @@ def test_tool_list_repeated_on_every_turn_is_sent_as_is_without_drop_params() -> assert hoisted.tools == (_TOP_LEVEL_TOOL, _function("exec", "old"), _function("exec", "new")) assert hoisted.input == [_USER_MESSAGE, _USER_MESSAGE] + + +@pytest.mark.parametrize("tools", [7, "exec", {"type": "function"}], ids=["int", "str", "dict"]) +def test_request_whose_top_level_tools_is_not_a_list_stays_as_sent_for_the_provider_to_refuse(tools: object) -> None: + request: Final = {"input": [_tool_list(_function("exec")), _USER_MESSAGE], "tools": tools} + + assert with_additional_tools_hoisted(request, drop_params=True) == request diff --git a/tests/unit/test_router/core/test_registry_characterization_model_info.py b/tests/unit/test_router/core/test_registry_characterization_model_info.py index d79aebf75b9..578183d7f02 100644 --- a/tests/unit/test_router/core/test_registry_characterization_model_info.py +++ b/tests/unit/test_router/core/test_registry_characterization_model_info.py @@ -273,6 +273,27 @@ def test_max_input_tokens_leave_out_blocked_deployments_and_other_teams(make_cor assert core.registry().group("mt", field="max_input_tokens", team_id="TA") == (2, 3) +def test_pinned_reasoning_efforts_are_each_routable_deployments_own_value_in_order(make_core: MakeCore) -> None: + core = make_core( + model_list=[ + deployment("pe", "p1", reasoning_effort="high"), + deployment("pe", "p2"), + deployment("pe", "p3", reasoning_effort="low"), + deployment("pe", "p4", reasoning_effort="minimal", model_info={"blocked": True}), + deployment("pe", "p5", reasoning_effort="medium", model_info={"team_id": "TA"}), + ] + ) + + assert core.registry().group("pe", field="pinned_reasoning_efforts") == ("high", None, "low") + assert core.registry().group("pe", field="pinned_reasoning_efforts", team_id="TA") == ( + "high", + None, + "low", + "medium", + ) + assert core.registry().group("not-configured", field="pinned_reasoning_efforts") == () + + def test_max_input_tokens_skip_a_value_a_saved_model_info_copied_from_the_cost_map(make_core: MakeCore) -> None: catalog_limit = GetModelCostMap.loaded_model_cost_map()["gpt-4o"]["max_input_tokens"] core = make_core( diff --git a/tests/unit/test_router/test_router_entry_points.py b/tests/unit/test_router/test_router_entry_points.py index c3d5d962123..06287ba7e0b 100644 --- a/tests/unit/test_router/test_router_entry_points.py +++ b/tests/unit/test_router/test_router_entry_points.py @@ -514,6 +514,81 @@ def test_attribute_errors_about_other_objects_pass_through_unchanged(router): assert _public_attribute_error(router, "_core_x", "x", other) is other +_MINTING_DEPLOYMENT = "dep-minting-org" +_OTHER_ORG_DEPLOYMENT = "dep-other-org" +_BOTH_ORGS = (_MINTING_DEPLOYMENT, _OTHER_ORG_DEPLOYMENT) +_GROUP_AFFINITY = ["encrypted_content_affinity"] + + +@pytest.mark.parametrize( + "affinity, target_ids, reasoning_stays", + [ + pytest.param({}, _BOTH_ORGS, False, id="no affinity, one target cannot decrypt"), + pytest.param({}, (_MINTING_DEPLOYMENT,), True, id="no affinity, the only target minted it"), + pytest.param( + {"optional_pre_call_checks": _GROUP_AFFINITY}, _BOTH_ORGS, True, id="global affinity, origin among targets" + ), + pytest.param( + {"optional_pre_call_checks": _GROUP_AFFINITY}, + (_OTHER_ORG_DEPLOYMENT,), + False, + id="global affinity, origin not among targets", + ), + pytest.param( + {"optional_pre_call_checks": [], "model_group_affinity_config": {"pair": _GROUP_AFFINITY}}, + _BOTH_ORGS, + True, + id="affinity on for this group", + ), + pytest.param( + {"optional_pre_call_checks": [], "model_group_affinity_config": {"other-group": _GROUP_AFFINITY}}, + _BOTH_ORGS, + False, + id="affinity on for another group only", + ), + ], +) +def test_decryptable_reasoning_keeps_encrypted_reasoning_only_when_the_router_sends_it_to_a_deployment_that_decrypts_it( + affinity, target_ids, reasoning_stays +): + from litellm.responses.utils import ResponsesAPIRequestUtils + + r = Router( + model_list=[ + { + "model_name": "pair", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_base": api_base, "api_key": "sk-pair"}, + "model_info": {"id": model_id}, + } + for api_base, model_id in ( + ("https://minting-org.test/v1", _MINTING_DEPLOYMENT), + ("https://other-org.test/v1", _OTHER_ORG_DEPLOYMENT), + ) + ], + **affinity, + ) + summary = [{"type": "summary_text", "text": "multiply 17 by 23"}] + user_turn = {"type": "message", "role": "user", "content": "What is 17*23?"} + reasoning = { + "type": "reasoning", + "id": "rs_minted", + "encrypted_content": ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id( + "gAAAAA-blob", _MINTING_DEPLOYMENT + ), + "summary": summary, + } + request_input = [user_turn, reasoning] + targets = tuple(r.registry().deployment(model_id) for model_id in target_ids) + try: + kept = r.registry().decryptable_reasoning("pair", request_input, targets) + finally: + r.retire() + + readable_only = {"type": "reasoning", "summary": summary} + assert kept == (user_turn, reasoning if reasoning_stays else readable_only) + assert request_input == [user_turn, reasoning], "the caller's input must not change" + + # Router-family helpers whose Router parameter became a RoutingContext keep the old spellings. diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index ab1f3c253ea..5cef303ee67 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -1864,13 +1864,15 @@ export interface paths { }; /** * Codex Passthrough Route - * @description Forward a client's ChatGPT subscription with separate LiteLLM authentication. + * @description Forward a client's ChatGPT subscription with separate LiteLLM authentication, listing and serving the + * key's other chat models through LiteLLM. */ get: operations["codex_passthrough_route_backend_api_codex__endpoint__get"]; put?: never; /** * Codex Passthrough Route - * @description Forward a client's ChatGPT subscription with separate LiteLLM authentication. + * @description Forward a client's ChatGPT subscription with separate LiteLLM authentication, listing and serving the + * key's other chat models through LiteLLM. */ post: operations["codex_passthrough_route_backend_api_codex__endpoint__post"]; delete?: never; @@ -55970,7 +55972,7 @@ export interface operations { }; }; responses: { - /** @description Unmodified Codex response or server-sent events */ + /** @description Codex response, the model catalog with the key's LiteLLM models, or server-sent events */ 200: { headers: { [name: string]: unknown; From 150f4b706b00f75d31898733f5e84eafa1bf3a90 Mon Sep 17 00:00:00 2001 From: "m.torres" <93216769+chuckiihub@users.noreply.github.com> Date: Fri, 6 Nov 2026 20:32:08 +0100 Subject: [PATCH 15/39] fix(router): count embedding batches per input in pre-call checks (#49851) * fix(router): count embedding batches per input in pre-call checks Embedding input lists went through the Responses input transform, which calls .get() on each item, so every list-input /embeddings call logged "failed to count tokens" and skipped the context window check. Lists of texts, token IDs or token ID lists now count as their longest input, since an embedding model's context window applies to each input on its own * fix(router): keep empty and boolean input lists on the Responses count path An empty list carries no embedding evidence, and counting it as a zero token batch skipped the instructions and tools of an empty Responses input. Boolean lists are not token IDs * fix(router): honor disable_token_counter for token ID embedding input Text inputs already counted as zero through token_counter when token counting is disabled, while token ID inputs were still measured by length and could be rejected. The router reads the flag through its dynamic config like its other process settings * refactor(router): type the embedding count helper with a read-only sequence and final locals * fix(router): count embedding batches per input on the media-free text projection The per-input count now runs on the text projection from the embedding media split, so data URIs a deployment embeds as media keep the media path and only text items are checked against the context window. A caption next to an image is checked on its own, and a text-only deployment still counts an image data URI as text * fix(router): keep embedding images out of the pre-call count only on image backends, and skip tokenizing inputs that fit by bytes Media leaves the embedding count only when every deployment in the group declares image embedding input, so a text-only backend still counts a data URI as text and an image backend outside the TPM media list no longer gets a false 400. A text no longer than the smallest window in UTF-8 bytes counts as its byte length without the tokenizer, since a token is never shorter than a byte. * fix(router): keep embedding images out of the pre-call count when any backend in the group embeds images Mark Cohere embed-english-light-v3.0 as embedding images, as its v3 siblings are. * fix(router): check embedding text batches only with an exact OpenAI tokenizer litellm's default tokenizer counts non-English text several times higher than non-OpenAI embedding providers do, so counting batches with it refused inputs the provider accepts. Text batches on those groups now stay unchecked as at the merge base, without the error log. Main's per-provider media split and the cost map are restored, so strings and embedding media keep main's path. * fix(router): count long embedding batches in slices so chat counts do not wait behind them A text batch is now tokenized in slices of 65,536 characters, each its own offloaded call, so the FIFO count limiter can serve a chat count between slices. The no-OpenAI-tokenizer rule for text lists moved into the pre-call check, so RoutingContext.count_pre_call_check_tokens counts a text list by its longest text. * fix(router): read embedding media off the event loop and wait on the count limiter in the fairness test --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../prompt_templates/tpm_media_utils.py | 6 +- litellm/router_config.py | 3 + litellm/router_rust/steps/precall.py | 200 ++++++++++++++--- tests/unit/test_router/core/scripted_host.py | 1 + ...uction_precall_characterization_precall.py | 46 ++++ tests/unit/test_router/test_router.py | 209 ++++++++++++++++++ 6 files changed, 429 insertions(+), 36 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/tpm_media_utils.py b/litellm/litellm_core_utils/prompt_templates/tpm_media_utils.py index a4cb52d689c..a3e2d2e15d9 100644 --- a/litellm/litellm_core_utils/prompt_templates/tpm_media_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/tpm_media_utils.py @@ -202,7 +202,7 @@ def _tpm_embedding_leaf(value: object) -> _TPMMediaProjection: part: Final = as_str_mapping(value) if part is not None and part.get("type") == "file": return _tpm_embedding_file_block(part) - if not isinstance(value, str) or not value.startswith("data:"): + if not isinstance(value, str) or is_plain_embedding_text(value): return _tpm_media_leaf(value) if not _is_base64_data_uri(value): return _TPMMediaProjection(value) @@ -210,6 +210,10 @@ def _tpm_embedding_leaf(value: object) -> _TPMMediaProjection: return _TPMMediaProjection("" if media else value, media) +def is_plain_embedding_text(value: object) -> bool: + return isinstance(value, str) and not value.startswith("data:") + + def _is_json_array(value: object) -> bool: return isinstance(value, (list, tuple)) diff --git a/litellm/router_config.py b/litellm/router_config.py index 36154af296c..1d38fe84d96 100644 --- a/litellm/router_config.py +++ b/litellm/router_config.py @@ -45,6 +45,7 @@ class RouterDynamicConfig: context_window_fallbacks: Sequence[object] | None content_policy_fallbacks: Sequence[object] | None drop_params: bool | None + disable_token_counter: bool provider_list: Sequence[str] expose_router_debug_in_errors: bool redact_user_api_key_info: bool | None @@ -102,6 +103,7 @@ def current_dynamic_config() -> RouterDynamicConfig: context_window_fallbacks=litellm.context_window_fallbacks, content_policy_fallbacks=litellm.content_policy_fallbacks, drop_params=litellm.drop_params, + disable_token_counter=litellm.disable_token_counter, provider_list=litellm.provider_list, expose_router_debug_in_errors=litellm.expose_router_debug_in_errors, redact_user_api_key_info=litellm.redact_user_api_key_info, @@ -130,6 +132,7 @@ _LIVE_SOURCES: Final = MappingProxyType( "context_window_fallbacks": "context_window_fallbacks", "content_policy_fallbacks": "content_policy_fallbacks", "drop_params": "drop_params", + "disable_token_counter": "disable_token_counter", "provider_list": "provider_list", "expose_router_debug_in_errors": "expose_router_debug_in_errors", "redact_user_api_key_info": "redact_user_api_key_info", diff --git a/litellm/router_rust/steps/precall.py b/litellm/router_rust/steps/precall.py index 227bb8de66f..c68c5fd5982 100644 --- a/litellm/router_rust/steps/precall.py +++ b/litellm/router_rust/steps/precall.py @@ -13,7 +13,9 @@ deployment records rather than a step, and those need not be in the Rust registr """ from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass from functools import partial +from itertools import accumulate, groupby from types import MappingProxyType from typing import TYPE_CHECKING, Final, Protocol @@ -21,6 +23,7 @@ from litellm._logging import verbose_router_logger from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.deployment_pricing import deployment_backend_models from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider +from litellm.litellm_core_utils.prompt_templates.tpm_media_utils import is_plain_embedding_text from litellm.litellm_core_utils.token_counter import ( offload_token_count, served_openai_models, @@ -38,6 +41,17 @@ if TYPE_CHECKING: # The chat params a deployment must take when the request sends them (`_PRE_CALL_CHECK_FILTERED_PARAMS`). _FILTERED_PARAMS: Final = frozenset({"response_format"}) +_CHARS_PER_OFFLOADED_COUNT: Final = 1 << 16 + + +@dataclass(frozen=True, slots=True) +class _TextBatch: + texts: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class _TokenIdBatch: + longest: int class PreCallRegistry(Protocol): @@ -59,35 +73,111 @@ def count_tokens( request_kwargs: Mapping[str, object] | None, tokenizer_model: str, deployments: Sequence[Mapping[str, object]] = (), + fits_within: int = 0, ) -> int: """``_count_pre_call_check_tokens``: messages, or Responses input normalized to messages, plus the tools and Anthropic ``system`` block the message list never carries. Embedding media an item of ``deployments`` reads as media is left out of the input: it never counts as text.""" from litellm.llms.anthropic.messages.native.utils import anthropic_system_to_openai_message - extras: Final[Mapping[str, object]] = request_kwargs if request_kwargs is not None else MappingProxyType({}) - raw_instructions: Final = extras.get("instructions") - instructions: Final = raw_instructions if isinstance(raw_instructions, str) else None - raw_tools: Final = extras.get("tools") - tools: Final = raw_tools if _is_list(raw_tools) and raw_tools else None + if messages is None and input is not None: + return _input_tokens(host, _text_input(input, deployments), request_kwargs, tokenizer_model, fits_within) + if messages is None: + raise ValueError("Either messages or input must be provided to count tokens") + extras: Final = _extras(request_kwargs) system_message: Final = anthropic_system_to_openai_message(extras.get("system")) counter: Final = host.invoke("token_counter") # pyright: ignore[reportUnknownMemberType] # the host's overloads name litellm's untyped returns - if messages is not None: - counted_messages: Final = (system_message, *_items(messages)) if system_message is not None else messages - return counter(model=tokenizer_model, messages=counted_messages, tools=tools) - if input is not None: - from litellm.responses.litellm_completion_transformation.transformation import ( - LiteLLMCompletionResponsesConfig, - ) + counted_messages: Final = (system_message, *_items(messages)) if system_message is not None else messages + return counter(model=tokenizer_model, messages=counted_messages, tools=_tools(extras)) - text_input, _ = split_embedding_input_tokens(input, partial(deployment_backend_models, deployments)) - input_messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( # pyright: ignore[reportUnknownMemberType] # its input type is untyped - input=text_input, # pyright: ignore[reportArgumentType] # str | list, as the request carries it - responses_api_request={"instructions": instructions} if instructions is not None else {}, - only_since_compaction=True, - ) - return counter(model=tokenizer_model, messages=input_messages, tools=tools) - raise ValueError("Either messages or input must be provided to count tokens") + +def _input_tokens( + host: "PythonHost", + text_input: object, + request_kwargs: Mapping[str, object] | None, + tokenizer_model: str, + fits_within: int, +) -> int: + from litellm.responses.litellm_completion_transformation.transformation import ( + LiteLLMCompletionResponsesConfig, + ) + + counter: Final = host.invoke("token_counter") # pyright: ignore[reportUnknownMemberType] # see count_tokens + match _embedding_batch(text_input): + case _TextBatch(texts): + return _longest_text_tokens(counter, tokenizer_model, texts, fits_within) + case _TokenIdBatch(longest): + return _token_id_count(host, longest) + case None: + pass + extras: Final = _extras(request_kwargs) + raw_instructions: Final = extras.get("instructions") + instructions: Final = raw_instructions if isinstance(raw_instructions, str) else None + input_messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( # pyright: ignore[reportUnknownMemberType] # its input type is untyped + input=text_input, # pyright: ignore[reportArgumentType] # str | list, as the request carries it + responses_api_request={"instructions": instructions} if instructions is not None else {}, + only_since_compaction=True, + ) + return counter(model=tokenizer_model, messages=input_messages, tools=_tools(extras)) + + +def _extras(request_kwargs: Mapping[str, object] | None) -> Mapping[str, object]: + return request_kwargs if request_kwargs is not None else MappingProxyType({}) + + +def _tools(extras: Mapping[str, object]) -> object: + raw_tools: Final = extras.get("tools") + return raw_tools if _is_list(raw_tools) and raw_tools else None + + +def _text_input(input: object, deployments: Sequence[Mapping[str, object]]) -> object: # the request field's name + text_input, _ = split_embedding_input_tokens(input, partial(deployment_backend_models, deployments)) + return text_input + + +def _embedding_batch(text_input: object) -> _TextBatch | _TokenIdBatch | None: + items: Final = _items(text_input) if _is_list(text_input) else () + if not items: + return None + texts: Final = tuple(item for item in items if isinstance(item, str)) + if len(texts) == len(items): + return _TextBatch(texts) + if all(_is_token_id(item) for item in items): + return _TokenIdBatch(len(items)) + if all(_is_token_id_list(item) for item in items): + return _TokenIdBatch(max(len(_items(item)) for item in items)) + return None + + +def _longest_text_tokens( + counter: Callable[..., int], tokenizer_model: str, texts: Sequence[str], fits_within: int +) -> int: + return max(_text_tokens(counter, tokenizer_model, text, fits_within) for text in texts) + + +def _offload_slices(texts: Sequence[str]) -> tuple[tuple[str, ...], ...]: + ends: Final = accumulate(len(text) for text in texts) + slices: Final = groupby( + zip(ends, texts, strict=True), key=lambda end_and_text: (end_and_text[0] - 1) // _CHARS_PER_OFFLOADED_COUNT + ) + return tuple(tuple(text for _, text in texts_in_slice) for _, texts_in_slice in slices) + + +def _text_tokens(counter: Callable[..., int], tokenizer_model: str, text: str, fits_within: int) -> int: + size: Final = len(text.encode(errors="surrogatepass")) + return size if size <= fits_within else counter(model=tokenizer_model, text=text) + + +def _is_token_id(value: object) -> bool: + return isinstance(value, int) and not isinstance(value, bool) + + +def _is_token_id_list(value: object) -> bool: + return _is_list(value) and all(_is_token_id(token) for token in _items(value)) + + +def _token_id_count(host: "PythonHost", count: int) -> int: + return 0 if host.dynamic().disable_token_counter else count def _max_input_tokens(registry: PreCallRegistry, model: str, deployment: Mapping[str, object]) -> int | None: @@ -246,16 +336,31 @@ class PreCallFacts: def _candidate_records(self) -> Sequence[Mapping[str, object]]: return tuple(self._record(id) for id in self._candidates) + def _fits_within(self) -> int: + return min((limit for limit in self._windows.values() if limit is not None), default=0) + + def _input_is_a_list(self) -> bool: + return self._messages is None and _is_list(self._input) + def _count(self) -> int | None: try: - return count_tokens( - self._host, - self._messages, - self._input, - self._request, - self._tokenizer_model(), - self._candidate_records(), - ) + tokenizer_model: Final = self._tokenizer_model() + if not self._input_is_a_list(): + return count_tokens( + self._host, + self._messages, + self._input, + self._request, + tokenizer_model, + self._candidate_records(), + self._fits_within(), + ) + text_input: Final = _text_input(self._input, self._candidate_records()) + match _embedding_batch(text_input): + case _TextBatch() if not tokenizer_model: + return 0 + case _: + return _input_tokens(self._host, text_input, self._request, tokenizer_model, self._fits_within()) except Exception as e: # noqa: BLE001 # best-effort: an uncountable prompt must not fail the request verbose_router_logger.error( "litellm.router.py::_acount_pre_call_check_tokens: failed to count tokens. Got - %s", e @@ -264,14 +369,39 @@ class PreCallFacts: async def _count_async(self) -> int | None: try: - return await offload_token_count(count_tokens)( - self._host, - self._messages, - self._input, - self._request, - self._tokenizer_model(), - self._candidate_records(), + tokenizer_model: Final = self._tokenizer_model() + if not self._input_is_a_list(): + return await offload_token_count(count_tokens)( + self._host, + self._messages, + self._input, + self._request, + tokenizer_model, + self._candidate_records(), + self._fits_within(), + ) + text_input: Final = ( + self._input + if all(map(is_plain_embedding_text, _items(self._input))) + else await offload_token_count(_text_input)(self._input, self._candidate_records()) ) + match _embedding_batch(text_input): + case _TextBatch() if not tokenizer_model: + return 0 + case _TextBatch(texts): + counter: Final = self._host.invoke("token_counter") # pyright: ignore[reportUnknownMemberType] # see count_tokens + fits_within: Final = self._fits_within() + count_slice: Final = offload_token_count(_longest_text_tokens) + return max( + [ + await count_slice(counter, tokenizer_model, texts_in_slice, fits_within) + for texts_in_slice in _offload_slices(texts) + ] + ) + case _: + return await offload_token_count(_input_tokens)( + self._host, text_input, self._request, tokenizer_model, self._fits_within() + ) except Exception as e: # noqa: BLE001 # best-effort: an uncountable prompt must not fail the request verbose_router_logger.error( "litellm.router.py::_acount_pre_call_check_tokens: failed to count tokens. Got - %s", e diff --git a/tests/unit/test_router/core/scripted_host.py b/tests/unit/test_router/core/scripted_host.py index 33164bd4408..066faad4db8 100644 --- a/tests/unit/test_router/core/scripted_host.py +++ b/tests/unit/test_router/core/scripted_host.py @@ -166,6 +166,7 @@ def dynamic_config(**changes: object) -> RouterDynamicConfig: # kwargs-ok: Rout context_window_fallbacks=None, content_policy_fallbacks=None, drop_params=None, + disable_token_counter=False, provider_list=litellm.provider_list, expose_router_debug_in_errors=False, redact_user_api_key_info=None, diff --git a/tests/unit/test_router/core/test_construction_precall_characterization_precall.py b/tests/unit/test_router/core/test_construction_precall_characterization_precall.py index 9cdb3f55d17..c9e04117f4d 100644 --- a/tests/unit/test_router/core/test_construction_precall_characterization_precall.py +++ b/tests/unit/test_router/core/test_construction_precall_characterization_precall.py @@ -27,6 +27,7 @@ PICKS: Final = 60 # three-way shuffles miss a deployment in 20 picks about once SHORT: Final = [{"role": "user", "content": "hi"}] LONG_TEXT: Final = " ".join(["word"] * 100) LONG: Final = [{"role": "user", "content": LONG_TEXT}] +OPENAI_EMBEDDER: Final = "text-embedding-3-small" EMBEDDING_IMAGE: Final = f"data:image/png;base64,{base64.b64encode(bytes(range(256)) * 1024).decode()}" IMAGE: Final = [ { @@ -86,6 +87,14 @@ def _small_and_big( ) +def _small_and_big_openai_embedders(make_core: MakeCore, small: int) -> RouterCore: + counted_as: Final = {"model_info": {"base_model": OPENAI_EMBEDDER}} + return make_core( + model_list=[windowed("g", "small", small, **counted_as), windowed("g", "big", 10000, **counted_as)], + enable_pre_call_checks=True, + ) + + def _json_mode_group(make_core: MakeCore, **arguments: object) -> RouterCore: # kwargs-ok: constructor arguments return make_core( model_list=[deployment("g", "old", model=NO_JSON_MODE_MODEL), deployment("g", "new", model=JSON_MODE_MODEL)], @@ -279,6 +288,43 @@ async def test_an_embedding_image_stays_out_of_the_context_window_on_a_backend_t assert await _picked(core, times=2, input=EMBEDDING_IMAGE) == {"image"} +@pytest.mark.asyncio +async def test_an_embedding_input_over_the_smaller_window_in_bytes_but_not_in_tokens_fits_it( + make_core: MakeCore, +) -> None: + short_words: Final = "word " * 8 + assert litellm.token_counter(model=OPENAI_EMBEDDER, text=short_words) <= 10 < len(short_words.encode()) + assert await _picked(_small_and_big_openai_embedders(make_core, small=10), input=["hi", short_words]) == { + "small", + "big", + } + + +@pytest.mark.asyncio +async def test_an_embedding_input_that_fits_the_window_in_characters_but_not_in_tokens_is_counted( + make_core: MakeCore, +) -> None: + dense: Final = "\ua66e" * 40 + assert len(dense) <= 50 < litellm.token_counter(model=OPENAI_EMBEDDER, text=dense) + assert await _picked(_small_and_big_openai_embedders(make_core, small=50), times=10, input=["hi", dense]) == {"big"} + + +@pytest.mark.asyncio +async def test_an_embedding_batch_is_never_refused_on_the_default_tokenizers_count(make_core: MakeCore) -> None: + dense: Final = "\ua66e" * 40 + assert litellm.token_counter(text=dense) > 50 + assert await _picked(_small_and_big(make_core, small=50), input=["hi", dense]) == {"small", "big"} + + +@pytest.mark.asyncio +async def test_an_embedding_input_with_a_lone_surrogate_is_counted(make_core: MakeCore) -> None: + unpaired: Final = "\ud800" + "word " * 20 + assert litellm.token_counter(model=OPENAI_EMBEDDER, text=unpaired) > 10 + assert await _picked(_small_and_big_openai_embedders(make_core, small=10), times=10, input=["hi", unpaired]) == { + "big" + } + + @pytest.mark.asyncio async def test_an_embedding_image_counts_as_text_on_a_text_only_backend(make_core: MakeCore) -> None: core: Final = make_core( diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 8f8676103b2..1b98cd4b4b6 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -4396,6 +4396,48 @@ def test_context_window_pre_call_check_counts_an_embedding_image_as_text_on_a_te ) +@pytest.mark.parametrize( + ("backend", "fits"), + [ + pytest.param("vertex_ai/gemini-embedding-2", True, id="image-backend-without-an-openai-tokenizer"), + pytest.param("text-embedding-3-small", False, id="text-only-backend-counts-the-image"), + ], +) +def test_context_window_pre_call_check_on_a_caption_next_to_an_embedding_image(backend: str, fits: bool, caplog): + caption: Final = "word " * 40 + router: Final = _embedding_context_router(backend, window=20) + deployments: Final = router.registry().deployments("embedder") + + with caplog.at_level(logging.ERROR, logger="LiteLLM Router"): + if fits: + kept: Final = router._pre_call_checks( + model="embedder", healthy_deployments=deployments, input=[caption, _EMBEDDING_IMAGE_INPUT] + ) + assert len(kept) == 1 + else: + with pytest.raises(litellm.ContextWindowExceededError): + router._pre_call_checks( + model="embedder", healthy_deployments=deployments, input=[caption, _EMBEDDING_IMAGE_INPUT] + ) + + assert "failed to count tokens" not in caplog.text + + +@pytest.mark.asyncio +async def test_an_async_pre_call_check_reads_embedding_media_off_the_event_loop(): + from tests.unit.litellm_core_utils.event_loop_lag import assert_loop_stayed_free, timed_with_loop_lags + + router: Final = _embedding_context_router("vertex_ai/gemini-embedding-2", window=20) + caption_and_images: Final = ["a red square", *[_EMBEDDING_IMAGE_INPUT] * 400] + + deployment, took, lags = await timed_with_loop_lags( + lambda: router.select("embedder", {}, input=caption_and_images) + ) + + assert deployment["litellm_params"]["model"] == "vertex_ai/gemini-embedding-2" + assert_loop_stayed_free(took, lags) + + async def test_a_wildcard_route_checks_the_context_window_with_the_requested_openai_models_tokenizer(): limit: Final = ( litellm.token_counter(model="openai/gpt-6.1-sol", messages=_MULTILINGUAL_PROMPT) @@ -4547,6 +4589,173 @@ async def test_aresponses_enforces_context_window_pre_call_check(): ) +def _embedding_router(max_input_tokens: int) -> litellm.Router: + return litellm.Router( + model_list=[ + { + "model_name": "embed", + "litellm_params": { + "model": "openai/text-embedding-3-small", + "api_key": "sk-test", + "mock_response": [0.1, 0.2], + }, + "model_info": {"max_input_tokens": max_input_tokens}, + } + ], + enable_pre_call_checks=True, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "embedding_input", + [ + pytest.param(["one two three four five six"] * 3, id="texts"), + pytest.param([list(range(6))] * 3, id="token-id-lists"), + pytest.param(list(range(10)), id="token-ids-at-the-limit"), + ], +) +async def test_aembedding_accepts_a_batch_whose_inputs_each_fit_the_context_window(embedding_input, caplog): + router: Final = _embedding_router(max_input_tokens=10) + + with caplog.at_level(logging.ERROR, logger="LiteLLM Router"): + response: Final = await router.aembedding(model="embed", input=embedding_input) + + assert response.data + assert "failed to count tokens" not in caplog.text + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "embedding_input", + [ + pytest.param(["one", " ".join(["word"] * 20)], id="texts"), + pytest.param(list(range(11)), id="token-ids"), + pytest.param([[1], list(range(11))], id="token-id-lists"), + ], +) +async def test_aembedding_rejects_a_batch_with_one_input_over_the_context_window(embedding_input): + router: Final = _embedding_router(max_input_tokens=10) + + with pytest.raises(litellm.ContextWindowExceededError): + await router.aembedding(model="embed", input=embedding_input) + + +def test_embedding_rejects_a_batch_with_one_input_over_the_context_window(): + router: Final = _embedding_router(max_input_tokens=10) + + with pytest.raises(litellm.ContextWindowExceededError): + router.embedding(model="embed", input=["one", " ".join(["word"] * 20)]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "embedding_input", + [ + pytest.param(["one", " ".join(["word"] * 20)], id="texts"), + pytest.param(list(range(11)), id="token-ids"), + pytest.param([[1], list(range(11))], id="token-id-lists"), + ], +) +async def test_aembedding_skips_the_context_window_check_when_token_counting_is_disabled(embedding_input, monkeypatch): + monkeypatch.setattr(litellm, "disable_token_counter", True) + router: Final = _embedding_router(max_input_tokens=10) + + response: Final = await router.aembedding(model="embed", input=embedding_input) + + assert response.data + + +async def _until_every_count_slot_is_taken() -> None: + from litellm.constants import TOKEN_COUNTER_MAX_CONCURRENT_COUNTS + from litellm.litellm_core_utils.token_counter import _count_offload_limiter_for_this_loop + + while _count_offload_limiter_for_this_loop().borrowed_tokens < TOKEN_COUNTER_MAX_CONCURRENT_COUNTS: + await asyncio.sleep(0) + + +@pytest.mark.asyncio +async def test_a_chat_pre_call_count_does_not_wait_behind_long_embedding_batches(): + from litellm.constants import TOKEN_COUNTER_MAX_CONCURRENT_COUNTS + + window: Final = {"max_input_tokens": 10000} + router: Final = litellm.Router( + model_list=[ + { + "model_name": "embed", + "litellm_params": { + "model": "openai/text-embedding-3-small", + "api_key": "sk-test", + "mock_response": [0.1], + }, + "model_info": window, + }, + { + "model_name": "chat", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test", "mock_response": "hi"}, + "model_info": window, + }, + ], + enable_pre_call_checks=True, + ) + batch: Final = ["word " * 2100] * 76 + batches: Final = tuple( + asyncio.create_task(router.aembedding(model="embed", input=batch)) + for _ in range(TOKEN_COUNTER_MAX_CONCURRENT_COUNTS) + ) + await asyncio.wait_for(_until_every_count_slot_is_taken(), timeout=30) + chat: Final = asyncio.create_task(router.acompletion(model="chat", messages=[{"role": "user", "content": "hi"}])) + + first_done, _ = await asyncio.wait((chat, *batches), return_when=asyncio.FIRST_COMPLETED) + await asyncio.gather(chat, *batches) + + assert first_done == {chat} + + +def test_a_routing_plugin_counts_an_embedding_text_batch_by_its_longest_text(): + long_text: Final = "word " * 4000 + context: Final = RouterContext(_embedding_router(max_input_tokens=10)) + + assert context.count_pre_call_check_tokens(None, ["one", long_text]) == litellm.token_counter(text=long_text) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("local_model_cost_map", "isolate_router_model_cost_state") +@pytest.mark.parametrize( + "prompt_kwargs", + [ + pytest.param({"instructions": "you are a helpful assistant. " * 20}, id="instructions"), + pytest.param( + { + "tools": [ + { + "type": "function", + "name": "lookup", + "description": "look up a record by its identifier. " * 20, + "parameters": {"type": "object", "properties": {"q": {"type": "string"}}}, + } + ] + }, + id="tools", + ), + ], +) +async def test_aresponses_with_empty_input_still_counts_its_other_prompt_content(prompt_kwargs): + router: Final = litellm.Router( + model_list=[ + { + "model_name": "small-ctx", + "litellm_params": {"model": "gpt-3.5-turbo", "mock_response": "hi"}, + "model_info": {"max_input_tokens": 50}, + } + ], + enable_pre_call_checks=True, + ) + + with pytest.raises(litellm.ContextWindowExceededError): + await router.aresponses(model="small-ctx", input=[], **prompt_kwargs) + + def test_get_deployment_model_info_base_model_flow(): """Test that get_deployment_model_info correctly handles the base model flow""" from unittest.mock import patch From eacb670b6ec8ea213742ebdb067e51cb6de7a2ae Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 6 Nov 2026 11:53:37 -0800 Subject: [PATCH 16/39] bump: litellm-enterprise 0.1.81 -> 0.1.83, litellm-proxy-extras 0.4.112 -> 0.4.114, litellm 1.109.0 -> 1.110.0 (#50437) --- enterprise/pyproject.toml | 4 ++-- litellm-proxy-extras/pyproject.toml | 4 ++-- pyproject.toml | 8 ++++---- uv.lock | 6 +++--- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/enterprise/pyproject.toml b/enterprise/pyproject.toml index 7db2de1f874..64fe03769b8 100644 --- a/enterprise/pyproject.toml +++ b/enterprise/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-enterprise" -version = "0.1.81" +version = "0.1.83" description = "Package for LiteLLM Enterprise features" readme = "README.md" requires-python = ">=3.9" @@ -26,7 +26,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.1.81" +version = "0.1.83" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-enterprise==", diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index a12e1e27cfc..1c0c708ae7a 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-proxy-extras" -version = "0.4.112" +version = "0.4.114" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." readme = "README.md" requires-python = ">=3.9" @@ -30,7 +30,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.4.112" +version = "0.4.114" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-proxy-extras==", diff --git a/pyproject.toml b/pyproject.toml index c53fbb4d888..e193eeee93f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm" -version = "1.109.0" +version = "1.110.0" description = "Library to easily interface with LLM API providers" readme = "README.md" requires-python = ">=3.10, <3.16" @@ -84,8 +84,8 @@ proxy = [ "mcp>=2.3.0,<3", "httpx2>=2.10.0,<3", "pydantic>=2.12.0,<3", - "litellm-proxy-extras==0.4.112", - "litellm-enterprise==0.1.81", + "litellm-proxy-extras==0.4.114", + "litellm-enterprise==0.1.83", "RestrictedPython>=8.5,<9.0", "rich>=13.9.4,<14.0", "InquirerPy>=0.3.4,<1.0", @@ -406,7 +406,7 @@ litellm-enterprise = { workspace = true } members = ["enterprise", "litellm-proxy-extras", "packaging/litellm-proxy"] [tool.commitizen] -version = "1.109.0" +version = "1.110.0" version_files = [ "pyproject.toml:^version", ] diff --git a/uv.lock b/uv.lock index b6a9a599ba2..63a075f5407 100644 --- a/uv.lock +++ b/uv.lock @@ -5020,7 +5020,7 @@ wheels = [ [[package]] name = "litellm" -version = "1.109.0" +version = "1.110.0" source = { editable = "." } dependencies = [ { name = "aiohttp" }, @@ -5547,7 +5547,7 @@ provides-extras = ["test"] [[package]] name = "litellm-enterprise" -version = "0.1.81" +version = "0.1.83" source = { editable = "enterprise" } [[package]] @@ -5557,7 +5557,7 @@ source = { editable = "packaging/litellm-proxy" } [[package]] name = "litellm-proxy-extras" -version = "0.4.112" +version = "0.4.114" source = { editable = "litellm-proxy-extras" } dependencies = [ { name = "psycopg" }, From 8013d5b8c130a467fb865ba7f6786300665f20b7 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 6 Nov 2026 12:04:03 -0800 Subject: [PATCH 17/39] chore(model_prices): absorb #50415 and #50332, add Perplexity gpt-5.5 cached >272k rate (#50436) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Adam Dalloul <47503782+Adam-Dalloul@users.noreply.github.com> Co-authored-by: halewang --- ...odel_prices_and_context_window_backup.json | 212 +++++++++++++----- model_prices_and_context_window.json | 212 +++++++++++++----- 2 files changed, 304 insertions(+), 120 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 4796ed3ac9c..b102d6e5a07 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -64213,6 +64213,39 @@ "supports_tool_choice": true, "supports_vision": false }, + "tencent/hy4": { + "cache_read_input_token_cost": 4.2e-08, + "comment": "TokenHub bills Hy4 at 50% of list until 2026-12-31 23:59:59 Beijing time; these are the list rates.", + "input_cost_per_token": 8.34e-07, + "input_cost_per_token_cache_hit": 4.2e-08, + "litellm_provider": "tencent", + "max_input_tokens": 960000, + "max_output_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 2.501e-06, + "source": "https://www.tencentcloud.com/document/product/1300/78937", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, + "tencent/hy4-preview": { + "cache_read_input_token_cost": 4.2e-08, + "comment": "TokenHub bills Hy4 preview at 50% of list until 2026-12-31 23:59:59 Beijing time; these are the list rates.", + "input_cost_per_token": 8.34e-07, + "input_cost_per_token_cache_hit": 4.2e-08, + "litellm_provider": "tencent", + "max_input_tokens": 960000, + "max_output_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 2.501e-06, + "source": "https://www.tencentcloud.com/document/product/1300/78937", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true + }, "tencent/minimax-m3": { "cache_creation_input_token_cost": 0.0, "cache_read_input_token_cost": 6e-08, @@ -70882,6 +70915,7 @@ "cache_read_input_token_cost": 5e-07, "input_cost_per_token_above_272k_tokens": 1e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, + "cache_read_input_token_cost_above_272k_tokens": 1e-06, "source": "https://docs.perplexity.ai/docs/agent-api/models" }, "perplexity/openai/gpt-5.4": { @@ -83107,13 +83141,16 @@ "mode": "chat", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "input_cost_per_token": 3e-07, - "output_cost_per_token": 1.2e-06, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 8e-07, "supports_system_messages": true, "supports_vision": true, "supports_reasoning": true, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/deepseek-v4-1-flash-1017" + "source": "https://empiriolabs.ai/models/deepseek-v4-1-flash-1017", + "cache_read_input_token_cost": 2e-08, + "supports_prompt_caching": true, + "supports_function_calling": true }, "empiriolabs/deepseek-v4-flash-0731": { "litellm_provider": "empiriolabs", @@ -83125,7 +83162,8 @@ "supports_system_messages": true, "supports_reasoning": true, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/deepseek-v4-flash-0731" + "source": "https://empiriolabs.ai/models/deepseek-v4-flash-0731", + "supports_function_calling": true }, "empiriolabs/deepseek-v4-flash-0731:variant1": { "litellm_provider": "empiriolabs", @@ -83137,7 +83175,8 @@ "supports_system_messages": true, "supports_reasoning": true, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/deepseek-v4-flash-0731" + "source": "https://empiriolabs.ai/models/deepseek-v4-flash-0731", + "supports_function_calling": true }, "empiriolabs/deepseek-v4-pro": { "litellm_provider": "empiriolabs", @@ -83149,7 +83188,8 @@ "supports_system_messages": true, "supports_reasoning": true, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/deepseek-v4-pro" + "source": "https://empiriolabs.ai/models/deepseek-v4-pro", + "supports_function_calling": true }, "empiriolabs/deepseek-v4-pro-0813": { "litellm_provider": "empiriolabs", @@ -83161,7 +83201,8 @@ "supports_system_messages": true, "supports_reasoning": true, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/deepseek-v4-pro-0813" + "source": "https://empiriolabs.ai/models/deepseek-v4-pro-0813", + "supports_function_calling": true }, "empiriolabs/deepseek-v4-pro:variant1": { "litellm_provider": "empiriolabs", @@ -83179,7 +83220,8 @@ "search_context_size_high": 0.02 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/deepseek-v4-pro" + "source": "https://empiriolabs.ai/models/deepseek-v4-pro", + "supports_function_calling": true }, "empiriolabs/deepseek-v4-pro:variant2": { "litellm_provider": "empiriolabs", @@ -83199,7 +83241,8 @@ }, "supports_prompt_caching": true, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/deepseek-v4-pro" + "source": "https://empiriolabs.ai/models/deepseek-v4-pro", + "supports_function_calling": true }, "empiriolabs/deepseek-v4-pro:variant3": { "litellm_provider": "empiriolabs", @@ -83211,7 +83254,8 @@ "supports_system_messages": true, "supports_reasoning": true, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/deepseek-v4-pro" + "source": "https://empiriolabs.ai/models/deepseek-v4-pro", + "supports_function_calling": true }, "empiriolabs/fugu-max": { "litellm_provider": "empiriolabs", @@ -83409,7 +83453,8 @@ }, "supports_prompt_caching": true, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/glm-5-2" + "source": "https://empiriolabs.ai/models/glm-5-2", + "supports_function_calling": true }, "empiriolabs/glm-5-2:variant2": { "litellm_provider": "empiriolabs", @@ -83427,7 +83472,8 @@ "search_context_size_high": 0.013 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/glm-5-2" + "source": "https://empiriolabs.ai/models/glm-5-2", + "supports_function_calling": true }, "empiriolabs/glm-5-3": { "litellm_provider": "empiriolabs", @@ -83481,7 +83527,8 @@ "supports_reasoning": true, "supports_prompt_caching": true, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/kimi-k2-6" + "source": "https://empiriolabs.ai/models/kimi-k2-6", + "supports_function_calling": true }, "empiriolabs/kimi-k2-7-code": { "litellm_provider": "empiriolabs", @@ -83500,7 +83547,8 @@ "search_context_size_high": 0.015 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/kimi-k2-7-code" + "source": "https://empiriolabs.ai/models/kimi-k2-7-code", + "supports_function_calling": true }, "empiriolabs/kimi-k2-7-code-highspeed": { "litellm_provider": "empiriolabs", @@ -83519,7 +83567,8 @@ "search_context_size_high": 0.015 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/kimi-k2-7-code-highspeed" + "source": "https://empiriolabs.ai/models/kimi-k2-7-code-highspeed", + "supports_function_calling": true }, "empiriolabs/kimi-k2-7-code:variant1": { "litellm_provider": "empiriolabs", @@ -83540,7 +83589,8 @@ }, "supports_prompt_caching": true, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/kimi-k2-7-code" + "source": "https://empiriolabs.ai/models/kimi-k2-7-code", + "supports_function_calling": true }, "empiriolabs/kimi-k3": { "litellm_provider": "empiriolabs", @@ -83559,7 +83609,8 @@ "search_context_size_high": 0.015 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/kimi-k3" + "source": "https://empiriolabs.ai/models/kimi-k3", + "supports_function_calling": true }, "empiriolabs/kimi-k3-1-pro": { "litellm_provider": "empiriolabs", @@ -83601,7 +83652,8 @@ }, "supports_response_schema": true, "supports_audio_input": true, - "source": "https://empiriolabs.ai/models/mimo-v2-6-flash" + "source": "https://empiriolabs.ai/models/mimo-v2-6-flash", + "supports_function_calling": true }, "empiriolabs/mimo-v2-6-pro": { "litellm_provider": "empiriolabs", @@ -83621,7 +83673,8 @@ }, "supports_response_schema": true, "supports_audio_input": true, - "source": "https://empiriolabs.ai/models/mimo-v2-6-pro" + "source": "https://empiriolabs.ai/models/mimo-v2-6-pro", + "supports_function_calling": true }, "empiriolabs/mimo-v2-6-pro-ultraspeed": { "litellm_provider": "empiriolabs", @@ -83635,7 +83688,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_audio_input": true, - "source": "https://empiriolabs.ai/models/mimo-v2-6-pro-ultraspeed" + "source": "https://empiriolabs.ai/models/mimo-v2-6-pro-ultraspeed", + "supports_function_calling": true }, "empiriolabs/minimax-m2-7": { "litellm_provider": "empiriolabs", @@ -83750,7 +83804,8 @@ "search_context_size_high": 0.14 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/mistral-small-4" + "source": "https://empiriolabs.ai/models/mistral-small-4", + "supports_function_calling": true }, "empiriolabs/muse-glimmer-30b": { "litellm_provider": "empiriolabs", @@ -83765,7 +83820,8 @@ "supports_reasoning": true, "supports_prompt_caching": true, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/muse-glimmer-30b" + "source": "https://empiriolabs.ai/models/muse-glimmer-30b", + "supports_function_calling": true }, "empiriolabs/muse-spark-1-1": { "litellm_provider": "empiriolabs", @@ -83831,7 +83887,8 @@ "supports_system_messages": true, "supports_vision": true, "supports_prompt_caching": true, - "source": "https://empiriolabs.ai/models/nova-lite-1-0" + "source": "https://empiriolabs.ai/models/nova-lite-1-0", + "supports_function_calling": true }, "empiriolabs/nova-lite-2": { "litellm_provider": "empiriolabs", @@ -83845,7 +83902,8 @@ "supports_vision": true, "supports_reasoning": true, "supports_prompt_caching": true, - "source": "https://empiriolabs.ai/models/nova-lite-2" + "source": "https://empiriolabs.ai/models/nova-lite-2", + "supports_function_calling": true }, "empiriolabs/nova-micro-1-0": { "litellm_provider": "empiriolabs", @@ -83857,7 +83915,8 @@ "cache_read_input_token_cost": 2.24e-08, "supports_system_messages": true, "supports_prompt_caching": true, - "source": "https://empiriolabs.ai/models/nova-micro-1-0" + "source": "https://empiriolabs.ai/models/nova-micro-1-0", + "supports_function_calling": true }, "empiriolabs/nova-pro-1-0": { "litellm_provider": "empiriolabs", @@ -83868,7 +83927,8 @@ "output_cost_per_token": 9.6e-06, "supports_system_messages": true, "supports_vision": true, - "source": "https://empiriolabs.ai/models/nova-pro-1-0" + "source": "https://empiriolabs.ai/models/nova-pro-1-0", + "supports_function_calling": true }, "empiriolabs/perplexity-advanced-deep-research": { "litellm_provider": "empiriolabs", @@ -83968,7 +84028,8 @@ "search_context_size_high": 0.01 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-5-122b-a10b" + "source": "https://empiriolabs.ai/models/qwen3-5-122b-a10b", + "supports_function_calling": true }, "empiriolabs/qwen3-5-27b": { "litellm_provider": "empiriolabs", @@ -83989,7 +84050,8 @@ "search_context_size_high": 0.01 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-5-27b" + "source": "https://empiriolabs.ai/models/qwen3-5-27b", + "supports_function_calling": true }, "empiriolabs/qwen3-5-35b-a3b": { "litellm_provider": "empiriolabs", @@ -84010,7 +84072,8 @@ "search_context_size_high": 0.01 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-5-35b-a3b" + "source": "https://empiriolabs.ai/models/qwen3-5-35b-a3b", + "supports_function_calling": true }, "empiriolabs/qwen3-5-397b-a17b": { "litellm_provider": "empiriolabs", @@ -84031,7 +84094,8 @@ "search_context_size_high": 0.01 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-5-397b-a17b" + "source": "https://empiriolabs.ai/models/qwen3-5-397b-a17b", + "supports_function_calling": true }, "empiriolabs/qwen3-5-4b": { "litellm_provider": "empiriolabs", @@ -84082,7 +84146,8 @@ "search_context_size_high": 0.015 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-5-flash" + "source": "https://empiriolabs.ai/models/qwen3-5-flash", + "supports_function_calling": true }, "empiriolabs/qwen3-5-flash:variant1": { "litellm_provider": "empiriolabs", @@ -84105,7 +84170,8 @@ "search_context_size_high": 0.01 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-5-flash" + "source": "https://empiriolabs.ai/models/qwen3-5-flash", + "supports_function_calling": true }, "empiriolabs/qwen3-5-omni-flash": { "litellm_provider": "empiriolabs", @@ -84123,7 +84189,8 @@ "search_context_size_high": 0.015 }, "supports_audio_input": true, - "source": "https://empiriolabs.ai/models/qwen3-5-omni-flash" + "source": "https://empiriolabs.ai/models/qwen3-5-omni-flash", + "supports_function_calling": true }, "empiriolabs/qwen3-5-omni-plus": { "litellm_provider": "empiriolabs", @@ -84142,7 +84209,8 @@ }, "supports_response_schema": true, "supports_audio_input": true, - "source": "https://empiriolabs.ai/models/qwen3-5-omni-plus" + "source": "https://empiriolabs.ai/models/qwen3-5-omni-plus", + "supports_function_calling": true }, "empiriolabs/qwen3-5-plus": { "litellm_provider": "empiriolabs", @@ -84151,8 +84219,8 @@ "max_output_tokens": 65536, "input_cost_per_token": 3.6e-07, "output_cost_per_token": 2.21e-06, - "input_cost_per_token_above_256k_tokens": 1.08e-06, - "output_cost_per_token_above_256k_tokens": 6.62e-06, + "input_cost_per_token_above_256k_tokens": 4.5e-07, + "output_cost_per_token_above_256k_tokens": 2.76e-06, "supports_system_messages": true, "supports_vision": true, "supports_reasoning": true, @@ -84163,7 +84231,8 @@ "search_context_size_high": 0.015 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-5-plus" + "source": "https://empiriolabs.ai/models/qwen3-5-plus", + "supports_function_calling": true }, "empiriolabs/qwen3-5-plus:variant1": { "litellm_provider": "empiriolabs", @@ -84186,7 +84255,8 @@ "search_context_size_high": 0.01 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-5-plus" + "source": "https://empiriolabs.ai/models/qwen3-5-plus", + "supports_function_calling": true }, "empiriolabs/qwen3-6-27b": { "litellm_provider": "empiriolabs", @@ -84205,7 +84275,8 @@ "search_context_size_high": 0.01 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-6-27b" + "source": "https://empiriolabs.ai/models/qwen3-6-27b", + "supports_function_calling": true }, "empiriolabs/qwen3-6-35b-a3b": { "litellm_provider": "empiriolabs", @@ -84219,7 +84290,8 @@ "supports_reasoning": true, "supports_prompt_caching": true, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-6-35b-a3b" + "source": "https://empiriolabs.ai/models/qwen3-6-35b-a3b", + "supports_function_calling": true }, "empiriolabs/qwen3-6-flash": { "litellm_provider": "empiriolabs", @@ -84240,7 +84312,8 @@ "search_context_size_high": 0.02 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-6-flash" + "source": "https://empiriolabs.ai/models/qwen3-6-flash", + "supports_function_calling": true }, "empiriolabs/qwen3-6-flash:variant1": { "litellm_provider": "empiriolabs", @@ -84261,7 +84334,8 @@ "search_context_size_high": 0.01 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-6-flash" + "source": "https://empiriolabs.ai/models/qwen3-6-flash", + "supports_function_calling": true }, "empiriolabs/qwen3-6-plus": { "litellm_provider": "empiriolabs", @@ -84282,7 +84356,8 @@ "search_context_size_high": 0.026 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-6-plus" + "source": "https://empiriolabs.ai/models/qwen3-6-plus", + "supports_function_calling": true }, "empiriolabs/qwen3-6-plus:variant1": { "litellm_provider": "empiriolabs", @@ -84303,7 +84378,8 @@ "search_context_size_high": 0.01 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-6-plus" + "source": "https://empiriolabs.ai/models/qwen3-6-plus", + "supports_function_calling": true }, "empiriolabs/qwen3-7-flash": { "litellm_provider": "empiriolabs", @@ -84330,7 +84406,8 @@ }, "supports_prompt_caching": true, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-7-flash" + "source": "https://empiriolabs.ai/models/qwen3-7-flash", + "supports_function_calling": true }, "empiriolabs/qwen3-7-flash:variant1": { "litellm_provider": "empiriolabs", @@ -84357,7 +84434,8 @@ }, "supports_prompt_caching": true, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-7-flash" + "source": "https://empiriolabs.ai/models/qwen3-7-flash", + "supports_function_calling": true }, "empiriolabs/qwen3-7-max": { "litellm_provider": "empiriolabs", @@ -84375,7 +84453,8 @@ "search_context_size_high": 0.02 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-7-max" + "source": "https://empiriolabs.ai/models/qwen3-7-max", + "supports_function_calling": true }, "empiriolabs/qwen3-7-max:variant1": { "litellm_provider": "empiriolabs", @@ -84393,7 +84472,8 @@ "search_context_size_high": 0.01 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-7-max" + "source": "https://empiriolabs.ai/models/qwen3-7-max", + "supports_function_calling": true }, "empiriolabs/qwen3-7-plus": { "litellm_provider": "empiriolabs", @@ -84414,7 +84494,8 @@ "search_context_size_high": 0.03 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-7-plus" + "source": "https://empiriolabs.ai/models/qwen3-7-plus", + "supports_function_calling": true }, "empiriolabs/qwen3-7-plus:variant1": { "litellm_provider": "empiriolabs", @@ -84438,7 +84519,8 @@ }, "supports_prompt_caching": true, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-7-plus" + "source": "https://empiriolabs.ai/models/qwen3-7-plus", + "supports_function_calling": true }, "empiriolabs/qwen3-8-27b": { "litellm_provider": "empiriolabs", @@ -84473,7 +84555,8 @@ "search_context_size_high": 0.02 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-8-flash" + "source": "https://empiriolabs.ai/models/qwen3-8-flash", + "supports_function_calling": true }, "empiriolabs/qwen3-8-max": { "litellm_provider": "empiriolabs", @@ -84492,7 +84575,8 @@ "search_context_size_high": 0.02 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-8-max" + "source": "https://empiriolabs.ai/models/qwen3-8-max", + "supports_function_calling": true }, "empiriolabs/qwen3-8-max-0902": { "litellm_provider": "empiriolabs", @@ -84511,7 +84595,8 @@ "search_context_size_high": 0.02 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-8-max-0902" + "source": "https://empiriolabs.ai/models/qwen3-8-max-0902", + "supports_function_calling": true }, "empiriolabs/qwen3-8-max:variant1": { "litellm_provider": "empiriolabs", @@ -84530,7 +84615,8 @@ "search_context_size_high": 0.01 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-8-max" + "source": "https://empiriolabs.ai/models/qwen3-8-max", + "supports_function_calling": true }, "empiriolabs/qwen3-8-omni-flash": { "litellm_provider": "empiriolabs", @@ -84550,7 +84636,8 @@ }, "supports_response_schema": true, "supports_audio_input": true, - "source": "https://empiriolabs.ai/models/qwen3-8-omni-flash" + "source": "https://empiriolabs.ai/models/qwen3-8-omni-flash", + "supports_function_calling": true }, "empiriolabs/seed-2-0-code": { "litellm_provider": "empiriolabs", @@ -84564,7 +84651,8 @@ "supports_system_messages": true, "supports_vision": true, "supports_reasoning": true, - "source": "https://empiriolabs.ai/models/seed-2-0-code" + "source": "https://empiriolabs.ai/models/seed-2-0-code", + "supports_function_calling": true }, "empiriolabs/seed-2-0-lite": { "litellm_provider": "empiriolabs", @@ -84579,7 +84667,8 @@ "supports_vision": true, "supports_reasoning": true, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/seed-2-0-lite" + "source": "https://empiriolabs.ai/models/seed-2-0-lite", + "supports_function_calling": true }, "empiriolabs/seed-2-0-mini": { "litellm_provider": "empiriolabs", @@ -84594,7 +84683,8 @@ "supports_vision": true, "supports_reasoning": true, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/seed-2-0-mini" + "source": "https://empiriolabs.ai/models/seed-2-0-mini", + "supports_function_calling": true }, "empiriolabs/seed-2-0-pro": { "litellm_provider": "empiriolabs", @@ -84609,7 +84699,8 @@ "supports_vision": true, "supports_reasoning": true, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/seed-2-0-pro" + "source": "https://empiriolabs.ai/models/seed-2-0-pro", + "supports_function_calling": true }, "empiriolabs/seed-2-1-turbo": { "litellm_provider": "empiriolabs", @@ -84622,7 +84713,8 @@ "supports_vision": true, "supports_reasoning": true, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/seed-2-1-turbo" + "source": "https://empiriolabs.ai/models/seed-2-1-turbo", + "supports_function_calling": true }, "empiriolabs/skylark-embedding-vision": { "litellm_provider": "empiriolabs", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 4796ed3ac9c..b102d6e5a07 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -64213,6 +64213,39 @@ "supports_tool_choice": true, "supports_vision": false }, + "tencent/hy4": { + "cache_read_input_token_cost": 4.2e-08, + "comment": "TokenHub bills Hy4 at 50% of list until 2026-12-31 23:59:59 Beijing time; these are the list rates.", + "input_cost_per_token": 8.34e-07, + "input_cost_per_token_cache_hit": 4.2e-08, + "litellm_provider": "tencent", + "max_input_tokens": 960000, + "max_output_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 2.501e-06, + "source": "https://www.tencentcloud.com/document/product/1300/78937", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true + }, + "tencent/hy4-preview": { + "cache_read_input_token_cost": 4.2e-08, + "comment": "TokenHub bills Hy4 preview at 50% of list until 2026-12-31 23:59:59 Beijing time; these are the list rates.", + "input_cost_per_token": 8.34e-07, + "input_cost_per_token_cache_hit": 4.2e-08, + "litellm_provider": "tencent", + "max_input_tokens": 960000, + "max_output_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 2.501e-06, + "source": "https://www.tencentcloud.com/document/product/1300/78937", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true + }, "tencent/minimax-m3": { "cache_creation_input_token_cost": 0.0, "cache_read_input_token_cost": 6e-08, @@ -70882,6 +70915,7 @@ "cache_read_input_token_cost": 5e-07, "input_cost_per_token_above_272k_tokens": 1e-05, "output_cost_per_token_above_272k_tokens": 4.5e-05, + "cache_read_input_token_cost_above_272k_tokens": 1e-06, "source": "https://docs.perplexity.ai/docs/agent-api/models" }, "perplexity/openai/gpt-5.4": { @@ -83107,13 +83141,16 @@ "mode": "chat", "max_input_tokens": 1000000, "max_output_tokens": 393216, - "input_cost_per_token": 3e-07, - "output_cost_per_token": 1.2e-06, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 8e-07, "supports_system_messages": true, "supports_vision": true, "supports_reasoning": true, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/deepseek-v4-1-flash-1017" + "source": "https://empiriolabs.ai/models/deepseek-v4-1-flash-1017", + "cache_read_input_token_cost": 2e-08, + "supports_prompt_caching": true, + "supports_function_calling": true }, "empiriolabs/deepseek-v4-flash-0731": { "litellm_provider": "empiriolabs", @@ -83125,7 +83162,8 @@ "supports_system_messages": true, "supports_reasoning": true, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/deepseek-v4-flash-0731" + "source": "https://empiriolabs.ai/models/deepseek-v4-flash-0731", + "supports_function_calling": true }, "empiriolabs/deepseek-v4-flash-0731:variant1": { "litellm_provider": "empiriolabs", @@ -83137,7 +83175,8 @@ "supports_system_messages": true, "supports_reasoning": true, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/deepseek-v4-flash-0731" + "source": "https://empiriolabs.ai/models/deepseek-v4-flash-0731", + "supports_function_calling": true }, "empiriolabs/deepseek-v4-pro": { "litellm_provider": "empiriolabs", @@ -83149,7 +83188,8 @@ "supports_system_messages": true, "supports_reasoning": true, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/deepseek-v4-pro" + "source": "https://empiriolabs.ai/models/deepseek-v4-pro", + "supports_function_calling": true }, "empiriolabs/deepseek-v4-pro-0813": { "litellm_provider": "empiriolabs", @@ -83161,7 +83201,8 @@ "supports_system_messages": true, "supports_reasoning": true, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/deepseek-v4-pro-0813" + "source": "https://empiriolabs.ai/models/deepseek-v4-pro-0813", + "supports_function_calling": true }, "empiriolabs/deepseek-v4-pro:variant1": { "litellm_provider": "empiriolabs", @@ -83179,7 +83220,8 @@ "search_context_size_high": 0.02 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/deepseek-v4-pro" + "source": "https://empiriolabs.ai/models/deepseek-v4-pro", + "supports_function_calling": true }, "empiriolabs/deepseek-v4-pro:variant2": { "litellm_provider": "empiriolabs", @@ -83199,7 +83241,8 @@ }, "supports_prompt_caching": true, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/deepseek-v4-pro" + "source": "https://empiriolabs.ai/models/deepseek-v4-pro", + "supports_function_calling": true }, "empiriolabs/deepseek-v4-pro:variant3": { "litellm_provider": "empiriolabs", @@ -83211,7 +83254,8 @@ "supports_system_messages": true, "supports_reasoning": true, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/deepseek-v4-pro" + "source": "https://empiriolabs.ai/models/deepseek-v4-pro", + "supports_function_calling": true }, "empiriolabs/fugu-max": { "litellm_provider": "empiriolabs", @@ -83409,7 +83453,8 @@ }, "supports_prompt_caching": true, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/glm-5-2" + "source": "https://empiriolabs.ai/models/glm-5-2", + "supports_function_calling": true }, "empiriolabs/glm-5-2:variant2": { "litellm_provider": "empiriolabs", @@ -83427,7 +83472,8 @@ "search_context_size_high": 0.013 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/glm-5-2" + "source": "https://empiriolabs.ai/models/glm-5-2", + "supports_function_calling": true }, "empiriolabs/glm-5-3": { "litellm_provider": "empiriolabs", @@ -83481,7 +83527,8 @@ "supports_reasoning": true, "supports_prompt_caching": true, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/kimi-k2-6" + "source": "https://empiriolabs.ai/models/kimi-k2-6", + "supports_function_calling": true }, "empiriolabs/kimi-k2-7-code": { "litellm_provider": "empiriolabs", @@ -83500,7 +83547,8 @@ "search_context_size_high": 0.015 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/kimi-k2-7-code" + "source": "https://empiriolabs.ai/models/kimi-k2-7-code", + "supports_function_calling": true }, "empiriolabs/kimi-k2-7-code-highspeed": { "litellm_provider": "empiriolabs", @@ -83519,7 +83567,8 @@ "search_context_size_high": 0.015 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/kimi-k2-7-code-highspeed" + "source": "https://empiriolabs.ai/models/kimi-k2-7-code-highspeed", + "supports_function_calling": true }, "empiriolabs/kimi-k2-7-code:variant1": { "litellm_provider": "empiriolabs", @@ -83540,7 +83589,8 @@ }, "supports_prompt_caching": true, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/kimi-k2-7-code" + "source": "https://empiriolabs.ai/models/kimi-k2-7-code", + "supports_function_calling": true }, "empiriolabs/kimi-k3": { "litellm_provider": "empiriolabs", @@ -83559,7 +83609,8 @@ "search_context_size_high": 0.015 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/kimi-k3" + "source": "https://empiriolabs.ai/models/kimi-k3", + "supports_function_calling": true }, "empiriolabs/kimi-k3-1-pro": { "litellm_provider": "empiriolabs", @@ -83601,7 +83652,8 @@ }, "supports_response_schema": true, "supports_audio_input": true, - "source": "https://empiriolabs.ai/models/mimo-v2-6-flash" + "source": "https://empiriolabs.ai/models/mimo-v2-6-flash", + "supports_function_calling": true }, "empiriolabs/mimo-v2-6-pro": { "litellm_provider": "empiriolabs", @@ -83621,7 +83673,8 @@ }, "supports_response_schema": true, "supports_audio_input": true, - "source": "https://empiriolabs.ai/models/mimo-v2-6-pro" + "source": "https://empiriolabs.ai/models/mimo-v2-6-pro", + "supports_function_calling": true }, "empiriolabs/mimo-v2-6-pro-ultraspeed": { "litellm_provider": "empiriolabs", @@ -83635,7 +83688,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_audio_input": true, - "source": "https://empiriolabs.ai/models/mimo-v2-6-pro-ultraspeed" + "source": "https://empiriolabs.ai/models/mimo-v2-6-pro-ultraspeed", + "supports_function_calling": true }, "empiriolabs/minimax-m2-7": { "litellm_provider": "empiriolabs", @@ -83750,7 +83804,8 @@ "search_context_size_high": 0.14 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/mistral-small-4" + "source": "https://empiriolabs.ai/models/mistral-small-4", + "supports_function_calling": true }, "empiriolabs/muse-glimmer-30b": { "litellm_provider": "empiriolabs", @@ -83765,7 +83820,8 @@ "supports_reasoning": true, "supports_prompt_caching": true, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/muse-glimmer-30b" + "source": "https://empiriolabs.ai/models/muse-glimmer-30b", + "supports_function_calling": true }, "empiriolabs/muse-spark-1-1": { "litellm_provider": "empiriolabs", @@ -83831,7 +83887,8 @@ "supports_system_messages": true, "supports_vision": true, "supports_prompt_caching": true, - "source": "https://empiriolabs.ai/models/nova-lite-1-0" + "source": "https://empiriolabs.ai/models/nova-lite-1-0", + "supports_function_calling": true }, "empiriolabs/nova-lite-2": { "litellm_provider": "empiriolabs", @@ -83845,7 +83902,8 @@ "supports_vision": true, "supports_reasoning": true, "supports_prompt_caching": true, - "source": "https://empiriolabs.ai/models/nova-lite-2" + "source": "https://empiriolabs.ai/models/nova-lite-2", + "supports_function_calling": true }, "empiriolabs/nova-micro-1-0": { "litellm_provider": "empiriolabs", @@ -83857,7 +83915,8 @@ "cache_read_input_token_cost": 2.24e-08, "supports_system_messages": true, "supports_prompt_caching": true, - "source": "https://empiriolabs.ai/models/nova-micro-1-0" + "source": "https://empiriolabs.ai/models/nova-micro-1-0", + "supports_function_calling": true }, "empiriolabs/nova-pro-1-0": { "litellm_provider": "empiriolabs", @@ -83868,7 +83927,8 @@ "output_cost_per_token": 9.6e-06, "supports_system_messages": true, "supports_vision": true, - "source": "https://empiriolabs.ai/models/nova-pro-1-0" + "source": "https://empiriolabs.ai/models/nova-pro-1-0", + "supports_function_calling": true }, "empiriolabs/perplexity-advanced-deep-research": { "litellm_provider": "empiriolabs", @@ -83968,7 +84028,8 @@ "search_context_size_high": 0.01 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-5-122b-a10b" + "source": "https://empiriolabs.ai/models/qwen3-5-122b-a10b", + "supports_function_calling": true }, "empiriolabs/qwen3-5-27b": { "litellm_provider": "empiriolabs", @@ -83989,7 +84050,8 @@ "search_context_size_high": 0.01 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-5-27b" + "source": "https://empiriolabs.ai/models/qwen3-5-27b", + "supports_function_calling": true }, "empiriolabs/qwen3-5-35b-a3b": { "litellm_provider": "empiriolabs", @@ -84010,7 +84072,8 @@ "search_context_size_high": 0.01 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-5-35b-a3b" + "source": "https://empiriolabs.ai/models/qwen3-5-35b-a3b", + "supports_function_calling": true }, "empiriolabs/qwen3-5-397b-a17b": { "litellm_provider": "empiriolabs", @@ -84031,7 +84094,8 @@ "search_context_size_high": 0.01 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-5-397b-a17b" + "source": "https://empiriolabs.ai/models/qwen3-5-397b-a17b", + "supports_function_calling": true }, "empiriolabs/qwen3-5-4b": { "litellm_provider": "empiriolabs", @@ -84082,7 +84146,8 @@ "search_context_size_high": 0.015 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-5-flash" + "source": "https://empiriolabs.ai/models/qwen3-5-flash", + "supports_function_calling": true }, "empiriolabs/qwen3-5-flash:variant1": { "litellm_provider": "empiriolabs", @@ -84105,7 +84170,8 @@ "search_context_size_high": 0.01 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-5-flash" + "source": "https://empiriolabs.ai/models/qwen3-5-flash", + "supports_function_calling": true }, "empiriolabs/qwen3-5-omni-flash": { "litellm_provider": "empiriolabs", @@ -84123,7 +84189,8 @@ "search_context_size_high": 0.015 }, "supports_audio_input": true, - "source": "https://empiriolabs.ai/models/qwen3-5-omni-flash" + "source": "https://empiriolabs.ai/models/qwen3-5-omni-flash", + "supports_function_calling": true }, "empiriolabs/qwen3-5-omni-plus": { "litellm_provider": "empiriolabs", @@ -84142,7 +84209,8 @@ }, "supports_response_schema": true, "supports_audio_input": true, - "source": "https://empiriolabs.ai/models/qwen3-5-omni-plus" + "source": "https://empiriolabs.ai/models/qwen3-5-omni-plus", + "supports_function_calling": true }, "empiriolabs/qwen3-5-plus": { "litellm_provider": "empiriolabs", @@ -84151,8 +84219,8 @@ "max_output_tokens": 65536, "input_cost_per_token": 3.6e-07, "output_cost_per_token": 2.21e-06, - "input_cost_per_token_above_256k_tokens": 1.08e-06, - "output_cost_per_token_above_256k_tokens": 6.62e-06, + "input_cost_per_token_above_256k_tokens": 4.5e-07, + "output_cost_per_token_above_256k_tokens": 2.76e-06, "supports_system_messages": true, "supports_vision": true, "supports_reasoning": true, @@ -84163,7 +84231,8 @@ "search_context_size_high": 0.015 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-5-plus" + "source": "https://empiriolabs.ai/models/qwen3-5-plus", + "supports_function_calling": true }, "empiriolabs/qwen3-5-plus:variant1": { "litellm_provider": "empiriolabs", @@ -84186,7 +84255,8 @@ "search_context_size_high": 0.01 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-5-plus" + "source": "https://empiriolabs.ai/models/qwen3-5-plus", + "supports_function_calling": true }, "empiriolabs/qwen3-6-27b": { "litellm_provider": "empiriolabs", @@ -84205,7 +84275,8 @@ "search_context_size_high": 0.01 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-6-27b" + "source": "https://empiriolabs.ai/models/qwen3-6-27b", + "supports_function_calling": true }, "empiriolabs/qwen3-6-35b-a3b": { "litellm_provider": "empiriolabs", @@ -84219,7 +84290,8 @@ "supports_reasoning": true, "supports_prompt_caching": true, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-6-35b-a3b" + "source": "https://empiriolabs.ai/models/qwen3-6-35b-a3b", + "supports_function_calling": true }, "empiriolabs/qwen3-6-flash": { "litellm_provider": "empiriolabs", @@ -84240,7 +84312,8 @@ "search_context_size_high": 0.02 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-6-flash" + "source": "https://empiriolabs.ai/models/qwen3-6-flash", + "supports_function_calling": true }, "empiriolabs/qwen3-6-flash:variant1": { "litellm_provider": "empiriolabs", @@ -84261,7 +84334,8 @@ "search_context_size_high": 0.01 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-6-flash" + "source": "https://empiriolabs.ai/models/qwen3-6-flash", + "supports_function_calling": true }, "empiriolabs/qwen3-6-plus": { "litellm_provider": "empiriolabs", @@ -84282,7 +84356,8 @@ "search_context_size_high": 0.026 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-6-plus" + "source": "https://empiriolabs.ai/models/qwen3-6-plus", + "supports_function_calling": true }, "empiriolabs/qwen3-6-plus:variant1": { "litellm_provider": "empiriolabs", @@ -84303,7 +84378,8 @@ "search_context_size_high": 0.01 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-6-plus" + "source": "https://empiriolabs.ai/models/qwen3-6-plus", + "supports_function_calling": true }, "empiriolabs/qwen3-7-flash": { "litellm_provider": "empiriolabs", @@ -84330,7 +84406,8 @@ }, "supports_prompt_caching": true, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-7-flash" + "source": "https://empiriolabs.ai/models/qwen3-7-flash", + "supports_function_calling": true }, "empiriolabs/qwen3-7-flash:variant1": { "litellm_provider": "empiriolabs", @@ -84357,7 +84434,8 @@ }, "supports_prompt_caching": true, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-7-flash" + "source": "https://empiriolabs.ai/models/qwen3-7-flash", + "supports_function_calling": true }, "empiriolabs/qwen3-7-max": { "litellm_provider": "empiriolabs", @@ -84375,7 +84453,8 @@ "search_context_size_high": 0.02 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-7-max" + "source": "https://empiriolabs.ai/models/qwen3-7-max", + "supports_function_calling": true }, "empiriolabs/qwen3-7-max:variant1": { "litellm_provider": "empiriolabs", @@ -84393,7 +84472,8 @@ "search_context_size_high": 0.01 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-7-max" + "source": "https://empiriolabs.ai/models/qwen3-7-max", + "supports_function_calling": true }, "empiriolabs/qwen3-7-plus": { "litellm_provider": "empiriolabs", @@ -84414,7 +84494,8 @@ "search_context_size_high": 0.03 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-7-plus" + "source": "https://empiriolabs.ai/models/qwen3-7-plus", + "supports_function_calling": true }, "empiriolabs/qwen3-7-plus:variant1": { "litellm_provider": "empiriolabs", @@ -84438,7 +84519,8 @@ }, "supports_prompt_caching": true, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-7-plus" + "source": "https://empiriolabs.ai/models/qwen3-7-plus", + "supports_function_calling": true }, "empiriolabs/qwen3-8-27b": { "litellm_provider": "empiriolabs", @@ -84473,7 +84555,8 @@ "search_context_size_high": 0.02 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-8-flash" + "source": "https://empiriolabs.ai/models/qwen3-8-flash", + "supports_function_calling": true }, "empiriolabs/qwen3-8-max": { "litellm_provider": "empiriolabs", @@ -84492,7 +84575,8 @@ "search_context_size_high": 0.02 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-8-max" + "source": "https://empiriolabs.ai/models/qwen3-8-max", + "supports_function_calling": true }, "empiriolabs/qwen3-8-max-0902": { "litellm_provider": "empiriolabs", @@ -84511,7 +84595,8 @@ "search_context_size_high": 0.02 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-8-max-0902" + "source": "https://empiriolabs.ai/models/qwen3-8-max-0902", + "supports_function_calling": true }, "empiriolabs/qwen3-8-max:variant1": { "litellm_provider": "empiriolabs", @@ -84530,7 +84615,8 @@ "search_context_size_high": 0.01 }, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/qwen3-8-max" + "source": "https://empiriolabs.ai/models/qwen3-8-max", + "supports_function_calling": true }, "empiriolabs/qwen3-8-omni-flash": { "litellm_provider": "empiriolabs", @@ -84550,7 +84636,8 @@ }, "supports_response_schema": true, "supports_audio_input": true, - "source": "https://empiriolabs.ai/models/qwen3-8-omni-flash" + "source": "https://empiriolabs.ai/models/qwen3-8-omni-flash", + "supports_function_calling": true }, "empiriolabs/seed-2-0-code": { "litellm_provider": "empiriolabs", @@ -84564,7 +84651,8 @@ "supports_system_messages": true, "supports_vision": true, "supports_reasoning": true, - "source": "https://empiriolabs.ai/models/seed-2-0-code" + "source": "https://empiriolabs.ai/models/seed-2-0-code", + "supports_function_calling": true }, "empiriolabs/seed-2-0-lite": { "litellm_provider": "empiriolabs", @@ -84579,7 +84667,8 @@ "supports_vision": true, "supports_reasoning": true, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/seed-2-0-lite" + "source": "https://empiriolabs.ai/models/seed-2-0-lite", + "supports_function_calling": true }, "empiriolabs/seed-2-0-mini": { "litellm_provider": "empiriolabs", @@ -84594,7 +84683,8 @@ "supports_vision": true, "supports_reasoning": true, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/seed-2-0-mini" + "source": "https://empiriolabs.ai/models/seed-2-0-mini", + "supports_function_calling": true }, "empiriolabs/seed-2-0-pro": { "litellm_provider": "empiriolabs", @@ -84609,7 +84699,8 @@ "supports_vision": true, "supports_reasoning": true, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/seed-2-0-pro" + "source": "https://empiriolabs.ai/models/seed-2-0-pro", + "supports_function_calling": true }, "empiriolabs/seed-2-1-turbo": { "litellm_provider": "empiriolabs", @@ -84622,7 +84713,8 @@ "supports_vision": true, "supports_reasoning": true, "supports_response_schema": true, - "source": "https://empiriolabs.ai/models/seed-2-1-turbo" + "source": "https://empiriolabs.ai/models/seed-2-1-turbo", + "supports_function_calling": true }, "empiriolabs/skylark-embedding-vision": { "litellm_provider": "empiriolabs", From de0b389205c8ef2f2a6c5a57791068f9a7cf7eca Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Fri, 6 Nov 2026 12:07:57 -0800 Subject: [PATCH 18/39] feat(router): add the Switchyard stage router as an auto router strategy (#50407) * feat(router): add Switchyard Stage Router strategy * feat(router): decision-model classifier and safer fallbacks for the Switchyard stage router Moves to nemo-switchyard 0.4.0a1, which adds decision-model classifiers and makes ModelCall.respond and fail async. The classifier is now a tagged union of an LLM judge and a decision model, and decision calls go through the router's adecisions so any decisions provider works. Algorithm misbehavior and library errors route to the capable model instead of failing the request, and the session id plus client session headers are always passed so capable holds apply without a classifier Co-Authored-By: Claude Opus 5.5 * fix(router): keep the Switchyard router inside the type-check gate The gate's environment did not install the optional switchyard extra, so every libsy symbol counted as Unknown. nemo-switchyard joins proxy-dev like the other optional runtime packages, and imports go through switchyard_rust.libsy, which ships py.typed. Step handling moves into helpers so no Final is rebound in the loop, and the dashboard API types are regenerated for the new routing fields Co-Authored-By: Claude Opus 5.5 * fix(router): route Switchyard on every endpoint and reject unsafe router saves The router now passes each request's call type to routing strategies, and Switchyard converts the request to chat messages with the existing converter for that call type before translating it. /v1/messages turns with tool or thinking blocks used to fall back to the capable model, and now route on their signals. The Anthropic converter can keep tool_result.is_error on an analysis copy, so Switchyard sees tool failures there, while requests sent to models never carry it Saves of a strategy router that lists itself now fail with a 400 instead of a 500 with a dead row, the capable and efficient models must differ, judge and decision calls are not retried before failing open, the Switchyard routing fields are classified for redaction, and the edit dialog returns the saved router instead of the stale one Co-Authored-By: Claude Opus 5.5 * fix(router): catch self-routing on patch and rename, and trim Switchyard tests The self-routing check now reads the stored params through the decrypting helper and runs when a patch carries no litellm_params, so a PATCH that points a router at itself or renames it onto one of its targets returns 400 instead of a 500 with a dead row. The Switchyard tests share one transcript builder and router fixture, and reformatting noise in unrelated test files is reverted Co-Authored-By: Claude Opus 5.5 * refactor(ui): make Switchyard a strategy in the existing auto router form Switchyard is now a card in the existing strategy picker. Picking it swaps the tier and classifier fields for the Switchyard fields, and the shared name, team and save button create it through one submit branch, the same flow as the other strategies. The separate add form, the extracted picker and the hidden-form swap are gone, and the edit dialog keeps its own Switchyard form The judge's session_affinity and message_hash_fallback settings are removed, since Switchyard rejects both for the stage router Co-Authored-By: Claude Opus 5.5 * fix(ui): accept partial tool semantics and show server errors on model edit The Switchyard tool semantics field rejected mappings that list only some categories, although the server accepts them. Model PATCH errors were reduced to a generic message, so it now goes through the shared API client like model create and surfaces the server text Co-Authored-By: Claude Opus 5.5 * feat(ui): show the routing strategy picker up front in the auto router form Every strategy was hidden behind an Advanced configuration link, so the quick setup read as the only option and never said it creates a heuristic router. The picker now always shows with Heuristics as the labelled default, the quick setup sits under it, and picking another strategy opens that strategy prefilled with the chosen models Co-Authored-By: Claude Opus 5.5 * fix(ui): show the server's reason when a model settings or API key update fails Both screens replaced the rejection with a fixed message, so a 400 such as a router routing to itself read only as a generic failure Co-Authored-By: Claude Opus 5.5 * feat(router): route Switchyard on the Rust router core main replaced the Python router core with the Rust one, which left Switchyard routers registered nowhere. Register them with the other pre-routing strategies, hand each strategy the endpoint the request came through, and evict Switchyard routers on reload and delete like the rest Co-Authored-By: Claude Opus 5.5 * fix(router): scope Switchyard sessions per API key and fix review gaps in the Switchyard form Switchyard session state is keyed by a hash of the caller's key hash and session id, and client session headers no longer reach Switchyard unscoped. The add form carries the quick setup models into Switchyard, sends the team and access groups, clears a stale tool semantics error, and the schema reads API-saved configs with nulls and structured decision instructions Co-Authored-By: Claude Opus 5.5 * test(router): make the Switchyard test folder a package * fix(router): read the Switchyard session key hash only from the proxy's own metadata * fix(proxy): keep nested nulls as null when masking deployment params * fix(router): send Switchyard judge calls with the caller's own metadata bucket so tag routing applies * fix(router): put Switchyard judge metadata in the bucket the judge endpoint reads --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- .github/workflows/test-unit.yml | 10 +- .../sensitive_data_masker.py | 6 +- .../chat/guardrail_translation/handler.py | 9 +- .../messages/adapters/transformation.py | 20 +- .../guardrail_translation/base_translation.py | 5 +- .../chat/guardrail_translation/handler.py | 2 +- .../guardrail_translation/handler.py | 2 +- litellm/proxy/_lazy_openapi_snapshot.json | 4 + .../auto_router_endpoints.py | 1 + .../model_management_endpoints.py | 31 +- litellm/router_api.py | 10 + litellm/router_rust/core.py | 10 +- litellm/router_rust/invoke/endpoints.py | 5 + litellm/router_rust/steps/prerouting.py | 41 +- litellm/router_rust/surface/registry.py | 1 + .../adaptive_router/adaptive_router.py | 1 + .../auto_router/auto_router.py | 1 + .../auto_router/auto_router_model_naming.py | 13 +- .../complexity_router/complexity_router.py | 1 + .../quality_router/quality_router.py | 1 + .../switchyard_router/__init__.py | 0 .../switchyard_router/config.py | 154 ++++++ .../switchyard_router/switchyard_router.py | 238 ++++++++++ .../switchyard_router/translation.py | 232 ++++++++++ litellm/types/litellm_params.py | 1 + litellm/types/llms/openai.py | 1 + litellm/types/router.py | 2 + litellm/types/utils.py | 9 +- pyproject.toml | 2 + .../test_anthropic_guardrail_handler.py | 38 ++ .../test_openai_endpoint_utils.py | 30 +- .../test_model_management_endpoints.py | 117 ++++- .../switchyard_router/__init__.py | 0 .../switchyard_router/test_config.py | 100 ++++ .../test_switchyard_router.py | 438 ++++++++++++++++++ .../test_router_tag_routing.py | 2 +- tests/unit/test_router/test_router.py | 10 +- tests/unit/types/test_litellm_params.py | 1 + .../app/(dashboard)/hooks/models/useModels.ts | 1 + .../components/AutoRouters/autoRouterRows.ts | 11 +- .../add_model/AutoRouterClassifierTabs.tsx | 20 +- .../add_model/BlockedReasonTooltip.tsx | 17 + .../add_model/SwitchyardConfigFields.tsx | 342 ++++++++++++++ .../add_model/SwitchyardRouterForm.tsx | 176 +++++++ .../add_auto_router_tab.integration.test.tsx | 88 +++- .../add_model/add_auto_router_tab.tsx | 188 ++++---- .../add_model/auto_router_strategies.ts | 11 +- .../handle_add_auto_router_submit.tsx | 27 ++ .../add_model/switchyard_config.test.ts | 132 ++++++ .../components/add_model/switchyard_config.ts | 102 ++++ ...dit_auto_router_modal.integration.test.tsx | 31 ++ .../edit_auto_router_modal.tsx | 43 +- .../model_info_view.integration.test.tsx | 12 + .../src/components/model_info_view.tsx | 2 +- .../src/components/networking.test.ts | 22 + .../src/components/networking.tsx | 31 +- .../update_model_credentials_modal.test.tsx | 12 + .../update_model_credentials_modal.tsx | 2 +- ui/litellm-dashboard/src/i18n/locales/en.json | 14 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 18 +- uv.lock | 24 +- 61 files changed, 2675 insertions(+), 200 deletions(-) create mode 100644 litellm/router_strategy/switchyard_router/__init__.py create mode 100644 litellm/router_strategy/switchyard_router/config.py create mode 100644 litellm/router_strategy/switchyard_router/switchyard_router.py create mode 100644 litellm/router_strategy/switchyard_router/translation.py create mode 100644 tests/unit/router_strategy/switchyard_router/__init__.py create mode 100644 tests/unit/router_strategy/switchyard_router/test_config.py create mode 100644 tests/unit/router_strategy/switchyard_router/test_switchyard_router.py create mode 100644 ui/litellm-dashboard/src/components/add_model/BlockedReasonTooltip.tsx create mode 100644 ui/litellm-dashboard/src/components/add_model/SwitchyardConfigFields.tsx create mode 100644 ui/litellm-dashboard/src/components/add_model/SwitchyardRouterForm.tsx create mode 100644 ui/litellm-dashboard/src/components/add_model/switchyard_config.test.ts create mode 100644 ui/litellm-dashboard/src/components/add_model/switchyard_config.ts diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 41198887cab..fb29a92e306 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -1035,9 +1035,9 @@ jobs: run: | diff -u model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then - .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra switchyard --extra saml --extra extra_proxy --extra proxy-runtime --extra utils else - .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project --no-install-package litellm-proxy + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra switchyard --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project --no-install-package litellm-proxy uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so uv run --no-sync python -c "import importlib.metadata; import litellm.rust_bridge._native; print(importlib.metadata.version('litellm'))" @@ -1294,7 +1294,7 @@ jobs: - name: Install dependencies if: steps.changes.outputs.decision != 'skip' run: | - .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra switchyard - name: Cache Prisma binaries if: steps.changes.outputs.decision != 'skip' @@ -1497,9 +1497,9 @@ jobs: RUST_BRIDGE_ARTIFACT: ${{ needs.rust-bridge.outputs.artifact }} run: | if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then - .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra switchyard --extra saml --extra extra_proxy --extra proxy-runtime --extra utils else - .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project --no-install-package litellm-proxy + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra switchyard --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project --no-install-package litellm-proxy uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so fi diff --git a/litellm/litellm_core_utils/sensitive_data_masker.py b/litellm/litellm_core_utils/sensitive_data_masker.py index 8c402fe9b44..f6bad7f611b 100644 --- a/litellm/litellm_core_utils/sensitive_data_masker.py +++ b/litellm/litellm_core_utils/sensitive_data_masker.py @@ -117,7 +117,9 @@ class SensitiveDataMasker: elif key_is_sensitive and isinstance(item, str): masked_items.append(self.mask_value(item)) else: - masked_items.append(item if isinstance(item, (int, float, bool, str, list)) else str(item)) + masked_items.append( + item if item is None or isinstance(item, (int, float, bool, str, list)) else str(item) + ) return masked_items def mask_dict( @@ -144,7 +146,7 @@ class SensitiveDataMasker: str_value = str(v) if v is not None else "" masked_data[k] = self.mask_value(str_value) else: - masked_data[k] = v if isinstance(v, (int, float, bool, str, list)) else str(v) + masked_data[k] = v if v is None or isinstance(v, (int, float, bool, str, list)) else str(v) except Exception: masked_data[k] = "" diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 8f1a08d2964..ca9529ba246 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -795,18 +795,21 @@ class AnthropicMessagesHandler(BaseTranslation): data: dict, # mutable-ok: the request body the adapter reads *, keep_cache_control: bool = False, + keep_tool_errors: bool = False, ) -> ChatCompletionRequest: """Translate Anthropic request to OpenAI chat completion format.""" ( chat_completion_compatible_request, _tool_name_mapping, - ) = LiteLLMAnthropicMessagesAdapter(keep_cache_control=keep_cache_control).translate_anthropic_to_openai( + ) = LiteLLMAnthropicMessagesAdapter( + keep_cache_control=keep_cache_control, keep_tool_errors=keep_tool_errors + ).translate_anthropic_to_openai( anthropic_message_request=cast(AnthropicMessagesRequest, scannable_tool_result_request(data)), preserve_midturn_system=True, ) return chat_completion_compatible_request - def get_structured_messages(self, data: dict) -> list[AllMessageValues] | None: + def get_structured_messages(self, data: dict, *, keep_tool_errors: bool = False) -> list[AllMessageValues] | None: """ Convert Anthropic messages request data to OpenAI-spec structured messages. @@ -815,7 +818,7 @@ class AnthropicMessagesHandler(BaseTranslation): messages: Final = data.get("messages") if messages is None: return None - chat_completion_compatible_request: Final = self._translate_to_openai(data) + chat_completion_compatible_request: Final = self._translate_to_openai(data, keep_tool_errors=keep_tool_errors) result: Final = cast( list[AllMessageValues], chat_completion_compatible_request.get("messages", []), diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index 43eaa37aba7..5e4fba6d706 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -8,7 +8,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast from pydantic import JsonValue, TypeAdapter import litellm -from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params from litellm.litellm_core_utils.hidden_params import get_hidden_params from litellm.llms.anthropic.messages.convertible_blocks import tool_result_text_part from litellm.llms.anthropic.messages.utils import ( @@ -434,8 +434,9 @@ _CLAUDE_OVER_REASONING_EFFORT_PROVIDERS: Final = frozenset({LlmProviders.KYMA.va class LiteLLMAnthropicMessagesAdapter: - def __init__(self, *, keep_cache_control: bool = False) -> None: + def __init__(self, *, keep_cache_control: bool = False, keep_tool_errors: bool = False) -> None: self.keep_cache_control: Final = keep_cache_control + self.keep_tool_errors: Final = keep_tool_errors ### FOR [BETA] `/v1/messages` endpoint support @@ -615,11 +616,7 @@ class LiteLLMAnthropicMessagesAdapter: self._add_cache_control_if_applicable(content, doc_obj, model) new_user_content_list.append(doc_obj) elif content.get("type") == "tool_result": - tool_result = ChatCompletionToolMessage( - role="tool", - tool_call_id=content.get("tool_use_id", ""), - content=self._tool_result_content(content.get("content")), - ) + tool_result = self._tool_message(content) self._add_cache_control_if_applicable(content, tool_result, model) tool_message_list.append(tool_result) @@ -1432,6 +1429,15 @@ class LiteLLMAnthropicMessagesAdapter: return None return anthropic_image_source_to_openai_url(image_source) + def _tool_message(self, block: object) -> ChatCompletionToolMessage: + fields: Final = as_str_mapping(block) or {} + is_error: Final = fields.get("is_error") + tool_call_id: Final = tool_use_id if isinstance(tool_use_id := fields.get("tool_use_id"), str) else "" + content: Final = self._tool_result_content(fields.get("content")) + if self.keep_tool_errors and isinstance(is_error, bool): + return ChatCompletionToolMessage(role="tool", tool_call_id=tool_call_id, content=content, is_error=is_error) + return ChatCompletionToolMessage(role="tool", tool_call_id=tool_call_id, content=content) + def _tool_result_content(self, raw_content: object) -> ToolResultContent: if isinstance(raw_content, str): return raw_content diff --git a/litellm/llms/base_llm/guardrail_translation/base_translation.py b/litellm/llms/base_llm/guardrail_translation/base_translation.py index 6399ded9555..71368f4ef61 100644 --- a/litellm/llms/base_llm/guardrail_translation/base_translation.py +++ b/litellm/llms/base_llm/guardrail_translation/base_translation.py @@ -257,11 +257,12 @@ class BaseTranslation(ABC): """ return None - def get_structured_messages(self, data: dict) -> list["AllMessageValues"] | None: + def get_structured_messages(self, data: dict, *, keep_tool_errors: bool = False) -> list["AllMessageValues"] | None: """ Convert request data to OpenAI-spec structured messages. - Override in subclasses for format-specific conversion. + Override in subclasses for format-specific conversion. ``keep_tool_errors`` copies a provider's + tool-failure marker onto tool messages as ``is_error``; only for copies that are never sent to a model. Returns None if no convertible content is found. """ diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index c5488bc8a98..26246cfd7b6 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -103,7 +103,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): delivers_ended_stream_rewrites = True assembles_streamed_response = True - def get_structured_messages(self, data: dict) -> list[AllMessageValues] | None: + def get_structured_messages(self, data: dict, *, keep_tool_errors: bool = False) -> list[AllMessageValues] | None: """ Convert chat completions request data to OpenAI-spec structured messages. diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index aa4dea196ff..20b7506ae0a 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -777,7 +777,7 @@ class OpenAIResponsesHandler(BaseTranslation): delivers_ended_stream_rewrites = True assembles_streamed_response = True - def get_structured_messages(self, data: dict) -> list[AllMessageValues] | None: + def get_structured_messages(self, data: dict, *, keep_tool_errors: bool = False) -> list[AllMessageValues] | None: """ Convert Responses API request data to OpenAI-spec structured messages. diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 137d6016402..b50e3955838 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -49950,6 +49950,10 @@ ], "title": "Content" }, + "is_error": { + "title": "Is Error", + "type": "boolean" + }, "role": { "const": "tool", "title": "Role", diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py index 42321fe1b1d..3c733d08a2e 100644 --- a/litellm/proxy/management_endpoints/auto_router_endpoints.py +++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py @@ -1112,6 +1112,7 @@ def _is_configured_pre_routing_strategy(llm_router: "Router", router_name: str) router_registry.plugins("complexity"), router_registry.plugins("adaptive"), router_registry.plugins("quality"), + router_registry.plugins("switchyard"), ) ) diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 2c7453ebb1c..b8b926a3057 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -142,6 +142,7 @@ from litellm.router_strategy.auto_router.auto_router_model_naming import ( count_capability_routers, gated_capabilities_of, is_complexity_router_model, + strategy_router_dependencies, validate_complexity_router_config_placement, validate_complexity_router_config_write, validate_strategy_router_model_write, @@ -355,6 +356,22 @@ async def get_db_model(model_id: str, prisma_client: PrismaClient) -> Deployment return deployment_pydantic_obj +def _self_routing_violation( + model_name: str | None, + incoming_params: GenericLiteLLMParams | None, + existing_params: GenericLiteLLMParams | None, +) -> str | None: + if not model_name: + return None + effective_params: Final = { + **(_decrypted_litellm_params(existing_params) if existing_params is not None else {}), + **(_dump_without_none(incoming_params) if incoming_params is not None else {}), + } + if any(dependency.model_name == model_name for dependency in strategy_router_dependencies(effective_params)): + return f"A strategy router cannot route to itself: {model_name} is listed as one of its own models" + return None + + def _strategy_router_write_violation( incoming_params: GenericLiteLLMParams | None, existing_params: GenericLiteLLMParams | None, @@ -374,6 +391,11 @@ def _strategy_router_write_violation( """ if incoming_params is None: return None + from litellm.router_strategy.switchyard_router.config import validate_switchyard_config_write + + switchyard_violation: Final = validate_switchyard_config_write(incoming_params.switchyard_router_config) + if switchyard_violation is not None: + return switchyard_violation config_violation: Final = validate_complexity_router_config_write( complexity_router_config=( _effective_complexity_router_config(incoming_params, existing_params) @@ -406,8 +428,11 @@ def _strategy_router_write_violation( def _raise_on_strategy_router_write_violation( incoming_params: GenericLiteLLMParams | None, existing_params: GenericLiteLLMParams | None, + model_name: str | None = None, ) -> None: - violation = _strategy_router_write_violation(incoming_params=incoming_params, existing_params=existing_params) + violation: Final = _self_routing_violation(model_name, incoming_params, existing_params) or ( + _strategy_router_write_violation(incoming_params=incoming_params, existing_params=existing_params) + ) if violation is None: return raise ProxyException( @@ -1751,6 +1776,7 @@ async def patch_model( _raise_on_strategy_router_write_violation( incoming_params=incoming_params, existing_params=remaining_params, + model_name=patch_data.model_name or db_model.model_name, ) written_team_id: Final = ( @@ -3120,6 +3146,7 @@ async def add_new_model( _raise_on_strategy_router_write_violation( incoming_params=model_params.litellm_params, existing_params=None, + model_name=model_params.model_name, ) _raise_if_rate_limits_required_but_missing( @@ -3325,6 +3352,7 @@ async def update_model( _raise_on_strategy_router_write_violation( incoming_params=model_params.litellm_params, existing_params=deployment.litellm_params, + model_name=model_params.model_name or deployment.model_name, ) renamed_to: Final = ( model_params.model_name @@ -4028,6 +4056,7 @@ async def clear_cache() -> ReconcileOutcome: llm_router.registry().plugins("complexity").pop(model_name, None) llm_router.registry().plugins("adaptive").pop(model_name, None) llm_router.registry().plugins("quality").pop(model_name, None) + llm_router.registry().plugins("switchyard").pop(model_name, None) # Reload only DB models. _add_deployment_locked, not add_deployment: this # coroutine already holds MODEL_RECONCILE_LOCK and asyncio.Lock is not diff --git a/litellm/router_api.py b/litellm/router_api.py index 1ade356f494..ca370488f5d 100644 --- a/litellm/router_api.py +++ b/litellm/router_api.py @@ -65,6 +65,7 @@ if TYPE_CHECKING: from litellm.router_strategy.budget_limiter import RouterBudgetLimiting from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter from litellm.router_strategy.quality_router.quality_router import QualityRouter + from litellm.router_strategy.switchyard_router.switchyard_router import SwitchyardRouter from litellm.router_utils.cooldown_cache import CooldownCache from litellm.router_utils.fallback_event_handlers import AttemptedFallbackTargets as _AttemptedFallbackTargets from litellm.router_utils.health_state_cache import DeploymentHealthCache @@ -106,6 +107,7 @@ if TYPE_CHECKING: | dict[str, list[TaggedPreRoutingStrategy[ComplexityRouter]]] | dict[str, list[TaggedPreRoutingStrategy[QualityRouter]]] | dict[str, list[TaggedPreRoutingStrategy[AdaptiveRouter]]] + | dict[str, list[TaggedPreRoutingStrategy[SwitchyardRouter]]] | list[RoutingPlugin] | RouterBudgetLimiting | DualCache @@ -171,6 +173,7 @@ PluginKind: TypeAlias = Literal[ "complexity", "quality", "adaptive", + "switchyard", "routing", "budget_limiter", "prompt_caching_pins", @@ -1106,6 +1109,11 @@ class RouterRegistry: self, kind: Literal["adaptive"] ) -> "dict[str, list[TaggedPreRoutingStrategy[AdaptiveRouter]]]": ... # mutable-ok: the live dict + @overload + def plugins( + self, kind: Literal["switchyard"] + ) -> "dict[str, list[TaggedPreRoutingStrategy[SwitchyardRouter]]]": ... # mutable-ok: the live dict + @overload def plugins(self, kind: Literal["routing"]) -> "list[RoutingPlugin]": ... # mutable-ok: the live list @@ -1132,6 +1140,8 @@ class RouterRegistry: return r._core_quality_routers if kind == "adaptive": return r._core_adaptive_routers + if kind == "switchyard": + return r._core_switchyard_routers if kind == "routing": return r._core_routing_plugins if kind == "budget_limiter": diff --git a/litellm/router_rust/core.py b/litellm/router_rust/core.py index 40ae7e0e554..3e222313f32 100644 --- a/litellm/router_rust/core.py +++ b/litellm/router_rust/core.py @@ -1177,6 +1177,7 @@ _HOSTED_STATE: Final = frozenset( "_core_auto_routers", "_core_complexity_routers", "_core_quality_routers", + "_core_switchyard_routers", "_core_adaptive_routers", "_core_routing_plugins", "_core_deployment_latency_map", @@ -1722,6 +1723,7 @@ class RustCore( run_plugins=partial(self._run_plugins, request_kwargs, messages) if self._hosted._core_routing_plugins # pyright: ignore[reportPrivateUsage] # the live plugin list else None, + call_type=None, ) def _core_log_retry( @@ -2715,7 +2717,7 @@ class RustCore( web_search: Final = not plan.sync and self._picks_async() and is_web_search_request(request) # The generic handler `_route_capability_pin` names: a call only some providers serve # drops the others, and a Decisions call carrying media skips the deployments refusing it. - handler: Final = plan.host_op.rsplit(".", 1)[-1] + handler: Final = plan.handler decisions_media: Final = handler in DECISIONS_MEDIA_CALL_TYPES and bool( request.get("audio") or request.get("videos") ) @@ -3113,6 +3115,7 @@ class RustCore( specific_deployment: object = _NO_INPUT, *, pass_through: bool = False, + call_type: str | None = None, ) -> object: """The host's pre-routing of one pick (router_rust.steps.prerouting): the model group to resolve instead of the step's ``model``, a custom strategy's deployment, or ``None``. ``pass_through`` @@ -3166,6 +3169,7 @@ class RustCore( fallback_depth=depth if isinstance(depth, int) else 0, router_strategy=normalize_strategy(r.routing_strategy), pass_through=pass_through, + call_type=call_type, enable_tag_filtering=r.enable_tag_filtering, match_any=r.tag_filtering_match_any, public_router=public if public is not None else self, @@ -5266,7 +5270,9 @@ class _Driver: async def pre_route(self, step: Mapping[str, object]) -> object: request: Final = self._request try: - return await self._core._pre_route(step, request, *self._plan.pick_input(request)) # pyright: ignore[reportPrivateUsage] # see above + return await self._core._pre_route( # pyright: ignore[reportPrivateUsage] # see above + step, request, *self._plan.pick_input(request), call_type=self._plan.handler + ) except Exception as error: # noqa: BLE001 # see select return self._pick_raised(error, logged=True) diff --git a/litellm/router_rust/invoke/endpoints.py b/litellm/router_rust/invoke/endpoints.py index e2c621ccb7d..7a5b52761e2 100644 --- a/litellm/router_rust/invoke/endpoints.py +++ b/litellm/router_rust/invoke/endpoints.py @@ -244,6 +244,11 @@ class Plan: # the request (`{**data, name: value, **kwargs}`). named: tuple[str, ...] = () + @property + def handler(self) -> str: + """The handler name the host op ends in, which route capabilities and pre-routing strategies key on.""" + return self.host_op.rsplit(".", 1)[-1] + @property def on_its_own_loop(self) -> bool: """``function_with_fallbacks`` runs these sync endpoints' whole walk on a private event loop.""" diff --git a/litellm/router_rust/steps/prerouting.py b/litellm/router_rust/steps/prerouting.py index 63d15a1a80a..47bd1e69046 100644 --- a/litellm/router_rust/steps/prerouting.py +++ b/litellm/router_rust/steps/prerouting.py @@ -197,9 +197,13 @@ class PreRoutingStrategies: def quality(self) -> StrategyRegistry: return self._hosted._core_quality_routers # pyright: ignore[reportPrivateUsage] # see above + @property + def switchyard(self) -> StrategyRegistry: + return self._hosted._core_switchyard_routers # pyright: ignore[reportPrivateUsage] # see above + def all(self) -> tuple[StrategyRegistry, ...]: """In the order ``_select_pre_routing_strategy`` searches them.""" - return (self.auto, self.complexity, self.adaptive, self.quality) + return (self.switchyard, self.auto, self.complexity, self.adaptive, self.quality) def any_registered(self) -> bool: return any(self.all()) @@ -211,6 +215,7 @@ class PreRoutingStrategies: self._hosted._core_quality_routers = {} # pyright: ignore[reportPrivateUsage] # see above self._hosted._core_complexity_routers = {} # pyright: ignore[reportPrivateUsage] # see above self._hosted._core_auto_routers = {} # pyright: ignore[reportPrivateUsage] # see above + self._hosted._core_switchyard_routers = {} # pyright: ignore[reportPrivateUsage] # see above def register(self, deployment: Deployment) -> None: """The end of ``_add_deployment``: an auto, complexity or quality marker registers its router @@ -222,6 +227,8 @@ class PreRoutingStrategies: self._init_complexity_router(deployment) case "quality": self._init_quality_router(deployment) + case "switchyard": + self._init_switchyard_router(deployment) case _: pass @@ -231,7 +238,7 @@ class PreRoutingStrategies: return model_name: Final = deployment.model_name tags: Final = _deployment_tags(deployment) - for registry in (self.auto, self.complexity, self.quality): + for registry in (self.auto, self.complexity, self.quality, self.switchyard): _ = _unregister(registry, model_name, tags) if _unregister(self.adaptive, model_name, tags): self._sync_adaptive_router_hooks() @@ -463,6 +470,32 @@ class PreRoutingStrategies: ) self._register(self.quality, deployment, router, "Quality-router") + def _init_switchyard_router(self, deployment: Deployment) -> None: + from litellm.router_strategy.switchyard_router.config import SwitchyardRouterConfig, build_algorithm + from litellm.router_strategy.switchyard_router.switchyard_router import SwitchyardRouter + from litellm.types.decisions import OpenAIDecisionResponse + from litellm.types.utils import ModelResponse + + config: Final = SwitchyardRouterConfig.model_validate(deployment.litellm_params.switchyard_router_config) + if deployment.model_name in (config.models.capable, config.models.efficient, config.models.judge): + raise ValueError("Switchyard cannot route to itself") + ctx: Final = self._ctx + + async def complete_judge(request: Mapping[str, object]) -> ModelResponse: + result: Final = await ctx.acompletion(**request) # pyright: ignore[reportAttributeAccessIssue] # the core's RoutingContext + if not isinstance(result, ModelResponse): + raise TypeError("Switchyard judge must return a non-streaming completion") + return result + + async def decide(request: Mapping[str, object]) -> OpenAIDecisionResponse: + result: Final = await ctx.adecisions(**request) # pyright: ignore[reportAttributeAccessIssue] # see above + if not isinstance(result, OpenAIDecisionResponse): + raise TypeError("Switchyard decision judge must return an OpenAI-shaped decision response") + return result + + router: Final = SwitchyardRouter(deployment.model_name, config, build_algorithm(config), complete_judge, decide) + self._register(self.switchyard, deployment, router, "Switchyard") + def _init_adaptive_router(self, deployment: Deployment) -> None: from litellm.router_strategy.adaptive_router.adaptive_router import AdaptiveRouter from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterPostCallHook @@ -741,6 +774,7 @@ class PreRouter: public_router: object, run_plugins: Callable[[str], Awaitable[None]] | None = None, pass_through: bool = False, + call_type: str | None = None, ) -> str | None: """The start of ``_async_get_available_deployment``: run the hook, record its pick and pin the tier's litellm_params. The model group to resolve, or ``None`` to resolve ``model`` itself. @@ -759,6 +793,7 @@ class PreRouter: match_any=match_any, public_router=public_router, run_plugins=run_plugins, + call_type=call_type, ) routed: Final = response.model if response is not None else model if response is not None: @@ -782,6 +817,7 @@ class PreRouter: match_any: bool, public_router: object, run_plugins: Callable[[str], Awaitable[None]] | None, + call_type: str | None, ) -> "PreRoutingHookResponse | None": _restore_client_output_ceiling(request_kwargs) _record_routing_decision(request_kwargs, None) @@ -861,6 +897,7 @@ class PreRouter: messages=routing_messages if routing_messages is not None else messages, # pyright: ignore[reportArgumentType] # the request's messages, as given input=input, # pyright: ignore[reportArgumentType] # the request's input, as given specific_deployment=False, + call_type=call_type, ) hooked: Final[PreRoutingHookResponse | None] = routed traced: Final = ( diff --git a/litellm/router_rust/surface/registry.py b/litellm/router_rust/surface/registry.py index 7c3abdcd45e..820db52d5a1 100644 --- a/litellm/router_rust/surface/registry.py +++ b/litellm/router_rust/surface/registry.py @@ -658,6 +658,7 @@ class HostRegistry(HostModelInfo): self._core_complexity_routers: dict[str, list[TaggedPreRoutingStrategy[object]]] = {} # mutable-ok: same self._core_adaptive_routers: dict[str, list[TaggedPreRoutingStrategy[object]]] = {} # mutable-ok: same self._core_quality_routers: dict[str, list[TaggedPreRoutingStrategy[object]]] = {} # mutable-ok: same + self._core_switchyard_routers: dict[str, list[TaggedPreRoutingStrategy[object]]] = {} # mutable-ok: same self.strategies: PreRoutingStrategies | None = None # The router's optional pre-call checks (deployment budgets, prompt caching pins); # RustCore installs its ``OptionalChecks`` (litellm/router_rust/steps/callbacks.py). diff --git a/litellm/router_strategy/adaptive_router/adaptive_router.py b/litellm/router_strategy/adaptive_router/adaptive_router.py index c3a65d6aa91..43869fdf832 100644 --- a/litellm/router_strategy/adaptive_router/adaptive_router.py +++ b/litellm/router_strategy/adaptive_router/adaptive_router.py @@ -184,6 +184,7 @@ class AdaptiveRouter: messages: list[dict[str, object]] | None = None, input: str | list | None = None, specific_deployment: bool | None = False, + call_type: str | None = None, ) -> PreRoutingHookResponse | None: """ Plugin entry point invoked by `Router.async_pre_routing_hook` when the diff --git a/litellm/router_strategy/auto_router/auto_router.py b/litellm/router_strategy/auto_router/auto_router.py index b92c53fa93c..68e8701eb23 100644 --- a/litellm/router_strategy/auto_router/auto_router.py +++ b/litellm/router_strategy/auto_router/auto_router.py @@ -204,6 +204,7 @@ class AutoRouter(CustomLogger): messages: list[dict[str, object]] | None = None, input: str | list | None = None, specific_deployment: bool | None = False, + call_type: str | None = None, ) -> Optional["PreRoutingHookResponse"]: """ This hook is called before the routing decision is made. diff --git a/litellm/router_strategy/auto_router/auto_router_model_naming.py b/litellm/router_strategy/auto_router/auto_router_model_naming.py index f5aeccd9305..47daa175c99 100644 --- a/litellm/router_strategy/auto_router/auto_router_model_naming.py +++ b/litellm/router_strategy/auto_router/auto_router_model_naming.py @@ -26,7 +26,7 @@ from litellm.router_strategy.complexity_router.config import ( AUTO_ROUTER_MODEL_PREFIX: Final = "auto_router/" -StrategyRouterKind = Literal["semantic", "complexity", "adaptive", "quality"] +StrategyRouterKind: TypeAlias = Literal["semantic", "complexity", "adaptive", "quality", "switchyard"] StrategyRouterDependencyRole: TypeAlias = Literal[ "tier", "default", "classifier", "embedding", "evaluation", "compactor" @@ -43,6 +43,7 @@ class StrategyRouterDependency: STRATEGY_ROUTER_PARAM_FIELDS: Final[frozenset[str]] = frozenset( { + "switchyard_router_config", "auto_router_config", "auto_router_config_path", "auto_router_default_model", @@ -57,6 +58,7 @@ STRATEGY_ROUTER_PARAM_FIELDS: Final[frozenset[str]] = frozenset( ) _REQUIRED_FIELD_GROUPS: Final[Mapping[StrategyRouterKind, tuple[tuple[str, ...], ...]]] = { + "switchyard": (("switchyard_router_config",),), "semantic": ( ("auto_router_config", "auto_router_config_path"), ("auto_router_default_model",), @@ -78,6 +80,8 @@ def classify_strategy_router_model(model: str) -> StrategyRouterKind | None: if not model.startswith(AUTO_ROUTER_MODEL_PREFIX): return None remainder: Final = model[len(AUTO_ROUTER_MODEL_PREFIX) :] + if remainder.startswith("switchyard"): + return "switchyard" if remainder.startswith("complexity_router"): return "complexity" if remainder.startswith("adaptive_router"): @@ -137,6 +141,13 @@ def strategy_router_dependencies( kind: Final = classify_strategy_router_model(str(litellm_params.get("model", ""))) if kind is None: return () + if kind == "switchyard": + models: Final = _mapping(_mapping(litellm_params.get("switchyard_router_config")).get("models")) + return ( + _named(models.get("capable"), "tier") + + _named(models.get("efficient"), "tier") + + _named(models.get("judge"), "classifier") + ) if kind == "semantic": return _named(litellm_params.get("auto_router_default_model"), "default") + _named( litellm_params.get("auto_router_embedding_model"), "embedding" diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 05dd4706b62..f004dfcc38d 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -4490,6 +4490,7 @@ class ComplexityRouter(CustomLogger): messages: list[dict[str, object]] | None = None, input: str | list | None = None, specific_deployment: bool | None = False, + call_type: str | None = None, ) -> PreRoutingHookResponse | None: """ Pre-routing hook called before the routing decision. diff --git a/litellm/router_strategy/quality_router/quality_router.py b/litellm/router_strategy/quality_router/quality_router.py index 2d9ad63e9de..fc8d5ab5fcb 100644 --- a/litellm/router_strategy/quality_router/quality_router.py +++ b/litellm/router_strategy/quality_router/quality_router.py @@ -330,6 +330,7 @@ class QualityRouter(CustomLogger): messages: list[dict[str, object]] | None = None, input: str | list | None = None, specific_deployment: bool | None = False, + call_type: str | None = None, ) -> Optional["PreRoutingHookResponse"]: """Try keyword override first; fall back to complexity-tier routing.""" from litellm.types.router import PreRoutingHookResponse diff --git a/litellm/router_strategy/switchyard_router/__init__.py b/litellm/router_strategy/switchyard_router/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/router_strategy/switchyard_router/config.py b/litellm/router_strategy/switchyard_router/config.py new file mode 100644 index 00000000000..7e0d3ad3a94 --- /dev/null +++ b/litellm/router_strategy/switchyard_router/config.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import TYPE_CHECKING, Annotated, Final, Literal, TypeAlias + +from pydantic import BaseModel, ConfigDict, Discriminator, Field, Tag, TypeAdapter, model_validator + +if TYPE_CHECKING: + from switchyard_rust.libsy import Algorithm, TaskClassifierConfig + + +class LlmClassifierConfig(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + kind: Literal["llm"] = "llm" + base_threshold: float = Field(default=0.5, ge=0, le=1) + threshold_step: float = Field(default=0.0, ge=0, le=1) + recent_turn_window: int | None = Field(default=None, gt=0) + max_output_tokens: int = Field(default=4096, gt=0) + response_format_type: Literal["json_schema", "json_object"] = "json_schema" + prompt: str | None = None + + +class DecisionClassifierConfig(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + kind: Literal["decision"] + cutoff: float = Field(default=0.6, ge=0, le=1) + instructions: str | Mapping[str, object] | None = None + evidence: Mapping[str, object] | None = None + + +_KIND: Final = TypeAdapter(Mapping[str, object]) + + +def _classifier_kind(value: object) -> str: + match value: + case LlmClassifierConfig() | DecisionClassifierConfig(): + return value.kind + case Mapping(): + return str(_KIND.validate_python(value).get("kind", "llm")) + case _: + return "llm" + + +ClassifierConfig: TypeAlias = Annotated[ + Annotated[LlmClassifierConfig, Tag("llm")] | Annotated[DecisionClassifierConfig, Tag("decision")], + Discriminator(_classifier_kind), +] + + +class StageOptions(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + picker: Literal["efficient_first", "capable_first"] = "efficient_first" + confidence_threshold: float | None = Field(default=None, ge=0, le=1) + recent_window: int | None = Field(default=None, gt=0) + tool_semantics: Mapping[str, tuple[str, ...]] | None = None + only_on_wrong_signal_escalation: bool = True + + +class StageModels(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid", str_strip_whitespace=True) + + capable: str = Field(min_length=1) + efficient: str = Field(min_length=1) + judge: str | None = Field(default=None, min_length=1) + + @model_validator(mode="after") + def reject_recursive_targets(self) -> StageModels: + if any(name.startswith("auto_router/") for name in (self.capable, self.efficient, self.judge or "")): + raise ValueError("Switchyard targets must be concrete model groups, not auto-router markers") + if self.capable == self.efficient: + raise ValueError("The capable and efficient models must be different model groups") + return self + + +class SwitchyardRouterConfig(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + algorithm: Literal["stage_router"] = "stage_router" + models: StageModels + options: StageOptions = Field(default_factory=StageOptions) + classifier: ClassifierConfig | None = None + + @model_validator(mode="after") + def validate_judge(self) -> SwitchyardRouterConfig: + if (self.classifier is None) != (self.models.judge is None): + raise ValueError("A judge model is required exactly when the classifier is enabled") + return self + + @property + def confidence_threshold(self) -> float: + if self.options.confidence_threshold is not None: + return self.options.confidence_threshold + return 0.7 if self.classifier is not None else 0.5 + + def runtime_models(self) -> Mapping[str, tuple[str, ...]]: + return { + "any": (self.models.capable, self.models.efficient), + "capable": (self.models.capable,), + "efficient": (self.models.efficient,), + **({"judge": (self.models.judge,)} if self.models.judge is not None else {}), + } + + +def _task_classifier(config: SwitchyardRouterConfig) -> TaskClassifierConfig | None: + from switchyard_rust.libsy import TaskClassifierConfig + + match config.classifier: + case None: + return None + case LlmClassifierConfig() as llm: + return TaskClassifierConfig( + base_threshold=llm.base_threshold, + threshold_step=llm.threshold_step, + recent_turn_window=llm.recent_turn_window, + max_output_tokens=llm.max_output_tokens, + response_format_type=llm.response_format_type, + prompt=llm.prompt, + ) + case DecisionClassifierConfig() as decision: + return TaskClassifierConfig.decision( + cutoff=decision.cutoff, + candidates={"capable": config.models.capable, "efficient": config.models.efficient}, + evidence=dict(decision.evidence or {}), + instructions=decision.instructions, + ) + + +def build_algorithm(config: SwitchyardRouterConfig) -> Algorithm: + from switchyard_rust.libsy import LlmFallback, stage_router + + classifier: Final = _task_classifier(config) + return stage_router( + picker=config.options.picker, + confidence_threshold=config.confidence_threshold, + recent_window=config.options.recent_window, + tool_semantics=dict(config.options.tool_semantics) if config.options.tool_semantics is not None else None, + only_on_wrong_signal_escalation=config.options.only_on_wrong_signal_escalation, + classifier=LlmFallback(config=classifier) if classifier is not None else None, + ) + + +def validate_switchyard_config_write(config: Mapping[str, object] | None) -> str | None: + if config is None: + return None + try: + build_algorithm(SwitchyardRouterConfig.model_validate(config)) + except ImportError: + return "Switchyard requires the optional dependency. Install litellm[switchyard] (nemo-switchyard==0.4.0a1)." + except (ValueError, TypeError) as error: + return f"Invalid switchyard_router_config: {error}" + return None diff --git a/litellm/router_strategy/switchyard_router/switchyard_router.py b/litellm/router_strategy/switchyard_router/switchyard_router.py new file mode 100644 index 00000000000..cb455bab337 --- /dev/null +++ b/litellm/router_strategy/switchyard_router/switchyard_router.py @@ -0,0 +1,238 @@ +from __future__ import annotations + +import hashlib +from collections.abc import Awaitable, Callable, Mapping, Sequence +from typing import TYPE_CHECKING, Final, Literal, TypeAlias, TypeVar + +from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY +from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs +from litellm.litellm_core_utils.internal_call_metadata import ( + effective_turn_off_message_logging, + forwarded_internal_call_metadata, + parent_session_kwargs, +) +from litellm.router_strategy.switchyard_router.config import ( + DecisionClassifierConfig, + LlmClassifierConfig, + SwitchyardRouterConfig, +) +from litellm.router_strategy.switchyard_router.translation import ( + chat_messages, + decision_request, + decision_response, + judge_request, + judge_response, + mapping, + request_to_switchyard, +) +from litellm.types.decisions import OpenAIDecisionResponse +from litellm.types.router import PreRoutingHookResponse +from litellm.types.utils import AUTOROUTER_CLASSIFIER_CALL_ORIGIN, ModelResponse, StandardLoggingRoutingDecision + +if TYPE_CHECKING: + from switchyard_rust.libsy import Algorithm, DecisionCall, ModelCall, RoutingOutcome + +JudgeCompletion: TypeAlias = Callable[[Mapping[str, object]], Awaitable[ModelResponse]] +DecisionCompletion: TypeAlias = Callable[[Mapping[str, object]], Awaitable[OpenAIDecisionResponse]] +FallbackReason: TypeAlias = Literal[ + "unsupported_request_type", + "unsupported_content", + "libsy_error", + "unexpected_judge_call", + "unexpected_decision_call", + "target_outside_config", + "no_decision", +] +_T = TypeVar("_T") +_SESSION_HEADER: Final = "x-switchyard-session-id" +_FORWARDED_HEADER_PREFIXES: Final = ("x-switchyard-", "x-codex-", "x-nemo-relay-", "x-dynamo-") +_FORWARDED_HEADERS: Final = frozenset( + {"x-session-id", "session-id", "thread-id", "x-task-id", "x-request-id", "x-client-request-id"} +) + + +async def _resolved(value: _T) -> _T: + return value + + +_SESSION_ALIASES: Final = frozenset( + {_SESSION_HEADER, "x-claude-code-session-id", "x-nemo-relay-session-id", "x-session-id", "session-id"} +) +_CODEX_TURN_METADATA_HEADER: Final = "x-codex-turn-metadata" + + +def _forwarded_header(name: str, session_scoped: bool) -> bool: + lowered: Final = name.lower() + if lowered in _SESSION_ALIASES or (lowered == _CODEX_TURN_METADATA_HEADER and not session_scoped): + return False + return lowered in _FORWARDED_HEADERS or lowered.startswith(_FORWARDED_HEADER_PREFIXES) + + +def _caller_key_hash(request_kwargs: Mapping[str, object]) -> str: + internal: Final = mapping(request_kwargs.get(get_metadata_variable_name_from_kwargs(request_kwargs)) or {}) + key: Final = internal.get("user_api_key_hash") + return str(key) if key is not None else "" + + +def _scoped_session(request_kwargs: Mapping[str, object]) -> str | None: + session: Final = parent_session_kwargs(request_kwargs).get("litellm_session_id") + if not session: + return None + return hashlib.sha256(f"{_caller_key_hash(request_kwargs)}\0{session}".encode()).hexdigest() + + +def switchyard_headers(request_kwargs: Mapping[str, object]) -> Mapping[str, str]: + proxy_request: Final = mapping(request_kwargs.get("proxy_server_request") or {}) + client: Final = mapping(proxy_request.get("headers") or {}) + session: Final = _scoped_session(request_kwargs) + forwarded: Final = { + name.lower(): value + for name, value in client.items() + if isinstance(value, str) and _forwarded_header(name, session is not None) + } + return {**forwarded, **({_SESSION_HEADER: session} if session else {})} + + +class SwitchyardRouter: + def __init__( + self, + model_name: str, + config: SwitchyardRouterConfig, + algorithm: Algorithm, + completion: JudgeCompletion, + decide: DecisionCompletion, + ) -> None: + self.model_name = model_name + self.config = config + self.algorithm = algorithm + self.completion = completion + self.decide = decide + + def _fallback( + self, reason: FallbackReason, messages: Sequence[Mapping[str, object]] | None + ) -> PreRoutingHookResponse: + return PreRoutingHookResponse( + model=self.config.models.capable, + messages=[dict(message) for message in messages] if messages is not None else None, + routing_decision={ + "router_type": "switchyard", + "router_model_name": self.model_name, + "routed_model": self.config.models.capable, + "switchyard_algorithm": self.config.algorithm, + "switchyard_fallback_reason": reason, + }, + ) + + def _internal_call( + self, + request: Mapping[str, object], + caller: Mapping[str, object], + bucket: Literal["metadata", "litellm_metadata"], + ) -> Mapping[str, object]: + caller_metadata: Final = mapping(caller.get(get_metadata_variable_name_from_kwargs(caller)) or {}) + return { + **request, + bucket: { + **forwarded_internal_call_metadata(caller_metadata, AUTOROUTER_CLASSIFIER_CALL_ORIGIN), + INTERNAL_CALL_ORIGIN_METADATA_KEY: AUTOROUTER_CLASSIFIER_CALL_ORIGIN, + }, + "proxy_server_request": {"body": request}, + "num_retries": 0, + "turn_off_message_logging": effective_turn_off_message_logging(caller), + **parent_session_kwargs(caller), + } + + def _decision( + self, pick: str, evidence: object, messages: Sequence[Mapping[str, object]] | None + ) -> PreRoutingHookResponse: + decision: Final[StandardLoggingRoutingDecision] = { + "router_type": "switchyard", + "router_model_name": self.model_name, + "routed_model": pick, + "switchyard_algorithm": self.config.algorithm, + "switchyard_evidence": evidence, + } + return PreRoutingHookResponse( + model=pick, + messages=[dict(message) for message in messages] if messages is not None else None, + routing_decision=decision, + ) + + async def _answer_judge(self, call: ModelCall, caller: Mapping[str, object]) -> None: + try: + converted: Final = judge_response( + await self.completion( + self._internal_call(judge_request(call.request, call.models[0]), caller, "metadata") + ) + ) + except Exception as error: # noqa: BLE001 # judge/provider failures are inputs to Switchyard's fall-open decision + await call.fail(error) + else: + await call.respond(_resolved(converted)) + + async def _answer_decision(self, call: DecisionCall, caller: Mapping[str, object]) -> None: + try: + answer: Final = decision_response( + await self.decide( + self._internal_call(decision_request(call.request, call.model), caller, "litellm_metadata") + ), + call.request, + ) + except Exception as error: # noqa: BLE001 # decision/provider failures are inputs to Switchyard's fall-open decision + await call.fail(error) + else: + await call.respond(_resolved(answer)) + + def _outcome( + self, outcome: RoutingOutcome, messages: Sequence[Mapping[str, object]] | None + ) -> PreRoutingHookResponse: + pick: Final = outcome.selected_model_ids[0] + if pick not in (self.config.models.capable, self.config.models.efficient): + return self._fallback("target_outside_config", messages) + metadata: Final = outcome.metadata + evidence: Final[object] = metadata.evidence if metadata is not None else None + return self._decision(pick, evidence, messages) + + async def async_pre_routing_hook( + self, + model: str, + request_kwargs: Mapping[str, object], + messages: Sequence[Mapping[str, object]] | None = None, + input: object = None, + specific_deployment: bool | None = False, + call_type: str | None = None, + ) -> PreRoutingHookResponse | None: + from switchyard_rust.libsy import LibsyError, Step + + if model != self.model_name: + return None + try: + chat: Final = chat_messages(call_type, model, messages, input, request_kwargs) + request: Final = request_to_switchyard(chat) if chat is not None else None + except (ValueError, KeyError, TypeError): + return self._fallback("unsupported_content", messages) + if request is None: + return self._fallback("unsupported_request_type", messages) + judge: Final = self.config.models.judge + try: + async for step in self.algorithm.run_stream( + request, self.config.runtime_models(), headers=switchyard_headers(request_kwargs) + ): + match step: + case Step.CallModel(call) if isinstance( + self.config.classifier, LlmClassifierConfig + ) and call.models[:1] == [judge]: + await self._answer_judge(call, request_kwargs) + case Step.CallDecision(call) if ( + isinstance(self.config.classifier, DecisionClassifierConfig) and call.model == judge + ): + await self._answer_decision(call, request_kwargs) + case Step.CallModel(): + return self._fallback("unexpected_judge_call", messages) + case Step.CallDecision(): + return self._fallback("unexpected_decision_call", messages) + case Step.Done(outcome): + return self._outcome(outcome, messages) + except LibsyError: + return self._fallback("libsy_error", messages) + return self._fallback("no_decision", messages) diff --git a/litellm/router_strategy/switchyard_router/translation.py b/litellm/router_strategy/switchyard_router/translation.py new file mode 100644 index 00000000000..246f2ba3e41 --- /dev/null +++ b/litellm/router_strategy/switchyard_router/translation.py @@ -0,0 +1,232 @@ +from __future__ import annotations + +import json +from collections.abc import Callable, Mapping, Sequence +from typing import TYPE_CHECKING, Final, Protocol + +from pydantic import TypeAdapter + +from litellm.types.decisions import ( + OpenAIChoiceAnswer, + OpenAIDecisionResponse, + systemone_choice_key, +) +from litellm.types.utils import CallTypes, ModelResponse + +if TYPE_CHECKING: + from switchyard_rust.libsy import LlmResponse + +_MAPPING: Final = TypeAdapter(Mapping[str, object]) +_SEQUENCE: Final = TypeAdapter(tuple[object, ...]) +_CALL_TYPES: Final = frozenset(call_type.value for call_type in CallTypes) + + +def mapping(value: object) -> Mapping[str, object]: + return _MAPPING.validate_python(value) + + +def sequence(value: object) -> tuple[object, ...]: + return _SEQUENCE.validate_python(value) + + +def text_content(value: object) -> tuple[Mapping[str, object], ...]: + if value is None: + return () + if isinstance(value, str): + return ({"type": "text", "text": value},) + blocks: Final = tuple(mapping(block) for block in sequence(value)) + if any(block.get("type") != "text" or not isinstance(block.get("text"), str) for block in blocks): + raise ValueError("Switchyard routing supports text content only; media and audio are unsupported") + return tuple({"type": "text", "text": block["text"]} for block in blocks) + + +def tool_call(value: object) -> Mapping[str, object]: + call: Final = mapping(value) + function: Final = mapping(call.get("function")) + arguments: Final = function.get("arguments") + if call.get("type") != "function" or not isinstance(arguments, str): + raise ValueError("Switchyard requires function calls with JSON arguments") + return { + "type": "tool_call", + "id": call["id"], + "name": function["name"], + "arguments": _MAPPING.validate_json(arguments), + } + + +def message_to_switchyard(value: object) -> Mapping[str, object]: + message: Final = mapping(value) + if message.get("audio") is not None: + raise ValueError("Switchyard routing does not support audio") + role: Final = message.get("role") + content: Final = text_content(message.get("content")) + if role == "tool": + return { + "role": "tool", + "content": ( + { + "type": "tool_result", + "tool_call_id": message["tool_call_id"], + "content": content, + "is_error": message.get("is_error"), + }, + ), + } + if role not in ("user", "system", "developer", "assistant"): + raise ValueError("Switchyard received an unsupported message role") + calls: Final = tuple(tool_call(call) for call in sequence(message.get("tool_calls") or ())) + return {"role": role, "content": (*content, *calls)} + + +class ChatConverter(Protocol): + def get_structured_messages( + self, + data: dict[str, object], # mutable-ok: matches the request converters' existing dict parameter + *, + keep_tool_errors: bool = False, + ) -> Sequence[object] | None: ... + + +def _instantiate(converter: Callable[[], ChatConverter]) -> ChatConverter: + return converter() + + +def chat_messages( + call_type: str | None, + model: str, + messages: Sequence[Mapping[str, object]] | None, + input: object, + request_kwargs: Mapping[str, object], +) -> tuple[Mapping[str, object], ...] | None: + from litellm.llms import load_guardrail_translation_mappings + + converter: Final = ( + load_guardrail_translation_mappings().get(CallTypes(call_type)) if call_type in _CALL_TYPES else None + ) + if converter is None: + return None + request: Final = { + **request_kwargs, + "model": model, + **({"messages": [dict(message) for message in messages]} if messages is not None else {}), + **({"input": input} if input is not None else {}), + } + return tuple(map(mapping, _instantiate(converter).get_structured_messages(request, keep_tool_errors=True) or ())) + + +def request_to_switchyard(messages: Sequence[Mapping[str, object]]) -> Mapping[str, object]: + if not messages: + raise ValueError("Switchyard requires at least one message") + return { + "model": "auto", + "messages": tuple(message_to_switchyard(message) for message in messages), + "stream": False, + } + + +def judge_message(value: object) -> Mapping[str, object]: + message: Final = mapping(value) + blocks: Final = text_content(message.get("content")) + return {"role": message["role"], "content": "".join(str(block["text"]) for block in blocks)} + + +def judge_request(request: Mapping[str, object], model: str) -> Mapping[str, object]: + output: Final = mapping(request.get("output") or {}) + sampling: Final = mapping(request.get("sampling") or {}) + options: Final = { + "max_tokens": output.get("max_output_tokens"), + "response_format": output.get("response_format"), + "temperature": sampling.get("temperature"), + "top_p": sampling.get("top_p"), + } + return { + "model": model, + "stream": False, + "messages": [ + judge_message(message) + for message in ( + *sequence(request.get("instructions") or ()), + *sequence(request.get("messages") or ()), + ) + ], + **{key: value for key, value in options.items() if value is not None}, + } + + +def judge_response(response: ModelResponse) -> LlmResponse.Agg: + from switchyard_rust.libsy import LlmResponse + + body: Final = _MAPPING.validate_json(response.model_dump_json(exclude_unset=True)) + choices: Final = tuple(mapping(choice) for choice in sequence(body["choices"])) + return LlmResponse.Agg( + { + "id": response.id, + "model": response.model, + "outputs": tuple(message_to_switchyard(choice["message"]) for choice in choices), + } + ) + + +def _decision_text(value: object) -> str: + return value if isinstance(value, str) else json.dumps(value) + + +def _decision_question(name: str, question: Mapping[str, object]) -> Mapping[str, object]: + kind: Final = mapping(question["kind"]) + if kind.get("type") != "choice": + raise ValueError("Switchyard decision routing asks choice questions only") + return { + "type": "choice", + "name": name, + "instructions": _decision_text(question.get("instructions")), + "choices": [ + { + "value": option["id"], + **({"description": _decision_text(option["description"])} if option.get("description") else {}), + } + for option in map(mapping, sequence(mapping(kind["data"])["options"])) + ], + } + + +def decision_request(request: Mapping[str, object], model: str) -> Mapping[str, object]: + questions: Final = mapping(request["questions"]) + return { + "model": model, + "input": _decision_text(request["context"]), + "questions": [_decision_question(name, mapping(question)) for name, question in questions.items()], + } + + +def _choice_value(answer: object) -> Mapping[str, object]: + if not isinstance(answer, OpenAIChoiceAnswer): + raise TypeError("The decision model refused or returned a non-choice answer") + return { + "type": "choice", + "data": { + "selected": systemone_choice_key(answer.choice), + "probabilities": {systemone_choice_key(p.value): p.probability for p in answer.probabilities}, + }, + } + + +def decision_response(response: OpenAIDecisionResponse, request: Mapping[str, object]) -> Mapping[str, object]: + names: Final = tuple(mapping(request["questions"])) + if len(response.answers) != len(names): + raise ValueError("The decision model did not answer every question") + return { + "id": None, + "model": response.model, + "answers": { + answer.name or name: { + "value": _choice_value(answer), + "provider_confidence": getattr(answer, "confidence", None), + } + for name, answer in zip(names, response.answers, strict=True) + }, + "usage": { + "input_tokens": response.usage.input_tokens, + "output_tokens": response.usage.output_tokens, + "total_tokens": response.usage.total_tokens, + }, + } diff --git a/litellm/types/litellm_params.py b/litellm/types/litellm_params.py index 9f63d27f5f3..c04430322c4 100644 --- a/litellm/types/litellm_params.py +++ b/litellm/types/litellm_params.py @@ -207,6 +207,7 @@ class DeploymentOptions: @dataclass(frozen=True, slots=True, kw_only=True) class SpecializedRouterOptions: + switchyard_router_config: Mapping[str, object] | None = None auto_router_config_path: str | None = None auto_router_config: str | None = None auto_router_default_model: str | None = None diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index c9739eee91d..378899b667d 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -957,6 +957,7 @@ class ChatCompletionToolMessage(TypedDict): role: Literal["tool"] content: str | Iterable[ToolMessageContentPart] # writable-ok: Pydantic warns on ReadOnly TypedDict fields tool_call_id: str + is_error: NotRequired[bool] # writable-ok: Pydantic warns on ReadOnly TypedDict fields class ChatCompletionFunctionMessage(TypedDict): diff --git a/litellm/types/router.py b/litellm/types/router.py index 3dcfc694003..c0b182f0f20 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -659,6 +659,7 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): tag_regex: list[str] | None = None # auto-router params + switchyard_router_config: Mapping[str, object] | None = None auto_router_config_path: str | None = None auto_router_config: str | None = None auto_router_default_model: str | None = None @@ -1461,6 +1462,7 @@ class PreRoutingStrategy(Protocol): messages: list[dict[str, Any]] | None = None, input: "str | list[Any] | None" = None, specific_deployment: bool | None = False, + call_type: str | None = None, ) -> "PreRoutingHookResponse | None": ... diff --git a/litellm/types/utils.py b/litellm/types/utils.py index b082198d3f2..755a0a8cc7a 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3382,7 +3382,10 @@ class StandardLoggingRoutingDecision(TypedDict, total=False): """Per-request provenance for a pre-routing strategy (auto-router) decision.""" router_model_name: str - router_type: ReadOnly[Literal["complexity", "adaptive", "quality", "semantic"]] + router_type: ReadOnly[Literal["complexity", "adaptive", "quality", "semantic", "switchyard"]] + switchyard_algorithm: ReadOnly[str] + switchyard_evidence: ReadOnly[object] + switchyard_fallback_reason: ReadOnly[str] router_config_id: ReadOnly[str] router_config_updated_at: ReadOnly[str] router_config_fingerprint: ReadOnly[str] @@ -3443,10 +3446,12 @@ class StandardLoggingKubernetesPodRouting(TypedDict): # so a redacted row stays explainable. `test_every_routing_decision_field_is_classified` # fails if a field is added to the record without being placed in one set or the other. PROMPT_QUOTING_ROUTING_DECISION_FIELDS: frozenset[str] = frozenset( - {"signals", "matched_keyword", "escalation_keyword", "classifier_crux"} + {"signals", "matched_keyword", "escalation_keyword", "classifier_crux", "switchyard_evidence"} ) DERIVED_ROUTING_DECISION_FIELDS: Final[frozenset[str]] = frozenset( { + "switchyard_algorithm", + "switchyard_fallback_reason", "router_model_name", "router_type", "router_config_id", diff --git a/pyproject.toml b/pyproject.toml index e193eeee93f..02834f604d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,6 +54,7 @@ Documentation = "https://docs.litellm.ai" # sub-packages and the opentelemetry trio move in lockstep, and grpcio is # supply-chain-pinned to a vetted, aged release. [project.optional-dependencies] +switchyard = ["nemo-switchyard==0.4.0a1"] proxy = [ "gunicorn>=23.0.0,<24.0", "uvicorn>=0.33.0,<1.0", @@ -268,6 +269,7 @@ proxy-dev = [ "opentelemetry-instrumentation-fastapi==0.66b1", "azure-identity==1.25.2", "a2a-sdk==1.1.0", + "nemo-switchyard==0.4.0a1", "backports-zstd==1.8.0; python_version < '3.14'", "httptools>=0.7.1,<0.8", "semver>=3.0.4,<4", diff --git a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index 8affe2e136e..b6229dbaec0 100644 --- a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -3458,3 +3458,41 @@ def test_reading_a_dropped_budget_for_a_guardrail_logs_no_drop_warning(caplog: p assert messages == [{"role": "user", "content": "hi"}] assert not [record for record in caplog.records if "Dropping" in record.getMessage()] + + +FAILED_TOOL_RESULT_REQUEST: Final = { + "model": "claude-sonnet", + "max_tokens": 16, + "messages": [ + {"role": "user", "content": "Rename the helper"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "edit0", "name": "Edit", "input": {"file_path": "a.py"}}], + }, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "edit0", "content": "String not found", "is_error": True} + ], + }, + ], +} + + +@pytest.mark.parametrize("keep_tool_errors", [True, False]) +def test_structured_messages_carry_tool_failures_only_when_asked(keep_tool_errors: bool) -> None: + messages: Final = AnthropicMessagesHandler().get_structured_messages( + FAILED_TOOL_RESULT_REQUEST, keep_tool_errors=keep_tool_errors + ) + assert messages is not None + (tool_message,) = [message for message in messages if message["role"] == "tool"] + assert tool_message.get("is_error") is (True if keep_tool_errors else None) + + +def test_request_sent_to_a_model_never_carries_tool_failure_marker() -> None: + from litellm.llms.anthropic.messages.adapters.transformation import LiteLLMAnthropicMessagesAdapter + + request, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( + anthropic_message_request=FAILED_TOOL_RESULT_REQUEST # pyright: ignore[reportArgumentType] # a JSON request in the Anthropic shape + ) + assert "is_error" not in json.dumps(request["messages"], default=str) diff --git a/tests/unit/proxy/common_utils/test_openai_endpoint_utils.py b/tests/unit/proxy/common_utils/test_openai_endpoint_utils.py index 3e3e89f8117..3eb901144a3 100644 --- a/tests/unit/proxy/common_utils/test_openai_endpoint_utils.py +++ b/tests/unit/proxy/common_utils/test_openai_endpoint_utils.py @@ -77,9 +77,7 @@ from litellm.proxy.common_utils.openai_endpoint_utils import ( ), ], ) -def test_remove_sensitive_info_from_deployment( - model_config: dict, expected_config: dict -): +def test_remove_sensitive_info_from_deployment(model_config: dict, expected_config: dict): sanitized_config = remove_sensitive_info_from_deployment(model_config) assert sanitized_config == expected_config @@ -110,10 +108,7 @@ def test_remove_sensitive_info_from_deployment_with_excluded_keys(): sanitized_config = remove_sensitive_info_from_deployment( copy.deepcopy(base_config), excluded_keys={"litellm_credentials_name"} ) - assert ( - sanitized_config["litellm_params"]["litellm_credentials_name"] - == "my-credential-name" - ) + assert sanitized_config["litellm_params"]["litellm_credentials_name"] == "my-credential-name" # access_token should still be masked (not in excluded_keys) assert sanitized_config["litellm_params"]["access_token"] != "token-12345" @@ -121,3 +116,24 @@ def test_remove_sensitive_info_from_deployment_with_excluded_keys(): # api_key should still be removed (popped) regardless of excluded_keys assert "api_key" not in sanitized_config["litellm_params"] + + +def test_remove_sensitive_info_from_deployment_keeps_nested_nulls(): + router_config = { + "models": {"capable": "strong", "efficient": "fast", "judge": None}, + "options": {"confidence_threshold": None, "tool_semantics": None}, + "fallbacks": [None, "fast"], + } + deployment = { + "model_name": "route", + "litellm_params": { + "model": "auto_router/switchyard", + "api_key": "sk-sensitive-key-123", + "switchyard_router_config": copy.deepcopy(router_config), + }, + } + + sanitized = remove_sensitive_info_from_deployment(deployment) + + assert "api_key" not in sanitized["litellm_params"] + assert sanitized["litellm_params"]["switchyard_router_config"] == router_config diff --git a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py index 32c9535b4c8..2fdb77af8f9 100644 --- a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py @@ -1085,7 +1085,7 @@ class TestClearCache: Test that clear_cache successfully clears router model caches and reloads models. """ mock_router = MagicMock() - plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}} + plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "switchyard": {}} mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__ mock_router.registry.return_value.all.return_value = ["openai/gpt-4o", "openai/gpt-4o-mini"] @@ -1125,7 +1125,7 @@ class TestClearCache: # entries are auto_router/* deployments (so their router-map entries should be # cleared for reload); the config-defined router is preserved. mock_router = MagicMock() - plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}} + plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "switchyard": {}} mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__ mock_router.registry.return_value.all.return_value = [ { @@ -1209,7 +1209,7 @@ class TestClearCache: ) mock_router = MagicMock() - plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}} + plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "switchyard": {}} mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__ mock_router.registry.return_value.all.return_value = [ { @@ -1270,7 +1270,7 @@ class TestClearCachePreservesConfigRouters: ) mock_router = MagicMock() - plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}} + plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "switchyard": {}} mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__ mock_router.registry.return_value.all.return_value = [ { @@ -1315,7 +1315,7 @@ class TestClearCachePreservesConfigRouters: from litellm.proxy.management_endpoints.model_management_endpoints import clear_cache mock_router = MagicMock() - plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}} + plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "switchyard": {}} mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__ mock_router.registry.return_value.all.return_value = [ { @@ -1346,15 +1346,15 @@ class TestClearCachePreservesConfigRouters: assert "shared-name" in plugins["complexity"] @pytest.mark.asyncio - async def test_db_quality_and_adaptive_routers_are_evicted(self): - """The auto_router/ prefix also covers quality_router/ and adaptive_router/. Their + async def test_db_quality_adaptive_and_switchyard_routers_are_evicted(self): + """The auto_router/ prefix also covers quality_router/, adaptive_router/ and switchyard. Their registry entries must be popped too, or reload's init raises 'already exists' (quality) or leaves a stale entry (adaptive). """ from litellm.proxy.management_endpoints.model_management_endpoints import clear_cache mock_router = MagicMock() - plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}} + plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "switchyard": {}} mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__ mock_router.registry.return_value.all.return_value = [ { @@ -1367,11 +1367,17 @@ class TestClearCachePreservesConfigRouters: "model_info": {"id": "db-a", "db_model": True}, "litellm_params": {"model": "auto_router/adaptive_router/a1"}, }, + { + "model_name": "s1", + "model_info": {"id": "db-s", "db_model": True}, + "litellm_params": {"model": "auto_router/switchyard"}, + }, ] plugins["auto"] = {} plugins["complexity"] = {} plugins["quality"] = {"q1": MagicMock()} plugins["adaptive"] = {"a1": MagicMock()} + plugins["switchyard"] = {"s1": MagicMock()} mock_config = MagicMock() mock_config.add_deployment_locked = AsyncMock( @@ -1389,6 +1395,7 @@ class TestClearCachePreservesConfigRouters: assert "q1" not in plugins["quality"] assert "a1" not in plugins["adaptive"] + assert "s1" not in plugins["switchyard"] class TestDeleteModelClearsRouterRegistry: @@ -1503,7 +1510,7 @@ class TestDeleteModelClearsRouterRegistry: mock_prisma.db.litellm_proxymodeltable.delete = AsyncMock(return_value=db_row) mock_router = MagicMock() - plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}} + plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "switchyard": {}} mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__ mock_router.apply.return_value = ApplyResult( version=1, @@ -11431,6 +11438,98 @@ async def test_add_new_model_reports_a_database_outage_during_the_insert_as_503( assert (exc_info.value.code, exc_info.value.type) == ("503", "no_db_connection") +@pytest.mark.parametrize("invalid", [ + {"algorithm": "random"}, + {"classifier": {}}, + {"options": {"capable_system_prompt": "ignored"}}, +]) +def test_switchyard_write_rejects_unloadable_configuration(invalid: dict[str, object]) -> None: + from litellm.proxy.management_endpoints.model_management_endpoints import _strategy_router_write_violation + from litellm.types.router import updateLiteLLMParams + + violation: Final = _strategy_router_write_violation( + incoming_params=updateLiteLLMParams( + model="auto_router/switchyard", + switchyard_router_config={"models": {"capable": "strong", "efficient": "fast"}, **invalid}, + ), + existing_params=None, + ) + assert violation is not None + assert "Invalid switchyard_router_config" in violation + + +SWITCHYARD_PARAMS: Final = { + "model": "auto_router/switchyard", + "switchyard_router_config": {"models": {"capable": "strong", "efficient": "fast"}}, +} + + +@pytest.mark.parametrize( + ("model_name", "incoming", "existing"), + [ + ("strong", SWITCHYARD_PARAMS, None), + ("fast", {"model": "auto_router/switchyard"}, SWITCHYARD_PARAMS), + ("fast", None, SWITCHYARD_PARAMS), + ( + "smart", + { + "model": "auto_router/complexity_router", + "complexity_router_config": {"tiers": {"SIMPLE": "smart", "COMPLEX": "strong"}}, + }, + None, + ), + ], +) +def test_strategy_router_that_lists_itself_is_rejected_before_saving( + model_name: str, incoming: dict[str, object] | None, existing: dict[str, object] | None +) -> None: + from litellm.proxy._types import ProxyException + from litellm.proxy.management_endpoints.model_management_endpoints import ( + _raise_on_strategy_router_write_violation, + ) + from litellm.types.router import updateLiteLLMParams + + with pytest.raises(ProxyException) as exc_info: + _raise_on_strategy_router_write_violation( + incoming_params=updateLiteLLMParams(**incoming) if incoming is not None else None, + existing_params=updateLiteLLMParams(**existing) if existing is not None else None, + model_name=model_name, + ) + assert exc_info.value.code == "400" + assert model_name in exc_info.value.message + + +def test_strategy_router_with_other_targets_saves() -> None: + from litellm.proxy.management_endpoints.model_management_endpoints import _self_routing_violation + from litellm.types.router import updateLiteLLMParams + + assert _self_routing_violation("route", updateLiteLLMParams(**SWITCHYARD_PARAMS), None) is None + + +def test_self_routing_is_caught_when_the_stored_model_is_encrypted(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy._types import ProxyException + from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper + from litellm.proxy.management_endpoints.model_management_endpoints import ( + _raise_on_strategy_router_write_violation, + ) + from litellm.types.router import updateLiteLLMParams + + monkeypatch.setenv("LITELLM_SALT_KEY", "sk-test-salt-key-for-encryption") + stored: Final = updateLiteLLMParams( + **{**SWITCHYARD_PARAMS, "model": encrypt_value_helper("auto_router/switchyard")} + ) + assert stored.model != "auto_router/switchyard" + with pytest.raises(ProxyException) as exc_info: + _raise_on_strategy_router_write_violation( + incoming_params=updateLiteLLMParams( + switchyard_router_config={"models": {"capable": "route", "efficient": "fast"}} + ), + existing_params=stored, + model_name="route", + ) + assert exc_info.value.code == "400" + + @pytest.mark.asyncio async def test_rename_rewrites_only_in_a_transaction_holding_every_locked_teams_advisory_lock(): """A team that starts listing the old name between the advisory snapshot and the row lock must not be diff --git a/tests/unit/router_strategy/switchyard_router/__init__.py b/tests/unit/router_strategy/switchyard_router/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/router_strategy/switchyard_router/test_config.py b/tests/unit/router_strategy/switchyard_router/test_config.py new file mode 100644 index 00000000000..c24023ddb79 --- /dev/null +++ b/tests/unit/router_strategy/switchyard_router/test_config.py @@ -0,0 +1,100 @@ +from typing import Final + +import pytest +from pydantic import ValidationError + +from litellm.router_strategy.switchyard_router.config import SwitchyardRouterConfig, validate_switchyard_config_write + + +@pytest.mark.parametrize("classifier", [None, {}]) +def test_default_confidence_depends_on_classifier(classifier: dict[str, object] | None) -> None: + config: Final = SwitchyardRouterConfig.model_validate( + { + "models": { + "capable": "strong", + "efficient": "fast", + **({"judge": "judge"} if classifier is not None else {}), + }, + "classifier": classifier, + } + ) + assert config.confidence_threshold == (0.7 if classifier is not None else 0.5) + + +@pytest.mark.parametrize("classifier", [None, {}]) +def test_explicit_confidence_is_preserved(classifier: dict[str, object] | None) -> None: + config: Final = SwitchyardRouterConfig.model_validate( + { + "models": { + "capable": "strong", + "efficient": "fast", + **({"judge": "judge"} if classifier is not None else {}), + }, + "classifier": classifier, + "options": {"confidence_threshold": 0.63, "picker": "capable_first"}, + } + ) + assert config.confidence_threshold == 0.63 + + +@pytest.mark.parametrize( + "option", ["escalation_note", "deescalation_note", "capable_system_prompt", "efficient_system_prompt", "unknown"] +) +def test_request_rewrite_and_unknown_options_are_rejected(option: str) -> None: + with pytest.raises(ValidationError, match=option): + SwitchyardRouterConfig.model_validate( + {"models": {"capable": "strong", "efficient": "fast"}, "options": {option: "ignored"}} + ) + + +@pytest.mark.parametrize( + "bad", + [ + {"algorithm": "random"}, + {"classifier": {}}, + {"options": {"confidence_threshold": 2}}, + {"models": {"capable": "", "efficient": "fast"}}, + {"models": {"capable": "same", "efficient": "same"}}, + {"models": {"capable": "strong", "efficient": "fast", "judge": "j"}, "classifier": {"session_affinity": True}}, + {"models": {"capable": "strong", "efficient": "fast", "judge": "judge"}}, + {"models": {"capable": "strong", "efficient": "fast", "judge": "j"}, "classifier": {"kind": "other"}}, + { + "models": {"capable": "strong", "efficient": "fast", "judge": "j"}, + "classifier": {"kind": "decision", "cutoff": 1.5}, + }, + { + "models": {"capable": "strong", "efficient": "fast", "judge": "j"}, + "classifier": {"kind": "decision", "base_threshold": 0.5}, + }, + { + "models": {"capable": "strong", "efficient": "fast", "judge": "j"}, + "classifier": {"kind": "llm", "cutoff": 0.5}, + }, + ], +) +def test_invalid_config_cannot_be_saved(bad: dict[str, object]) -> None: + violation: Final = validate_switchyard_config_write({"models": {"capable": "strong", "efficient": "fast"}, **bad}) + assert violation is not None + assert "Invalid switchyard_router_config" in violation + + +def test_missing_package_rejects_save_with_install_instructions() -> None: + import sys + from unittest.mock import patch + + with patch.dict(sys.modules, {"switchyard_rust.libsy": None}): + violation: Final = validate_switchyard_config_write({"models": {"capable": "strong", "efficient": "fast"}}) + assert violation is not None + assert "litellm[switchyard]" in violation + + +@pytest.mark.parametrize( + "classifier", [{"kind": "decision"}, {"kind": "decision", "cutoff": 0.4, "evidence": {"a": 1}}] +) +def test_decision_classifier_config_can_be_saved(classifier: dict[str, object]) -> None: + assert ( + validate_switchyard_config_write( + {"models": {"capable": "strong", "efficient": "fast", "judge": "jev"}, "classifier": classifier} + ) + is None + ) diff --git a/tests/unit/router_strategy/switchyard_router/test_switchyard_router.py b/tests/unit/router_strategy/switchyard_router/test_switchyard_router.py new file mode 100644 index 00000000000..8a26b4610db --- /dev/null +++ b/tests/unit/router_strategy/switchyard_router/test_switchyard_router.py @@ -0,0 +1,438 @@ +import asyncio +import json +from itertools import chain +from collections.abc import AsyncIterator, Mapping +from typing import Final + +import pytest + +import litellm +from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs +from litellm.integrations.custom_logger import CustomLogger +from litellm.router_api import ChangeSet +from litellm.router_strategy.switchyard_router.config import SwitchyardRouterConfig, build_algorithm +from litellm.router_strategy.switchyard_router.switchyard_router import SwitchyardRouter +from litellm.types.decisions import OpenAIDecisionResponse +from litellm.types.router import Deployment +from litellm.types.utils import ModelResponse + +libsy = pytest.importorskip("switchyard_rust.libsy") + +HELLO: Final = [{"role": "user", "content": "Hello"}] + + +async def unexpected_judge(request: Mapping[str, object]) -> ModelResponse: + raise AssertionError("This route must not call an LLM judge") + + +async def unexpected_decision(request: Mapping[str, object]) -> OpenAIDecisionResponse: + raise AssertionError("This route must not call a decision model") + + +def config( + *, classifier: Mapping[str, object] | None = None, picker: str = "efficient_first", threshold: float | None = None +) -> SwitchyardRouterConfig: + return SwitchyardRouterConfig.model_validate( + { + "models": { + "capable": "strong", + "efficient": "fast", + **({"judge": "judge"} if classifier is not None else {}), + }, + "options": {"picker": picker, **({"confidence_threshold": threshold} if threshold is not None else {})}, + **({"classifier": classifier} if classifier is not None else {}), + } + ) + + +def router( + route_config: SwitchyardRouterConfig, + *, + completion=unexpected_judge, + decide=unexpected_decision, + algorithm: object = None, +) -> SwitchyardRouter: + return SwitchyardRouter("route", route_config, algorithm or build_algorithm(route_config), completion, decide) + + +async def route( + route_config: SwitchyardRouterConfig, + messages: object, + *, + call_type: str | None = "acompletion", + request_kwargs: Mapping[str, object] | None = None, + **router_kwargs: object, +) -> tuple[str, Mapping[str, object]]: + result: Final = await router(route_config, **router_kwargs).async_pre_routing_hook( # pyright: ignore[reportArgumentType] # test doubles + "route", request_kwargs or {}, messages, call_type=call_type + ) + assert result is not None + assert result.routing_decision is not None + return result.model, result.routing_decision + + +def anthropic_turn(call_id: str, tool: str, result: str, marked: bool) -> tuple[dict[str, object], ...]: + error: Final = {"is_error": True} if marked else {} + return ( + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "Next step", "signature": "sig"}, + {"type": "tool_use", "id": call_id, "name": tool, "input": {"file_path": "a.py"}}, + ], + }, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": call_id, "content": result, **error}]}, + ) + + +def responses_turn(call_id: str, tool: str, result: str, marked: bool) -> tuple[dict[str, object], ...]: + return ( + {"type": "function_call", "call_id": call_id, "name": tool, "arguments": '{"file_path":"a.py"}'}, + {"type": "function_call_output", "call_id": call_id, "output": result}, + ) + + +def chat_turn(call_id: str, tool: str, result: str, marked: bool) -> tuple[dict[str, object], ...]: + call: Final = {"id": call_id, "type": "function", "function": {"name": tool, "arguments": '{"file_path":"a.py"}'}} + return ( + {"role": "assistant", "content": None, "tool_calls": [call]}, + {"role": "tool", "tool_call_id": call_id, "content": result}, + ) + + +TURN_FORMATS: Final = {"anthropic_messages": anthropic_turn, "aresponses": responses_turn, "acompletion": chat_turn} + + +def transcript(endpoint: str, tool: str, results: tuple[str, ...], *, marked: bool = False) -> list[dict[str, object]]: + turn: Final = TURN_FORMATS[endpoint] + turns: Final = chain.from_iterable( + turn(f"call{index}", tool, result, marked) for index, result in enumerate(results) + ) + return [{"role": "user", "content": "Fix the helper"}, *turns] + + +def llm_router( + *, route_config: Mapping[str, object], judge_fails: bool = False, num_retries: int = 0 +) -> litellm.Router: + def deployment(name: str, reply: object) -> Mapping[str, object]: + return { + "model_name": name, + "litellm_params": {"model": f"openai/{name}", "api_key": "k", "mock_response": reply}, + } + + return litellm.Router( + num_retries=num_retries, + model_list=[ + deployment("strong", "strong"), + deployment("fast", "fast"), + *((deployment("judge", Exception("judge down")),) if judge_fails else ()), + switchyard_deployment(route_config), + ], + ) + + +def switchyard_deployment(route_config: Mapping[str, object]) -> dict[str, object]: + return { + "model_name": "route", + "litellm_params": {"model": "auto_router/switchyard", "switchyard_router_config": route_config}, + "model_info": {"id": "route-id"}, + } + + +@pytest.mark.asyncio +async def test_critical_tool_error_selects_capable_without_rewriting_messages() -> None: + messages: Final = transcript("acompletion", "bash", ("CUDA out of memory",)) + result: Final = await router(config()).async_pre_routing_hook("route", {}, messages, call_type="acompletion") + assert result is not None + assert result.model == "strong" + assert result.messages == messages + + +async def routed_reply(proxy: litellm.Router, endpoint: str, edits: tuple[str, ...]) -> object: + match endpoint: + case "anthropic_messages": + message: Final = await proxy.aanthropic_messages( + model="route", messages=transcript(endpoint, "Edit", edits), max_tokens=16 + ) + return message["content"][0]["text"] # pyright: ignore[reportIndexIssue] # non-streaming response + case "aresponses": + response: Final = await proxy.aresponses(model="route", input=transcript(endpoint, "Edit", edits)) + return response.output[0].content[0].text # pyright: ignore[reportAttributeAccessIssue] # non-streaming response + case _: + chat: Final = await proxy.acompletion(model="route", messages=transcript(endpoint, "Edit", edits)) + return chat.choices[0].message.content # pyright: ignore[reportAttributeAccessIssue] # non-streaming response + + +@pytest.mark.asyncio +@pytest.mark.parametrize("endpoint", ["acompletion", "anthropic_messages", "aresponses"]) +async def test_router_routes_agent_traffic_from_every_endpoint(endpoint: str) -> None: + proxy: Final = llm_router(route_config={"models": {"capable": "strong", "efficient": "fast"}}) + assert await routed_reply(proxy, endpoint, ("Edit applied",) * 3) == "fast" + + +@pytest.mark.asyncio +async def test_router_follows_an_edited_switchyard_router_and_forgets_a_deleted_one() -> None: + proxy: Final = llm_router(route_config={"models": {"capable": "strong", "efficient": "fast"}}) + assert await routed_reply(proxy, "acompletion", ("Edit applied",) * 3) == "fast" + + swapped: Final = switchyard_deployment({"models": {"capable": "fast", "efficient": "strong"}}) + _ = proxy.apply(ChangeSet(upsert=[Deployment(**swapped)])) + assert await routed_reply(proxy, "acompletion", ("Edit applied",) * 3) == "strong" + + _ = proxy.apply(ChangeSet(delete=["route-id"])) + with pytest.raises(litellm.BadRequestError): + await proxy.acompletion(model="route", messages=HELLO) + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("marked", "expected"), [(True, "strong"), (False, "fast")]) +async def test_anthropic_tool_failure_marker_reaches_switchyard(marked: bool, expected: str) -> None: + failures: Final = transcript("anthropic_messages", "Edit", ("String to replace not found.",) * 4, marked=marked) + model, decision = await route(config(), failures, call_type="anthropic_messages") + assert model == expected + assert "switchyard_fallback_reason" not in decision + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("request_kwargs", "expected"), + [ + ({"litellm_session_id": "s1"}, ["strong", "strong", "strong", "fast"]), + ( + {"proxy_server_request": {"headers": {"X-Session-Id": "s1", "session-id": "s1"}}}, + ["strong", "fast", "fast", "fast"], + ), + ({}, ["strong", "fast", "fast", "fast"]), + ], +) +async def test_capable_hold_follows_the_session(request_kwargs: Mapping[str, object], expected: list[str]) -> None: + switchyard: Final = router(config()) + turns: Final = (("CUDA out of memory",), *((("All tests passed",) * count) for count in (2, 3, 4))) + picks: Final = [await pick(switchyard, request_kwargs, results) for results in turns] + assert picks == expected + + +async def pick(switchyard: SwitchyardRouter, request_kwargs: Mapping[str, object], results: tuple[str, ...]) -> str: + result: Final = await switchyard.async_pre_routing_hook( + "route", request_kwargs, transcript("acompletion", "bash", results), call_type="acompletion" + ) + assert result is not None + return result.model + + +@pytest.mark.asyncio +async def test_a_session_hold_never_crosses_api_keys() -> None: + switchyard: Final = router(config()) + owner: Final = {"litellm_session_id": "shared", "metadata": {"user_api_key_hash": "owner"}} + other: Final = {"litellm_session_id": "shared", "metadata": {"user_api_key_hash": "other"}} + assert await pick(switchyard, owner, ("CUDA out of memory",)) == "strong" + assert await pick(switchyard, other, ("All tests passed",) * 2) == "fast" + assert await pick(switchyard, owner, ("All tests passed",) * 2) == "strong" + + +@pytest.mark.asyncio +async def test_a_client_metadata_key_hash_cannot_join_another_keys_session() -> None: + switchyard: Final = router(config()) + owner: Final = {"litellm_session_id": "shared", "litellm_metadata": {"user_api_key_hash": "owner"}} + spoofer: Final = { + "litellm_session_id": "shared", + "metadata": {"user_api_key_hash": "owner"}, + "litellm_metadata": {"user_api_key_hash": "spoofer"}, + } + assert await pick(switchyard, owner, ("CUDA out of memory",)) == "strong" + assert await pick(switchyard, spoofer, ("All tests passed",) * 2) == "fast" + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("p_solve", "expected"), [(0.0, "strong"), (1.0, "fast")]) +async def test_llm_judge_verdict_selects_the_target(p_solve: float, expected: str) -> None: + async def judge(request: Mapping[str, object]) -> ModelResponse: + assert "autorouter_classifier" in str(request["metadata"]) + verdict: Final = {"crux": "c", "primary_rule": "none", "capability_boundary": "unmatched", "p_solve": p_solve} + return ModelResponse(choices=[{"message": {"role": "assistant", "content": json.dumps(verdict)}}]) + + model, _ = await route(config(classifier={}), HELLO, completion=judge) + assert model == expected + + +def decision_answer(advantage: float) -> OpenAIDecisionResponse: + return OpenAIDecisionResponse.model_validate( + { + "model": "jev", + "answers": [ + { + "type": "choice", + "name": "route", + "choice": "advantage", + "probabilities": [ + {"value": "advantage", "probability": advantage}, + {"value": "no_advantage", "probability": 1 - advantage}, + ], + } + ], + "usage": {"input_tokens": 1, "output_tokens": 0, "total_tokens": 1}, + } + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("parent_bucket", ["metadata", "litellm_metadata"]) +@pytest.mark.parametrize("classifier", [{}, {"kind": "decision"}]) +async def test_judge_calls_carry_the_callers_tags_and_origin_where_their_endpoint_reads_them( + parent_bucket: str, classifier: Mapping[str, object] +) -> None: + seen: Final[list[Mapping[str, object]]] = [] # mutable-ok: collects the judge request + + async def judge(request: Mapping[str, object]) -> ModelResponse: + seen.append(request) + verdict: Final = {"crux": "c", "primary_rule": "none", "capability_boundary": "unmatched", "p_solve": 1.0} + return ModelResponse(choices=[{"message": {"role": "assistant", "content": json.dumps(verdict)}}]) + + async def decide(request: Mapping[str, object]) -> OpenAIDecisionResponse: + seen.append(request) + return decision_answer(0.0) + + caller: Final = {parent_bucket: {"tags": ["eu-only"], "inherited_tags": ["team-eu"]}} + await route( + config(classifier=classifier, threshold=1.0), HELLO, completion=judge, decide=decide, request_kwargs=caller + ) + (request,) = seen + read_by_endpoint: Final = "litellm_metadata" if classifier else "metadata" + assert set(request) & {"metadata", "litellm_metadata"} == {read_by_endpoint} + metadata: Final = request[read_by_endpoint] + assert metadata["tags"] == ["eu-only"] # pyright: ignore[reportIndexIssue] # the judge request's metadata dict + assert metadata["inherited_tags"] == ["team-eu"] # pyright: ignore[reportIndexIssue] # same + assert metadata["internal_call_origin"] == "autorouter_classifier" # pyright: ignore[reportIndexIssue] # same + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("advantage", "expected"), [(0.61, "strong"), (0.6, "fast")]) +async def test_decision_model_advantage_must_exceed_the_cutoff(advantage: float, expected: str) -> None: + async def decide(request: Mapping[str, object]) -> OpenAIDecisionResponse: + (question,) = request["questions"] # pyright: ignore[reportGeneralTypeIssues] # the decisions request shape + assert {choice["value"] for choice in question["choices"]} == {"advantage", "no_advantage"} + return decision_answer(advantage) + + model, decision = await route( + config(classifier={"kind": "decision", "cutoff": 0.6}, threshold=1.0), HELLO, decide=decide + ) + assert model == expected + assert decision["switchyard_evidence"]["score"] == advantage + + +async def judge_down(request: Mapping[str, object]) -> ModelResponse: + raise ConnectionError("judge down") + + +async def decision_refused(request: Mapping[str, object]) -> OpenAIDecisionResponse: + return OpenAIDecisionResponse.model_validate( + { + "model": "jev", + "answers": [{"type": "refusal", "name": "route"}], + "usage": {"input_tokens": 1, "output_tokens": 0, "total_tokens": 1}, + } + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("classifier", "router_kwargs", "picker", "expected"), + [ + ({}, {"completion": judge_down}, "efficient_first", "fast"), + ({}, {"completion": judge_down}, "capable_first", "strong"), + ({"kind": "decision"}, {"decide": decision_refused}, "efficient_first", "fast"), + ], +) +async def test_classifier_failure_falls_open_to_the_picker_default( + classifier: Mapping[str, object], router_kwargs: Mapping[str, object], picker: str, expected: str +) -> None: + model, decision = await route(config(classifier=classifier, picker=picker, threshold=1.0), HELLO, **router_kwargs) + assert model == expected + assert "switchyard_fallback_reason" not in decision + + +class EndsWithoutDecision: + async def _steps(self) -> AsyncIterator[object]: + for step in (): + yield step + + def run_stream(self, *args: object, **kwargs: object) -> AsyncIterator[object]: + return self._steps() + + +class RaisesLibsyError: + async def _steps(self) -> AsyncIterator[object]: + raise libsy.LibsyError("algorithm failed") + yield + + def run_stream(self, *args: object, **kwargs: object) -> AsyncIterator[object]: + return self._steps() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("route_config", "messages", "call_type", "algorithm", "reason"), + [ + ( + config(), + [{"role": "user", "content": [{"type": "input_audio"}]}], + "acompletion", + None, + "unsupported_content", + ), + (config(), HELLO, None, None, "unsupported_request_type"), + (config(), HELLO, "acompletion", EndsWithoutDecision(), "no_decision"), + (config(), HELLO, "acompletion", RaisesLibsyError(), "libsy_error"), + ( + config(classifier={}, threshold=1.0), + HELLO, + "acompletion", + build_algorithm(config(classifier={"kind": "decision"}, threshold=1.0)), + "unexpected_decision_call", + ), + ( + config(classifier={"kind": "decision"}, threshold=1.0), + HELLO, + "acompletion", + build_algorithm(config(classifier={}, threshold=1.0)), + "unexpected_judge_call", + ), + ], +) +async def test_anything_switchyard_cannot_route_goes_to_capable( + route_config: SwitchyardRouterConfig, messages: object, call_type: str | None, algorithm: object, reason: str +) -> None: + model, decision = await route(route_config, messages, call_type=call_type, algorithm=algorithm) + assert model == "strong" + assert decision["switchyard_fallback_reason"] == reason + + +@pytest.mark.asyncio +async def test_failing_judge_is_called_once_even_when_the_router_retries(monkeypatch: pytest.MonkeyPatch) -> None: + attempts: Final[list[str]] = [] + + class CountAttempts(CustomLogger): + def log_pre_api_call(self, model: str, messages: object, kwargs: dict[str, object]) -> None: + attempts.append(model) + + monkeypatch.setattr(litellm, "callbacks", [CountAttempts()]) + proxy: Final = llm_router( + route_config={ + "models": {"capable": "strong", "efficient": "fast", "judge": "judge"}, + "classifier": {}, + "options": {"confidence_threshold": 1.0}, + }, + judge_fails=True, + num_retries=3, + ) + await proxy.acompletion(model="route", messages=HELLO) + assert attempts.count("judge") == 1 + + +@pytest.mark.asyncio +async def test_judge_cancellation_propagates() -> None: + async def cancelled(request: Mapping[str, object]) -> ModelResponse: + raise asyncio.CancelledError() + + with pytest.raises(asyncio.CancelledError): + await route(config(classifier={}), HELLO, completion=cancelled) diff --git a/tests/unit/router_strategy/test_router_tag_routing.py b/tests/unit/router_strategy/test_router_tag_routing.py index 0d3876c64a3..e916e5b235d 100644 --- a/tests/unit/router_strategy/test_router_tag_routing.py +++ b/tests/unit/router_strategy/test_router_tag_routing.py @@ -3023,7 +3023,7 @@ class _RewriteToTierStrategy: self.rewrite_to = rewrite_to async def async_pre_routing_hook( - self, model, request_kwargs, messages=None, input=None, specific_deployment=False + self, model, request_kwargs, messages=None, input=None, specific_deployment=False, call_type=None ): from litellm.types.router import PreRoutingHookResponse diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 1b98cd4b4b6..ca61fcd0905 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -9006,7 +9006,7 @@ class TestConsumedRequestTagsStamp: self.rewrite_to = rewrite_to async def async_pre_routing_hook( - self, model, request_kwargs, messages=None, input=None, specific_deployment=False + self, model, request_kwargs, messages=None, input=None, specific_deployment=False, call_type=None ): from litellm.types.router import PreRoutingHookResponse @@ -9097,7 +9097,7 @@ class TestClaudeCodeSubagentSessionRouterBinding: self.routed_model = routed_model async def async_pre_routing_hook( - self, model, request_kwargs, messages=None, input=None, specific_deployment=False + self, model, request_kwargs, messages=None, input=None, specific_deployment=False, call_type=None ): from litellm.types.router import PreRoutingHookResponse @@ -10151,7 +10151,7 @@ class TestModelGroupAliasReachesPreRoutingStrategies: class _RewriteStrategy: async def async_pre_routing_hook( - self, model, request_kwargs, messages=None, input=None, specific_deployment=False + self, model, request_kwargs, messages=None, input=None, specific_deployment=False, call_type=None ): from litellm.types.router import PreRoutingHookResponse @@ -10268,7 +10268,7 @@ class TestTeamPublicNameReachesPreRoutingStrategies: self.rewrite_to = rewrite_to async def async_pre_routing_hook( - self, model, request_kwargs, messages=None, input=None, specific_deployment=False + self, model, request_kwargs, messages=None, input=None, specific_deployment=False, call_type=None ): from litellm.types.router import PreRoutingHookResponse @@ -10505,7 +10505,7 @@ class TestAutoRouterCompressionDecoupling: self.received_messages: list[dict] | None = None async def async_pre_routing_hook( - self, model, request_kwargs, messages=None, input=None, specific_deployment=False + self, model, request_kwargs, messages=None, input=None, specific_deployment=False, call_type=None ): from litellm.types.router import PreRoutingHookResponse diff --git a/tests/unit/types/test_litellm_params.py b/tests/unit/types/test_litellm_params.py index d71791b6f11..aae5dfe6472 100644 --- a/tests/unit/types/test_litellm_params.py +++ b/tests/unit/types/test_litellm_params.py @@ -168,6 +168,7 @@ OPTION_NAMES: Final = ( "tag_regex", "max_file_size_mb", "silent_model", + "switchyard_router_config", "auto_router_config_path", "auto_router_config", "auto_router_default_model", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts index f9304bab0d2..d13bc3d3181 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts @@ -109,6 +109,7 @@ export interface AutoRouterDeployment extends AutoRouterCandidateDeployment { model?: string | null; base_model?: string | null; custom_llm_provider?: string | null; + switchyard_router_config?: unknown; complexity_router_config?: unknown; complexity_router_default_model?: string | null; auto_router_config?: unknown; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/autoRouterRows.ts b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/autoRouterRows.ts index 2cdc13606bb..0fe320a318a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/autoRouterRows.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AutoRouters/autoRouterRows.ts @@ -87,6 +87,14 @@ const configManaged = (label: string, config: Record): Presenta /** How each strategy renders itself, given its own config object. */ const PRESENTERS: Record) => Presentation> = { + switchyard: (config) => ({ + typeLabel: "Switchyard", + targets: dedupe( + [asRecord(config.models).capable, asRecord(config.models).efficient].filter( + (model): model is string => typeof model === "string", + ), + ), + }), complexity: (config) => ({ typeLabel: complexityTypeLabel(config), targets: dedupe(Object.values(asRecord(config.tiers)).flatMap(normalizeTierModels)), @@ -131,7 +139,8 @@ export const toAutoRouterRow = ( canDelete: canDelete && mayActOnRow, editBlockedReason, createdAt: info.created_at ?? undefined, - defaultModel: (params[strategy.defaultModelKey] as string | null | undefined) ?? null, + defaultModel: + strategy.kind === "switchyard" ? null : (params[strategy.defaultModelKey] as string | null | undefined) ?? null, deployment, ...PRESENTERS[strategy.kind](asRecord(params[strategy.configKey])), }; diff --git a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx index bd8b4eb9854..0dd80854312 100644 --- a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx +++ b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx @@ -135,6 +135,7 @@ interface AutoRouterClassifierTabsProps { children: React.ReactNode; judgeModel?: React.ReactNode; decisionModel?: React.ReactNode; + switchyard?: { selected: boolean; onSelect: () => void; fields: React.ReactNode }; } const AutoRouterClassifierTabs: React.FC = ({ @@ -143,6 +144,7 @@ const AutoRouterClassifierTabs: React.FC = ({ children, judgeModel, decisionModel, + switchyard, }) => { const { t } = useTranslation(); const id = useId(); @@ -159,14 +161,16 @@ const AutoRouterClassifierTabs: React.FC = ({ jev: "jev", custom: "custom", }; - const family = familyByType[classifierType]; + const family = switchyard?.selected ? "switchyard" : familyByType[classifierType]; const hasCustomTiers = Boolean(value.custom_tier_set); const changeType = (next: ClassifierType) => { if (next !== classifierType) onChange(transitionClassifierType(value, next)); }; const changeFamily = (next: unknown) => { + if (next === "switchyard") return switchyard?.onSelect(); if (next === family) return; + if (next === familyByType[classifierType]) return onChange(value); if (next === "heuristics") changeType("heuristic"); if (next === "llm") changeType("llm"); if (next === "jev") changeType("jev"); @@ -207,7 +211,11 @@ const AutoRouterClassifierTabs: React.FC = ({ - + {[ { value: "heuristics", @@ -220,6 +228,9 @@ const AutoRouterClassifierTabs: React.FC = ({ label: t("addModel.autoRouterClassifierTabs.families.decisionModel"), description: t("addModel.autoRouterClassifierTabs.families.decisionModelDescription"), }, + ...(switchyard + ? [{ value: "switchyard", label: "Switchyard", description: "Route by the conversation’s tool history." }] + : []), ].map((option) => ( + {family === "switchyard" && switchyard?.fields} {family === "jev" && (decisionModel ?? (
@@ -355,7 +367,7 @@ const AutoRouterClassifierTabs: React.FC = ({

)} - {hasCustomTiers && ( + {family !== "switchyard" && hasCustomTiers && (

{t("addModel.autoRouterClassifierTabs.restoreStandardTiers")}

)} {availability.data?.error && !availability.isChecking && ( @@ -372,7 +384,7 @@ const AutoRouterClassifierTabs: React.FC = ({

)} - {children} + {family !== "switchyard" && children} ); }; diff --git a/ui/litellm-dashboard/src/components/add_model/BlockedReasonTooltip.tsx b/ui/litellm-dashboard/src/components/add_model/BlockedReasonTooltip.tsx new file mode 100644 index 00000000000..59588cf478f --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/BlockedReasonTooltip.tsx @@ -0,0 +1,17 @@ +import React from "react"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; + +const BlockedReasonTooltip: React.FC<{ reason: string | null; children: React.ReactElement }> = ({ + reason, + children, +}) => + reason === null ? ( + children + ) : ( + + + {reason} + + ); + +export default BlockedReasonTooltip; diff --git a/ui/litellm-dashboard/src/components/add_model/SwitchyardConfigFields.tsx b/ui/litellm-dashboard/src/components/add_model/SwitchyardConfigFields.tsx new file mode 100644 index 00000000000..2043e392928 --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/SwitchyardConfigFields.tsx @@ -0,0 +1,342 @@ +"use client"; + +import { useId, useState } from "react"; +import { Input } from "@/components/ui/input"; +import { Textarea } from "@/components/ui/textarea"; +import { Slider } from "@/components/ui/slider"; +import { Switch } from "@/components/ui/switch"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import ModelChoiceCombobox, { type ModelChoice } from "./ModelChoiceCombobox"; +import { + stageConfidence, + toggleClassifier, + setClassifierKind, + parseToolSemantics, + type SwitchyardConfig, +} from "./switchyard_config"; + +interface Props { + value: SwitchyardConfig; + onChange: (value: SwitchyardConfig) => void; + models: ModelChoice[]; + decisionModels: ModelChoice[]; + onToolSemanticsError: (error: string | null) => void; +} + +const isStructured = (instructions: unknown): instructions is Record => + typeof instructions === "object" && instructions !== null; +const instructionsText = (instructions: string | Record | null | undefined): string => + isStructured(instructions) ? JSON.stringify(instructions, null, 2) : instructions ?? ""; + +export default function SwitchyardConfigFields({ + value, + onChange, + models, + decisionModels, + onToolSemanticsError, +}: Props) { + const id = useId(); + const [toolSemantics, setToolSemantics] = useState( + value.options.tool_semantics ? JSON.stringify(value.options.tool_semantics, null, 2) : "", + ); + const classifier = value.classifier; + const llm = classifier?.kind === "llm" ? classifier : undefined; + const decision = classifier?.kind === "decision" ? classifier : undefined; + const setOption = (key: "recent_window", text: string) => + onChange({ ...value, options: { ...value.options, [key]: text === "" ? undefined : Number(text) } }); + const setClassifierNumber = ( + key: "base_threshold" | "threshold_step" | "max_output_tokens" | "recent_turn_window", + text: string, + ) => { + if (llm) onChange({ ...value, classifier: { ...llm, [key]: text === "" ? undefined : Number(text) } }); + }; + const changeToolSemantics = (text: string) => { + setToolSemantics(text); + try { + const parsed = parseToolSemantics(text); + onChange({ ...value, options: { ...value.options, tool_semantics: parsed } }); + onToolSemanticsError(null); + } catch { + onToolSemanticsError("Tool semantics must be JSON mapping observe, mutate, plan, or new to arrays of tool names"); + } + }; + return ( +
+
+
+ + Heuristic + +

+ Stage Router +

+
+
+ + + {value.options.picker === "capable_first" && ( +

+ Switchyard has not benchmarked this bias, so it has no calibrated confidence thresholds. +

+ )} +
+ {(["capable", "efficient"] as const).map((tier) => ( +
+ + onChange({ ...value, models: { ...value.models, [tier]: model ?? "" } })} + choices={models} + placeholder="Select a model" + ariaInvalid={undefined} + ariaDescribedBy={undefined} + /> +
+ ))} +
+
+
+
+ +

+ Ask a judge model when the tool history does not give a confident routing decision. +

+
+ onChange(toggleClassifier(value, enabled))} + /> +
+ {classifier && ( +
+
+ + +
+
+ + onChange({ ...value, models: { ...value.models, judge: judge ?? "" } })} + choices={decision ? decisionModels : models} + placeholder={decision ? "Select a decision model" : "Select a judge model"} + ariaInvalid={undefined} + ariaDescribedBy={undefined} + /> +
+
+ )} +
+
+
+ + Confidence threshold + + {stageConfidence(value).toFixed(2)} +
+ + onChange({ + ...value, + options: { ...value.options, confidence_threshold: Array.isArray(next) ? next[0] : next }, + }) + } + /> +

+ Below this confidence, use {classifier ? "the judge model" : "the bias’s default model"}. Critical errors can + still select the capable model. +

+
+
+ Advanced settings +
+

Stage

+
+ + setOption("recent_window", event.target.value)} + /> +
+
+ +