From ce0dbe60dfe9bea323d94c34d170a49ab737680e Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 09:36:32 -0700 Subject: [PATCH 01/20] chore(cost-map): add fireworks minimax m3.1 flash priority prices from the prices api (#47346) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- .../model_prices_and_context_window_backup.json | 16 ++++++++++++++++ model_prices_and_context_window.json | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index bb0cdd51d9c..e32eb4ba747 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -76100,5 +76100,21 @@ "supported_output_modalities": [ "image" ] + }, + "fireworks_ai/accounts/fireworks/models/minimax-m3p1-flash": { + "cache_read_input_token_cost_priority": 2.25e-08, + "input_cost_per_token_priority": 2.25e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token_priority": 9e-07, + "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index bb0cdd51d9c..e32eb4ba747 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -76100,5 +76100,21 @@ "supported_output_modalities": [ "image" ] + }, + "fireworks_ai/accounts/fireworks/models/minimax-m3p1-flash": { + "cache_read_input_token_cost_priority": 2.25e-08, + "input_cost_per_token_priority": 2.25e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token_priority": 9e-07, + "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true } } From 3eeee3e76118783671bd7c9dd44a192eedf917ad Mon Sep 17 00:00:00 2001 From: Alex Ker Date: Mon, 19 Oct 2026 12:40:13 -0400 Subject: [PATCH 02/20] fix(baseten): list current Model APIs instead of retired 2023 deployment IDs (#46945) * fix(baseten): list current Model APIs instead of retired 2023 deployment IDs `baseten/*` wildcard expansion and `/v1/models` read a hardcoded set of three 2023 deployment IDs (Falcon 7B / WizardLM / MosaicML) that no longer exist, while the model cost map already carries current Baseten entries. - Populate `baseten_models` from the model cost map, like fireworks_ai. - Add `BasetenConfig.get_models()` against the Model API `/v1/models` endpoint and register it in `ProviderConfigManager.get_provider_model_info`, so `check_provider_endpoint: true` returns the live catalog. - Remove the retired deployment IDs from `constants.py`. Co-Authored-By: Claude Opus 5.5 * fix(baseten): mark retired Model APIs deprecated and skip them in listings 15 Baseten entries in the model cost map are no longer servable: the Model API returns 410 "model version ... has been deprecated" (DeepSeek-V3.1 returns 400 deactivated), and none appear in `inference.baseten.co/v1/models`. - Set `deprecation_date` on those entries (kept for spend tracking). - Skip entries past their `deprecation_date` when building `baseten_models`, so `baseten/*` expansion only lists models that still serve. Co-Authored-By: Claude Opus 5.5 * test(baseten): assert the custom api_base model listing on the returned models The custom api_base test only inspected the patched client's call args, which trips the TQ002 mock-echo gate. Serve the custom URL with respx and assert on the models that come back instead, so a wrong host or a broken parse both fail. * fix(baseten): drop newly retired models on cost map reload and fake discovery at the HTTP boundary A cost map refresh only ever added to the Baseten set, so a model that gained a past deprecation_date after boot stayed listed under baseten/* until restart. Discard it on reload. Tests now serve the catalog with respx instead of patching our own HTTP client, the isinstance registration test is gone since the get_valid_models test already proves discovery end to end, and new test locals carry Final. --------- Co-authored-by: AlexKer Co-authored-by: Claude Opus 5.5 Co-authored-by: ryan-crabbe-berri --- litellm/__init__.py | 18 ++- litellm/constants.py | 8 -- litellm/llms/baseten/chat.py | 36 +++++- ...odel_prices_and_context_window_backup.json | 15 +++ litellm/utils.py | 2 + model_prices_and_context_window.json | 15 +++ .../baseten/chat/test_baseten_completions.py | 109 +++++++++++++++++- 7 files changed, 191 insertions(+), 12 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index cc5fbc1e72e..b3e80bfece5 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -7,6 +7,7 @@ _apply_process_defaults() ### INIT VARIABLES ######################### import threading import os +import datetime from collections.abc import Mapping, Sequence from typing import ( @@ -62,7 +63,6 @@ from litellm.constants import ( modelscope_models, empower_models, together_ai_models, - baseten_models, WANDB_MODELS, REPEATED_STREAMING_CHUNK_LIMIT, request_timeout, @@ -624,6 +624,7 @@ bedrock_models: Set = set() bedrock_converse_models: Set = set(BEDROCK_CONVERSE_MODELS) fal_ai_models: Set = set() fireworks_ai_models: Set = set() +baseten_models: Set[str] = set() fireworks_ai_embedding_models: Set = set() deepinfra_models: Set = set() perplexity_models: Set = set() @@ -742,6 +743,16 @@ def is_openai_finetune_model(key: str) -> bool: return key.startswith("ft:") and not key.count(":") > 1 +def _is_past_deprecation_date(value: Mapping[str, object]) -> bool: + deprecation_date: Final = value.get("deprecation_date") + if not isinstance(deprecation_date, str): + return False + try: + return datetime.date.fromisoformat(deprecation_date) <= datetime.date.today() + except ValueError: + return False + + def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, object]]) -> None: for key, value in tuple(model_cost_map.items()): if value.get("litellm_provider") == "openai" and not is_openai_finetune_model(key): @@ -842,6 +853,11 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje # ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params. if "-to-" not in key and "fireworks-ai-default" not in key: fireworks_ai_models.add(key) + elif value.get("litellm_provider") == "baseten": + if _is_past_deprecation_date(value): + baseten_models.discard(key) + else: + baseten_models.add(key) elif value.get("litellm_provider") == "fireworks_ai-embedding-models": # ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params. if "-to-" not in key: diff --git a/litellm/constants.py b/litellm/constants.py index da95b97036a..5473a3c6390 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1256,14 +1256,6 @@ together_ai_models: Final[set] = set( # supports all together ai models, just pass in the model id e.g. completion(model="together_computer/replit_code_3b",...) -baseten_models: Final[set] = set( - [ - "qvv0xeq", - "q841o8w", - "31dxrj3", - ] -) # FALCON 7B # WizardLM # Mosaic ML - featherless_ai_models: Final[set] = set( [ "featherless-ai/Qwerky-72B", diff --git a/litellm/llms/baseten/chat.py b/litellm/llms/baseten/chat.py index e92f47c41c0..381d487c36d 100644 --- a/litellm/llms/baseten/chat.py +++ b/litellm/llms/baseten/chat.py @@ -1,6 +1,10 @@ from typing import Final +import litellm from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig +from litellm.secret_managers.main import get_secret_str + +BASETEN_MODEL_API_BASE: Final = "https://inference.baseten.co/v1" class BasetenConfig(OpenAIGPTConfig): @@ -89,11 +93,39 @@ class BasetenConfig(OpenAIGPTConfig): Get the OpenAI compatible provider info for Baseten """ # Default to Model API - default_api_base: Final = "https://inference.baseten.co/v1" + default_api_base: Final = BASETEN_MODEL_API_BASE default_api_key: Final = api_key or "BASETEN_API_KEY" return default_api_base, default_api_key + @staticmethod + def get_api_key(api_key: str | None = None) -> str | None: + return api_key or get_secret_str("BASETEN_API_KEY") + + @staticmethod + def get_api_base(api_base: str | None = None) -> str | None: + return api_base or get_secret_str("BASETEN_API_BASE") or BASETEN_MODEL_API_BASE + + def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]: + """ + Calls Baseten's Model API `/v1/models` endpoint and returns the live catalog. + """ + resolved_api_key: Final = self.get_api_key(api_key) + if resolved_api_key is None: + raise ValueError( + "BASETEN_API_KEY is not set. Please set the environment variable, to query Baseten's `/models` endpoint." + ) + resolved_api_base: Final = (self.get_api_base(api_base) or BASETEN_MODEL_API_BASE).rstrip("/") + response: Final = litellm.module_level_client.get( # pyright: ignore[reportUnknownMemberType] # untyped params + url=f"{resolved_api_base.removesuffix('/v1')}/v1/models", + headers={"Authorization": f"Bearer {resolved_api_key}"}, + ) + if response.status_code != 200: + raise ValueError( + f"Failed to fetch models from Baseten. Status code: {response.status_code}, Response: {response.text}" + ) + return ["baseten/" + model["id"] for model in response.json()["data"]] + @staticmethod def is_dedicated_deployment(model: str) -> bool: """ @@ -118,4 +150,4 @@ class BasetenConfig(OpenAIGPTConfig): return f"https://model-{model_id}.api.baseten.co/environments/production/sync/v1" else: # Use Model API - return "https://inference.baseten.co/v1" + return BASETEN_MODEL_API_BASE diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index e32eb4ba747..e0ace9c4f39 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -33684,24 +33684,28 @@ "output_cost_per_token": 1.2e-06 }, "baseten/MiniMaxAI/MiniMax-M2.5": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 3e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 1.2e-06 }, "baseten/nvidia/Nemotron-120B-A12B": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 3e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 7.5e-07 }, "baseten/zai-org/GLM-5": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 9.5e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 3.15e-06 }, "baseten/zai-org/GLM-4.7": { + "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 1.2e-07, "input_cost_per_token": 6e-07, "litellm_provider": "baseten", @@ -33719,24 +33723,28 @@ "supports_vision": false }, "baseten/zai-org/GLM-4.6": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 6e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 2.2e-06 }, "baseten/moonshotai/Kimi-K2.5": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 6e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 3e-06 }, "baseten/moonshotai/Kimi-K2-Thinking": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 6e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 2.5e-06 }, "baseten/moonshotai/Kimi-K2-Instruct-0905": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 6e-07, "litellm_provider": "baseten", "mode": "chat", @@ -33760,12 +33768,14 @@ "supports_vision": false }, "baseten/deepseek-ai/DeepSeek-V3.1": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 5e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 1.5e-06 }, "baseten/deepseek-ai/DeepSeek-V3-0324": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 7.7e-07, "litellm_provider": "baseten", "mode": "chat", @@ -72125,6 +72135,7 @@ "supports_vision": true }, "baseten/moonshotai/Kimi-K2.6": { + "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 1.6e-07, "input_cost_per_token": 9.5e-07, "litellm_provider": "baseten", @@ -72142,6 +72153,7 @@ "supports_vision": true }, "baseten/moonshotai/Kimi-K2.7-Code": { + "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 1.6e-07, "input_cost_per_token": 9.5e-07, "litellm_provider": "baseten", @@ -72193,6 +72205,7 @@ "supports_vision": false }, "baseten/thinkingmachines/inkling": { + "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 1.7e-07, "input_cost_per_token": 1e-06, "litellm_provider": "baseten", @@ -72210,6 +72223,7 @@ "supports_vision": true }, "baseten/thinkingmachines/inkling-small": { + "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 5e-07, "litellm_provider": "baseten", @@ -72278,6 +72292,7 @@ "supports_vision": false }, "baseten/deepseek-ai/DeepSeek-V4-Pro": { + "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 1.45e-07, "input_cost_per_token": 1.74e-06, "litellm_provider": "baseten", diff --git a/litellm/utils.py b/litellm/utils.py index 66d78a03075..ff2c9012ec8 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9704,6 +9704,8 @@ class ProviderConfigManager: ) -> BaseLLMModelInfo | None: if LlmProviders.FIREWORKS_AI == provider: return litellm.FireworksAIConfig() + elif LlmProviders.BASETEN == provider: + return litellm.BasetenConfig() elif LlmProviders.OPENAI == provider: return litellm.OpenAIGPTConfig() elif LlmProviders.GEMINI == provider: diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e32eb4ba747..e0ace9c4f39 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -33684,24 +33684,28 @@ "output_cost_per_token": 1.2e-06 }, "baseten/MiniMaxAI/MiniMax-M2.5": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 3e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 1.2e-06 }, "baseten/nvidia/Nemotron-120B-A12B": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 3e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 7.5e-07 }, "baseten/zai-org/GLM-5": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 9.5e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 3.15e-06 }, "baseten/zai-org/GLM-4.7": { + "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 1.2e-07, "input_cost_per_token": 6e-07, "litellm_provider": "baseten", @@ -33719,24 +33723,28 @@ "supports_vision": false }, "baseten/zai-org/GLM-4.6": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 6e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 2.2e-06 }, "baseten/moonshotai/Kimi-K2.5": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 6e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 3e-06 }, "baseten/moonshotai/Kimi-K2-Thinking": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 6e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 2.5e-06 }, "baseten/moonshotai/Kimi-K2-Instruct-0905": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 6e-07, "litellm_provider": "baseten", "mode": "chat", @@ -33760,12 +33768,14 @@ "supports_vision": false }, "baseten/deepseek-ai/DeepSeek-V3.1": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 5e-07, "litellm_provider": "baseten", "mode": "chat", "output_cost_per_token": 1.5e-06 }, "baseten/deepseek-ai/DeepSeek-V3-0324": { + "deprecation_date": "2026-10-16", "input_cost_per_token": 7.7e-07, "litellm_provider": "baseten", "mode": "chat", @@ -72125,6 +72135,7 @@ "supports_vision": true }, "baseten/moonshotai/Kimi-K2.6": { + "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 1.6e-07, "input_cost_per_token": 9.5e-07, "litellm_provider": "baseten", @@ -72142,6 +72153,7 @@ "supports_vision": true }, "baseten/moonshotai/Kimi-K2.7-Code": { + "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 1.6e-07, "input_cost_per_token": 9.5e-07, "litellm_provider": "baseten", @@ -72193,6 +72205,7 @@ "supports_vision": false }, "baseten/thinkingmachines/inkling": { + "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 1.7e-07, "input_cost_per_token": 1e-06, "litellm_provider": "baseten", @@ -72210,6 +72223,7 @@ "supports_vision": true }, "baseten/thinkingmachines/inkling-small": { + "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 5e-07, "litellm_provider": "baseten", @@ -72278,6 +72292,7 @@ "supports_vision": false }, "baseten/deepseek-ai/DeepSeek-V4-Pro": { + "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 1.45e-07, "input_cost_per_token": 1.74e-06, "litellm_provider": "baseten", diff --git a/tests/unit/llms/baseten/chat/test_baseten_completions.py b/tests/unit/llms/baseten/chat/test_baseten_completions.py index 7cac16cca1c..4f135a1a106 100644 --- a/tests/unit/llms/baseten/chat/test_baseten_completions.py +++ b/tests/unit/llms/baseten/chat/test_baseten_completions.py @@ -1,8 +1,14 @@ import os from typing import Final -import pytest from unittest.mock import patch + +import httpx +import pytest +import respx + +import litellm from litellm.llms.baseten.chat import BasetenConfig +from litellm.utils import get_valid_models class TestBasetenRouting: @@ -50,6 +56,107 @@ class TestBasetenModelAPI: assert api_key == "test-key" +class TestBasetenModelDiscovery: + """Test Baseten model listing for `baseten/*` wildcard expansion and /v1/models""" + + def test_models_by_provider_comes_from_registry(self): + """Static listing uses the model cost map, not the retired 2023 deployment IDs.""" + baseten_models: Final = litellm.models_by_provider["baseten"] + servable_registry_models: Final = { + key + for key, value in litellm.model_cost.items() + if value.get("litellm_provider") == "baseten" and not litellm._is_past_deprecation_date(value) + } + + assert servable_registry_models + assert servable_registry_models <= set(baseten_models) + for retired_id in ("qvv0xeq", "q841o8w", "31dxrj3"): + assert retired_id not in baseten_models + + def test_models_by_provider_skips_deprecated_models(self): + """Retired Model APIs stay in the cost map for spend tracking but are not listed.""" + assert litellm.model_cost["baseten/zai-org/GLM-5"]["deprecation_date"] + assert "baseten/zai-org/GLM-5" not in litellm.models_by_provider["baseten"] + + def test_cost_map_reload_drops_a_newly_retired_model(self): + """A refreshed cost map that marks a listed model retired removes it without a restart.""" + model: Final = "baseten/acme/reload-probe" + try: + litellm.add_known_models({model: {"litellm_provider": "baseten"}}) + assert model in litellm.models_by_provider["baseten"] + + litellm.add_known_models({model: {"litellm_provider": "baseten", "deprecation_date": "2000-01-01"}}) + + assert model not in litellm.models_by_provider["baseten"] + finally: + litellm.baseten_models.discard(model) + litellm.add_known_models({}) + + @pytest.mark.parametrize( + "value, expected", + [ + ({}, False), + ({"deprecation_date": "2000-01-01"}, True), + ({"deprecation_date": "2999-01-01"}, False), + ({"deprecation_date": "not-a-date"}, False), + ], + ) + def test_is_past_deprecation_date(self, value, expected): + assert litellm._is_past_deprecation_date(value) is expected + + def test_get_models_calls_model_api(self, respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.get("https://inference.baseten.co/v1/models").mock( + return_value=httpx.Response( + 200, + json={ + "object": "list", + "data": [ + {"id": "zai-org/GLM-5", "object": "model"}, + {"id": "moonshotai/Kimi-K2.5", "object": "model"}, + ], + }, + ) + ) + + models: Final = BasetenConfig().get_models(api_key="test-key") + + assert models == ["baseten/zai-org/GLM-5", "baseten/moonshotai/Kimi-K2.5"] + assert route.calls.last.request.headers["Authorization"] == "Bearer test-key" + + def test_get_models_respects_custom_api_base(self, respx_mock: respx.MockRouter) -> None: + respx_mock.get("https://custom.baseten.example/v1/models").mock( + return_value=httpx.Response(200, json={"data": [{"id": "zai-org/GLM-5"}]}) + ) + + models: Final = BasetenConfig().get_models(api_key="test-key", api_base="https://custom.baseten.example/v1/") + + assert models == ["baseten/zai-org/GLM-5"] + + @patch.dict(os.environ, {}, clear=True) + def test_get_models_requires_api_key(self): + with pytest.raises(ValueError, match="BASETEN_API_KEY"): + BasetenConfig().get_models() + + def test_get_models_raises_on_error_status(self, respx_mock: respx.MockRouter) -> None: + respx_mock.get("https://inference.baseten.co/v1/models").mock( + return_value=httpx.Response(403, text="forbidden") + ) + + with pytest.raises(ValueError, match="403"): + BasetenConfig().get_models(api_key="test-key") + + @patch.dict(os.environ, {"BASETEN_API_KEY": "test-key"}) + def test_get_valid_models_check_provider_endpoint(self, respx_mock: respx.MockRouter) -> None: + """check_provider_endpoint=True returns the live catalog instead of the static list.""" + respx_mock.get("https://inference.baseten.co/v1/models").mock( + return_value=httpx.Response(200, json={"data": [{"id": "brand-new/model"}]}) + ) + + models: Final = get_valid_models(check_provider_endpoint=True, custom_llm_provider="baseten") + + assert models == ["baseten/brand-new/model"] + + class TestBasetenReasoningEffort: def test_reasoning_effort_passthrough_with_drop_params(self): config: Final = BasetenConfig() From 6fc60bc00d7e22402eee404719c52f0d72fce95f Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 09:58:19 -0700 Subject: [PATCH 03/20] feat(router): discover kubernetes pods behind a headless service api_base (#44342) Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 7 + litellm/router.py | 51 +- .../router_utils/kubernetes_pod_discovery.py | 462 ++++++ litellm/types/litellm_params.py | 1 + litellm/types/router.py | 1 + .../router_code_coverage.py | 6 +- .../test_kubernetes_pod_discovery.py | 1266 +++++++++++++++++ tests/unit/types/test_litellm_params.py | 1 + .../panels/AddModelPanel.integration.test.tsx | 19 + .../add_model/advanced_settings.test.tsx | 31 +- .../add_model/advanced_settings.tsx | 14 + 11 files changed, 1851 insertions(+), 8 deletions(-) create mode 100644 litellm/router_utils/kubernetes_pod_discovery.py create mode 100644 tests/unit/router_utils/test_kubernetes_pod_discovery.py diff --git a/litellm/constants.py b/litellm/constants.py index 5473a3c6390..db49a40a37b 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -40,6 +40,13 @@ class ServerStreamingClassification(str, Enum): SERVER_STREAMING_CLASSIFICATION_MARKER: Final = ServerStreamingClassification.MARKER DEFER_PYDANTIC_BUILD: Final = os.getenv("DEFER_PYDANTIC_BUILD", "true") in ("true", "1", "on") DEFAULT_HEALTH_CHECK_PROMPT: Final = str(os.getenv("DEFAULT_HEALTH_CHECK_PROMPT", "test from litellm")) +KUBERNETES_POD_DISCOVERY_REFRESH_INTERVAL_SECONDS: Final = get_env_float( + "KUBERNETES_POD_DISCOVERY_REFRESH_INTERVAL_SECONDS", 5.0 +) +KUBERNETES_POD_DISCOVERY_IDLE_EVICTION_SECONDS: Final = get_env_float( + "KUBERNETES_POD_DISCOVERY_IDLE_EVICTION_SECONDS", 300.0 +) +KUBERNETES_POD_DISCOVERY_DNS_TIMEOUT_SECONDS: Final = 2.0 AZURE_DEFAULT_RESPONSES_API_VERSION: Final = str(os.getenv("AZURE_DEFAULT_RESPONSES_API_VERSION", "preview")) AZURE_OPENAI_AUDIO_PROVIDERS: Final = frozenset({"azure", "azure_ai"}) ROUTER_MAX_FALLBACKS: Final = get_env_int("ROUTER_MAX_FALLBACKS", 5) diff --git a/litellm/router.py b/litellm/router.py index 6e9ffc5341e..23b516ea573 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -266,6 +266,13 @@ from litellm.router_utils.handle_error import ( send_llm_exception_alert, ) from litellm.router_utils.health_state_cache import DeploymentHealthCache +from litellm.router_utils.kubernetes_pod_discovery import ( + KubernetesPodDiscovery, + async_resolve_pods_after, + async_resolve_pods_after_bound, + resolve_pods_after, + resolve_pods_after_bound, +) from litellm.router_utils.pre_call_checks.continuation_prefill_check import ( MID_STREAM_CONTINUATION_MARKER, ContinuationOutput, @@ -1289,6 +1296,7 @@ class Router: cache_config: Final[dict[str, Any]] = {} self.client_ttl = client_ttl + self.kubernetes_pod_discovery: Final = KubernetesPodDiscovery() if redis_url is not None or (redis_host is not None and redis_port is not None): cache_type = "redis" @@ -13512,6 +13520,13 @@ class Router: client = self.cache.get_cache(key=cache_key, local_only=True, parent_otel_span=parent_otel_span) return client elif client_type == "async": + if ( + deployment["litellm_params"].get( # pyright: ignore[reportUnknownMemberType] # legacy mapping + "kubernetes_pod_discovery" + ) + is True + ): + return None if kwargs.get("stream") is True: cache_key = f"{model_id}_stream_async_client" client = self.cache.get_cache(key=cache_key, local_only=True, parent_otel_span=parent_otel_span) @@ -13521,6 +13536,13 @@ class Router: client = self.cache.get_cache(key=cache_key, local_only=True, parent_otel_span=parent_otel_span) return client else: + if ( + deployment["litellm_params"].get( # pyright: ignore[reportUnknownMemberType] # legacy mapping + "kubernetes_pod_discovery" + ) + is True + ): + return None if kwargs.get("stream") is True: cache_key = f"{model_id}_stream_client" client = self.cache.get_cache(key=cache_key, parent_otel_span=parent_otel_span) @@ -14550,7 +14572,7 @@ class Router: Router._pop_effort_from_nested_carrier(request_kwargs, "output_config") Router._pop_effort_from_nested_carrier(request_kwargs, "reasoning") - async def async_get_available_deployment( + async def _async_get_available_deployment_unresolved( # noqa: ANN202 # legacy selector result remains inferred self, model: str, request_kwargs: dict, @@ -14587,7 +14609,7 @@ class Router: and self.routing_strategy != "latency-based-routing" and self.routing_strategy != "least-busy" ): # prevent regressions for other routing strategies, that don't have async get available deployments implemented. - return self.get_available_deployment( + return self._get_available_deployment_unresolved( model=model, messages=messages, input=input, @@ -14726,6 +14748,12 @@ class Router: ) raise e + async_get_available_deployment = ( # pyright: ignore[reportUnknownVariableType] # legacy selector return + async_resolve_pods_after( + _async_get_available_deployment_unresolved # pyright: ignore[reportUnknownArgumentType] # legacy type + ) + ) + async def async_get_available_deployment_for_pass_through( self, model: str, @@ -15492,7 +15520,7 @@ class Router: } return cast(StandardLoggingRoutingDecision, kept) # cast-ok: dropping optional keys preserves the type - def get_available_deployment( + def _get_available_deployment_unresolved( # noqa: ANN202 # legacy selector result remains inferred self, model: str, messages: list[dict[str, str]] | None = None, @@ -15665,6 +15693,12 @@ class Router: phase_event(_DEPLOYMENT_SELECTED_EVENT, pick_attributes) return deployment + get_available_deployment = ( # pyright: ignore[reportUnknownVariableType] # legacy selector return + resolve_pods_after( + _get_available_deployment_unresolved # pyright: ignore[reportUnknownArgumentType] # legacy annotations + ) + ) + def get_available_deployment_for_pass_through( self, model: str, @@ -16086,15 +16120,22 @@ class Router: CustomRoutingStrategy: litellm.router.CustomRoutingStrategyBase """ + strategy: Final = CustomRoutingStrategy setattr( self, "get_available_deployment", - CustomRoutingStrategy.get_available_deployment, + resolve_pods_after_bound( + strategy.get_available_deployment, # pyright: ignore[reportUnknownArgumentType] # legacy stub + self.kubernetes_pod_discovery, + ), ) setattr( self, "async_get_available_deployment", - CustomRoutingStrategy.async_get_available_deployment, + async_resolve_pods_after_bound( + strategy.async_get_available_deployment, # pyright: ignore[reportUnknownArgumentType] # legacy stub + self.kubernetes_pod_discovery, + ), ) def _reset_custom_routing_strategy(self) -> None: diff --git a/litellm/router_utils/kubernetes_pod_discovery.py b/litellm/router_utils/kubernetes_pod_discovery.py new file mode 100644 index 00000000000..7b98a52a4a0 --- /dev/null +++ b/litellm/router_utils/kubernetes_pod_discovery.py @@ -0,0 +1,462 @@ +import asyncio +import concurrent.futures +import ipaddress +import socket +import threading +import time +import urllib.request +from collections.abc import Awaitable, Callable, Coroutine, Mapping, Sequence +from concurrent.futures import Future +from dataclasses import dataclass, replace +from functools import wraps +from types import MappingProxyType +from typing import ( + Concatenate, + Final, + ParamSpec, + Protocol, + TypeAlias, + TypeVar, + cast, # noqa: TID251 # typeshed lacks proxy_bypass_environment +) + +import httpx + +from litellm._logging import verbose_router_logger +from litellm.constants import ( + KUBERNETES_POD_DISCOVERY_DNS_TIMEOUT_SECONDS, + KUBERNETES_POD_DISCOVERY_IDLE_EVICTION_SECONDS, + KUBERNETES_POD_DISCOVERY_REFRESH_INTERVAL_SECONDS, +) + +_SocketAddress: TypeAlias = tuple[str, int] | tuple[str, int, int, int] | tuple[int, bytes] +_AddressInfo: TypeAlias = tuple[socket.AddressFamily, socket.SocketKind, int, str, _SocketAddress] +_SyncLookup: TypeAlias = Future[list[_AddressInfo]] +_AsyncLookup: TypeAlias = asyncio.Future[list[_AddressInfo]] +_CacheKey: TypeAlias = tuple[str, int | None] +_DeploymentT: Final = TypeVar("_DeploymentT") + + +@dataclass(frozen=True, slots=True) +class _PodSet: + ips: tuple[str, ...] + resolved_at: float + last_used_at: float + cursor: int = 0 + + +def _lookup_outcome( + done: _SyncLookup | _AsyncLookup, +) -> Sequence[_AddressInfo] | OSError: + try: + return done.result() + except OSError as error: + return error + except BaseException as error: # noqa: BLE001 # a cancelled or crashed lookup is a failed refresh, never a crash + return OSError(str(error)) + + +class KubernetesPodDiscovery: + def __init__( + self, + refresh_interval_seconds: float = KUBERNETES_POD_DISCOVERY_REFRESH_INTERVAL_SECONDS, + idle_eviction_seconds: float = KUBERNETES_POD_DISCOVERY_IDLE_EVICTION_SECONDS, + proxy_environment: Callable[[], Mapping[str, str]] = urllib.request.getproxies_environment, + clock: Callable[[], float] = time.monotonic, + dns_timeout_seconds: float = KUBERNETES_POD_DISCOVERY_DNS_TIMEOUT_SECONDS, + dns_executor: concurrent.futures.Executor | None = None, + ) -> None: + self.refresh_interval_seconds = refresh_interval_seconds + self.idle_eviction_seconds = idle_eviction_seconds + self.dns_timeout_seconds = dns_timeout_seconds + self._dns_executor = dns_executor or concurrent.futures.ThreadPoolExecutor( + thread_name_prefix="litellm-k8s-pod-dns" + ) + self.proxy_environment = proxy_environment + self.clock = clock + self._cache: Mapping[_CacheKey, _PodSet] = MappingProxyType({}) + self._refreshing: frozenset[_CacheKey] = frozenset() + self._timed_out: frozenset[_CacheKey] = frozenset() + self._https_warning_hosts: frozenset[str] = frozenset() + self._proxy_warning_hosts: frozenset[str] = frozenset() + self._lock = threading.Lock() + + def _begin_refresh(self, key: _CacheKey, now: float) -> tuple[_PodSet | None, bool]: + with self._lock: + self._cache = MappingProxyType( + { + cache_key: pod_set + for cache_key, pod_set in self._cache.items() + if cache_key in self._refreshing or now - pod_set.last_used_at <= self.idle_eviction_seconds + } + ) + cached: Final = self._cache.get(key) + refreshing: Final = key in self._refreshing + if cached is not None and (now - cached.resolved_at < self.refresh_interval_seconds or refreshing): + return cached, False + if refreshing: + return None, False + self._refreshing = self._refreshing | {key} + return None, True + + def resolve_deployment(self, deployment: _DeploymentT) -> _DeploymentT: + eligible: Final = self._eligible(deployment) + if eligible is None: + return deployment + url, key, deployment_mapping = eligible + host, port = key + now: Final = self.clock() + cached, should_refresh = self._begin_refresh(key, now) + if cached is not None: + return self._deployment_with_cached_ip(deployment, deployment_mapping, key, url, now) + if not should_refresh: + return deployment + + lookup: Final = self._dns_executor.submit( + socket.getaddrinfo, host, port, type=socket.SOCK_STREAM, flags=socket.AI_ADDRCONFIG + ) + lookup.add_done_callback(lambda done: self._finish_refresh(key, done)) + try: + return self._apply_lookup( + deployment, deployment_mapping, key, url, self._await_lookup(key, host, lookup), now + ) + finally: + self._end_refresh(key) + + async def async_resolve_deployment(self, deployment: _DeploymentT) -> _DeploymentT: + eligible: Final = self._eligible(deployment) + if eligible is None: + return deployment + url, key, deployment_mapping = eligible + host, port = key + now: Final = self.clock() + cached, should_refresh = self._begin_refresh(key, now) + if cached is not None: + return self._deployment_with_cached_ip(deployment, deployment_mapping, key, url, now) + if not should_refresh: + return deployment + + lookup: Final = asyncio.ensure_future( + asyncio.get_running_loop().getaddrinfo(host, port, type=socket.SOCK_STREAM, flags=socket.AI_ADDRCONFIG) + ) + lookup.add_done_callback(lambda done: self._finish_refresh(key, done)) + try: + result: Final = await self._await_async_lookup(key, host, lookup) + return self._apply_lookup(deployment, deployment_mapping, key, url, result, now) + finally: + self._end_refresh(key) + + def _eligible(self, deployment: object) -> tuple[httpx.URL, _CacheKey, Mapping[str, object]] | None: + if not isinstance(deployment, Mapping): + return None + typed_deployment: Final = cast( # cast-ok: runtime Mapping check precedes typed lookup + Mapping[str, object], deployment + ) + target: Final = self._target(typed_deployment) + if target is None: + return None + url, host, port, target_deployment = target + if url.scheme == "https": + self._warn_https_once(host) + return None + if url.scheme != "http" or self._is_ip_literal(host): + return None + return (url, (host, port), target_deployment) + + @staticmethod + def _target( + deployment_mapping: Mapping[str, object], + ) -> tuple[httpx.URL, str, int | None, Mapping[str, object]] | None: + raw_params: Final = deployment_mapping.get("litellm_params") + if not isinstance(raw_params, Mapping): + return None + params: Final = cast( # cast-ok: runtime Mapping validation precedes the string field checks + Mapping[str, object], raw_params + ) + if params.get("kubernetes_pod_discovery") is not True: + return None + raw_api_base: Final = params.get("api_base") + if not isinstance(raw_api_base, str): + return None + try: + url: Final = httpx.URL(raw_api_base) + except httpx.InvalidURL: + return None + return (url, url.host, url.port, deployment_mapping) + + def _await_lookup(self, key: _CacheKey, host: str, lookup: _SyncLookup) -> Sequence[_AddressInfo] | OSError: + try: + return lookup.result(timeout=self.dns_timeout_seconds) + except (TimeoutError, concurrent.futures.TimeoutError): + return self._abandon_lookup(key, host, lookup) + except OSError as error: + return error + + async def _await_async_lookup( + self, key: _CacheKey, host: str, lookup: _AsyncLookup + ) -> Sequence[_AddressInfo] | OSError: + try: + return await asyncio.wait_for(asyncio.shield(lookup), timeout=self.dns_timeout_seconds) + except (TimeoutError, asyncio.TimeoutError): + return self._abandon_lookup(key, host, lookup) + except asyncio.CancelledError: + self._abandon_lookup(key, host, lookup) + raise + except OSError as error: + return error + + def _abandon_lookup( + self, + key: _CacheKey, + host: str, + lookup: _SyncLookup | _AsyncLookup, + ) -> Sequence[_AddressInfo] | OSError: + with self._lock: + if lookup.done(): + return _lookup_outcome(lookup) + self._timed_out = self._timed_out | {key} + return socket.gaierror(socket.EAI_AGAIN, f"DNS lookup for {host} exceeded {self.dns_timeout_seconds}s") + + def _end_refresh(self, key: _CacheKey) -> None: + with self._lock: + if key not in self._timed_out: + self._refreshing = self._refreshing - {key} + + def _finish_refresh( + self, + key: _CacheKey, + done: _SyncLookup | _AsyncLookup, + ) -> None: + with self._lock: + timed_out: Final = key in self._timed_out + self._timed_out = self._timed_out - {key} + self._refreshing = self._refreshing - {key} if timed_out else self._refreshing + late_result: Final = _lookup_outcome(done) if timed_out else None + if late_result is None: + return + if not isinstance(late_result, OSError): + self._store(key, self._pod_ips(late_result), self.clock()) + elif isinstance(late_result, socket.gaierror) and self._is_authoritative_no_pods(late_result): + self._store(key, (), self.clock()) + + def _apply_lookup( + self, + deployment: _DeploymentT, + deployment_mapping: Mapping[str, object], + key: _CacheKey, + url: httpx.URL, + result: Sequence[_AddressInfo] | OSError, + now: float, + ) -> _DeploymentT: + if isinstance(result, OSError): + if isinstance(result, socket.gaierror) and self._is_authoritative_no_pods(result): + self._store(key, (), self.clock()) + return deployment + self._stamp_failed_refresh(key, self.clock()) + verbose_router_logger.debug("Kubernetes pod discovery DNS refresh failed for %s: %s", key[0], result) + return self._deployment_with_cached_ip(deployment, deployment_mapping, key, url, now) + ips: Final = self._pod_ips(result) + self._store(key, ips, self.clock()) + if not ips: + return deployment + return self._deployment_with_cached_ip(deployment, deployment_mapping, key, url, now) + + @staticmethod + def _is_ip_literal(host: str) -> bool: + try: + ipaddress.ip_address(host) + except ValueError: + return False + return True + + @staticmethod + def _is_authoritative_no_pods(error: socket.gaierror) -> bool: + no_pod_errors: Final = (socket.EAI_NONAME, socket.EAI_NODATA) + return error.errno in no_pod_errors + + @staticmethod + def _pod_ip(record: _AddressInfo) -> str | None: + address: Final = record[4][0] + return address if isinstance(address, str) else None + + @classmethod + def _pod_ips(cls, records: Sequence[_AddressInfo]) -> tuple[str, ...]: + addresses: Final = tuple(cls._pod_ip(record) for record in records) + return tuple(sorted({address for address in addresses if address is not None})) + + def _store(self, key: _CacheKey, ips: tuple[str, ...], resolved_at: float) -> None: + with self._lock: + cached: Final = self._cache.get(key) + cursor: Final = cached.cursor % len(ips) if cached is not None and cached.ips and ips else 0 + self._cache = MappingProxyType( + { + **self._cache, + key: _PodSet(ips=ips, resolved_at=resolved_at, last_used_at=resolved_at, cursor=cursor), + } + ) + + def _stamp_failed_refresh(self, key: _CacheKey, resolved_at: float) -> None: + with self._lock: + cached: Final = self._cache.get(key) + stamped: Final = ( + replace(cached, resolved_at=resolved_at) + if cached is not None + else _PodSet(ips=(), resolved_at=resolved_at, last_used_at=resolved_at) + ) + self._cache = MappingProxyType({**self._cache, key: stamped}) + + def _next_ip(self, key: _CacheKey, now: float) -> str | None: + with self._lock: + cached: Final = self._cache.get(key) + if cached is None or not cached.ips: + return None + ip: Final = cached.ips[cached.cursor] + next_cursor: Final = (cached.cursor + 1) % len(cached.ips) + self._cache = MappingProxyType( + { + **self._cache, + key: replace(cached, cursor=next_cursor, last_used_at=max(cached.last_used_at, now)), + } + ) + return ip + + def _deployment_with_cached_ip( + self, + deployment: _DeploymentT, + deployment_mapping: Mapping[str, object], + key: _CacheKey, + url: httpx.URL, + now: float, + ) -> _DeploymentT: + ip: Final = self._next_ip(key, now) + if ip is None: + return deployment + if self._proxy_bypasses_only_service_host(key, ip): + self._warn_proxy_once(key[0]) + return deployment + params: Final = deployment_mapping.get("litellm_params") + if not isinstance(params, Mapping): + return deployment + typed_params: Final = cast( # cast-ok: runtime Mapping validation precedes the copied provider parameters + Mapping[str, object], params + ) + return cast( # cast-ok: resolving produces a copied dict for the selector's Mapping type + _DeploymentT, + { + **deployment_mapping, + "litellm_params": {**typed_params, "api_base": str(url.copy_with(host=ip))}, + }, + ) + + def _proxy_bypasses_only_service_host(self, key: _CacheKey, ip: str) -> bool: + proxies: Final = self.proxy_environment() + if not (proxies.get("http") or proxies.get("all")): + return False + proxy_bypass_environment: Final = cast( # cast-ok: stdlib helper accepts the supplied proxy mapping + Callable[[str, dict[str, str]], bool], + urllib.request.proxy_bypass_environment, # pyright: ignore[reportAttributeAccessIssue] # typeshed gap + ) + no_proxy: Final = proxies.get("no", "") + entries: Final = tuple(entry.strip() for entry in no_proxy.split(",")) + host: Final = key[0] + port: Final = key[1] if key[1] is not None else 80 + bracketed_ip: Final = f"[{ip}]" if ipaddress.ip_address(ip).version == 6 else ip + service_bypassed: Final = "*" in entries or any( + proxy_bypass_environment(candidate, dict(proxies)) for candidate in (host, f"{host}:{port}") + ) + pod_bypassed: Final = any( + proxy_bypass_environment(candidate, dict(proxies)) for candidate in (ip, f"{bracketed_ip}:{port}") + ) + return service_bypassed and not pod_bypassed + + def _warn_https_once(self, host: str) -> None: + with self._lock: + should_warn: Final = host not in self._https_warning_hosts + if should_warn: + self._https_warning_hosts = self._https_warning_hosts | {host} + if should_warn: + verbose_router_logger.warning( + "Kubernetes pod discovery is disabled for HTTPS host %s because substituting an IP " + "breaks TLS hostname verification", + host, + ) + + def _warn_proxy_once(self, host: str) -> None: + with self._lock: + should_warn: Final = host not in self._proxy_warning_hosts + if should_warn: + self._proxy_warning_hosts = self._proxy_warning_hosts | {host} + if should_warn: + verbose_router_logger.warning( + "Kubernetes pod discovery keeps the service hostname %s: NO_PROXY exempts it but not its pod IPs", + host, + ) + + +class _HasPodDiscovery(Protocol): + kubernetes_pod_discovery: KubernetesPodDiscovery + + +_P: Final = ParamSpec("_P") +_R: Final = TypeVar("_R") +_S: Final = TypeVar("_S", bound=_HasPodDiscovery) + + +def resolve_pods_after( + fn: Callable[Concatenate[_S, _P], _R], +) -> Callable[Concatenate[_S, _P], _R]: + @wraps(fn) + def wrapped( + self: _S, + *args: _P.args, + **kwargs: _P.kwargs, # kwargs-ok: ParamSpec forwards the wrapped call signature + ) -> _R: + deployment: Final = fn(self, *args, **kwargs) + return self.kubernetes_pod_discovery.resolve_deployment(deployment) + + return wrapped + + +def async_resolve_pods_after( + fn: Callable[Concatenate[_S, _P], Awaitable[_R]], +) -> Callable[Concatenate[_S, _P], Coroutine[object, object, _R]]: + @wraps(fn) + async def wrapped( + self: _S, + *args: _P.args, + **kwargs: _P.kwargs, # kwargs-ok: ParamSpec forwards the wrapped call signature + ) -> _R: + deployment: Final = await fn(self, *args, **kwargs) + return await self.kubernetes_pod_discovery.async_resolve_deployment(deployment) + + return wrapped + + +def resolve_pods_after_bound( + fn: Callable[_P, _R], + discovery: KubernetesPodDiscovery, +) -> Callable[_P, _R]: + @wraps(fn) + def wrapped( + *args: _P.args, + **kwargs: _P.kwargs, # kwargs-ok: ParamSpec forwards the wrapped call signature + ) -> _R: + deployment: Final = fn(*args, **kwargs) + return discovery.resolve_deployment(deployment) + + return wrapped + + +def async_resolve_pods_after_bound( + fn: Callable[_P, Awaitable[_R]], + discovery: KubernetesPodDiscovery, +) -> Callable[_P, Coroutine[object, object, _R]]: + @wraps(fn) + async def wrapped( + *args: _P.args, + **kwargs: _P.kwargs, # kwargs-ok: ParamSpec forwards the wrapped call signature + ) -> _R: + deployment: Final = await fn(*args, **kwargs) + return await discovery.async_resolve_deployment(deployment) + + return wrapped diff --git a/litellm/types/litellm_params.py b/litellm/types/litellm_params.py index 174d07a2cea..b05d25462aa 100644 --- a/litellm/types/litellm_params.py +++ b/litellm/types/litellm_params.py @@ -183,6 +183,7 @@ class DeploymentOptions: tpm: int | None = None itpm: int | None = None otpm: int | None = None + kubernetes_pod_discovery: bool | None = None default_api_key_rpm_limit: int | None = None default_api_key_tpm_limit: int | None = None max_parallel_requests: int | None = None diff --git a/litellm/types/router.py b/litellm/types/router.py index 1bd1c2e5502..68dbc701c9e 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -720,6 +720,7 @@ class LiteLLMParamsTypedDict(TypedDict, total=False): use_chat_completions_api: bool | None ## PASS-THROUGH ENDPOINTS ## use_in_pass_through: bool | None + kubernetes_pod_discovery: ReadOnly[bool | None] litellm_credential_name: str | None ## UNIFIED PROJECT/REGION ## region_name: str | None diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index 681476456c2..4ea242258f6 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -42,9 +42,11 @@ def get_all_functions_called_in_tests(base_dir): print("dir_path: ", dir_path) for root, _, files in os.walk(dir_path): for file in files: - if file.endswith(".py") and ("router" in file.lower() or test_dir == "unit"): + file_path = os.path.join(root, file) + if file.endswith(".py") and ( + "router" in os.path.relpath(file_path, dir_path).lower() or test_dir == "unit" + ): print("file: ", file) - file_path = os.path.join(root, file) with open(file_path, "r", encoding="utf-8") as f: try: tree = ast.parse(f.read()) diff --git a/tests/unit/router_utils/test_kubernetes_pod_discovery.py b/tests/unit/router_utils/test_kubernetes_pod_discovery.py new file mode 100644 index 00000000000..b3fc6eede62 --- /dev/null +++ b/tests/unit/router_utils/test_kubernetes_pod_discovery.py @@ -0,0 +1,1266 @@ +import asyncio +import copy +import json +import logging +import re +import socket +import threading +from collections.abc import Callable, Mapping +from concurrent.futures import Future, ThreadPoolExecutor +from itertools import chain, count, repeat +from typing import Final, TypeAlias, cast + +import httpx +import pytest +import respx +from openai import AsyncOpenAI, OpenAI + +import litellm +from litellm.router import CustomRoutingStrategyBase, Router +from litellm.router_utils.kubernetes_pod_discovery import KubernetesPodDiscovery + +_SERVICE_URL: Final = "http://vllm-headless.ns.svc.cluster.local:8000/v1" +_SocketAddress: TypeAlias = tuple[str, int] | tuple[str, int, int, int] +_AddrInfo: TypeAlias = tuple[socket.AddressFamily, socket.SocketKind, int, str, _SocketAddress] +_NO_POD_ERRNOS: Final = tuple(sorted((socket.EAI_NONAME, socket.EAI_NODATA))) +_CHAT_RESPONSE: Final = { + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1, + "model": "my-model", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "ok"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, +} + + +def _deployment(api_base: str = _SERVICE_URL, kubernetes_pod_discovery: bool | None = True) -> dict[str, object]: + params: Final = { + "model": "openai/my-model", + "api_base": api_base, + "api_key": "fake", + **({"kubernetes_pod_discovery": kubernetes_pod_discovery} if kubernetes_pod_discovery is not None else {}), + } + return { + "model_name": "gpu-model", + "litellm_params": params, + "model_info": {"id": "registered-id"}, + } + + +class _ModelListRoutingStrategy(CustomRoutingStrategyBase): + def __init__(self, router: Router) -> None: + self._router: Final = router + + def get_available_deployment(self, *args: object, **kwargs: object) -> Mapping[str, object]: + return self._router.model_list[0] + + async def async_get_available_deployment(self, *args: object, **kwargs: object) -> Mapping[str, object]: + return self._router.model_list[0] + + +def _record(ip: str, port: int) -> _AddrInfo: + address: Final = (ip, port, 0, 0) if ":" in ip else (ip, port) + family: Final = socket.AF_INET6 if ":" in ip else socket.AF_INET + return (family, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", address) + + +def _records(*ips: str, port: int = 8000) -> list[_AddrInfo]: + return [_record(ip, port) for ip in ips] + + +def _clock(ticks: tuple[float, ...]) -> Callable[[], float]: + timestamps: Final = iter(ticks) + + def monotonic() -> float: + return next(timestamps) + + return monotonic + + +def _proxy_environment(no_proxy: str) -> Callable[[], Mapping[str, str]]: + return lambda: {"http": "http://proxy.example:8080", "no": no_proxy} + + +def _empty_proxy_environment() -> Mapping[str, str]: + return {} + + +def _stub_sync_dns(monkeypatch: pytest.MonkeyPatch, *ips: str) -> None: + def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + return _records(*ips, port=int(port or 8000)) + + monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo) + + +def _api_base(deployment: Mapping[str, object]) -> str: + params: Final = deployment["litellm_params"] + assert isinstance(params, Mapping) + api_base: Final = params["api_base"] + assert isinstance(api_base, str) + return api_base + + +def _request_body(content: bytes) -> Mapping[str, object]: + body: Final = json.loads(content) + assert isinstance(body, dict) + return cast(Mapping[str, object], body) + + +def test_sync_resolution_round_robins_sorted_ips_without_mutating_deployment( + monkeypatch: pytest.MonkeyPatch, +) -> None: + original: Final = _deployment() + before: Final = copy.deepcopy(original) + lookups: Final = count() + + def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(lookups) + assert type == socket.SOCK_STREAM + return _records("10.0.0.3", "10.0.0.1", "10.0.0.2", "10.0.0.1") + + monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery( + refresh_interval_seconds=30, + proxy_environment=_empty_proxy_environment, + ) + + results: Final = tuple(discovery.resolve_deployment(original) for _ in range(3)) + + assert tuple(_api_base(result) for result in results) == ( + "http://10.0.0.1:8000/v1", + "http://10.0.0.2:8000/v1", + "http://10.0.0.3:8000/v1", + ) + assert original == before + assert results[0] is not original + assert results[0]["model_info"] is original["model_info"] + + +@pytest.mark.asyncio +async def test_async_resolution_round_robins_sorted_ips_and_preserves_url_path( + monkeypatch: pytest.MonkeyPatch, +) -> None: + original: Final = _deployment(api_base=f"{_SERVICE_URL}/custom?version=1") + before: Final = copy.deepcopy(original) + loop: Final = asyncio.get_running_loop() + lookups: Final = count() + + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(lookups) + return _records("10.0.0.3", "10.0.0.1", "10.0.0.2", port=int(port or 0)) + + monkeypatch.setattr(loop, "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery( + refresh_interval_seconds=30, + proxy_environment=_empty_proxy_environment, + ) + + results: Final = ( + await discovery.async_resolve_deployment(original), + await discovery.async_resolve_deployment(original), + await discovery.async_resolve_deployment(original), + ) + + assert tuple(_api_base(result) for result in results) == ( + "http://10.0.0.1:8000/v1/custom?version=1", + "http://10.0.0.2:8000/v1/custom?version=1", + "http://10.0.0.3:8000/v1/custom?version=1", + ) + assert original == before + assert results[0] is not original + assert results[0]["model_info"] is original["model_info"] + + +def test_refresh_reuses_cached_set_until_interval_then_uses_new_pods( + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock: Final = _clock((10.0, 10.0, 14.99, 15.0, 15.0)) + responses: Final = iter((_records("10.0.0.1", "10.0.0.2"), _records("10.0.0.3"))) + lookups: Final = count() + + def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(lookups) + return list(next(responses)) + + monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery( + refresh_interval_seconds=5, + clock=clock, + proxy_environment=_empty_proxy_environment, + ) + + first: Final = _api_base(discovery.resolve_deployment(_deployment())) + cached: Final = _api_base(discovery.resolve_deployment(_deployment())) + refreshed: Final = _api_base(discovery.resolve_deployment(_deployment())) + + assert (first, cached, refreshed) == ( + "http://10.0.0.1:8000/v1", + "http://10.0.0.2:8000/v1", + "http://10.0.0.3:8000/v1", + ) + assert next(lookups) == 2 + + +def test_refresh_carries_round_robin_cursor_into_new_ip_set(monkeypatch: pytest.MonkeyPatch) -> None: + clock: Final = _clock((0.0, 0.0, 5.0, 5.0)) + responses: Final = iter((_records("10.0.0.1", "10.0.0.2", "10.0.0.3"), _records("10.0.0.4", "10.0.0.5"))) + + def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + return list(next(responses)) + + monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery( + refresh_interval_seconds=5, + clock=clock, + proxy_environment=_empty_proxy_environment, + ) + + first: Final = _api_base(discovery.resolve_deployment(_deployment())) + refreshed: Final = _api_base(discovery.resolve_deployment(_deployment())) + + assert (first, refreshed) == ("http://10.0.0.1:8000/v1", "http://10.0.0.5:8000/v1") + + +def test_idle_cache_eviction_restarts_round_robin_cursor(monkeypatch: pytest.MonkeyPatch) -> None: + clock: Final = _clock((0.0, 0.0, 0.0, 11.0, 11.0)) + lookups: Final = count() + + def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(lookups) + return _records("10.0.0.1", "10.0.0.2", "10.0.0.3", port=int(port or 0)) + + monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery( + refresh_interval_seconds=5, + idle_eviction_seconds=10, + clock=clock, + proxy_environment=_empty_proxy_environment, + ) + + first: Final = _api_base(discovery.resolve_deployment(_deployment())) + second: Final = _api_base(discovery.resolve_deployment(_deployment())) + after_eviction: Final = _api_base(discovery.resolve_deployment(_deployment())) + + assert (first, second, after_eviction) == ( + "http://10.0.0.1:8000/v1", + "http://10.0.0.2:8000/v1", + "http://10.0.0.1:8000/v1", + ) + assert next(lookups) == 2 + + +def test_active_pod_set_survives_idle_eviction_when_refresh_interval_is_longer( + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock_values: Final = iter(chain((0.0, 0.0, 200.0, 400.0, 550.0), repeat(550.0))) + + def clock() -> float: + return next(clock_values) + + lookups: Final = count() + + def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(lookups) + return _records("10.0.0.1", "10.0.0.2", "10.0.0.3", port=int(port or 0)) + + monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery( + refresh_interval_seconds=600, + idle_eviction_seconds=300, + clock=clock, + proxy_environment=_empty_proxy_environment, + ) + + api_bases: Final = tuple(_api_base(discovery.resolve_deployment(_deployment())) for _ in range(4)) + + assert api_bases == ( + "http://10.0.0.1:8000/v1", + "http://10.0.0.2:8000/v1", + "http://10.0.0.3:8000/v1", + "http://10.0.0.1:8000/v1", + ) + assert next(lookups) == 1 + + +def test_slow_dns_refresh_does_not_move_last_use_backward(monkeypatch: pytest.MonkeyPatch) -> None: + clock_values: Final = iter(chain((0.0, 250.0), repeat(500.0))) + + def clock() -> float: + return next(clock_values) + + lookups: Final = count() + + def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(lookups) + return _records("10.0.0.1", "10.0.0.2", "10.0.0.3", port=int(port or 0)) + + monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery( + refresh_interval_seconds=600, + idle_eviction_seconds=300, + clock=clock, + proxy_environment=_empty_proxy_environment, + ) + + api_bases: Final = tuple(_api_base(discovery.resolve_deployment(_deployment())) for _ in range(2)) + + assert api_bases == ("http://10.0.0.1:8000/v1", "http://10.0.0.2:8000/v1") + assert next(lookups) == 1 + + +def test_proxy_bypassing_only_service_hostname_keeps_api_base_and_warns_once( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + _stub_sync_dns(monkeypatch, "10.244.1.3") + discovery: Final = KubernetesPodDiscovery(proxy_environment=_proxy_environment(".svc.cluster.local")) + original: Final = _deployment() + + with caplog.at_level(logging.WARNING): + first: Final = discovery.resolve_deployment(original) + second: Final = discovery.resolve_deployment(original) + + assert first is original + assert second is original + assert _api_base(first) == _SERVICE_URL + assert sum("NO_PROXY exempts it but not its pod IPs" in record.getMessage() for record in caplog.records) == 1 + + +def test_port_qualified_no_proxy_service_keeps_api_base_and_warns_once( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + _stub_sync_dns(monkeypatch, "10.244.1.3") + discovery: Final = KubernetesPodDiscovery( + proxy_environment=_proxy_environment("vllm-headless.ns.svc.cluster.local:8000") + ) + original: Final = _deployment() + + with caplog.at_level(logging.WARNING): + result: Final = discovery.resolve_deployment(original) + + assert result is original + assert _api_base(result) == _SERVICE_URL + assert sum("NO_PROXY exempts it but not its pod IPs" in record.getMessage() for record in caplog.records) == 1 + + +def test_port_qualified_no_proxy_service_and_pod_substitutes_pod_ip(monkeypatch: pytest.MonkeyPatch) -> None: + _stub_sync_dns(monkeypatch, "10.244.1.3") + discovery: Final = KubernetesPodDiscovery( + proxy_environment=_proxy_environment("vllm-headless.ns.svc.cluster.local:8000,10.244.1.3:8000") + ) + + result: Final = discovery.resolve_deployment(_deployment()) + + assert _api_base(result) == "http://10.244.1.3:8000/v1" + + +def test_port_qualified_no_proxy_other_service_port_substitutes_pod_ip( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _stub_sync_dns(monkeypatch, "10.244.1.3") + discovery: Final = KubernetesPodDiscovery( + proxy_environment=_proxy_environment("vllm-headless.ns.svc.cluster.local:9000") + ) + + result: Final = discovery.resolve_deployment(_deployment()) + + assert _api_base(result) == "http://10.244.1.3:8000/v1" + + +def test_proxy_with_pod_cidr_in_no_proxy_keeps_service_hostname(monkeypatch: pytest.MonkeyPatch) -> None: + _stub_sync_dns(monkeypatch, "10.244.1.3") + discovery: Final = KubernetesPodDiscovery(proxy_environment=_proxy_environment(".svc.cluster.local,10.244.0.0/16")) + + result: Final = discovery.resolve_deployment(_deployment()) + + assert _api_base(result) == _SERVICE_URL + + +def test_proxy_with_exact_pod_ip_in_no_proxy_keeps_pod_ip_substitution(monkeypatch: pytest.MonkeyPatch) -> None: + _stub_sync_dns(monkeypatch, "10.244.1.3") + discovery: Final = KubernetesPodDiscovery(proxy_environment=_proxy_environment(".svc.cluster.local,10.244.1.3")) + + result: Final = discovery.resolve_deployment(_deployment()) + + assert _api_base(result) == "http://10.244.1.3:8000/v1" + + +def test_proxy_with_empty_no_proxy_keeps_pod_ip_substitution(monkeypatch: pytest.MonkeyPatch) -> None: + _stub_sync_dns(monkeypatch, "10.244.1.3") + discovery: Final = KubernetesPodDiscovery(proxy_environment=_proxy_environment("")) + + result: Final = discovery.resolve_deployment(_deployment()) + + assert _api_base(result) == "http://10.244.1.3:8000/v1" + + +def test_empty_proxy_environment_keeps_pod_ip_substitution(monkeypatch: pytest.MonkeyPatch) -> None: + _stub_sync_dns(monkeypatch, "10.244.1.3") + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment) + + result: Final = discovery.resolve_deployment(_deployment()) + + assert _api_base(result) == "http://10.244.1.3:8000/v1" + + +@pytest.mark.parametrize("no_pods_errno", _NO_POD_ERRNOS) +def test_authoritative_no_pods_uses_service_url_and_transient_failure_keeps_cached_pods( + monkeypatch: pytest.MonkeyPatch, no_pods_errno: int +) -> None: + clock: Final = _clock((0.0, 0.0, 0.0, 0.0, 5.0, 5.0)) + missing: Final = _deployment() + no_pod_lookups: Final = count() + + def missing_getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(no_pod_lookups) + raise socket.gaierror(no_pods_errno, "missing") + + monkeypatch.setattr(socket, "getaddrinfo", missing_getaddrinfo) + discovery_without_pods: Final = KubernetesPodDiscovery( + refresh_interval_seconds=5, + clock=clock, + proxy_environment=_empty_proxy_environment, + ) + + assert discovery_without_pods.resolve_deployment(missing) is missing + assert discovery_without_pods.resolve_deployment(missing) is missing + assert _api_base(missing) == _SERVICE_URL + assert next(no_pod_lookups) == 1 + + responses: Final = iter((_records("10.0.0.4"), socket.gaierror(socket.EAI_AGAIN, "temporary"))) + + def transient_getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + response: Final = next(responses) + if isinstance(response, socket.gaierror): + raise response + return list(response) + + monkeypatch.setattr(socket, "getaddrinfo", transient_getaddrinfo) + discovery_with_pods: Final = KubernetesPodDiscovery( + refresh_interval_seconds=5, + clock=clock, + proxy_environment=_empty_proxy_environment, + ) + first: Final = _api_base(discovery_with_pods.resolve_deployment(_deployment())) + after_failure: Final = _api_base(discovery_with_pods.resolve_deployment(_deployment())) + + assert (first, after_failure) == ("http://10.0.0.4:8000/v1", "http://10.0.0.4:8000/v1") + + +@pytest.mark.parametrize( + ("api_base", "flag"), + ( + pytest.param(_SERVICE_URL, None, id="flag-absent"), + pytest.param(_SERVICE_URL, False, id="flag-false"), + pytest.param("https://vllm-headless.ns.svc.cluster.local:8000/v1", True, id="https"), + pytest.param("http://10.0.0.9:8000/v1", True, id="ip-literal"), + pytest.param("http://[zzzz]", True, id="malformed-url"), + ), +) +def test_unsupported_deployments_are_returned_unchanged_without_dns( + monkeypatch: pytest.MonkeyPatch, api_base: str, flag: bool | None +) -> None: + deployment: Final = _deployment(api_base=api_base, kubernetes_pod_discovery=flag) + lookups: Final = count() + + def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(lookups) + return [] + + monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment) + + assert discovery.resolve_deployment(deployment) is deployment + assert next(lookups) == 0 + + +@pytest.mark.asyncio +async def test_non_mapping_deployments_are_returned_unchanged() -> None: + deployment: Final = object() + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment) + + assert discovery.resolve_deployment(deployment) is deployment + assert await discovery.async_resolve_deployment(deployment) is deployment + + +def test_https_discovery_warning_is_emitted_once_per_host(caplog: pytest.LogCaptureFixture) -> None: + caplog.set_level(logging.WARNING) + deployment: Final = _deployment(api_base="https://vllm-headless.ns.svc.cluster.local:8000/v1") + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment) + + results: Final = ( + discovery.resolve_deployment(deployment), + discovery.resolve_deployment(deployment), + ) + warnings: Final = tuple( + record + for record in caplog.records + if "Kubernetes pod discovery is disabled for HTTPS host" in record.getMessage() + ) + + assert results == (deployment, deployment) + assert all(result is deployment for result in results) + assert len(warnings) == 1 + + +def test_ipv6_pod_address_is_bracketed_in_url(monkeypatch: pytest.MonkeyPatch) -> None: + def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + return _records("2001:db8::2") + + monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment) + + result: Final = discovery.resolve_deployment(_deployment()) + + assert _api_base(result) == "http://[2001:db8::2]:8000/v1" + + +def test_transient_dns_failure_on_cold_cache_waits_for_the_refresh_interval(monkeypatch: pytest.MonkeyPatch) -> None: + clock: Final = _clock((0.0, 0.0, 1.0, 6.0, 6.0, 6.0, 6.0, 6.0)) + lookups: Final = count() + + def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(lookups) + raise socket.gaierror(socket.EAI_AGAIN, "temporary") + + monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery( + refresh_interval_seconds=5, + clock=clock, + proxy_environment=_empty_proxy_environment, + ) + deployment: Final = _deployment() + + within_interval: Final = tuple(discovery.resolve_deployment(deployment) for _ in range(2)) + lookups_within_interval: Final = next(lookups) + after_interval: Final = discovery.resolve_deployment(deployment) + + assert all(result is deployment for result in within_interval) + assert after_interval is deployment + assert (lookups_within_interval, next(lookups)) == (1, 3) + + +def test_slow_sync_dns_lookup_falls_back_to_the_service_url_within_the_deadline( + monkeypatch: pytest.MonkeyPatch, +) -> None: + release: Final = threading.Event() + + def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + release.wait() + return _records("10.0.0.1", port=int(port or 8000)) + + monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment, dns_timeout_seconds=0.01) + deployment: Final = _deployment() + + during_slow_lookup: Final = discovery.resolve_deployment(deployment) + release.set() + + assert during_slow_lookup is deployment + assert _api_base(during_slow_lookup) == _SERVICE_URL + + +@pytest.mark.asyncio +async def test_slow_async_dns_lookup_falls_back_to_the_service_url_within_the_deadline( + monkeypatch: pytest.MonkeyPatch, +) -> None: + release: Final = asyncio.Event() + + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + await release.wait() + return _records("10.0.0.1", port=int(port or 8000)) + + monkeypatch.setattr(asyncio.get_running_loop(), "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment, dns_timeout_seconds=0.01) + deployment: Final = _deployment() + + during_slow_lookup: Final = await discovery.async_resolve_deployment(deployment) + release.set() + + assert during_slow_lookup is deployment + assert _api_base(during_slow_lookup) == _SERVICE_URL + + +def _stuck_then_ten_clock() -> Callable[[], float]: + ticks: Final = chain((0.0, 0.0), repeat(10.0)) + + def monotonic() -> float: + return next(ticks) + + return monotonic + + +def test_timed_out_sync_lookup_is_not_resubmitted_and_its_late_pods_are_used(monkeypatch: pytest.MonkeyPatch) -> None: + release: Final = threading.Event() + lookups: Final = count() + + def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(lookups) + release.wait() + return _records("10.0.0.1", port=int(port or 8000)) + + monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo) + dns_executor: Final = ThreadPoolExecutor(max_workers=1) + discovery: Final = KubernetesPodDiscovery( + refresh_interval_seconds=5, + clock=_stuck_then_ten_clock(), + proxy_environment=_empty_proxy_environment, + dns_timeout_seconds=0.01, + dns_executor=dns_executor, + ) + deployment: Final = _deployment() + + timed_out: Final = discovery.resolve_deployment(deployment) + after_interval: Final = discovery.resolve_deployment(deployment) + release.set() + dns_executor.shutdown(wait=True) + late: Final = _api_base(discovery.resolve_deployment(deployment)) + + assert (timed_out is deployment, after_interval is deployment) == (True, True) + assert late == "http://10.0.0.1:8000/v1" + assert next(lookups) == 1 + + +@pytest.mark.asyncio +async def test_timed_out_async_lookup_is_not_resubmitted_and_its_late_pods_are_used( + monkeypatch: pytest.MonkeyPatch, +) -> None: + release: Final = asyncio.Event() + lookups: Final = count() + + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(lookups) + await release.wait() + return _records("10.0.0.1", port=int(port or 8000)) + + monkeypatch.setattr(asyncio.get_running_loop(), "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery( + refresh_interval_seconds=5, + clock=_stuck_then_ten_clock(), + proxy_environment=_empty_proxy_environment, + dns_timeout_seconds=0.01, + ) + deployment: Final = _deployment() + + timed_out: Final = await discovery.async_resolve_deployment(deployment) + after_interval: Final = await discovery.async_resolve_deployment(deployment) + release.set() + for _ in range(5): + await asyncio.sleep(0) + late: Final = _api_base(await discovery.async_resolve_deployment(deployment)) + + assert (timed_out is deployment, after_interval is deployment) == (True, True) + assert late == "http://10.0.0.1:8000/v1" + assert next(lookups) == 1 + + +def test_late_authoritative_no_pods_after_a_timed_out_refresh_clears_the_cached_pods( + monkeypatch: pytest.MonkeyPatch, +) -> None: + release: Final = threading.Event() + lookups: Final = count() + + def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + if next(lookups) == 0: + return _records("10.0.0.1", port=int(port or 8000)) + release.wait() + raise socket.gaierror(socket.EAI_NONAME, "gone") + + monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo) + dns_executor: Final = ThreadPoolExecutor(max_workers=1) + discovery: Final = KubernetesPodDiscovery( + refresh_interval_seconds=5, + clock=_stuck_then_ten_clock(), + proxy_environment=_empty_proxy_environment, + dns_timeout_seconds=0.01, + dns_executor=dns_executor, + ) + deployment: Final = _deployment() + + first: Final = _api_base(discovery.resolve_deployment(deployment)) + during_timed_out_refresh: Final = _api_base(discovery.resolve_deployment(deployment)) + release.set() + dns_executor.shutdown(wait=True) + after_late_no_pods: Final = discovery.resolve_deployment(deployment) + + assert (first, during_timed_out_refresh) == ("http://10.0.0.1:8000/v1", "http://10.0.0.1:8000/v1") + assert after_late_no_pods is deployment + assert next(lookups) == 2 + + +@pytest.mark.asyncio +async def test_cancelled_request_does_not_resubmit_the_lookup_it_abandoned(monkeypatch: pytest.MonkeyPatch) -> None: + release: Final = asyncio.Event() + lookups: Final = count() + + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(lookups) + await release.wait() + return _records("10.0.0.1", port=int(port or 8000)) + + monkeypatch.setattr(asyncio.get_running_loop(), "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery( + refresh_interval_seconds=5, + clock=_stuck_then_ten_clock(), + proxy_environment=_empty_proxy_environment, + dns_timeout_seconds=60, + ) + deployment: Final = _deployment() + + disconnected: Final = asyncio.create_task(discovery.async_resolve_deployment(deployment)) + for _ in range(3): + await asyncio.sleep(0) + disconnected.cancel() + cancelled: Final = await asyncio.gather(disconnected, return_exceptions=True) + after_interval: Final = await discovery.async_resolve_deployment(deployment) + release.set() + for _ in range(5): + await asyncio.sleep(0) + late: Final = _api_base(await discovery.async_resolve_deployment(deployment)) + + assert isinstance(cancelled[0], asyncio.CancelledError) + assert after_interval is deployment + assert late == "http://10.0.0.1:8000/v1" + assert next(lookups) == 1 + + +def _ipv4_only_host_records(port: int, flags: int) -> list[_AddrInfo]: + """A resolver on a host with no IPv6 address: dual-stack records only when AI_ADDRCONFIG is absent.""" + if flags & socket.AI_ADDRCONFIG: + return _records("10.0.0.1", "10.0.0.2", port=port) + return _records("10.0.0.1", "fd00::1", "10.0.0.2", "fd00::2", port=port) + + +def test_dual_stack_service_on_ipv4_only_host_rotates_only_ipv4_pods(monkeypatch: pytest.MonkeyPatch) -> None: + def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + return _ipv4_only_host_records(int(port or 8000), flags) + + monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment) + + hosts: Final = tuple(httpx.URL(_api_base(discovery.resolve_deployment(_deployment()))).host for _ in range(4)) + + assert hosts == ("10.0.0.1", "10.0.0.2", "10.0.0.1", "10.0.0.2") + + +@pytest.mark.asyncio +async def test_dual_stack_service_on_ipv4_only_host_rotates_only_ipv4_pods_async( + monkeypatch: pytest.MonkeyPatch, +) -> None: + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + return _ipv4_only_host_records(int(port or 8000), flags) + + monkeypatch.setattr(asyncio.get_running_loop(), "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment) + + hosts: Final = tuple( + [httpx.URL(_api_base(await discovery.async_resolve_deployment(_deployment()))).host for _ in range(4)] + ) + + assert hosts == ("10.0.0.1", "10.0.0.2", "10.0.0.1", "10.0.0.2") + + +@pytest.mark.asyncio +async def test_concurrent_async_refresh_uses_cached_ips_while_refresh_is_in_flight( + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock: Final = _clock((0.0, 0.0, 5.0, 5.0, 5.0, 5.0, 5.0)) + loop: Final = asyncio.get_running_loop() + refresh_started: Final = asyncio.Event() + finish_refresh: Final = asyncio.Event() + lookups: Final = count() + + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + lookup_number: Final = next(lookups) + if lookup_number == 0: + return _records("10.0.0.1", port=int(port or 0)) + if lookup_number > 1: + return _records("10.0.0.3", port=int(port or 0)) + refresh_started.set() + await finish_refresh.wait() + return _records("10.0.0.2", port=int(port or 0)) + + monkeypatch.setattr(loop, "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery( + refresh_interval_seconds=5, + clock=clock, + proxy_environment=_empty_proxy_environment, + ) + first: Final = _api_base(await discovery.async_resolve_deployment(_deployment())) + refresh_task: Final = asyncio.create_task(discovery.async_resolve_deployment(_deployment())) + await refresh_started.wait() + concurrent: Final = _api_base(await discovery.async_resolve_deployment(_deployment())) + finish_refresh.set() + refreshed: Final = _api_base(await refresh_task) + + assert (first, concurrent, refreshed) == ( + "http://10.0.0.1:8000/v1", + "http://10.0.0.1:8000/v1", + "http://10.0.0.2:8000/v1", + ) + assert next(lookups) == 2 + + +@pytest.mark.parametrize("cache_existing", (False, True), ids=("cold-cache", "stale-cache")) +def test_concurrent_sync_refresh_uses_cached_ips_or_service_url_while_in_flight( + monkeypatch: pytest.MonkeyPatch, + cache_existing: bool, +) -> None: + ticks: Final = (0.0, 0.0, 5.0, 5.0, 5.0) if cache_existing else (0.0, 0.0, 0.0) + clock: Final = _clock(ticks) + refresh_started: Final = threading.Event() + finish_refresh: Final = threading.Event() + lookups: Final = count() + + def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + lookup_number: Final = next(lookups) + if cache_existing and lookup_number == 0: + return _records("10.0.0.1", "10.0.0.2", port=int(port or 0)) + refresh_started.set() + assert finish_refresh.wait(timeout=3) + return _records("10.0.0.3", port=int(port or 0)) + + monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery( + refresh_interval_seconds=5, + clock=clock, + proxy_environment=_empty_proxy_environment, + ) + if cache_existing: + assert _api_base(discovery.resolve_deployment(_deployment())) == "http://10.0.0.1:8000/v1" + + refresh_result: Final = Future[dict[str, object]]() + + def refresh() -> None: + refresh_result.set_result(discovery.resolve_deployment(_deployment())) + + thread: Final = threading.Thread(target=refresh, daemon=True) + thread.start() + try: + assert refresh_started.wait(timeout=2) + concurrent: Final = discovery.resolve_deployment(_deployment()) + assert _api_base(concurrent) == ("http://10.0.0.2:8000/v1" if cache_existing else _SERVICE_URL) + assert next(lookups) == (2 if cache_existing else 1) + finally: + finish_refresh.set() + thread.join(timeout=2) + assert not thread.is_alive() + assert _api_base(refresh_result.result(timeout=2)) == "http://10.0.0.3:8000/v1" + + +@pytest.mark.asyncio +async def test_router_sends_pod_hosts_without_forwarding_discovery_flag( + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock: Final = _clock((0.0, 0.0, 0.0, 0.0, 0.0, 5.0, 5.0)) + loop: Final = asyncio.get_running_loop() + async_lookups: Final = count() + sync_lookups: Final = count() + + async def async_getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(async_lookups) + return _records("10.0.0.1", "10.0.0.2", port=int(port or 0)) + + def sync_getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(sync_lookups) + return _records("10.0.0.1", "10.0.0.2", port=int(port or 0)) + + monkeypatch.setattr(loop, "getaddrinfo", async_getaddrinfo) + monkeypatch.setattr(socket, "getaddrinfo", sync_getaddrinfo) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + with respx.mock(assert_all_called=True) as respx_mock: + route: Final = respx_mock.post( + url__regex=re.compile( + r"http://(?:10\.0\.0\.1|10\.0\.0\.2|vllm-headless\.ns\.svc\.cluster\.local):8000/v1/chat/completions" + ) + ).mock(return_value=httpx.Response(200, json=_CHAT_RESPONSE)) + discovery_enabled_router: Final = Router(model_list=[_deployment()]) + control_router: Final = Router(model_list=[_deployment(kubernetes_pod_discovery=None)]) + discovery_enabled_router.kubernetes_pod_discovery = KubernetesPodDiscovery( + clock=clock, + proxy_environment=_empty_proxy_environment, + ) + cached_async_client: Final = AsyncOpenAI(api_key="fake", base_url=_SERVICE_URL) + cached_sync_client: Final = OpenAI(api_key="fake", base_url=_SERVICE_URL) + discovery_enabled_router.cache.set_cache( + key="registered-id_async_client", + value=cached_async_client, + local_only=True, + ) + discovery_enabled_router.cache.set_cache( + key="registered-id_client", + value=cached_sync_client, + local_only=True, + ) + for _ in range(4): + await discovery_enabled_router.acompletion( + model="gpu-model", messages=[{"role": "user", "content": "hello"}] + ) + discovery_enabled_router.completion(model="gpu-model", messages=[{"role": "user", "content": "hello"}]) + await control_router.acompletion(model="gpu-model", messages=[{"role": "user", "content": "hello"}]) + + hosts: Final = tuple(call.request.url.host for call in route.calls) + bodies: Final = tuple(_request_body(call.request.content) for call in route.calls) + await cached_async_client.close() + cached_sync_client.close() + + assert hosts == ( + "10.0.0.1", + "10.0.0.2", + "10.0.0.1", + "10.0.0.2", + "10.0.0.1", + "vllm-headless.ns.svc.cluster.local", + ) + assert all("kubernetes_pod_discovery" not in body for body in bodies) + assert next(async_lookups) == 1 + assert next(sync_lookups) == 1 + + +@pytest.mark.asyncio +async def test_custom_routing_strategy_resolves_pod_hosts_for_sync_and_async_requests( + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock: Final = _clock((0.0,) * 40) + loop: Final = asyncio.get_running_loop() + async_lookups: Final = count() + sync_lookups: Final = count() + + async def async_getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(async_lookups) + return _records("10.0.0.1", "10.0.0.2", port=int(port or 0)) + + def sync_getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(sync_lookups) + return _records("10.0.0.1", "10.0.0.2", port=int(port or 0)) + + monkeypatch.setattr(loop, "getaddrinfo", async_getaddrinfo) + monkeypatch.setattr(socket, "getaddrinfo", sync_getaddrinfo) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + with respx.mock(assert_all_called=True) as respx_mock: + route: Final = respx_mock.post( + url__regex=re.compile( + r"http://(?:10\.0\.0\.1|10\.0\.0\.2|vllm-headless\.ns\.svc\.cluster\.local):8000/v1/chat/completions" + ) + ).mock(return_value=httpx.Response(200, json=_CHAT_RESPONSE)) + async_router: Final = Router(model_list=[_deployment()]) + sync_router: Final = Router(model_list=[_deployment()]) + async_router.kubernetes_pod_discovery = KubernetesPodDiscovery( + clock=clock, + proxy_environment=_empty_proxy_environment, + ) + sync_router.kubernetes_pod_discovery = KubernetesPodDiscovery( + clock=clock, + proxy_environment=_empty_proxy_environment, + ) + async_router.set_custom_routing_strategy(_ModelListRoutingStrategy(async_router)) + sync_router.set_custom_routing_strategy(_ModelListRoutingStrategy(sync_router)) + for _ in range(4): + await async_router.acompletion(model="gpu-model", messages=[{"role": "user", "content": "hello"}]) + for _ in range(4): + sync_router.completion(model="gpu-model", messages=[{"role": "user", "content": "hello"}]) + hosts: Final = tuple(call.request.url.host for call in route.calls) + + assert hosts == ("10.0.0.1", "10.0.0.2") * 4 + assert next(async_lookups) == 1 + assert next(sync_lookups) == 1 + + +@pytest.mark.asyncio +async def test_unresolved_router_selectors_keep_service_hostname() -> None: + router: Final = Router(model_list=[_deployment()]) + + sync_selected: Final = router._get_available_deployment_unresolved(model="gpu-model", request_kwargs={}) + async_selected: Final = await router._async_get_available_deployment_unresolved( + model="gpu-model", + request_kwargs={}, + ) + + assert _api_base(sync_selected) == _SERVICE_URL + assert _api_base(async_selected) == _SERVICE_URL + + +@pytest.mark.asyncio +async def test_router_async_retry_uses_next_discovered_pod(monkeypatch: pytest.MonkeyPatch) -> None: + clock: Final = _clock((0.0,) * 8) + loop: Final = asyncio.get_running_loop() + attempts: Final = count() + + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + return _records("10.0.0.1", "10.0.0.2", port=int(port or 0)) + + def response_for(request: httpx.Request) -> httpx.Response: + attempt: Final = next(attempts) + status_code: Final = 500 if attempt == 0 else 200 + body: Final = {"error": {"message": "retryable", "type": "server_error"}} if attempt == 0 else _CHAT_RESPONSE + return httpx.Response(status_code, json=body, request=request) + + monkeypatch.setattr(loop, "getaddrinfo", getaddrinfo) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + with respx.mock(assert_all_called=True) as respx_mock: + route: Final = respx_mock.post( + url__regex=re.compile(r"http://(?:10\.0\.0\.1|10\.0\.0\.2):8000/v1/chat/completions") + ).mock(side_effect=response_for) + router: Final = Router(model_list=[_deployment()], num_retries=1) + router.kubernetes_pod_discovery = KubernetesPodDiscovery( + clock=clock, + proxy_environment=_empty_proxy_environment, + ) + + response: Final = await router.acompletion( + model="gpu-model", + messages=[{"role": "user", "content": "hello"}], + ) + hosts: Final = tuple(call.request.url.host for call in route.calls) + + assert response.choices[0].message.content == "ok" + assert hosts == ("10.0.0.1", "10.0.0.2") + + +def test_router_sync_retry_uses_next_discovered_pod(monkeypatch: pytest.MonkeyPatch) -> None: + clock: Final = _clock((0.0,) * 8) + attempts: Final = count() + + def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + return _records("10.0.0.1", "10.0.0.2", port=int(port or 0)) + + def response_for(request: httpx.Request) -> httpx.Response: + attempt: Final = next(attempts) + status_code: Final = 500 if attempt == 0 else 200 + body: Final = {"error": {"message": "retryable", "type": "server_error"}} if attempt == 0 else _CHAT_RESPONSE + return httpx.Response(status_code, json=body, request=request) + + monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + with respx.mock(assert_all_called=True) as respx_mock: + route: Final = respx_mock.post( + url__regex=re.compile(r"http://(?:10\.0\.0\.1|10\.0\.0\.2):8000/v1/chat/completions") + ).mock(side_effect=response_for) + router: Final = Router(model_list=[_deployment()], num_retries=1) + router.kubernetes_pod_discovery = KubernetesPodDiscovery( + clock=clock, + proxy_environment=_empty_proxy_environment, + ) + + response: Final = router.completion( + model="gpu-model", + messages=[{"role": "user", "content": "hello"}], + ) + hosts: Final = tuple(call.request.url.host for call in route.calls) + + assert response.choices[0].message.content == "ok" + assert hosts == ("10.0.0.1", "10.0.0.2") diff --git a/tests/unit/types/test_litellm_params.py b/tests/unit/types/test_litellm_params.py index aef43a15a72..ce763378656 100644 --- a/tests/unit/types/test_litellm_params.py +++ b/tests/unit/types/test_litellm_params.py @@ -131,6 +131,7 @@ OPTION_NAMES: Final = ( "use_litellm_proxy", "use_chat_completions_api", "use_in_pass_through", + "kubernetes_pod_discovery", "allowed_openai_params", "fallbacks", "context_window_fallback_dict", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx index 99656df3b0e..43b9a5541db 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx @@ -132,6 +132,7 @@ const alwaysMounted = { const advancedOpenExtras = { guardrails: undefined, + kubernetes_pod_discovery: undefined, tags: undefined, use_in_pass_through: undefined, vector_store_ids: undefined, @@ -304,6 +305,24 @@ describe("AddModelPanel submit payload contract", () => { }); }); + it("submits Kubernetes pod discovery in litellm_params", async () => { + const { user, openAdvanced, fillRequired, submit } = await setup(); + await fillRequired(); + await openAdvanced(); + await user.click(screen.getByRole("switch", { name: "Kubernetes pod discovery" })); + await submit(); + + expect(lastCreatedModel()).toStrictEqual({ + model_name: "gpt-4o", + litellm_params: { + ...alwaysMounted, + ...advancedOpenExtras, + kubernetes_pod_discovery: true, + }, + model_info: { ...baseModelInfo }, + }); + }); + it("drops a collapsed section's keys and the value typed into it", async () => { const { user, openAdvanced, closeAdvanced, fillRequired, submit } = await setup(); await fillRequired(); diff --git a/ui/litellm-dashboard/src/components/add_model/advanced_settings.test.tsx b/ui/litellm-dashboard/src/components/add_model/advanced_settings.test.tsx index 15a13219d18..e3d98492cb9 100644 --- a/ui/litellm-dashboard/src/components/add_model/advanced_settings.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/advanced_settings.test.tsx @@ -1,6 +1,8 @@ import { act, fireEvent, render, waitFor, screen } from "@testing-library/react"; +import { useFormContext } from "react-hook-form"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { MountedFormHost } from "../../../tests/mounted-form-host"; +import type { MountedFormValues } from "../common_components/MountedFormField"; import AdvancedSettings from "./advanced_settings"; const mockUsePtuCostAttributionEnabled = vi.fn(); @@ -11,7 +13,16 @@ vi.mock("@/app/(dashboard)/hooks/uiSettings/usePtuCostAttributionEnabled", () => const PTU_LABELS = ["PTU Count", "Calculated Cost per PTU / Hour (USD)", "PTU Effective From (UTC)"]; -const renderAdvancedSettings = () => +const KubernetesPodDiscoveryValue = () => { + const form = useFormContext(); + return ( + + {String(form.watch("kubernetes_pod_discovery") === true)} + + ); +}; + +const renderAdvancedSettings = (includeDiscoveryValue = false) => render( tagsList={{}} accessToken="test-token" /> + {includeDiscoveryValue && } , ); @@ -34,6 +46,23 @@ describe("AdvancedSettings", () => { renderAdvancedSettings(); }); + it("updates the Kubernetes pod discovery form value when enabled", async () => { + renderAdvancedSettings(true); + act(() => { + fireEvent.click(screen.getByText("Advanced Settings")); + }); + + const toggle = await screen.findByRole("switch", { name: "Kubernetes pod discovery" }); + const formValue = screen.getByRole("status", { name: "Kubernetes pod discovery form value" }); + expect(toggle).not.toBeChecked(); + expect(formValue).toHaveTextContent("false"); + + fireEvent.click(toggle); + + expect(toggle).toBeChecked(); + expect(formValue).toHaveTextContent("true"); + }); + it("should render tags list", async () => { renderAdvancedSettings(); fireEvent.click(screen.getByText("Advanced Settings")); diff --git a/ui/litellm-dashboard/src/components/add_model/advanced_settings.tsx b/ui/litellm-dashboard/src/components/add_model/advanced_settings.tsx index 6025db8e2e2..b17d38c5586 100644 --- a/ui/litellm-dashboard/src/components/add_model/advanced_settings.tsx +++ b/ui/litellm-dashboard/src/components/add_model/advanced_settings.tsx @@ -457,6 +457,20 @@ const AdvancedSettings: React.FC = ({ )} + + {(control) => ( + + )} + + Date: Mon, 19 Oct 2026 09:58:19 -0700 Subject: [PATCH 04/20] feat(router): keep a session on one kubernetes pod (#44393) Requests with an explicit session id stick to one discovered pod via rendezvous hashing Requests without a session id keep round robin Pod routing records are included in logs, spend logs, and traces Resolves LIT-9168 Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 1 + litellm/integrations/opentelemetry.py | 12 +- litellm/litellm_core_utils/core_helpers.py | 51 +- litellm/litellm_core_utils/litellm_logging.py | 13 +- litellm/proxy/_types.py | 2 + litellm/proxy/litellm_pre_call_utils.py | 10 +- .../spend_tracking/spend_tracking_utils.py | 22 +- litellm/router.py | 10 + .../router_utils/kubernetes_pod_discovery.py | 220 ++++- litellm/types/utils.py | 8 + .../gcs_pubsub/spend_logs_payload.json | 2 +- tests/unit/integrations/test_opentelemetry.py | 29 +- .../test_litellm_logging.py | 98 +- .../test_spend_management_endpoints.py | 13 +- .../test_spend_tracking_utils.py | 85 +- .../unit/proxy/test_litellm_pre_call_utils.py | 7 +- .../test_kubernetes_pod_discovery.py | 876 +++++++++++++++++- tests/unit/test_router/test_router.py | 88 ++ .../src/components/logs/constants.ts | 6 + .../LogDetailContent.integration.test.tsx | 31 + .../logs/detail/LogDetailContent.tsx | 12 +- .../src/components/logs/types.ts | 11 +- 22 files changed, 1541 insertions(+), 66 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index db49a40a37b..c7c1576bc20 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -47,6 +47,7 @@ KUBERNETES_POD_DISCOVERY_IDLE_EVICTION_SECONDS: Final = get_env_float( "KUBERNETES_POD_DISCOVERY_IDLE_EVICTION_SECONDS", 300.0 ) KUBERNETES_POD_DISCOVERY_DNS_TIMEOUT_SECONDS: Final = 2.0 +KUBERNETES_POD_ROUTING_KEY: Final = "kubernetes_pod_routing" AZURE_DEFAULT_RESPONSES_API_VERSION: Final = str(os.getenv("AZURE_DEFAULT_RESPONSES_API_VERSION", "preview")) AZURE_OPENAI_AUDIO_PROVIDERS: Final = frozenset({"azure", "azure_ai"}) ROUTER_MAX_FALLBACKS: Final = get_env_int("ROUTER_MAX_FALLBACKS", 5) diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index fb0723289ee..90392d9a45d 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -13,6 +13,7 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger +from litellm.constants import KUBERNETES_POD_ROUTING_KEY from litellm.integrations._types.open_inference import ( OpenInferenceSpanKindValues, ) @@ -50,6 +51,7 @@ from litellm.types.utils import ( Function, LLMResponseTypes, StandardCallbackDynamicParams, + StandardLoggingMetadata, StandardLoggingPayload, ) @@ -2420,9 +2422,15 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): ############################################# ############ LLM CALL METADATA ############## ############################################# - metadata: Final = standard_logging_payload["metadata"] + metadata: Final = cast( # cast-ok: payload metadata follows the standard logging contract + StandardLoggingMetadata, standard_logging_payload["metadata"] + ) for key, value in metadata.items(): - self.safe_set_attribute(span=span, key=f"metadata.{key}", value=value) + self.safe_set_attribute( + span=span, + key=f"metadata.{key}", + value=safe_dumps(value) if key == KUBERNETES_POD_ROUTING_KEY and value is not None else value, + ) decision: Final = metadata.get("routing_decision") span.set_attributes(routing_decision_attributes(decision)) diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index e706bcadef8..65d0089304b 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -11,13 +11,14 @@ import httpx from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_logger +from litellm.constants import KUBERNETES_POD_ROUTING_KEY from litellm.litellm_core_utils.llm_response_utils.get_headers import get_provider_request_id from litellm.types.llms.openai import AllMessageValues, OpenAIChatCompletionFinishReason if TYPE_CHECKING: from opentelemetry.trace import Span as _Span - from litellm.types.utils import ModelResponseStream + from litellm.types.utils import ModelResponseStream, StandardLoggingKubernetesPodRouting Span = _Span | Any else: @@ -385,6 +386,54 @@ def proxy_stamped_used_client_oauth_token(metadata: object, litellm_params: Mapp return metadata.get("used_client_oauth_token") if isinstance(metadata, Mapping) else None +def proxy_stamped_kubernetes_pod_routing( + metadata: object, + litellm_params: Mapping[str, object] | None, +) -> "StandardLoggingKubernetesPodRouting | None": + litellm_metadata: Final = litellm_params.get("litellm_metadata") if litellm_params is not None else None + litellm_routing: Final = _standard_logging_kubernetes_pod_routing(litellm_metadata) + if litellm_routing is not None: + return litellm_routing + return _standard_logging_kubernetes_pod_routing(metadata) + + +def _standard_logging_kubernetes_pod_routing( + metadata: object, +) -> "StandardLoggingKubernetesPodRouting | None": + if not isinstance(metadata, Mapping): + return None + metadata_mapping: Final = cast( # cast-ok: routing metadata is a string-keyed mapping + Mapping[str, object], metadata + ) + routing: Final = metadata_mapping.get(KUBERNETES_POD_ROUTING_KEY) + if not isinstance(routing, Mapping): + return None + routing_mapping: Final = cast( # cast-ok: pod routing records use string keys + Mapping[str, object], routing + ) + service_host: Final = routing_mapping.get("service_host") + pod_ip: Final = routing_mapping.get("pod_ip") + pod_count: Final = routing_mapping.get("pod_count") + selection: Final = routing_mapping.get("selection") + if ( + not isinstance(service_host, str) + or not isinstance(pod_ip, str) + or not isinstance(pod_count, int) + or isinstance(pod_count, bool) + or pod_count < 1 + ): + return None + if selection in ("round_robin", "session_affinity", "session_affinity_retry"): + routing_record: Final[StandardLoggingKubernetesPodRouting] = { + "service_host": service_host, + "pod_ip": pod_ip, + "pod_count": pod_count, + "selection": selection, + } + return routing_record + return None + + def get_litellm_metadata_from_kwargs(kwargs: dict): """ Helper to get litellm metadata from all litellm request kwargs diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 025741b18b2..0ac216c2029 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -43,6 +43,7 @@ from litellm.constants import ( DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT, EMPTY_MAPPING, INTERNAL_CALLBACK_NAMES, + KUBERNETES_POD_ROUTING_KEY, PROVIDER_REQUEST_ID_HEADERS, REDACTED_BY_LITELLM, ) @@ -75,6 +76,7 @@ from litellm.litellm_core_utils.classifier_logging import ( from litellm.litellm_core_utils.core_helpers import ( get_provider_response_headers_from_hidden_params, is_expected_client_error, + proxy_stamped_kubernetes_pod_routing, proxy_stamped_used_client_oauth_token, reconstruct_model_name, set_response_cost_in_hidden_params, @@ -298,7 +300,9 @@ else: _GENERIC_API_LOGGER_CLS: Final = GenericAPILogger _in_memory_loggers: Final[list[CustomLogger]] = [] -_STANDARD_LOGGING_METADATA_RESOLVED_KEYS: Final[frozenset[str]] = frozenset(("used_client_oauth_token", "usage_object")) +_STANDARD_LOGGING_METADATA_RESOLVED_KEYS: Final[frozenset[str]] = frozenset( + ("used_client_oauth_token", "usage_object", KUBERNETES_POD_ROUTING_KEY) +) _STANDARD_LOGGING_METADATA_KEYS: Final[frozenset[str]] = ( frozenset(StandardLoggingMetadata.__annotations__.keys()) - _STANDARD_LOGGING_METADATA_RESOLVED_KEYS ) @@ -6275,7 +6279,7 @@ class StandardLoggingPayloadSetup: @staticmethod def get_standard_logging_metadata( metadata: Mapping[str, object] | None, - litellm_params: dict | None = None, + litellm_params: dict[str, object] | None = None, prompt_integration: str | None = None, applied_guardrails: list[str] | None = None, mcp_tool_call_metadata: StandardLoggingMCPToolCall | None = None, @@ -6315,6 +6319,8 @@ class StandardLoggingPayloadSetup: prompt_integration=prompt_integration, ) + kubernetes_pod_routing: Final = proxy_stamped_kubernetes_pod_routing(metadata, litellm_params) + # Initialize with default values clean_metadata = StandardLoggingMetadata( user_api_key_hash=None, @@ -6351,6 +6357,7 @@ class StandardLoggingPayloadSetup: user_api_key_auth_metadata=None, team_alias=None, team_id=None, + kubernetes_pod_routing=kubernetes_pod_routing, used_client_oauth_token=resolve_used_client_oauth_token( proxy_stamped_used_client_oauth_token(metadata, litellm_params), custom_llm_provider, @@ -7418,6 +7425,7 @@ def get_standard_logging_metadata( user_api_key_auth_metadata=None, team_alias=None, team_id=None, + kubernetes_pod_routing=None, used_client_oauth_token=None, ) if isinstance(metadata, dict): @@ -7491,6 +7499,7 @@ def create_dummy_standard_logging_payload() -> StandardLoggingPayload: requester_ip_address="127.0.0.1", requester_metadata=None, user_api_key_end_user_id="test_end_user", + kubernetes_pod_routing=None, ) hidden_params: Final = StandardLoggingHiddenParams( diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 8ff5d1a1002..c2cdd47304a 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -105,6 +105,7 @@ from litellm.types.utils import ( ProviderField, StandardCallbackDynamicParams, StandardLoggingGuardrailInformation, + StandardLoggingKubernetesPodRouting, StandardLoggingMCPToolCall, StandardLoggingModelInformation, StandardLoggingPayloadErrorInformation, @@ -3682,6 +3683,7 @@ class SpendLogsMetadata(TypedDict): mcp_tool_call_metadata: StandardLoggingMCPToolCall | None vector_store_request_metadata: list[StandardLoggingVectorStoreRequest] | None routing_decision: StandardLoggingRoutingDecision | None + kubernetes_pod_routing: ReadOnly[StandardLoggingKubernetesPodRouting | None] internal_call_origin: InternalCallOrigin | None litellm_roi_estimator: ReadOnly[NotRequired[bool | None]] guardrail_information: list[StandardLoggingGuardrailInformation] | None diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index ce2e3cd30c4..06033f092dd 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1672,11 +1672,9 @@ class LiteLLMProxyRequestSetup: metadata_from_headers["agent_id"] = agent_id_from_header verbose_proxy_logger.debug("Extracted agent_id from header: %s", agent_id_from_header) - spend_log_session_id: Final = ( + caller_session_beside_trace: Final = ( _caller_session_beside_trace_header(normalized_headers, request_body, _metadata_variable_name) - if litellm.request_correlation_in_logs - and chain_id is not None - and chain_id == normalized_headers.get("x-litellm-trace-id") + if chain_id is not None and chain_id == normalized_headers.get("x-litellm-trace-id") else None ) if chain_id: @@ -1738,8 +1736,8 @@ class LiteLLMProxyRequestSetup: if isinstance(data[_metadata_variable_name], dict): data[_metadata_variable_name].update( - {**metadata_from_headers, CALLER_SESSION_ID_METADATA_KEY: spend_log_session_id} - if spend_log_session_id + {**metadata_from_headers, CALLER_SESSION_ID_METADATA_KEY: caller_session_beside_trace} + if caller_session_beside_trace else metadata_from_headers ) return data diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index d1303b012c8..78818cde266 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -19,6 +19,7 @@ from litellm.constants import ( CLI_SESSION_KEY_PREFIX, EMPTY_MAPPING, INTERNAL_CALL_ORIGIN_METADATA_KEY, + KUBERNETES_POD_ROUTING_KEY, LITELLM_PROXY_MASTER_KEY_ALIAS, LITELLM_TRUNCATED_PAYLOAD_FIELD, LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, @@ -43,6 +44,7 @@ from litellm.litellm_core_utils.classifier_logging import ( ) from litellm.litellm_core_utils.core_helpers import ( get_litellm_metadata_from_kwargs, + proxy_stamped_kubernetes_pod_routing, proxy_stamped_used_client_oauth_token, reconstruct_model_name, ) @@ -76,6 +78,7 @@ from litellm.types.utils import ( CostBreakdown, LlmProviders, StandardLoggingGuardrailInformation, + StandardLoggingKubernetesPodRouting, StandardLoggingMCPToolCall, StandardLoggingModelInformation, StandardLoggingPayload, @@ -178,6 +181,7 @@ _STAMPED_METADATA_KEYS: Final = frozenset( "autorouter_savings_estimate", "autorouter_baseline_observation", "used_client_oauth_token", + KUBERNETES_POD_ROUTING_KEY, "litellm_roi_estimator", "user_api_key_user_email", ) @@ -210,6 +214,7 @@ def _get_spend_logs_metadata( router_metadata: SpendLogsRouterMetadata | None = None, azure_spillover: AzureSpillover | None = None, used_client_oauth_token: bool | None = None, + kubernetes_pod_routing: StandardLoggingKubernetesPodRouting | None = None, ) -> SpendLogsMetadata: if metadata is None: return SpendLogsMetadata( @@ -257,6 +262,7 @@ def _get_spend_logs_metadata( router_metadata=router_metadata, azure_spillover=azure_spillover, used_client_oauth_token=used_client_oauth_token, + kubernetes_pod_routing=None, ) verbose_proxy_logger.debug("getting payload for SpendLogs, available keys in metadata: %s", [*metadata]) @@ -271,6 +277,7 @@ def _get_spend_logs_metadata( router_metadata=router_metadata, azure_spillover=azure_spillover, used_client_oauth_token=used_client_oauth_token, + kubernetes_pod_routing=kubernetes_pod_routing, litellm_roi_estimator=metadata.get("litellm_roi_estimator") is True, user_api_key_user_email=_stored_user_email(metadata), ) @@ -557,6 +564,16 @@ def get_logging_payload( # standardize this function to be used across, s3, dynamoDB, langfuse logging litellm_params: Final = kwargs.get("litellm_params", {}) metadata: Final = get_litellm_metadata_from_kwargs(kwargs) + routing_litellm_params: Final[Mapping[str, object] | None] = ( + litellm_params if isinstance(litellm_params, Mapping) else None + ) + routing_metadata: Final[object] = ( + routing_litellm_params.get("metadata") if routing_litellm_params is not None else None + ) + kubernetes_pod_routing: Final = proxy_stamped_kubernetes_pod_routing( + routing_metadata, + routing_litellm_params, + ) completion_start_time: Final = kwargs.get("completion_start_time", end_time) call_type: Final = kwargs.get("call_type") cache_hit: Final = kwargs.get("cache_hit", False) @@ -591,7 +608,9 @@ def get_logging_payload( id = get_spend_logs_id(call_type or "acompletion", response_obj_dict, kwargs, metadata) standard_logging_payload: Final = cast(StandardLoggingPayload | None, kwargs.get("standard_logging_object", None)) - end_user_id = get_end_user_id_for_cost_tracking(litellm_params) + end_user_id = get_end_user_id_for_cost_tracking( + dict(litellm_params) if isinstance(litellm_params, Mapping) else litellm_params + ) api_key = metadata.get("user_api_key", "") @@ -769,6 +788,7 @@ def get_logging_payload( used_client_oauth_token=resolve_used_client_oauth_token( proxy_stamped_used_client_oauth_token(litellm_params.get("metadata"), litellm_params), custom_llm_provider ), + kubernetes_pod_routing=kubernetes_pod_routing, azure_spillover=azure_spillover( response_headers=kwargs.get("response_headers") if isinstance(kwargs.get("response_headers"), Mapping) diff --git a/litellm/router.py b/litellm/router.py index 23b516ea573..113565fecaa 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -68,6 +68,7 @@ from litellm.constants import ( DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIER, DEFAULT_MAX_LRU_CACHE_SIZE, INTERNAL_CALL_ORIGIN_METADATA_KEY, + KUBERNETES_POD_ROUTING_KEY, MID_STREAM_FAILURE_LOG_WAIT_SECONDS, OUTPUT_TOKEN_CEILING_PARAMS, PROMPT_CACHE_PIN_CACHE_MAX_SIZE, @@ -4606,6 +4607,7 @@ class Router: deployment_that_just_failed: Final = _deployment_that_just_failed( _FALLBACK_HOP_ADAPTER.validate_python(kwargs).get(metadata_variable_name) ) + routing: Final = deployment.get(KUBERNETES_POD_ROUTING_KEY) kwargs.setdefault(metadata_variable_name, {}).update( { @@ -4613,8 +4615,16 @@ class Router: "model_info": model_info, "api_base": deployment_api_base, "deployment_model_name": deployment_model_name, + KUBERNETES_POD_ROUTING_KEY: routing, } ) + other_bucket: Final = "metadata" if metadata_variable_name == "litellm_metadata" else "litellm_metadata" + other_bucket_metadata: Final = kwargs.get(other_bucket) + if isinstance(other_bucket_metadata, Mapping) and KUBERNETES_POD_ROUTING_KEY in other_bucket_metadata: + kwargs[other_bucket] = { # rebind-ok: this method updates the router's request kwargs in place + **other_bucket_metadata, + KUBERNETES_POD_ROUTING_KEY: routing, + } # A retry/fallback reuses this same kwargs dict for the next deployment. # Refund and clear any reservation the previous deployment attempt left diff --git a/litellm/router_utils/kubernetes_pod_discovery.py b/litellm/router_utils/kubernetes_pod_discovery.py index 7b98a52a4a0..05ba7be0344 100644 --- a/litellm/router_utils/kubernetes_pod_discovery.py +++ b/litellm/router_utils/kubernetes_pod_discovery.py @@ -1,5 +1,7 @@ import asyncio import concurrent.futures +import hashlib +import inspect import ipaddress import socket import threading @@ -13,6 +15,7 @@ from types import MappingProxyType from typing import ( Concatenate, Final, + Literal, ParamSpec, Protocol, TypeAlias, @@ -24,9 +27,12 @@ import httpx from litellm._logging import verbose_router_logger from litellm.constants import ( + CALLER_SESSION_ID_METADATA_KEY, KUBERNETES_POD_DISCOVERY_DNS_TIMEOUT_SECONDS, KUBERNETES_POD_DISCOVERY_IDLE_EVICTION_SECONDS, KUBERNETES_POD_DISCOVERY_REFRESH_INTERVAL_SECONDS, + KUBERNETES_POD_ROUTING_KEY, + SESSION_ID_GENERATED_METADATA_KEY, ) _SocketAddress: TypeAlias = tuple[str, int] | tuple[str, int, int, int] | tuple[int, bytes] @@ -34,9 +40,22 @@ _AddressInfo: TypeAlias = tuple[socket.AddressFamily, socket.SocketKind, int, st _SyncLookup: TypeAlias = Future[list[_AddressInfo]] _AsyncLookup: TypeAlias = asyncio.Future[list[_AddressInfo]] _CacheKey: TypeAlias = tuple[str, int | None] +_PodSelection: TypeAlias = tuple[str, int, Literal["round_robin", "session_affinity", "session_affinity_retry"]] _DeploymentT: Final = TypeVar("_DeploymentT") +def _retry_count_from_metadata(metadata: object) -> int | None: + if not isinstance(metadata, Mapping): + return None + metadata_mapping: Final = cast( # cast-ok: request metadata is a string-keyed mapping + Mapping[str, object], metadata + ) + retry_count: Final = metadata_mapping.get("request_retry_count") + if type(retry_count) is int and retry_count >= 0: + return retry_count + return None + + @dataclass(frozen=True, slots=True) class _PodSet: ips: tuple[str, ...] @@ -99,16 +118,24 @@ class KubernetesPodDiscovery: self._refreshing = self._refreshing | {key} return None, True - def resolve_deployment(self, deployment: _DeploymentT) -> _DeploymentT: + def resolve_deployment( + self, + deployment: _DeploymentT, + request_kwargs: Mapping[str, object] | None = None, + ) -> _DeploymentT: eligible: Final = self._eligible(deployment) if eligible is None: return deployment url, key, deployment_mapping = eligible host, port = key now: Final = self.clock() + session_id: Final = self._session_id(request_kwargs) + retry_count: Final = self._request_retry_count(request_kwargs) cached, should_refresh = self._begin_refresh(key, now) if cached is not None: - return self._deployment_with_cached_ip(deployment, deployment_mapping, key, url, now) + return self._deployment_with_cached_ip( + deployment, deployment_mapping, key, url, now, session_id, retry_count + ) if not should_refresh: return deployment @@ -118,21 +145,36 @@ class KubernetesPodDiscovery: lookup.add_done_callback(lambda done: self._finish_refresh(key, done)) try: return self._apply_lookup( - deployment, deployment_mapping, key, url, self._await_lookup(key, host, lookup), now + deployment, + deployment_mapping, + key, + url, + self._await_lookup(key, host, lookup), + now, + session_id, + retry_count, ) finally: self._end_refresh(key) - async def async_resolve_deployment(self, deployment: _DeploymentT) -> _DeploymentT: + async def async_resolve_deployment( + self, + deployment: _DeploymentT, + request_kwargs: Mapping[str, object] | None = None, + ) -> _DeploymentT: eligible: Final = self._eligible(deployment) if eligible is None: return deployment url, key, deployment_mapping = eligible host, port = key now: Final = self.clock() + session_id: Final = self._session_id(request_kwargs) + retry_count: Final = self._request_retry_count(request_kwargs) cached, should_refresh = self._begin_refresh(key, now) if cached is not None: - return self._deployment_with_cached_ip(deployment, deployment_mapping, key, url, now) + return self._deployment_with_cached_ip( + deployment, deployment_mapping, key, url, now, session_id, retry_count + ) if not should_refresh: return deployment @@ -142,10 +184,56 @@ class KubernetesPodDiscovery: lookup.add_done_callback(lambda done: self._finish_refresh(key, done)) try: result: Final = await self._await_async_lookup(key, host, lookup) - return self._apply_lookup(deployment, deployment_mapping, key, url, result, now) + return self._apply_lookup(deployment, deployment_mapping, key, url, result, now, session_id, retry_count) finally: self._end_refresh(key) + @staticmethod + def _session_id(request_kwargs: Mapping[str, object] | None) -> str | None: + if request_kwargs is None: + return None + metadata_dicts: Final = tuple( + cast(Mapping[str, object], metadata) # cast-ok: runtime dict check precedes metadata lookup + for metadata in (request_kwargs.get("litellm_metadata"), request_kwargs.get("metadata")) + if isinstance(metadata, dict) + ) + if any(metadata.get(SESSION_ID_GENERATED_METADATA_KEY) for metadata in metadata_dicts): + return None + caller_session_id: Final = next( + ( + caller_session + for metadata in metadata_dicts + if (caller_session := metadata.get(CALLER_SESSION_ID_METADATA_KEY)) is not None and caller_session != "" + ), + None, + ) + if caller_session_id is not None: + return str(caller_session_id) + top_level_session_id: Final = request_kwargs.get("litellm_session_id") + if top_level_session_id is not None and top_level_session_id != "": + return str(top_level_session_id) + metadata_session_id: Final = next( + ( + session_id + for metadata in metadata_dicts + if (session_id := metadata.get("session_id")) is not None and session_id != "" + ), + None, + ) + if metadata_session_id is not None: + return str(metadata_session_id) + return None + + @staticmethod + def _request_retry_count(request_kwargs: Mapping[str, object] | None) -> int: + if request_kwargs is None: + return 0 + retry_counts: Final = tuple( + _retry_count_from_metadata(request_kwargs.get(metadata_key)) + for metadata_key in ("litellm_metadata", "metadata") + ) + return next((retry_count for retry_count in retry_counts if retry_count is not None), 0) + def _eligible(self, deployment: object) -> tuple[httpx.URL, _CacheKey, Mapping[str, object]] | None: if not isinstance(deployment, Mapping): return None @@ -247,6 +335,8 @@ class KubernetesPodDiscovery: url: httpx.URL, result: Sequence[_AddressInfo] | OSError, now: float, + session_id: str | None, + retry_count: int, ) -> _DeploymentT: if isinstance(result, OSError): if isinstance(result, socket.gaierror) and self._is_authoritative_no_pods(result): @@ -254,12 +344,14 @@ class KubernetesPodDiscovery: return deployment self._stamp_failed_refresh(key, self.clock()) verbose_router_logger.debug("Kubernetes pod discovery DNS refresh failed for %s: %s", key[0], result) - return self._deployment_with_cached_ip(deployment, deployment_mapping, key, url, now) + return self._deployment_with_cached_ip( + deployment, deployment_mapping, key, url, now, session_id, retry_count + ) ips: Final = self._pod_ips(result) self._store(key, ips, self.clock()) if not ips: return deployment - return self._deployment_with_cached_ip(deployment, deployment_mapping, key, url, now) + return self._deployment_with_cached_ip(deployment, deployment_mapping, key, url, now, session_id, retry_count) @staticmethod def _is_ip_literal(host: str) -> bool: @@ -305,20 +397,55 @@ class KubernetesPodDiscovery: ) self._cache = MappingProxyType({**self._cache, key: stamped}) - def _next_ip(self, key: _CacheKey, now: float) -> str | None: + def _next_ip( + self, + key: _CacheKey, + now: float, + session_id: str | None, + retry_count: int, + ) -> _PodSelection | None: + session_hash_prefix: Final = ( + hashlib.blake2b(f"{session_id}\x00".encode(), digest_size=8) if session_id is not None else None + ) + + def session_rank(address: str) -> int: + if session_hash_prefix is None: + return 0 + h: Final = session_hash_prefix.copy() + h.update(address.encode()) + return int.from_bytes(h.digest(), "big") + with self._lock: cached: Final = self._cache.get(key) if cached is None or not cached.ips: return None - ip: Final = cached.ips[cached.cursor] - next_cursor: Final = (cached.cursor + 1) % len(cached.ips) + pod_count: Final = len(cached.ips) + ranked_ips: Final = ( + tuple( + sorted( + cached.ips, + key=lambda address: (-session_rank(address), address), + ) + ) + if session_id is not None + else () + ) + ip: Final = cached.ips[cached.cursor] if session_id is None else ranked_ips[retry_count % pod_count] + next_cursor: Final = (cached.cursor + 1) % len(cached.ips) if session_id is None else cached.cursor + selection: Final = ( + "round_robin" + if session_id is None + else "session_affinity_retry" + if retry_count > 0 + else "session_affinity" + ) self._cache = MappingProxyType( { **self._cache, key: replace(cached, cursor=next_cursor, last_used_at=max(cached.last_used_at, now)), } ) - return ip + return ip, pod_count, selection def _deployment_with_cached_ip( self, @@ -327,10 +454,13 @@ class KubernetesPodDiscovery: key: _CacheKey, url: httpx.URL, now: float, + session_id: str | None, + retry_count: int, ) -> _DeploymentT: - ip: Final = self._next_ip(key, now) - if ip is None: + selection: Final = self._next_ip(key, now, session_id, retry_count) + if selection is None: return deployment + ip, pod_count, selection_kind = selection if self._proxy_bypasses_only_service_host(key, ip): self._warn_proxy_once(key[0]) return deployment @@ -345,6 +475,12 @@ class KubernetesPodDiscovery: { **deployment_mapping, "litellm_params": {**typed_params, "api_base": str(url.copy_with(host=ip))}, + KUBERNETES_POD_ROUTING_KEY: { + "service_host": key[0], + "pod_ip": ip, + "pod_count": pod_count, + "selection": selection_kind, + }, }, ) @@ -402,9 +538,39 @@ _R: Final = TypeVar("_R") _S: Final = TypeVar("_S", bound=_HasPodDiscovery) +def _request_kwargs_from_call( + signature: inspect.Signature, + *args: object, + **kwargs: object, # kwargs-ok: forwards the call to Signature.bind_partial +) -> Mapping[str, object] | None: + try: + bound_arguments: Final = cast( # cast-ok: Signature arguments use string parameter names + Mapping[str, object], + signature.bind_partial(*args, **kwargs).arguments, + ) + request_kwargs: Final = bound_arguments.get( + "request_kwargs", + kwargs.get("request_kwargs"), + ) + except TypeError: + return _mapping_request_kwargs(kwargs.get("request_kwargs")) + return _mapping_request_kwargs(request_kwargs) + + +def _mapping_request_kwargs(request_kwargs: object) -> Mapping[str, object] | None: + if not isinstance(request_kwargs, Mapping): + return None + return cast( # cast-ok: router selector request kwargs use string keys + Mapping[str, object], + request_kwargs, + ) + + def resolve_pods_after( fn: Callable[Concatenate[_S, _P], _R], ) -> Callable[Concatenate[_S, _P], _R]: + signature: Final = inspect.signature(fn) + @wraps(fn) def wrapped( self: _S, @@ -412,7 +578,10 @@ def resolve_pods_after( **kwargs: _P.kwargs, # kwargs-ok: ParamSpec forwards the wrapped call signature ) -> _R: deployment: Final = fn(self, *args, **kwargs) - return self.kubernetes_pod_discovery.resolve_deployment(deployment) + return self.kubernetes_pod_discovery.resolve_deployment( + deployment, + _request_kwargs_from_call(signature, self, *args, **kwargs), + ) return wrapped @@ -420,6 +589,8 @@ def resolve_pods_after( def async_resolve_pods_after( fn: Callable[Concatenate[_S, _P], Awaitable[_R]], ) -> Callable[Concatenate[_S, _P], Coroutine[object, object, _R]]: + signature: Final = inspect.signature(fn) + @wraps(fn) async def wrapped( self: _S, @@ -427,7 +598,10 @@ def async_resolve_pods_after( **kwargs: _P.kwargs, # kwargs-ok: ParamSpec forwards the wrapped call signature ) -> _R: deployment: Final = await fn(self, *args, **kwargs) - return await self.kubernetes_pod_discovery.async_resolve_deployment(deployment) + return await self.kubernetes_pod_discovery.async_resolve_deployment( + deployment, + _request_kwargs_from_call(signature, self, *args, **kwargs), + ) return wrapped @@ -436,13 +610,18 @@ def resolve_pods_after_bound( fn: Callable[_P, _R], discovery: KubernetesPodDiscovery, ) -> Callable[_P, _R]: + signature: Final = inspect.signature(fn) + @wraps(fn) def wrapped( *args: _P.args, **kwargs: _P.kwargs, # kwargs-ok: ParamSpec forwards the wrapped call signature ) -> _R: deployment: Final = fn(*args, **kwargs) - return discovery.resolve_deployment(deployment) + return discovery.resolve_deployment( + deployment, + _request_kwargs_from_call(signature, *args, **kwargs), + ) return wrapped @@ -451,12 +630,17 @@ def async_resolve_pods_after_bound( fn: Callable[_P, Awaitable[_R]], discovery: KubernetesPodDiscovery, ) -> Callable[_P, Coroutine[object, object, _R]]: + signature: Final = inspect.signature(fn) + @wraps(fn) async def wrapped( *args: _P.args, **kwargs: _P.kwargs, # kwargs-ok: ParamSpec forwards the wrapped call signature ) -> _R: deployment: Final = await fn(*args, **kwargs) - return await discovery.async_resolve_deployment(deployment) + return await discovery.async_resolve_deployment( + deployment, + _request_kwargs_from_call(signature, *args, **kwargs), + ) return wrapped diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 419c2624111..fd1d58adafb 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3242,6 +3242,13 @@ class StandardLoggingRoutingDecision(TypedDict, total=False): tier_litellm_params: Mapping[str, object] # writable-ok: Pydantic warns on ReadOnly TypedDict fields +class StandardLoggingKubernetesPodRouting(TypedDict): + service_host: ReadOnly[str] + pod_ip: ReadOnly[str] + pod_count: ReadOnly[int] + selection: ReadOnly[Literal["round_robin", "session_affinity", "session_affinity_retry"]] + + # Fields whose values quote the caller's prompt. Dropped when an operator turns message # logging off. Every other field aggregates the prompt without reproducing it and is kept, # so a redacted row stays explainable. `test_every_routing_decision_field_is_classified` @@ -3308,6 +3315,7 @@ class StandardLoggingMetadata(StandardLoggingUserAPIKeyMetadata): mcp_tool_call_metadata: StandardLoggingMCPToolCall | None vector_store_request_metadata: list[StandardLoggingVectorStoreRequest] | None routing_decision: StandardLoggingRoutingDecision | None + kubernetes_pod_routing: ReadOnly[StandardLoggingKubernetesPodRouting | None] applied_guardrails: list[str] | None usage_object: dict | None cold_storage_object_key: str | None # S3/GCS object key for cold storage retrieval diff --git a/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json b/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json index c6cf8ee7ee4..cfdf707c977 100644 --- a/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json +++ b/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json @@ -11,7 +11,7 @@ "user": "", "team_id": "", "organization_id": "", - "metadata": "{\"actor_agent_id\": null, \"target_agent_id\": null, \"billing_agent_id\": null, \"agent_execution_mode\": null, \"verified_human_user_id\": null, \"used_client_oauth_token\": null, \"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"batch_successful_requests\": null, \"batch_failed_requests\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"litellm_roi_estimator\": false, \"router_metadata\": null, \"autorouter_savings_estimate\": null, \"autorouter_baseline_observation\": null, \"azure_spillover\": null, \"guardrail_information\": null, \"compression_savings\": null, \"litellm_gateway_injected_cache\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_user_email\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"user_agent\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null}", + "metadata": "{\"actor_agent_id\": null, \"target_agent_id\": null, \"billing_agent_id\": null, \"agent_execution_mode\": null, \"verified_human_user_id\": null, \"used_client_oauth_token\": null, \"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"batch_successful_requests\": null, \"batch_failed_requests\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"litellm_roi_estimator\": false, \"router_metadata\": null, \"autorouter_savings_estimate\": null, \"autorouter_baseline_observation\": null, \"azure_spillover\": null, \"guardrail_information\": null, \"compression_savings\": null, \"litellm_gateway_injected_cache\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_user_email\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"user_agent\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null, \"kubernetes_pod_routing\": null}", "cache_key": "Cache OFF", "spend": 0.00022500000000000002, "total_tokens": 30, diff --git a/tests/unit/integrations/test_opentelemetry.py b/tests/unit/integrations/test_opentelemetry.py index b40c1a96500..9cc06ccaafc 100644 --- a/tests/unit/integrations/test_opentelemetry.py +++ b/tests/unit/integrations/test_opentelemetry.py @@ -51,7 +51,7 @@ from litellm.proxy import proxy_server from litellm.proxy._types import SpanAttributes from litellm.types.services import ServiceLoggerPayload, ServiceTypes from collections.abc import AsyncIterator -from litellm.constants import LOGGING_WORKER_MAX_TIME_PER_COROUTINE +from litellm.constants import KUBERNETES_POD_ROUTING_KEY, LOGGING_WORKER_MAX_TIME_PER_COROUTINE from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.types.utils import ChatCompletionMessageToolCall, Function from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome @@ -363,6 +363,33 @@ def test_v1_routing_diagnostics_add_scalar_attributes_and_preserve_legacy_metada assert finished.attributes["metadata.routing_decision"] == str(decision) +def test_v1_pod_routing_none_matches_other_none_metadata_attributes() -> None: + exporter: Final = InMemorySpanExporter() + provider: Final = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + otel: Final = OpenTelemetry(tracer_provider=provider) + with provider.get_tracer(__name__).start_as_current_span("inference") as span: + otel.set_attributes( + span, + { + "model": "answer", + "standard_logging_object": { + "id": "routing-call", + "call_type": "acompletion", + "metadata": { + KUBERNETES_POD_ROUTING_KEY: None, + "other_none": None, + }, + }, + }, + {"model": "answer", "choices": [], "usage": {}}, + ) + (finished,) = exporter.get_finished_spans() + routing_attribute: Final = finished.attributes["metadata.kubernetes_pod_routing"] + assert routing_attribute == finished.attributes["metadata.other_none"] + assert routing_attribute != "null" + + @pytest.mark.parametrize("decision", ["custom", 42, ["custom"]]) def test_v1_malformed_routing_metadata_preserves_inference_attributes(decision: object) -> None: exporter: Final = InMemorySpanExporter() diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 8d546f3be59..d1258e43154 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -30,7 +30,12 @@ from litellm._internal_context import in_post_response_phase from litellm._logging import session_id_var, trace_id_var, verbose_logger from litellm._service_logger import ServiceLogging from litellm.caching.caching import DualCache -from litellm.constants import LOGGING_WORKER_MAX_TIME_PER_COROUTINE, REDACTED_BY_LITELLM, SENTRY_PII_DENYLIST +from litellm.constants import ( + KUBERNETES_POD_ROUTING_KEY, + LOGGING_WORKER_MAX_TIME_PER_COROUTINE, + REDACTED_BY_LITELLM, + SENTRY_PII_DENYLIST, +) from litellm.cost_calculator import ocr_batch_cost from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils import litellm_logging @@ -5713,6 +5718,97 @@ def test_get_standard_logging_object_payload_takes_used_client_oauth_token_from_ assert payload["metadata"]["used_client_oauth_token"] is expected +@pytest.mark.parametrize( + ("metadata", "litellm_metadata", "expected"), + [ + ( + { + "kubernetes_pod_routing": { + "service_host": "vllm-headless.ns.svc.cluster.local", + "pod_ip": "10.0.0.1", + "pod_count": 3, + "selection": "round_robin", + } + }, + {}, + { + "service_host": "vllm-headless.ns.svc.cluster.local", + "pod_ip": "10.0.0.1", + "pod_count": 3, + "selection": "round_robin", + }, + ), + ( + {}, + { + "kubernetes_pod_routing": { + "service_host": "vllm-headless.ns.svc.cluster.local", + "pod_ip": "10.0.0.2", + "pod_count": 3, + "selection": "session_affinity", + } + }, + { + "service_host": "vllm-headless.ns.svc.cluster.local", + "pod_ip": "10.0.0.2", + "pod_count": 3, + "selection": "session_affinity", + }, + ), + ( + { + "kubernetes_pod_routing": { + "service_host": "vllm-headless.ns.svc.cluster.local", + "pod_ip": "10.0.0.1", + "pod_count": 3, + "selection": "round_robin", + } + }, + { + "kubernetes_pod_routing": { + "service_host": "vllm-headless.ns.svc.cluster.local", + "pod_ip": "10.0.0.3", + "pod_count": 3, + "selection": "session_affinity_retry", + } + }, + { + "service_host": "vllm-headless.ns.svc.cluster.local", + "pod_ip": "10.0.0.3", + "pod_count": 3, + "selection": "session_affinity_retry", + }, + ), + ], +) +def test_standard_logging_payload_resolves_kubernetes_pod_routing_from_metadata_buckets( + logging_obj: LitellmLogging, + metadata: dict[str, object], + litellm_metadata: dict[str, object], + expected: dict[str, object], +) -> None: + from datetime import datetime + + from litellm.litellm_core_utils.litellm_logging import get_standard_logging_object_payload + + now: Final = datetime(2026, 1, 1) + payload: Final = get_standard_logging_object_payload( + kwargs={ + "model": "gpt-4o", + "messages": [], + "litellm_params": {"metadata": metadata, "litellm_metadata": litellm_metadata}, + }, + init_response_obj={}, + start_time=now, + end_time=now, + logging_obj=logging_obj, + status="success", + ) + + assert payload is not None + assert payload["metadata"][KUBERNETES_POD_ROUTING_KEY] == expected + + def test_get_standard_logging_object_payload_carries_matched_access_groups(logging_obj): """Access groups stamped at auth time reach the logging payload, so integrations see what a request billed.""" from datetime import datetime diff --git a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py index 47cfa2df117..ed8ae2db8b1 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py @@ -3748,7 +3748,18 @@ class TestSpendLogsPayload: } ) - differences = _compare_nested_dicts(payload, expected_payload, ignore_keys=ignored_keys) + expected_payload_with_pod_routing: Final[SpendLogsPayload] = { + **expected_payload, + "metadata": expected_payload["metadata"].replace( + '"routing_decision": null, ', + '"routing_decision": null, "kubernetes_pod_routing": null, ', + ), + } + differences = _compare_nested_dicts( + payload, + expected_payload_with_pod_routing, + ignore_keys=ignored_keys, + ) if differences: pytest.fail(f"Dictionary mismatch: {differences}") diff --git a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py index 262357fe1bd..34e17b3cda8 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py @@ -14,7 +14,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm import litellm.constants as litellm_constants import litellm.proxy.spend_tracking.spend_tracking_utils as spend_tracking_utils -from litellm.constants import LITELLM_TRUNCATED_PAYLOAD_FIELD, LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, LITTELM_CLI_SERVICE_ACCOUNT_NAME, LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, MAX_SPEND_LOG_MODEL_NAME_LENGTH, REDACTED_BY_LITELM_STRING, SESSION_ID_GENERATED_METADATA_KEY, SESSION_ID_OMITTED_METADATA_KEY, UNKNOWN_MODEL_SPEND_LOG_MODEL +from litellm.constants import KUBERNETES_POD_ROUTING_KEY, LITELLM_TRUNCATED_PAYLOAD_FIELD, LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, LITTELM_CLI_SERVICE_ACCOUNT_NAME, LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, MAX_SPEND_LOG_MODEL_NAME_LENGTH, REDACTED_BY_LITELM_STRING, SESSION_ID_GENERATED_METADATA_KEY, SESSION_ID_OMITTED_METADATA_KEY, UNKNOWN_MODEL_SPEND_LOG_MODEL from litellm.constants import CALLER_SESSION_ID_METADATA_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY, MANAGED_BACKGROUND_RESPONSE_METADATA_KEY from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup from litellm.llms.base_llm.ocr.transformation import OCRResponse, OCRUsageInfo @@ -76,6 +76,89 @@ def _get_additional_usage_values_for_usage(usage: litellm.Usage) -> dict: return metadata["additional_usage_values"] +@pytest.mark.parametrize( + ("metadata", "litellm_metadata", "expected"), + [ + ( + { + "kubernetes_pod_routing": { + "service_host": "vllm-headless.ns.svc.cluster.local", + "pod_ip": "10.0.0.1", + "pod_count": 3, + "selection": "round_robin", + } + }, + {}, + { + "service_host": "vllm-headless.ns.svc.cluster.local", + "pod_ip": "10.0.0.1", + "pod_count": 3, + "selection": "round_robin", + }, + ), + ( + {}, + { + "kubernetes_pod_routing": { + "service_host": "vllm-headless.ns.svc.cluster.local", + "pod_ip": "10.0.0.2", + "pod_count": 3, + "selection": "session_affinity", + } + }, + { + "service_host": "vllm-headless.ns.svc.cluster.local", + "pod_ip": "10.0.0.2", + "pod_count": 3, + "selection": "session_affinity", + }, + ), + ( + { + "kubernetes_pod_routing": { + "service_host": "vllm-headless.ns.svc.cluster.local", + "pod_ip": "10.0.0.1", + "pod_count": 3, + "selection": "round_robin", + } + }, + { + "kubernetes_pod_routing": { + "service_host": "vllm-headless.ns.svc.cluster.local", + "pod_ip": "10.0.0.3", + "pod_count": 3, + "selection": "session_affinity_retry", + } + }, + { + "service_host": "vllm-headless.ns.svc.cluster.local", + "pod_ip": "10.0.0.3", + "pod_count": 3, + "selection": "session_affinity_retry", + }, + ), + ], +) +def test_get_logging_payload_includes_kubernetes_pod_routing( + metadata: dict[str, object], + litellm_metadata: dict[str, object], + expected: dict[str, object], +) -> None: + now: Final = datetime.datetime(2026, 1, 1, tzinfo=timezone.utc) + payload: Final = get_logging_payload( + kwargs={ + "model": "gpt-4o-mini", + "litellm_params": {"metadata": metadata, "litellm_metadata": litellm_metadata}, + }, + response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[]), + start_time=now, + end_time=now, + ) + + assert json.loads(payload["metadata"])[KUBERNETES_POD_ROUTING_KEY] == expected + assert _get_spend_logs_metadata(None)["kubernetes_pod_routing"] is None + + @pytest.mark.parametrize("store_prompts,redact", [(True, False), (False, False), (True, True)]) def test_classifier_audit_spend_storage_obeys_privacy_and_truncation(monkeypatch, store_prompts, redact): from litellm.proxy import proxy_server diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 93202d93566..2943fe79979 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -3309,12 +3309,11 @@ def test_add_litellm_metadata_from_request_headers_x_litellm_trace_id_sets_chain assert data["litellm_trace_id"] == "foo" -@pytest.mark.parametrize(("correlation_in_logs", "expected_spend_log_session"), [(True, "convo-1"), (False, None)]) +@pytest.mark.parametrize("correlation_in_logs", [True, False], ids=["correlation-on", "correlation-off"]) @pytest.mark.parametrize("session_in_metadata", [False, True], ids=["top-level", "metadata"]) -def test_x_litellm_trace_id_keeps_the_body_session_for_spend_logs_only( +def test_x_litellm_trace_id_keeps_the_body_session_beside_it_whatever_the_log_setting( monkeypatch: pytest.MonkeyPatch, correlation_in_logs: bool, - expected_spend_log_session: str | None, session_in_metadata: bool, ): monkeypatch.setattr(litellm, "request_correlation_in_logs", correlation_in_logs) @@ -3325,7 +3324,7 @@ def test_x_litellm_trace_id_keeps_the_body_session_for_spend_logs_only( LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers( headers={"x-litellm-trace-id": "req-trace-1"}, data=data, _metadata_variable_name="metadata" ) - assert metadata.get(CALLER_SESSION_ID_METADATA_KEY) == expected_spend_log_session + assert metadata.get(CALLER_SESSION_ID_METADATA_KEY) == "convo-1" assert data["litellm_session_id"] == "req-trace-1" assert metadata["session_id"] == "req-trace-1" assert data["litellm_trace_id"] == "req-trace-1" diff --git a/tests/unit/router_utils/test_kubernetes_pod_discovery.py b/tests/unit/router_utils/test_kubernetes_pod_discovery.py index b3fc6eede62..001b8863b17 100644 --- a/tests/unit/router_utils/test_kubernetes_pod_discovery.py +++ b/tests/unit/router_utils/test_kubernetes_pod_discovery.py @@ -1,5 +1,6 @@ import asyncio import copy +import hashlib import json import logging import re @@ -16,10 +17,22 @@ import respx from openai import AsyncOpenAI, OpenAI import litellm +from litellm.constants import ( + CALLER_SESSION_ID_METADATA_KEY, + KUBERNETES_POD_ROUTING_KEY, + SESSION_ID_GENERATED_METADATA_KEY, +) from litellm.router import CustomRoutingStrategyBase, Router -from litellm.router_utils.kubernetes_pod_discovery import KubernetesPodDiscovery +from litellm.router_utils.kubernetes_pod_discovery import ( + KubernetesPodDiscovery, + async_resolve_pods_after, + async_resolve_pods_after_bound, + resolve_pods_after, + resolve_pods_after_bound, +) _SERVICE_URL: Final = "http://vllm-headless.ns.svc.cluster.local:8000/v1" +_THREE_POD_CHAT_PATTERN: Final = re.compile(r"http://(?:10\.0\.0\.1|10\.0\.0\.2|10\.0\.0\.3):8000/v1/chat/completions") _SocketAddress: TypeAlias = tuple[str, int] | tuple[str, int, int, int] _AddrInfo: TypeAlias = tuple[socket.AddressFamily, socket.SocketKind, int, str, _SocketAddress] _NO_POD_ERRNOS: Final = tuple(sorted((socket.EAI_NONAME, socket.EAI_NODATA))) @@ -91,6 +104,14 @@ def _empty_proxy_environment() -> Mapping[str, str]: return {} +def _patch_router_discovery( + monkeypatch: pytest.MonkeyPatch, + *discoveries: KubernetesPodDiscovery, +) -> None: + discovery_iter: Final = iter(discoveries) + monkeypatch.setattr(litellm.router, "KubernetesPodDiscovery", lambda: next(discovery_iter)) + + def _stub_sync_dns(monkeypatch: pytest.MonkeyPatch, *ips: str) -> None: def getaddrinfo( host: str | None, @@ -113,12 +134,88 @@ def _api_base(deployment: Mapping[str, object]) -> str: return api_base +def _ranked_session_ips(session_id: str) -> tuple[str, ...]: + ips: Final = ("10.0.0.1", "10.0.0.2", "10.0.0.3") + return tuple( + sorted( + ips, + key=lambda ip: ( + -int.from_bytes(hashlib.blake2b(f"{session_id}\x00{ip}".encode(), digest_size=8).digest(), "big"), + ip, + ), + ) + ) + + +def test_session_hash_ranking_matches_full_hash_for_large_session_id( + monkeypatch: pytest.MonkeyPatch, +) -> None: + ips: Final = ("10.0.0.1", "10.0.0.2", "10.0.0.3") + _stub_sync_dns(monkeypatch, *ips) + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment) + + for session_id in ("s1", "session-2", "session-3"): + expected: Final = _ranked_session_ips(session_id) + actual: Final = tuple( + httpx.URL( + _api_base( + discovery.resolve_deployment( + _deployment(), + {"metadata": {"session_id": session_id, "request_retry_count": retry_count}}, + ) + ) + ).host + for retry_count in range(len(ips)) + ) + assert actual == expected + + large_ips: Final = tuple(f"10.1.0.{index}" for index in range(1, 101)) + _stub_sync_dns(monkeypatch, *large_ips) + large_discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment) + large_session_id: Final = "x" * (1024 * 1024) + + result: Final = large_discovery.resolve_deployment( + _deployment(), + {"metadata": {"session_id": large_session_id}}, + ) + + assert httpx.URL(_api_base(result)).host in large_ips + + def _request_body(content: bytes) -> Mapping[str, object]: body: Final = json.loads(content) assert isinstance(body, dict) return cast(Mapping[str, object], body) +def _cache_async_client(router: Router) -> AsyncOpenAI: + client: Final = AsyncOpenAI(api_key="fake", base_url=_SERVICE_URL) + router.cache.set_cache(key="registered-id_async_client", value=client, local_only=True) + return client + + +def _cache_sync_client(router: Router) -> OpenAI: + client: Final = OpenAI(api_key="fake", base_url=_SERVICE_URL) + router.cache.set_cache(key="registered-id_client", value=client, local_only=True) + return client + + +async def _router_acompletion(router: Router, **request_kwargs: object) -> None: + await router.acompletion( + model="gpu-model", + messages=[{"role": "user", "content": "hello"}], + **request_kwargs, + ) + + +def _router_completion(router: Router, **request_kwargs: object) -> None: + router.completion( + model="gpu-model", + messages=[{"role": "user", "content": "hello"}], + **request_kwargs, + ) + + def test_sync_resolution_round_robins_sorted_ips_without_mutating_deployment( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -1050,19 +1147,17 @@ async def test_router_sends_pod_hosts_without_forwarding_discovery_flag( monkeypatch.setattr(loop, "getaddrinfo", async_getaddrinfo) monkeypatch.setattr(socket, "getaddrinfo", sync_getaddrinfo) monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - with respx.mock(assert_all_called=True) as respx_mock: route: Final = respx_mock.post( url__regex=re.compile( r"http://(?:10\.0\.0\.1|10\.0\.0\.2|vllm-headless\.ns\.svc\.cluster\.local):8000/v1/chat/completions" ) ).mock(return_value=httpx.Response(200, json=_CHAT_RESPONSE)) + discovery_enabled: Final = KubernetesPodDiscovery(clock=clock, proxy_environment=_empty_proxy_environment) + control: Final = KubernetesPodDiscovery(clock=clock, proxy_environment=_empty_proxy_environment) + _patch_router_discovery(monkeypatch, discovery_enabled, control) discovery_enabled_router: Final = Router(model_list=[_deployment()]) control_router: Final = Router(model_list=[_deployment(kubernetes_pod_discovery=None)]) - discovery_enabled_router.kubernetes_pod_discovery = KubernetesPodDiscovery( - clock=clock, - proxy_environment=_empty_proxy_environment, - ) cached_async_client: Final = AsyncOpenAI(api_key="fake", base_url=_SERVICE_URL) cached_sync_client: Final = OpenAI(api_key="fake", base_url=_SERVICE_URL) discovery_enabled_router.cache.set_cache( @@ -1100,6 +1195,506 @@ async def test_router_sends_pod_hosts_without_forwarding_discovery_flag( assert next(sync_lookups) == 1 +@pytest.mark.asyncio +async def test_router_session_requests_stick_to_one_pod_for_sync_and_async( + monkeypatch: pytest.MonkeyPatch, +) -> None: + loop: Final = asyncio.get_running_loop() + async_lookups: Final = count() + sync_lookups: Final = count() + + async def async_getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(async_lookups) + return _records("10.0.0.1", "10.0.0.2", "10.0.0.3", port=int(port or 0)) + + def sync_getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(sync_lookups) + return _records("10.0.0.1", "10.0.0.2", "10.0.0.3", port=int(port or 0)) + + monkeypatch.setattr(loop, "getaddrinfo", async_getaddrinfo) + monkeypatch.setattr(socket, "getaddrinfo", sync_getaddrinfo) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock(assert_all_called=True) as respx_mock: + route: Final = respx_mock.post(url__regex=_THREE_POD_CHAT_PATTERN).mock( + return_value=httpx.Response(200, json=_CHAT_RESPONSE) + ) + async_discovery: Final = KubernetesPodDiscovery(clock=lambda: 0.0, proxy_environment=_empty_proxy_environment) + sync_discovery: Final = KubernetesPodDiscovery(clock=lambda: 0.0, proxy_environment=_empty_proxy_environment) + _patch_router_discovery(monkeypatch, async_discovery, sync_discovery) + async_router: Final = Router(model_list=[_deployment()]) + sync_router: Final = Router(model_list=[_deployment()]) + async_client: Final = _cache_async_client(async_router) + sync_client: Final = _cache_sync_client(sync_router) + for _ in range(6): + await _router_acompletion(async_router, metadata={"session_id": "s1"}) + for _ in range(6): + _router_completion(sync_router, metadata={"session_id": "s1"}) + hosts: Final = tuple(call.request.url.host for call in route.calls) + await async_client.close() + sync_client.close() + + assert len(set(hosts[:6])) == 1 + assert len(set(hosts[6:])) == 1 + assert next(async_lookups) == 1 + assert next(sync_lookups) == 1 + + +@pytest.mark.asyncio +async def test_router_session_mapping_repeats_across_pods( + monkeypatch: pytest.MonkeyPatch, +) -> None: + loop: Final = asyncio.get_running_loop() + lookups: Final = count() + + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(lookups) + return _records("10.0.0.1", "10.0.0.2", "10.0.0.3", port=int(port or 0)) + + monkeypatch.setattr(loop, "getaddrinfo", getaddrinfo) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + session_ids: Final = tuple(f"session-{index}" for index in range(30)) + with respx.mock(assert_all_called=True) as respx_mock: + route: Final = respx_mock.post(url__regex=_THREE_POD_CHAT_PATTERN).mock( + return_value=httpx.Response(200, json=_CHAT_RESPONSE) + ) + discovery: Final = KubernetesPodDiscovery(clock=lambda: 0.0, proxy_environment=_empty_proxy_environment) + _patch_router_discovery(monkeypatch, discovery) + router: Final = Router(model_list=[_deployment()]) + client: Final = _cache_async_client(router) + for session_id in session_ids: + await _router_acompletion(router, metadata={"session_id": session_id}) + await _router_acompletion(router) + for session_id in session_ids: + await _router_acompletion(router, metadata={"session_id": session_id}) + hosts: Final = tuple(call.request.url.host for call in route.calls) + await client.close() + + first_pass: Final = hosts[: len(session_ids)] + second_pass: Final = hosts[len(session_ids) + 1 :] + assert len(set(first_pass)) > 1 + assert hosts[len(session_ids)] == "10.0.0.1" + assert second_pass == first_pass + assert next(lookups) == 1 + + +@pytest.mark.asyncio +async def test_generated_session_id_uses_round_robin_pod_selection( + monkeypatch: pytest.MonkeyPatch, +) -> None: + loop: Final = asyncio.get_running_loop() + + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + return _records("10.0.0.1", "10.0.0.2", "10.0.0.3", port=int(port or 0)) + + monkeypatch.setattr(loop, "getaddrinfo", getaddrinfo) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock(assert_all_called=True) as respx_mock: + route: Final = respx_mock.post(url__regex=_THREE_POD_CHAT_PATTERN).mock( + return_value=httpx.Response(200, json=_CHAT_RESPONSE) + ) + discovery: Final = KubernetesPodDiscovery(clock=lambda: 0.0, proxy_environment=_empty_proxy_environment) + _patch_router_discovery(monkeypatch, discovery) + router: Final = Router(model_list=[_deployment()]) + client: Final = _cache_async_client(router) + for _ in range(6): + await _router_acompletion( + router, + metadata={ + "session_id": "generated-session", + SESSION_ID_GENERATED_METADATA_KEY: True, + }, + ) + hosts: Final = tuple(call.request.url.host for call in route.calls) + await client.close() + + assert hosts == ("10.0.0.1", "10.0.0.2", "10.0.0.3") * 2 + + +@pytest.mark.asyncio +async def test_empty_session_id_uses_round_robin_pod_selection( + monkeypatch: pytest.MonkeyPatch, +) -> None: + loop: Final = asyncio.get_running_loop() + + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + return _records("10.0.0.1", "10.0.0.2", "10.0.0.3", port=int(port or 0)) + + monkeypatch.setattr(loop, "getaddrinfo", getaddrinfo) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock(assert_all_called=True) as respx_mock: + route: Final = respx_mock.post(url__regex=_THREE_POD_CHAT_PATTERN).mock( + return_value=httpx.Response(200, json=_CHAT_RESPONSE) + ) + discovery: Final = KubernetesPodDiscovery(clock=lambda: 0.0, proxy_environment=_empty_proxy_environment) + _patch_router_discovery(monkeypatch, discovery) + router: Final = Router(model_list=[_deployment()]) + client: Final = _cache_async_client(router) + for _ in range(6): + await _router_acompletion(router, metadata={"session_id": ""}) + hosts: Final = tuple(call.request.url.host for call in route.calls) + await client.close() + + assert hosts == ("10.0.0.1", "10.0.0.2", "10.0.0.3") * 2 + + +@pytest.mark.asyncio +async def test_session_requests_do_not_advance_round_robin_cursor( + monkeypatch: pytest.MonkeyPatch, +) -> None: + loop: Final = asyncio.get_running_loop() + + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + return _records("10.0.0.1", "10.0.0.2", "10.0.0.3", port=int(port or 0)) + + monkeypatch.setattr(loop, "getaddrinfo", getaddrinfo) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock(assert_all_called=True) as respx_mock: + route: Final = respx_mock.post(url__regex=_THREE_POD_CHAT_PATTERN).mock( + return_value=httpx.Response(200, json=_CHAT_RESPONSE) + ) + discovery: Final = KubernetesPodDiscovery(clock=lambda: 0.0, proxy_environment=_empty_proxy_environment) + _patch_router_discovery(monkeypatch, discovery) + router: Final = Router(model_list=[_deployment()]) + client: Final = _cache_async_client(router) + await _router_acompletion(router) + await _router_acompletion(router, metadata={"session_id": "s1"}) + await _router_acompletion(router, metadata={"session_id": "s1"}) + await _router_acompletion(router) + await _router_acompletion(router, metadata={"session_id": "s1"}) + await _router_acompletion(router) + hosts: Final = tuple(call.request.url.host for call in route.calls) + await client.close() + + assert tuple(hosts[index] for index in (0, 3, 5)) == ( + "10.0.0.1", + "10.0.0.2", + "10.0.0.3", + ) + + +@pytest.mark.asyncio +async def test_session_pod_membership_change_remaps_only_affected_sessions( + monkeypatch: pytest.MonkeyPatch, +) -> None: + loop: Final = asyncio.get_running_loop() + lookups: Final = count() + + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + lookup_number: Final = next(lookups) + ips: Final = ("10.0.0.1", "10.0.0.2", "10.0.0.3") if lookup_number == 0 else ("10.0.0.2", "10.0.0.3") + return _records(*ips, port=int(port or 0)) + + monkeypatch.setattr(loop, "getaddrinfo", getaddrinfo) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + session_ids: Final = tuple(f"session-{index}" for index in range(60)) + with respx.mock(assert_all_called=True) as respx_mock: + route: Final = respx_mock.post(url__regex=_THREE_POD_CHAT_PATTERN).mock( + return_value=httpx.Response(200, json=_CHAT_RESPONSE) + ) + discovery: Final = KubernetesPodDiscovery( + refresh_interval_seconds=10, + clock=_clock((0.0,) * 62 + (10.0,) * 10), + proxy_environment=_empty_proxy_environment, + ) + _patch_router_discovery(monkeypatch, discovery) + router: Final = Router(model_list=[_deployment()]) + client: Final = _cache_async_client(router) + for session_id in session_ids: + await _router_acompletion(router, metadata={"session_id": session_id}) + initial_hosts: Final = tuple(call.request.url.host for call in route.calls) + removed_session_ids: Final = tuple( + session_id for session_id, host in zip(session_ids, initial_hosts) if host == "10.0.0.1" + ) + surviving_session_ids: Final = tuple( + session_id for session_id, host in zip(session_ids, initial_hosts) if host == "10.0.0.2" + ) + assert removed_session_ids + assert len(surviving_session_ids) >= 2 + await _router_acompletion(router) + await _router_acompletion(router, metadata={"session_id": surviving_session_ids[0]}) + await _router_acompletion(router, metadata={"session_id": removed_session_ids[0]}) + await _router_acompletion(router, metadata={"session_id": surviving_session_ids[1]}) + hosts: Final = tuple(call.request.url.host for call in route.calls) + await client.close() + + assert hosts[60] == "10.0.0.1" + assert hosts[61] == "10.0.0.2" + assert hosts[62] in ("10.0.0.2", "10.0.0.3") + assert hosts[63] == "10.0.0.2" + assert next(lookups) == 2 + + +@pytest.mark.asyncio +async def test_custom_routing_strategy_keeps_sync_and_async_sessions_on_one_pod( + monkeypatch: pytest.MonkeyPatch, +) -> None: + loop: Final = asyncio.get_running_loop() + async_lookups: Final = count() + sync_lookups: Final = count() + + async def async_getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(async_lookups) + return _records("10.0.0.1", "10.0.0.2", "10.0.0.3", port=int(port or 0)) + + def sync_getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(sync_lookups) + return _records("10.0.0.1", "10.0.0.2", "10.0.0.3", port=int(port or 0)) + + monkeypatch.setattr(loop, "getaddrinfo", async_getaddrinfo) + monkeypatch.setattr(socket, "getaddrinfo", sync_getaddrinfo) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock(assert_all_called=True) as respx_mock: + route: Final = respx_mock.post(url__regex=_THREE_POD_CHAT_PATTERN).mock( + return_value=httpx.Response(200, json=_CHAT_RESPONSE) + ) + async_discovery: Final = KubernetesPodDiscovery( + clock=lambda: 0.0, + proxy_environment=_empty_proxy_environment, + ) + sync_discovery: Final = KubernetesPodDiscovery( + clock=lambda: 0.0, + proxy_environment=_empty_proxy_environment, + ) + _patch_router_discovery(monkeypatch, async_discovery, sync_discovery) + async_router: Final = Router(model_list=[_deployment()]) + sync_router: Final = Router(model_list=[_deployment()]) + async_router.set_custom_routing_strategy(_ModelListRoutingStrategy(async_router)) + sync_router.set_custom_routing_strategy(_ModelListRoutingStrategy(sync_router)) + async_client: Final = _cache_async_client(async_router) + sync_client: Final = _cache_sync_client(sync_router) + for _ in range(6): + await _router_acompletion(async_router, metadata={"session_id": "custom-session"}) + for _ in range(6): + _router_completion(sync_router, metadata={"session_id": "custom-session"}) + hosts: Final = tuple(call.request.url.host for call in route.calls) + await async_client.close() + sync_client.close() + + assert len(set(hosts[:6])) == 1 + assert len(set(hosts[6:])) == 1 + assert next(async_lookups) == 1 + assert next(sync_lookups) == 1 + + +@pytest.mark.asyncio +async def test_top_level_litellm_session_id_keeps_requests_on_one_pod( + monkeypatch: pytest.MonkeyPatch, +) -> None: + loop: Final = asyncio.get_running_loop() + + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + return _records("10.0.0.1", "10.0.0.2", "10.0.0.3", port=int(port or 0)) + + monkeypatch.setattr(loop, "getaddrinfo", getaddrinfo) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock(assert_all_called=True) as respx_mock: + route: Final = respx_mock.post(url__regex=_THREE_POD_CHAT_PATTERN).mock( + return_value=httpx.Response(200, json=_CHAT_RESPONSE) + ) + discovery: Final = KubernetesPodDiscovery(clock=lambda: 0.0, proxy_environment=_empty_proxy_environment) + _patch_router_discovery(monkeypatch, discovery) + router: Final = Router(model_list=[_deployment()]) + client: Final = _cache_async_client(router) + for _ in range(6): + await _router_acompletion(router, litellm_session_id="s1") + hosts: Final = tuple(call.request.url.host for call in route.calls) + await client.close() + + assert len(set(hosts)) == 1 + + +def test_caller_session_beside_trace_header_keeps_the_pod_across_trace_ids(monkeypatch: pytest.MonkeyPatch) -> None: + _stub_sync_dns(monkeypatch, "10.0.0.1", "10.0.0.2", "10.0.0.3") + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment) + caller_session_id: Final = "chat-session-a" + caller_pod: Final = _ranked_session_ips(caller_session_id)[0] + trace_ids: Final = tuple( + trace_id + for trace_id in (f"trace-{turn}" for turn in range(20)) + if _ranked_session_ips(trace_id)[0] != caller_pod + )[:3] + assert len(trace_ids) == 3 + + pods: Final = tuple( + httpx.URL( + _api_base( + discovery.resolve_deployment( + _deployment(), + { + "litellm_session_id": trace_id, + "metadata": {CALLER_SESSION_ID_METADATA_KEY: caller_session_id}, + }, + ) + ) + ).host + for trace_id in trace_ids + ) + + assert pods == (caller_pod, caller_pod, caller_pod) + + +def test_proxy_headers_with_trace_and_session_id_pin_the_pod_without_log_correlation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup + + monkeypatch.setattr(litellm, "request_correlation_in_logs", False) + _stub_sync_dns(monkeypatch, "10.0.0.1", "10.0.0.2", "10.0.0.3") + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment) + caller_session_id: Final = "chat-session-a" + caller_pod: Final = _ranked_session_ips(caller_session_id)[0] + trace_ids: Final = tuple( + trace_id + for trace_id in (f"trace-{turn}" for turn in range(20)) + if _ranked_session_ips(trace_id)[0] != caller_pod + )[:3] + assert len(trace_ids) == 3 + + def resolve_turn(trace_id: str) -> str: + data: Final = LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers( + {"x-litellm-trace-id": trace_id, "x-litellm-session-id": caller_session_id}, + {"model": "gpu-model", "metadata": {}}, + "metadata", + ) + return httpx.URL(_api_base(discovery.resolve_deployment(_deployment(), data))).host + + assert tuple(resolve_turn(trace_id) for trace_id in trace_ids) == (caller_pod, caller_pod, caller_pod) + + +def test_top_level_litellm_session_id_precedes_metadata_session_id(monkeypatch: pytest.MonkeyPatch) -> None: + _stub_sync_dns(monkeypatch, "10.0.0.1", "10.0.0.2", "10.0.0.3") + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment) + top_level_session_id: Final = "s1" + metadata_session_id: Final = "metadata-session" + top_level_kwargs: Final = {"litellm_session_id": top_level_session_id} + metadata_kwargs: Final = {"metadata": {"session_id": metadata_session_id}} + combined_kwargs: Final = { + "litellm_session_id": top_level_session_id, + "metadata": {"session_id": metadata_session_id}, + } + + assert _ranked_session_ips(top_level_session_id)[0] != _ranked_session_ips(metadata_session_id)[0] + + top_level_pod: Final = httpx.URL(_api_base(discovery.resolve_deployment(_deployment(), top_level_kwargs))).host + metadata_pod: Final = httpx.URL(_api_base(discovery.resolve_deployment(_deployment(), metadata_kwargs))).host + combined_pod: Final = httpx.URL(_api_base(discovery.resolve_deployment(_deployment(), combined_kwargs))).host + + assert top_level_pod != metadata_pod + assert combined_pod == top_level_pod + + +@pytest.mark.asyncio +async def test_generated_metadata_ignores_top_level_litellm_session_id( + monkeypatch: pytest.MonkeyPatch, +) -> None: + loop: Final = asyncio.get_running_loop() + + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + return _records("10.0.0.1", "10.0.0.2", "10.0.0.3", port=int(port or 0)) + + monkeypatch.setattr(loop, "getaddrinfo", getaddrinfo) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock(assert_all_called=True) as respx_mock: + route: Final = respx_mock.post(url__regex=_THREE_POD_CHAT_PATTERN).mock( + return_value=httpx.Response(200, json=_CHAT_RESPONSE) + ) + discovery: Final = KubernetesPodDiscovery(clock=lambda: 0.0, proxy_environment=_empty_proxy_environment) + _patch_router_discovery(monkeypatch, discovery) + router: Final = Router(model_list=[_deployment()]) + client: Final = _cache_async_client(router) + for _ in range(6): + await _router_acompletion( + router, + litellm_session_id="s1", + metadata={ + "session_id": "s1", + SESSION_ID_GENERATED_METADATA_KEY: True, + }, + ) + hosts: Final = tuple(call.request.url.host for call in route.calls) + await client.close() + + assert hosts == ("10.0.0.1", "10.0.0.2", "10.0.0.3") * 2 + + @pytest.mark.asyncio async def test_custom_routing_strategy_resolves_pod_hosts_for_sync_and_async_requests( monkeypatch: pytest.MonkeyPatch, @@ -1134,23 +1729,23 @@ async def test_custom_routing_strategy_resolves_pod_hosts_for_sync_and_async_req monkeypatch.setattr(loop, "getaddrinfo", async_getaddrinfo) monkeypatch.setattr(socket, "getaddrinfo", sync_getaddrinfo) monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - with respx.mock(assert_all_called=True) as respx_mock: route: Final = respx_mock.post( url__regex=re.compile( r"http://(?:10\.0\.0\.1|10\.0\.0\.2|vllm-headless\.ns\.svc\.cluster\.local):8000/v1/chat/completions" ) ).mock(return_value=httpx.Response(200, json=_CHAT_RESPONSE)) + async_discovery: Final = KubernetesPodDiscovery( + clock=clock, + proxy_environment=_empty_proxy_environment, + ) + sync_discovery: Final = KubernetesPodDiscovery( + clock=clock, + proxy_environment=_empty_proxy_environment, + ) + _patch_router_discovery(monkeypatch, async_discovery, sync_discovery) async_router: Final = Router(model_list=[_deployment()]) sync_router: Final = Router(model_list=[_deployment()]) - async_router.kubernetes_pod_discovery = KubernetesPodDiscovery( - clock=clock, - proxy_environment=_empty_proxy_environment, - ) - sync_router.kubernetes_pod_discovery = KubernetesPodDiscovery( - clock=clock, - proxy_environment=_empty_proxy_environment, - ) async_router.set_custom_routing_strategy(_ModelListRoutingStrategy(async_router)) sync_router.set_custom_routing_strategy(_ModelListRoutingStrategy(sync_router)) for _ in range(4): @@ -1202,16 +1797,13 @@ async def test_router_async_retry_uses_next_discovered_pod(monkeypatch: pytest.M monkeypatch.setattr(loop, "getaddrinfo", getaddrinfo) monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - with respx.mock(assert_all_called=True) as respx_mock: route: Final = respx_mock.post( url__regex=re.compile(r"http://(?:10\.0\.0\.1|10\.0\.0\.2):8000/v1/chat/completions") ).mock(side_effect=response_for) + discovery: Final = KubernetesPodDiscovery(clock=clock, proxy_environment=_empty_proxy_environment) + _patch_router_discovery(monkeypatch, discovery) router: Final = Router(model_list=[_deployment()], num_retries=1) - router.kubernetes_pod_discovery = KubernetesPodDiscovery( - clock=clock, - proxy_environment=_empty_proxy_environment, - ) response: Final = await router.acompletion( model="gpu-model", @@ -1245,16 +1837,13 @@ def test_router_sync_retry_uses_next_discovered_pod(monkeypatch: pytest.MonkeyPa monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo) monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - with respx.mock(assert_all_called=True) as respx_mock: route: Final = respx_mock.post( url__regex=re.compile(r"http://(?:10\.0\.0\.1|10\.0\.0\.2):8000/v1/chat/completions") ).mock(side_effect=response_for) + discovery: Final = KubernetesPodDiscovery(clock=clock, proxy_environment=_empty_proxy_environment) + _patch_router_discovery(monkeypatch, discovery) router: Final = Router(model_list=[_deployment()], num_retries=1) - router.kubernetes_pod_discovery = KubernetesPodDiscovery( - clock=clock, - proxy_environment=_empty_proxy_environment, - ) response: Final = router.completion( model="gpu-model", @@ -1264,3 +1853,240 @@ def test_router_sync_retry_uses_next_discovered_pod(monkeypatch: pytest.MonkeyPa assert response.choices[0].message.content == "ok" assert hosts == ("10.0.0.1", "10.0.0.2") + + +@pytest.mark.parametrize( + ("litellm_retry_count", "metadata_retry_count", "expected_rank"), + [ + (0, None, 0), + (None, None, 0), + (True, None, 0), + ("1", None, 0), + (-1, None, 0), + (1, None, 1), + (True, 1, 1), + (1, 0, 1), + ], +) +def test_session_retry_count_selects_ranked_pod( + monkeypatch: pytest.MonkeyPatch, + litellm_retry_count: object, + metadata_retry_count: object, + expected_rank: int, +) -> None: + _stub_sync_dns(monkeypatch, "10.0.0.1", "10.0.0.2", "10.0.0.3") + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment) + request_kwargs: Final = { + "litellm_metadata": { + "session_id": "s1", + **({"request_retry_count": litellm_retry_count} if litellm_retry_count is not None else {}), + }, + "metadata": ({"request_retry_count": metadata_retry_count} if metadata_retry_count is not None else {}), + } + + resolved: Final = discovery.resolve_deployment(_deployment(), request_kwargs) + + assert httpx.URL(_api_base(resolved)).host == _ranked_session_ips("s1")[expected_rank] + + +@pytest.mark.asyncio +async def test_router_session_retry_uses_next_rendezvous_pod(monkeypatch: pytest.MonkeyPatch) -> None: + loop: Final = asyncio.get_running_loop() + attempts: Final = count() + + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + return _records("10.0.0.1", "10.0.0.2", "10.0.0.3", port=int(port or 0)) + + def response_for(request: httpx.Request) -> httpx.Response: + attempt: Final = next(attempts) + status_code: Final = 500 if attempt == 0 else 200 + body: Final = {"error": {"message": "retryable", "type": "server_error"}} if attempt == 0 else _CHAT_RESPONSE + return httpx.Response(status_code, json=body, request=request) + + monkeypatch.setattr(loop, "getaddrinfo", getaddrinfo) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + session_id: Final = "retry-session" + with respx.mock(assert_all_called=True) as respx_mock: + route: Final = respx_mock.post(url__regex=_THREE_POD_CHAT_PATTERN).mock(side_effect=response_for) + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment) + _patch_router_discovery(monkeypatch, discovery) + router: Final = Router(model_list=[_deployment()], num_retries=1) + + response: Final = await router.acompletion( + model="gpu-model", + messages=[{"role": "user", "content": "hello"}], + metadata={"session_id": session_id}, + ) + hosts: Final = tuple(call.request.url.host for call in route.calls) + + assert response.choices[0].message.content == "ok" + assert hosts == _ranked_session_ips(session_id)[:2] + + +@pytest.mark.asyncio +async def test_positional_request_kwargs_reach_all_pod_resolver_decorators( + monkeypatch: pytest.MonkeyPatch, +) -> None: + loop: Final = asyncio.get_running_loop() + + async def async_getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + return _records("10.0.0.1", "10.0.0.2", "10.0.0.3", port=int(port or 0)) + + def sync_getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + return _records("10.0.0.1", "10.0.0.2", "10.0.0.3", port=int(port or 0)) + + class _Selector: + def __init__(self, discovery: KubernetesPodDiscovery) -> None: + self.kubernetes_pod_discovery: Final = discovery + + @resolve_pods_after + def sync(self, deployment: dict[str, object], request_kwargs: Mapping[str, object]) -> dict[str, object]: + return deployment + + @async_resolve_pods_after + async def async_call( + self, + deployment: dict[str, object], + request_kwargs: Mapping[str, object], + ) -> dict[str, object]: + return deployment + + class _BoundSelector: + def sync(self, deployment: dict[str, object], request_kwargs: Mapping[str, object]) -> dict[str, object]: + return deployment + + async def async_call( + self, + deployment: dict[str, object], + request_kwargs: Mapping[str, object], + ) -> dict[str, object]: + return deployment + + monkeypatch.setattr(loop, "getaddrinfo", async_getaddrinfo) + monkeypatch.setattr(socket, "getaddrinfo", sync_getaddrinfo) + session_id: Final = next( + candidate + for candidate in (f"positional-{index}" for index in range(100)) + if _ranked_session_ips(candidate)[0] != "10.0.0.1" + ) + request_kwargs: Final = {"metadata": {"session_id": session_id}} + sync_selector: Final = _Selector(KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment)) + async_selector: Final = _Selector(KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment)) + bound_selector: Final = _BoundSelector() + bound_sync: Final = resolve_pods_after_bound( + bound_selector.sync, + KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment), + ) + bound_async: Final = async_resolve_pods_after_bound( + bound_selector.async_call, + KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment), + ) + + sync_result: Final = sync_selector.sync(_deployment(), request_kwargs) + async_result: Final = await async_selector.async_call(_deployment(), request_kwargs) + bound_sync_result: Final = bound_sync(_deployment(), request_kwargs) + bound_async_result: Final = await bound_async(_deployment(), request_kwargs) + + assert ( + tuple( + httpx.URL(_api_base(result)).host + for result in (sync_result, async_result, bound_sync_result, bound_async_result) + ) + == (_ranked_session_ips(session_id)[0],) * 4 + ) + + +def test_direct_session_id_keyword_does_not_enable_session_affinity(monkeypatch: pytest.MonkeyPatch) -> None: + _stub_sync_dns(monkeypatch, "10.0.0.1", "10.0.0.2", "10.0.0.3") + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment) + request_kwargs: Final = {"session_id": "s1", "messages": [{"role": "user", "content": "hello"}]} + + selected_ips: Final = tuple( + httpx.URL(_api_base(discovery.resolve_deployment(_deployment(), request_kwargs))).host for _ in range(3) + ) + + assert selected_ips == ("10.0.0.1", "10.0.0.2", "10.0.0.3") + + +@pytest.mark.parametrize( + ("request_kwargs", "selection"), + [ + ({}, "round_robin"), + ({"metadata": {"session_id": "s1"}}, "session_affinity"), + ({"metadata": {"session_id": "s1", "request_retry_count": 1}}, "session_affinity_retry"), + ], +) +def test_resolved_deployment_records_pod_selection( + monkeypatch: pytest.MonkeyPatch, + request_kwargs: dict[str, object], + selection: str, +) -> None: + _stub_sync_dns(monkeypatch, "10.0.0.1", "10.0.0.2", "10.0.0.3") + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment) + + resolved: Final = discovery.resolve_deployment(_deployment(), request_kwargs) + pod_ip: Final = httpx.URL(_api_base(resolved)).host + + assert resolved[KUBERNETES_POD_ROUTING_KEY] == { + "service_host": httpx.URL(_SERVICE_URL).host, + "pod_ip": pod_ip, + "pod_count": 3, + "selection": selection, + } + + +@pytest.mark.parametrize("fallback", ["https", "no_proxy", "empty_ips", "dns_failure"]) +def test_unresolved_deployment_does_not_record_pod_selection( + monkeypatch: pytest.MonkeyPatch, + fallback: str, +) -> None: + if fallback == "empty_ips": + _stub_sync_dns(monkeypatch) + if fallback == "dns_failure": + + def failing_getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + raise socket.gaierror(socket.EAI_AGAIN, "temporary failure") + + monkeypatch.setattr(socket, "getaddrinfo", failing_getaddrinfo) + if fallback == "no_proxy": + _stub_sync_dns(monkeypatch, "10.0.0.1") + + proxy_environment: Final = ( + _proxy_environment("vllm-headless.ns.svc.cluster.local") if fallback == "no_proxy" else _empty_proxy_environment + ) + api_base: Final = "https://vllm-headless.ns.svc.cluster.local:8000/v1" if fallback == "https" else _SERVICE_URL + discovery: Final = KubernetesPodDiscovery(proxy_environment=proxy_environment) + deployment: Final = _deployment(api_base=api_base) + + resolved: Final = discovery.resolve_deployment(deployment) + + assert resolved is deployment + assert KUBERNETES_POD_ROUTING_KEY not in resolved diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 1dec4de45f4..6403f00eb50 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -9978,6 +9978,94 @@ def test_update_kwargs_with_deployment_model_info_in_metadata(): assert model_info["output_cost_per_token"] == 0.0015 +def test_update_kwargs_with_deployment_clears_pod_routing_on_non_discovery_fallback() -> None: + from litellm.constants import KUBERNETES_POD_ROUTING_KEY + + routing_record: Final = { + "service_host": "vllm-headless.ns.svc.cluster.local", + "pod_ip": "10.0.0.1", + "pod_count": 3, + "selection": "session_affinity", + } + discovery_deployment: Final = { + "model_name": "gpt-4o-mini", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "fake"}, + "model_info": {"id": "discovery-id"}, + KUBERNETES_POD_ROUTING_KEY: routing_record, + } + fallback_deployment: Final = { + "model_name": "gpt-4o-mini", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "fake"}, + "model_info": {"id": "fallback-id"}, + } + router: Final = litellm.Router(model_list=[discovery_deployment]) + kwargs: Final = {"metadata": {}} + + router._update_kwargs_with_deployment(deployment=discovery_deployment, kwargs=kwargs) + assert kwargs["metadata"][KUBERNETES_POD_ROUTING_KEY] == routing_record + + router._update_kwargs_with_deployment(deployment=fallback_deployment, kwargs=kwargs) + assert kwargs["metadata"][KUBERNETES_POD_ROUTING_KEY] is None + + +def test_update_kwargs_with_deployment_synchronizes_existing_litellm_metadata_pod_routing() -> None: + from litellm.constants import KUBERNETES_POD_ROUTING_KEY + + routing_record: Final = { + "service_host": "vllm-headless.ns.svc.cluster.local", + "pod_ip": "10.0.0.1", + "pod_count": 3, + "selection": "session_affinity", + } + spoofed_record: Final = { + "service_host": "vllm-headless.ns.svc.cluster.local", + "pod_ip": "10.0.0.9", + "pod_count": 3, + "selection": "round_robin", + } + deployment: Final = { + "model_name": "gpt-4o-mini", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "fake"}, + "model_info": {"id": "discovery-id"}, + KUBERNETES_POD_ROUTING_KEY: routing_record, + } + original_litellm_metadata: Final = {KUBERNETES_POD_ROUTING_KEY: spoofed_record} + kwargs: Final = {"metadata": {}, "litellm_metadata": original_litellm_metadata} + router: Final = litellm.Router(model_list=[deployment]) + + router._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs, function_name="completion") + + assert kwargs["metadata"][KUBERNETES_POD_ROUTING_KEY] == routing_record + assert kwargs["litellm_metadata"][KUBERNETES_POD_ROUTING_KEY] == routing_record + assert kwargs["litellm_metadata"] is not original_litellm_metadata + assert original_litellm_metadata[KUBERNETES_POD_ROUTING_KEY] == spoofed_record + + +def test_update_kwargs_with_non_discovery_deployment_clears_other_pod_routing_bucket() -> None: + from litellm.constants import KUBERNETES_POD_ROUTING_KEY + + previous_record: Final = { + "service_host": "vllm-headless.ns.svc.cluster.local", + "pod_ip": "10.0.0.1", + "pod_count": 3, + "selection": "session_affinity", + } + deployment: Final = { + "model_name": "gpt-4o-mini", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "fake"}, + "model_info": {"id": "fallback-id"}, + } + original_litellm_metadata: Final = {KUBERNETES_POD_ROUTING_KEY: previous_record} + kwargs: Final = {"metadata": {}, "litellm_metadata": original_litellm_metadata} + router: Final = litellm.Router(model_list=[deployment]) + + router._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs, function_name="completion") + + assert kwargs["metadata"][KUBERNETES_POD_ROUTING_KEY] is None + assert kwargs["litellm_metadata"][KUBERNETES_POD_ROUTING_KEY] is None + assert original_litellm_metadata[KUBERNETES_POD_ROUTING_KEY] == previous_record + + def test_combine_fallback_usage(): """Test that _combine_fallback_usage merges partial and fallback usage.""" from litellm.router import Router diff --git a/ui/litellm-dashboard/src/components/logs/constants.ts b/ui/litellm-dashboard/src/components/logs/constants.ts index 9474c54d6b6..44231370521 100644 --- a/ui/litellm-dashboard/src/components/logs/constants.ts +++ b/ui/litellm-dashboard/src/components/logs/constants.ts @@ -33,6 +33,12 @@ export const CREDENTIAL_LABELS: Record = { false: "Configured key", }; +export const KUBERNETES_POD_ROUTING_LABELS = { + round_robin: "Round robin", + session_affinity: "Session affinity", + session_affinity_retry: "Session affinity (retry)", +} as const; + export const QUICK_SELECT_OPTIONS: { label: string; value: number; unit: string }[] = [ { label: "Last Minute", value: 1, unit: "minutes" }, { label: "Last 15 Minutes", value: 15, unit: "minutes" }, diff --git a/ui/litellm-dashboard/src/components/logs/detail/LogDetailContent.integration.test.tsx b/ui/litellm-dashboard/src/components/logs/detail/LogDetailContent.integration.test.tsx index 6c2569710e6..7f39a0d8fe5 100644 --- a/ui/litellm-dashboard/src/components/logs/detail/LogDetailContent.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/logs/detail/LogDetailContent.integration.test.tsx @@ -431,6 +431,37 @@ describe("LogDetailContent", () => { expect(screen.queryByText("Configured key")).not.toBeInTheDocument(); }); + it.each([ + { selection: "round_robin", label: "Round robin" }, + { selection: "session_affinity", label: "Session affinity" }, + { selection: "session_affinity_retry", label: "Session affinity (retry)" }, + ] as const)("shows the selected Kubernetes pod and routing mode", ({ selection, label }) => { + render( + , + ); + + expect(screen.getByText("Pod")).toBeInTheDocument(); + expect(screen.getByText(`10.0.0.2 · ${label} · 3 pods`)).toBeInTheDocument(); + }); + + it("omits the Pod row when routing metadata is absent", () => { + render(); + + expect(screen.queryByText("Pod")).not.toBeInTheDocument(); + }); + it("should display guardrail label when guardrail data exists", () => { render( + {podRouting && typeof podRouting === "object" && ( + + {podRouting.pod_ip} + {" · "} + {KUBERNETES_POD_ROUTING_LABELS[podRouting.selection]} + {" · "} + {podRouting.pod_count} pods + + )} {logEntry.requester_ip_address && ( {logEntry.requester_ip_address} )} diff --git a/ui/litellm-dashboard/src/components/logs/types.ts b/ui/litellm-dashboard/src/components/logs/types.ts index ce4a72a6d38..dc5fa4aaf5b 100644 --- a/ui/litellm-dashboard/src/components/logs/types.ts +++ b/ui/litellm-dashboard/src/components/logs/types.ts @@ -10,6 +10,13 @@ export const LOGS_SORT_FIELD_MAP = { export type LogsSortField = keyof typeof LOGS_SORT_FIELD_MAP; +export type KubernetesPodRouting = { + service_host: string; + pod_ip: string; + pod_count: number; + selection: "round_robin" | "session_affinity" | "session_affinity_retry"; +}; + export type LogEntry = { request_id: string; litellm_call_id?: string | null; @@ -29,7 +36,9 @@ export type LogEntry = { user?: string; end_user?: string; custom_llm_provider?: string; - metadata?: Record; + metadata?: Record & { + kubernetes_pod_routing?: KubernetesPodRouting | null; + }; cache_hit: string; cache_key?: string; request_tags?: Record; From f8f6a01caec7f43f0d9dfa919097d865247ed064 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 09:58:20 -0700 Subject: [PATCH 05/20] feat(ui): show kubernetes pods on the model page (#44666) Show discovered Kubernetes pods on the model page for admins Resolves LIT-9168 Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/_types.py | 1 + .../model_management_endpoints.py | 88 ++++++++ .../router_utils/kubernetes_pod_discovery.py | 31 +++ .../endpointaudit/coverage_allowlist.txt | 1 + tests/unit/proxy/auth/test_route_checks.py | 54 +++++ .../test_model_management_endpoints.py | 207 ++++++++++++++++++ .../test_kubernetes_pod_discovery.py | 172 +++++++++++++++ .../KubernetesPodsCard.integration.test.tsx | 118 ++++++++++ .../model_info/KubernetesPodsCard.tsx | 89 ++++++++ .../src/components/model_info_view.test.tsx | 88 ++++++++ .../src/components/model_info_view.tsx | 9 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 63 ++++++ 12 files changed, 920 insertions(+), 1 deletion(-) create mode 100644 ui/litellm-dashboard/src/components/model_info/KubernetesPodsCard.integration.test.tsx create mode 100644 ui/litellm-dashboard/src/components/model_info/KubernetesPodsCard.tsx diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index c2cdd47304a..bc32978d477 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -668,6 +668,7 @@ class LiteLLMRoutes(enum.Enum): "/model/update", "/model/delete", "/model/info", + "/model/{model_id}/kubernetes_pods", "/jwt/key/mapping/new", "/jwt/key/mapping/update", "/jwt/key/mapping/delete", diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 7460522f53e..7eae193dd8a 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -168,6 +168,19 @@ if TYPE_CHECKING: from prisma import types as prisma_types router: Final = APIRouter() + + +class KubernetesPodsResponse(BaseModel): + model_config = ConfigDict(frozen=True) + + model_id: str + service_host: str + port: int | None + pod_ips: tuple[str, ...] + pod_count: int + error: str | None + + CLEARABLE_LITELLM_PARAMS: Final = frozenset({"cache_control_injection_points", "litellm_credential_name"}) NULL_CLEARABLE_LITELLM_PARAMS: Final = frozenset((*SPECIAL_MODEL_INFO_PARAMS, *CLEARABLE_LITELLM_PARAMS)) @@ -1163,6 +1176,77 @@ def update_db_model( return prisma_compatible_model_dict +@router.get( + "/model/{model_id}/kubernetes_pods", + response_model=KubernetesPodsResponse, + tags=["model management"], + dependencies=[Depends(user_api_key_auth)], +) +async def get_kubernetes_pods( + model_id: str, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> KubernetesPodsResponse: + if user_api_key_dict.user_role not in ( + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + ): + raise ProxyException( + message="Only proxy admins can view Kubernetes pods", + type=ProxyErrorTypes.auth_error.value, + code=status.HTTP_403_FORBIDDEN, + param=None, + ) + + from litellm.proxy.proxy_server import llm_router + + if llm_router is None: + raise ProxyException( + message=CommonProxyErrors.no_llm_router.value, + type=ProxyErrorTypes.internal_server_error.value, + code=status.HTTP_500_INTERNAL_SERVER_ERROR, + param=None, + ) + + deployment: Final[Mapping[str, object] | None] = _OPTIONAL_JSON_OBJECT.validate_python( + llm_router.get_model_info(id=model_id) # pyright: ignore[reportUnknownMemberType] # validated below + ) + if deployment is None: + raise ProxyException( + message=f"Model with id {model_id} not found", + type=ProxyErrorTypes.not_found_error.value, + code=status.HTTP_404_NOT_FOUND, + param="model_id", + ) + + lookup: Final = await llm_router.kubernetes_pod_discovery.async_lookup_pods(deployment) + if lookup is None: + raw_litellm_params: Final = deployment.get("litellm_params") + litellm_params: Final[Mapping[str, object] | None] = ( + _JSON_OBJECT.validate_python(raw_litellm_params) if isinstance(raw_litellm_params, Mapping) else None + ) + discovery_enabled: Final = litellm_params is not None and litellm_params.get("kubernetes_pod_discovery") is True + raise ProxyException( + message=( + "Kubernetes pod discovery is enabled for this model, but its api_base is not an http:// " + "service hostname, so no pods are resolved" + if discovery_enabled + else "Kubernetes pod discovery is not enabled for this model" + ), + type=ProxyErrorTypes.bad_request_error.value, + code=status.HTTP_400_BAD_REQUEST, + param="model_id", + ) + + return KubernetesPodsResponse( + model_id=model_id, + service_host=lookup.service_host, + port=lookup.port, + pod_ips=tuple(lookup.pod_ips), + pod_count=len(lookup.pod_ips), + error=lookup.error, + ) + + @router.patch( "/model/{model_id}/update", tags=["model management"], @@ -3257,6 +3341,10 @@ _deduplicate_litellm_router_models: Final = deduplicate_litellm_router_models _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) +_OPTIONAL_JSON_OBJECT: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter( + Mapping[str, object] | None, + config=ConfigDict(hide_input_in_errors=True), +) def model_info_as_mapping(model_info: object) -> Mapping[str, object] | None: diff --git a/litellm/router_utils/kubernetes_pod_discovery.py b/litellm/router_utils/kubernetes_pod_discovery.py index 05ba7be0344..34d99db1f6e 100644 --- a/litellm/router_utils/kubernetes_pod_discovery.py +++ b/litellm/router_utils/kubernetes_pod_discovery.py @@ -44,6 +44,14 @@ _PodSelection: TypeAlias = tuple[str, int, Literal["round_robin", "session_affin _DeploymentT: Final = TypeVar("_DeploymentT") +@dataclass(frozen=True, slots=True) +class KubernetesPodLookup: + service_host: str + port: int | None + pod_ips: tuple[str, ...] + error: str | None + + def _retry_count_from_metadata(metadata: object) -> int | None: if not isinstance(metadata, Mapping): return None @@ -188,6 +196,18 @@ class KubernetesPodDiscovery: finally: self._end_refresh(key) + async def async_lookup_pods(self, deployment: object) -> KubernetesPodLookup | None: + eligible: Final = self._eligible(deployment) + if eligible is None: + return None + _, (host, port), _ = eligible + result: Final = await self._async_getaddrinfo(host, port) + if isinstance(result, OSError): + if isinstance(result, socket.gaierror) and self._is_authoritative_no_pods(result): + return KubernetesPodLookup(service_host=host, port=port, pod_ips=(), error=None) + return KubernetesPodLookup(service_host=host, port=port, pod_ips=(), error=str(result)) + return KubernetesPodLookup(service_host=host, port=port, pod_ips=self._pod_ips(result), error=None) + @staticmethod def _session_id(request_kwargs: Mapping[str, object] | None) -> str | None: if request_kwargs is None: @@ -293,6 +313,17 @@ class KubernetesPodDiscovery: except OSError as error: return error + async def _async_getaddrinfo(self, host: str, port: int | None) -> Sequence[_AddressInfo] | OSError: + lookup: Final = asyncio.get_running_loop().getaddrinfo( + host, port, type=socket.SOCK_STREAM, flags=socket.AI_ADDRCONFIG + ) + try: + return await asyncio.wait_for(lookup, timeout=self.dns_timeout_seconds) + except (TimeoutError, asyncio.TimeoutError): + return socket.gaierror(socket.EAI_AGAIN, f"DNS lookup for {host} exceeded {self.dns_timeout_seconds}s") + except OSError as error: + return error + def _abandon_lookup( self, key: _CacheKey, diff --git a/terraform/provider/tools/endpointaudit/coverage_allowlist.txt b/terraform/provider/tools/endpointaudit/coverage_allowlist.txt index 3a45eef3606..00f8f3b6031 100644 --- a/terraform/provider/tools/endpointaudit/coverage_allowlist.txt +++ b/terraform/provider/tools/endpointaudit/coverage_allowlist.txt @@ -74,6 +74,7 @@ GET /guardrails/ui/major_airlines GET /guardrails/ui/provider_specific_params GET /key/aliases GET /model/deprecations +GET /model/{model_id}/kubernetes_pods GET /search_tools/ui/available_providers GET /team/available GET /team/metadata_schema diff --git a/tests/unit/proxy/auth/test_route_checks.py b/tests/unit/proxy/auth/test_route_checks.py index a519a4e576a..6de54aa4af6 100644 --- a/tests/unit/proxy/auth/test_route_checks.py +++ b/tests/unit/proxy/auth/test_route_checks.py @@ -2472,6 +2472,7 @@ def test_proxy_admin_viewer_can_access_settings_read_endpoints(route): ADMIN_VIEWER_REPORTED_GET_ROUTES = [ "/health/latest", "/credentials", + "/model/model-pods-1/kubernetes_pods", "/v1/mcp/network/client-ip", "/claude-code/plugins", "/policy/templates", @@ -3615,6 +3616,59 @@ def test_internal_user_still_blocked_from_another_users_info(): assert "key not allowed to access this user's info" in str(exc_info.value.detail) +def test_internal_user_cannot_access_kubernetes_pod_discovery_route() -> None: + route: Final = "/model/model-pods-1/kubernetes_pods" + user_obj: Final = LiteLLM_UserTable( + user_id="internal_user", + user_email="user@example.com", + user_role=LitellmUserRoles.INTERNAL_USER.value, + ) + valid_token: Final = UserAPIKeyAuth( + user_id="internal_user", + user_role=LitellmUserRoles.INTERNAL_USER.value, + ) + request: Final = MagicMock(spec=Request) + request.method = "GET" + request.query_params = {} + + assert RouteChecks.is_management_route(route=route) + with pytest.raises(Exception, match="Only proxy admin can be used"): + RouteChecks.non_proxy_admin_allowed_routes_check( + user_obj=user_obj, + _user_role=LitellmUserRoles.INTERNAL_USER.value, + route=route, + request=request, + valid_token=valid_token, + request_data={}, + ) + + +def test_proxy_admin_viewer_can_access_kubernetes_pod_discovery_route() -> None: + route: Final = "/model/model-pods-1/kubernetes_pods" + user_obj: Final = LiteLLM_UserTable( + user_id="admin_viewer", + user_email="viewer@example.com", + user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY.value, + ) + valid_token: Final = UserAPIKeyAuth( + user_id="admin_viewer", + user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY.value, + ) + request: Final = MagicMock(spec=Request) + request.method = "GET" + request.query_params = {} + + assert RouteChecks.is_management_route(route=route) + RouteChecks.non_proxy_admin_allowed_routes_check( + user_obj=user_obj, + _user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY.value, + route=route, + request=request, + valid_token=valid_token, + request_data={}, + ) + + @pytest.mark.parametrize( "route", [ diff --git a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py index 7cd1107abce..bc714ad1cbc 100644 --- a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py @@ -2,6 +2,7 @@ import inspect import asyncio import contextlib import json +import socket from collections.abc import Iterator, Mapping from types import SimpleNamespace from typing import Dict, Final, Optional @@ -27,11 +28,13 @@ from litellm.proxy._types import ( ) from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper from litellm.proxy.management_endpoints.model_management_endpoints import ( + KubernetesPodsResponse, ModelManagementAuthChecks, _get_team_deployments, _raise_if_rate_limits_required_but_missing, clear_cache, delete_team_models, + get_kubernetes_pods, patch_model, update_model, ) @@ -44,6 +47,210 @@ async def _passthrough_row(update_data): return update_data +def _kubernetes_pod_router( + discovery_enabled: bool | None = True, + api_base: str = "http://vllm-headless.ns.svc.cluster.local:8000/v1", +) -> Router: + return Router( + model_list=[ + { + "model_name": "gpu-llama", + "litellm_params": { + "model": "openai/test-model", + "api_base": api_base, + "api_key": "not-in-response", + **({"kubernetes_pod_discovery": discovery_enabled} if discovery_enabled is not None else {}), + }, + "model_info": {"id": "model-pods-1"}, + } + ], + ignore_invalid_deployments=True, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "user_role", + [LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY], +) +async def test_get_kubernetes_pods_returns_sorted_read_only_response( + monkeypatch: pytest.MonkeyPatch, + user_role: LitellmUserRoles, +) -> None: + from litellm.proxy import proxy_server + + router: Final = _kubernetes_pod_router() + loop: Final = asyncio.get_running_loop() + + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[tuple[socket.AddressFamily, socket.SocketKind, int, str, tuple[str, int]]]: + return [ + (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", ("10.0.0.3", 8000)), + (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", ("10.0.0.1", 8000)), + (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", ("10.0.0.1", 8000)), + ] + + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(loop, "getaddrinfo", getaddrinfo) + + response: Final = await get_kubernetes_pods( + model_id="model-pods-1", + user_api_key_dict=UserAPIKeyAuth(user_id="proxy-admin", user_role=user_role), + ) + + assert isinstance(response, KubernetesPodsResponse) + assert response.model_dump() == { + "model_id": "model-pods-1", + "service_host": "vllm-headless.ns.svc.cluster.local", + "port": 8000, + "pod_ips": ("10.0.0.1", "10.0.0.3"), + "pod_count": 2, + "error": None, + } + + +@pytest.mark.asyncio +async def test_get_kubernetes_pods_returns_dns_error_without_exposing_deployment( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy import proxy_server + + router: Final = _kubernetes_pod_router() + loop: Final = asyncio.get_running_loop() + + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[tuple[socket.AddressFamily, socket.SocketKind, int, str, tuple[str, int]]]: + raise OSError("temporary DNS failure") + + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(loop, "getaddrinfo", getaddrinfo) + + response: Final = await get_kubernetes_pods( + model_id="model-pods-1", + user_api_key_dict=UserAPIKeyAuth(user_id="proxy-admin", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + + assert response.model_dump() == { + "model_id": "model-pods-1", + "service_host": "vllm-headless.ns.svc.cluster.local", + "port": 8000, + "pod_ips": (), + "pod_count": 0, + "error": "temporary DNS failure", + } + + +@pytest.mark.asyncio +async def test_get_kubernetes_pods_returns_404_for_unknown_model(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "llm_router", _kubernetes_pod_router()) + + with pytest.raises(ProxyException) as error: + await get_kubernetes_pods( + model_id="unknown", + user_api_key_dict=UserAPIKeyAuth(user_id="proxy-admin", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + + assert error.value.code == "404" + assert error.value.type == "not_found_error" + assert error.value.param == "model_id" + + +@pytest.mark.asyncio +async def test_get_kubernetes_pods_returns_500_without_router(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "llm_router", None) + + with pytest.raises(ProxyException) as error: + await get_kubernetes_pods( + model_id="model-pods-1", + user_api_key_dict=UserAPIKeyAuth(user_id="proxy-admin", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + + assert error.value.code == "500" + assert error.value.type == "internal_server_error" + + +@pytest.mark.asyncio +async def test_get_kubernetes_pods_returns_400_when_discovery_is_disabled( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "llm_router", _kubernetes_pod_router(discovery_enabled=None)) + + with pytest.raises(ProxyException) as error: + await get_kubernetes_pods( + model_id="model-pods-1", + user_api_key_dict=UserAPIKeyAuth(user_id="proxy-admin", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + + assert error.value.code == "400" + assert error.value.message == "Kubernetes pod discovery is not enabled for this model" + assert error.value.param == "model_id" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "api_base", + [ + "https://vllm-headless.ns.svc.cluster.local:8000/v1", + "http://192.0.2.10:8000/v1", + ], +) +async def test_get_kubernetes_pods_returns_400_for_ineligible_api_base( + monkeypatch: pytest.MonkeyPatch, + api_base: str, +) -> None: + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "llm_router", _kubernetes_pod_router(api_base=api_base)) + expected_message: Final = ( + "Kubernetes pod discovery is enabled for this model, but its api_base is not an http:// " + "service hostname, so no pods are resolved" + ) + + with pytest.raises(ProxyException) as error: + await get_kubernetes_pods( + model_id="model-pods-1", + user_api_key_dict=UserAPIKeyAuth(user_id="proxy-admin", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + + assert error.value.code == "400" + assert error.value.message == expected_message + assert error.value.param == "model_id" + + +@pytest.mark.asyncio +async def test_get_kubernetes_pods_rejects_internal_users(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "llm_router", _kubernetes_pod_router()) + + with pytest.raises(ProxyException) as error: + await get_kubernetes_pods( + model_id="model-pods-1", + user_api_key_dict=UserAPIKeyAuth(user_id="internal-user", user_role=LitellmUserRoles.INTERNAL_USER), + ) + + assert error.value.code == "403" + assert error.value.type == "auth_error" + + @pytest.mark.asyncio @pytest.mark.parametrize("model", ["openai/test-model", "auto_router/complexity_router"]) @pytest.mark.parametrize("team_role", ["admin", "user"]) diff --git a/tests/unit/router_utils/test_kubernetes_pod_discovery.py b/tests/unit/router_utils/test_kubernetes_pod_discovery.py index 001b8863b17..115e2a49bc5 100644 --- a/tests/unit/router_utils/test_kubernetes_pod_discovery.py +++ b/tests/unit/router_utils/test_kubernetes_pod_discovery.py @@ -87,6 +87,178 @@ def _records(*ips: str, port: int = 8000) -> list[_AddrInfo]: return [_record(ip, port) for ip in ips] +@pytest.mark.asyncio +async def test_async_lookup_pods_returns_sorted_unique_addresses(monkeypatch: pytest.MonkeyPatch) -> None: + loop: Final = asyncio.get_running_loop() + + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + assert host == "vllm-headless.ns.svc.cluster.local" + assert port == 8000 + assert type == socket.SOCK_STREAM + return _records("10.0.0.3", "10.0.0.1", "10.0.0.2", "10.0.0.1") + + monkeypatch.setattr(loop, "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment) + + lookup: Final = await discovery.async_lookup_pods(_deployment()) + + assert lookup is not None + assert lookup.service_host == "vllm-headless.ns.svc.cluster.local" + assert lookup.port == 8000 + assert lookup.pod_ips == ("10.0.0.1", "10.0.0.2", "10.0.0.3") + assert lookup.error is None + + +@pytest.mark.asyncio +async def test_async_lookup_pods_treats_authoritative_no_pods_as_empty( + monkeypatch: pytest.MonkeyPatch, +) -> None: + loop: Final = asyncio.get_running_loop() + + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + raise socket.gaierror(socket.EAI_NONAME, "no ready pods") + + monkeypatch.setattr(loop, "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment) + + lookup: Final = await discovery.async_lookup_pods(_deployment()) + + assert lookup is not None + assert lookup.pod_ips == () + assert lookup.error is None + + +@pytest.mark.asyncio +async def test_async_lookup_pods_returns_non_authoritative_dns_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + loop: Final = asyncio.get_running_loop() + + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + raise OSError("temporary DNS failure") + + monkeypatch.setattr(loop, "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment) + + lookup: Final = await discovery.async_lookup_pods(_deployment()) + + assert lookup is not None + assert lookup.pod_ips == () + assert lookup.error == "temporary DNS failure" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("api_base", "discovery_enabled"), + [ + (_SERVICE_URL, False), + ("https://vllm-headless.ns.svc.cluster.local:8000/v1", True), + ("http://10.0.0.4:8000/v1", True), + ], +) +async def test_async_lookup_pods_skips_ineligible_deployments( + monkeypatch: pytest.MonkeyPatch, + api_base: str, + discovery_enabled: bool, +) -> None: + loop: Final = asyncio.get_running_loop() + lookups: Final = count() + + async def getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(lookups) + return _records("10.0.0.1") + + monkeypatch.setattr(loop, "getaddrinfo", getaddrinfo) + discovery: Final = KubernetesPodDiscovery(proxy_environment=_empty_proxy_environment) + + lookup: Final = await discovery.async_lookup_pods( + _deployment(api_base=api_base, kubernetes_pod_discovery=discovery_enabled) + ) + + assert lookup is None + assert next(lookups) == 0 + + +@pytest.mark.asyncio +async def test_async_lookup_pods_does_not_cache_or_advance_round_robin_cursor( + monkeypatch: pytest.MonkeyPatch, +) -> None: + loop: Final = asyncio.get_running_loop() + async_lookups: Final = count() + sync_lookups: Final = count() + + async def async_getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(async_lookups) + return _records("10.0.0.9") + + def sync_getaddrinfo( + host: str | None, + port: str | int | None, + family: int = 0, + type: int = 0, + proto: int = 0, + flags: int = 0, + ) -> list[_AddrInfo]: + next(sync_lookups) + return _records("10.0.0.1", "10.0.0.2") + + monkeypatch.setattr(loop, "getaddrinfo", async_getaddrinfo) + monkeypatch.setattr(socket, "getaddrinfo", sync_getaddrinfo) + discovery: Final = KubernetesPodDiscovery( + refresh_interval_seconds=30, + proxy_environment=_empty_proxy_environment, + ) + + first_lookup: Final = await discovery.async_lookup_pods(_deployment()) + first_resolution: Final = discovery.resolve_deployment(_deployment()) + second_lookup: Final = await discovery.async_lookup_pods(_deployment()) + second_resolution: Final = discovery.resolve_deployment(_deployment()) + + assert first_lookup is not None + assert first_lookup.pod_ips == ("10.0.0.9",) + assert second_lookup is not None + assert second_lookup.pod_ips == ("10.0.0.9",) + assert _api_base(first_resolution) == "http://10.0.0.1:8000/v1" + assert _api_base(second_resolution) == "http://10.0.0.2:8000/v1" + assert next(async_lookups) == 2 + assert next(sync_lookups) == 1 + + def _clock(ticks: tuple[float, ...]) -> Callable[[], float]: timestamps: Final = iter(ticks) diff --git a/ui/litellm-dashboard/src/components/model_info/KubernetesPodsCard.integration.test.tsx b/ui/litellm-dashboard/src/components/model_info/KubernetesPodsCard.integration.test.tsx new file mode 100644 index 00000000000..3725b1181ad --- /dev/null +++ b/ui/litellm-dashboard/src/components/model_info/KubernetesPodsCard.integration.test.tsx @@ -0,0 +1,118 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import userEvent from "@testing-library/user-event"; +import { renderWithProviders, screen, testQueryClient } from "@/../tests/test-utils"; +import KubernetesPodsCard from "./KubernetesPodsCard"; + +const podResponse = { + model_id: "model-pods-1", + service_host: "vllm-headless.ns.svc.cluster.local", + port: 8000, + pod_ips: ["10.0.0.1", "10.0.0.2", "10.0.0.3"], + pod_count: 3, + error: null, +}; + +const lookupErrorResponse = { + ...podResponse, + pod_ips: [], + pod_count: 0, + error: "temporary DNS failure", +}; + +const network = vi.fn(); + +describe("KubernetesPodsCard", () => { + beforeEach(() => { + testQueryClient.clear(); + network.mockReset().mockResolvedValue(Response.json(podResponse)); + vi.stubGlobal("fetch", network); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + testQueryClient.clear(); + }); + + it("shows the service host, ready count, and all pod IPs", async () => { + renderWithProviders(); + + expect(await screen.findByText("vllm-headless.ns.svc.cluster.local:8000")).toBeInTheDocument(); + expect(screen.getByText("3 ready pods")).toBeInTheDocument(); + expect(screen.getByText("10.0.0.1")).toBeInTheDocument(); + expect(screen.getByText("10.0.0.2")).toBeInTheDocument(); + expect(screen.getByText("10.0.0.3")).toBeInTheDocument(); + }); + + it("shows an empty state when no pods are ready", async () => { + network.mockResolvedValueOnce( + Response.json({ + ...podResponse, + pod_ips: [], + pod_count: 0, + }), + ); + + renderWithProviders(); + + expect(await screen.findByText("No ready pods")).toBeInTheDocument(); + expect(screen.queryByText("0 ready pods")).not.toBeInTheDocument(); + }); + + it("shows a pod lookup error", async () => { + network.mockResolvedValueOnce(Response.json(lookupErrorResponse)); + + renderWithProviders(); + + expect(await screen.findByRole("alert")).toHaveTextContent("temporary DNS failure"); + expect(screen.queryByText("0 ready pods")).not.toBeInTheDocument(); + }); + + it("shows an API error", async () => { + network.mockResolvedValueOnce(Response.json({ detail: "DNS lookup unavailable" }, { status: 503 })); + + renderWithProviders(); + + expect(await screen.findByRole("alert")).toHaveTextContent("DNS lookup unavailable"); + }); + + it("shows the API message for an ineligible Kubernetes service host", async () => { + const message = + "Kubernetes pod discovery is enabled for this model, but its api_base is not an http:// " + + "service hostname, so no pods are resolved"; + network.mockResolvedValueOnce( + Response.json( + { + error: { + message, + type: "bad_request_error", + param: "model_id", + code: "400", + }, + }, + { status: 400 }, + ), + ); + + renderWithProviders(); + + expect(await screen.findByRole("alert")).toHaveTextContent(message); + }); + + it("refreshes the pod lookup when requested", async () => { + const user: ReturnType = userEvent.setup(); + renderWithProviders(); + + expect(await screen.findByText("10.0.0.1")).toBeInTheDocument(); + network.mockResolvedValueOnce( + Response.json({ + ...podResponse, + pod_ips: ["10.0.0.4"], + pod_count: 1, + }), + ); + await user.click(screen.getByRole("button", { name: "Refresh" })); + + expect(await screen.findByText("10.0.0.4")).toBeInTheDocument(); + expect(network).toHaveBeenCalledTimes(2); + }); +}); diff --git a/ui/litellm-dashboard/src/components/model_info/KubernetesPodsCard.tsx b/ui/litellm-dashboard/src/components/model_info/KubernetesPodsCard.tsx new file mode 100644 index 00000000000..054cd36254b --- /dev/null +++ b/ui/litellm-dashboard/src/components/model_info/KubernetesPodsCard.tsx @@ -0,0 +1,89 @@ +"use client"; + +import { $api } from "@/lib/http/api"; +import { Button } from "@/components/ui/button"; +import { Card } from "@/components/ui/card"; +import type { components } from "@/lib/http/schema"; + +interface KubernetesPodsCardProps { + modelId: string; +} + +interface KubernetesPodsCardContentProps { + data: components["schemas"]["KubernetesPodsResponse"] | undefined; + isError: boolean; + isPending: boolean; + apiErrorMessage: string; +} + +function KubernetesPodsCardContent({ data, isError, isPending, apiErrorMessage }: KubernetesPodsCardContentProps) { + const hasData = data !== undefined && !isError; + const hasLookupError = hasData && data.error !== null; + const showEmptyState = hasData && data.error === null && data.pod_ips.length === 0; + const showPodIps = hasData && data.error === null && data.pod_ips.length > 0; + const podCount = hasData && !hasLookupError ? data.pod_count : null; + + return ( + <> + {podCount !== null && podCount > 0 &&

{podCount} ready pods

} + {isPending && ( +

+ Loading Kubernetes pods +

+ )} + {isError && ( +

+ {apiErrorMessage} +

+ )} + {data && hasLookupError && ( +

+ {data.error} +

+ )} + {data && showEmptyState &&

No ready pods

} + {data && showPodIps && ( +
    + {data.pod_ips.map((ip) => ( +
  • {ip}
    • + ))} +
+ )} + + ); +} + +export default function KubernetesPodsCard({ modelId }: KubernetesPodsCardProps) { + const { data, error, isError, isPending, refetch } = $api.useQuery( + "get", + "/model/{model_id}/kubernetes_pods", + { params: { path: { model_id: modelId } } }, + { refetchInterval: 10_000 }, + ); + const apiErrorMessage = + error instanceof Error ? error.message : JSON.stringify(error) ?? "Failed to load Kubernetes pods"; + + return ( + +
+
+

Kubernetes pods

+ {data && ( +

+ {data.port === null ? data.service_host : `${data.service_host}:${data.port}`} +

+ )} +
+ +
+ + + ); +} diff --git a/ui/litellm-dashboard/src/components/model_info_view.test.tsx b/ui/litellm-dashboard/src/components/model_info_view.test.tsx index 6e7bc9a86d7..2e85fab5b76 100644 --- a/ui/litellm-dashboard/src/components/model_info_view.test.tsx +++ b/ui/litellm-dashboard/src/components/model_info_view.test.tsx @@ -6,6 +6,9 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import ModelInfoView from "./model_info_view"; import { toast } from "@/lib/toast"; import * as networking from "./networking"; +vi.mock("./model_info/KubernetesPodsCard", () => ({ + default: ({ modelId }: { modelId: string }) =>
{modelId}
, +})); vi.mock( "@/app/(dashboard)/hooks/autoRouter/useComplexityScorerDefaults", async () => await import("../../tests/mocks/complexityScorerDefaults"), @@ -196,6 +199,91 @@ describe("ModelInfoView", () => { }); }); + it("shows Kubernetes pods for discovery-enabled models", async () => { + mockUseModelsInfo.mockReturnValue({ + data: { + data: [ + { + ...defaultModelData, + litellm_params: { ...defaultModelData.litellm_params, kubernetes_pod_discovery: true }, + }, + ], + }, + isLoading: false, + error: null, + }); + + render(, { wrapper }); + + expect(await screen.findByTestId("kubernetes-pods-card")).toHaveTextContent("123"); + }); + + it("shows Kubernetes pods for admin viewers", async () => { + mockUseModelsInfo.mockReturnValue({ + data: { + data: [ + { + ...defaultModelData, + litellm_params: { ...defaultModelData.litellm_params, kubernetes_pod_discovery: true }, + }, + ], + }, + isLoading: false, + error: null, + }); + + render(, { wrapper }); + + expect(await screen.findByTestId("kubernetes-pods-card")).toHaveTextContent("123"); + }); + + it("hides Kubernetes pods for non-admin users", async () => { + mockUseModelsInfo.mockReturnValue({ + data: { + data: [ + { + ...defaultModelData, + litellm_params: { ...defaultModelData.litellm_params, kubernetes_pod_discovery: true }, + }, + ], + }, + isLoading: false, + error: null, + }); + + render(, { wrapper }); + + expect(await screen.findByText("Model Settings")).toBeInTheDocument(); + expect(screen.queryByTestId("kubernetes-pods-card")).not.toBeInTheDocument(); + }); + + it("hides Kubernetes pods when discovery is disabled", async () => { + mockUseModelsInfo.mockReturnValue({ + data: { + data: [ + { + ...defaultModelData, + litellm_params: { ...defaultModelData.litellm_params, kubernetes_pod_discovery: false }, + }, + ], + }, + isLoading: false, + error: null, + }); + + render(, { wrapper }); + + expect(await screen.findByText("Model Settings")).toBeInTheDocument(); + expect(screen.queryByTestId("kubernetes-pods-card")).not.toBeInTheDocument(); + }); + + it("hides Kubernetes pods when discovery is not configured", async () => { + render(, { wrapper }); + + expect(await screen.findByText("Model Settings")).toBeInTheDocument(); + expect(screen.queryByTestId("kubernetes-pods-card")).not.toBeInTheDocument(); + }); + it("should display loading state when model data is loading", () => { mockUseModelsInfo.mockReturnValue({ data: null, diff --git a/ui/litellm-dashboard/src/components/model_info_view.tsx b/ui/litellm-dashboard/src/components/model_info_view.tsx index ae34b17d0e8..74c1e498594 100644 --- a/ui/litellm-dashboard/src/components/model_info_view.tsx +++ b/ui/litellm-dashboard/src/components/model_info_view.tsx @@ -23,7 +23,7 @@ import { isComplexityRouter as isComplexityRouterParams, } from "./add_model/auto_router_strategies"; import { canEditAutoRouter, canModifyModel } from "@/utils/modelPermissions"; -import { teamsUserCanAssign } from "@/utils/roles"; +import { isProxyAdminTierRole, teamsUserCanAssign } from "@/utils/roles"; import { useTeams } from "@/app/(dashboard)/hooks/teams/useTeams"; import DeleteResourceModal from "./common_components/DeleteResourceModal"; import EditAutoRouterModal from "./edit_auto_router/edit_auto_router_modal"; @@ -50,6 +50,7 @@ import ModelInfoEditForm, { type ModelEditSubmitValues, type TouchedPricingField import { Tag } from "./tag_management/types"; import { getDisplayModelName } from "./view_model/model_name_display"; import { Dialog, DialogContent, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog"; +import KubernetesPodsCard from "./model_info/KubernetesPodsCard"; interface ModelInfoViewProps { modelId: string; @@ -680,6 +681,12 @@ export default function ModelInfoView({ + {modelData?.litellm_params?.kubernetes_pod_discovery === true && isProxyAdminTierRole(userRole ?? "") && ( +
+ +
+ )} + {/* Settings Card */}
diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index a111ed1d773..d697ccac98d 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -10187,6 +10187,23 @@ export interface paths { patch?: never; trace?: never; }; + "/model/{model_id}/kubernetes_pods": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Kubernetes Pods */ + get: operations["get_kubernetes_pods_model__model_id__kubernetes_pods_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/model/{model_id}/update": { parameters: { query?: never; @@ -33764,6 +33781,21 @@ export interface components { */ tier: string; }; + /** KubernetesPodsResponse */ + KubernetesPodsResponse: { + /** Error */ + error: string | null; + /** Model Id */ + model_id: string; + /** Pod Count */ + pod_count: number; + /** Pod Ips */ + pod_ips: string[]; + /** Port */ + port: number | null; + /** Service Host */ + service_host: string; + }; /** LLMV2Calibration */ LLMV2Calibration: { capable: components["schemas"]["LLMV2ProbabilityCalibration"]; @@ -65038,6 +65070,37 @@ export interface operations { }; }; }; + get_kubernetes_pods_model__model_id__kubernetes_pods_get: { + parameters: { + query?: never; + header?: never; + path: { + model_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["KubernetesPodsResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; patch_model_model__model_id__update_patch: { parameters: { query?: never; From c82e8244a57862feecfb48218bdd20c72c2afe6d Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:02:56 +0000 Subject: [PATCH 06/20] fix(count_tokens)!: count tools the way DeepSeek V4 and Qwen3.5 chat templates render them (#47173) * fix(count_tokens): count tools the way DeepSeek V4 and Qwen3.5 chat templates render them * refactor(count_tokens): hold the tool counting wiring for a follow-up commit * refactor(count_tokens): hold the chat template tool sections for a follow-up commit * fix(count_tokens): count tools the way DeepSeek V4 and Qwen3.5 chat templates render them * fix(count_tokens): skip tool_choice and non-function tools for chat template families, leave out Fireworks V4.1 spelling * fix(count_tokens): count Anthropic tools in the shape the messages bridge sends them A missing description is no longer rendered as null and strict is counted when set, matching what DeepSeek V4 and Qwen3.5 deployments receive. Adds tests that pin each template's schema rendering and its system turn. * test(count_tokens): check the prompt caching threshold agrees with token_counter for DeepSeek V4 and Qwen3.5 tools * fix(count_tokens): skip Anthropic web search tools and count unmapped Anthropic tool keys for chat template families * test(token_counter): cover chat-template tool counts across count routes, budgets, ITPM and context edits Integration cells check that DeepSeek V4 and Qwen3.5 tools are counted the way their chat templates render them on /v1/messages/count_tokens, /utils/token_counter, /v1/responses/input_tokens, strict budget reservation, project ITPM on chat, messages and responses, and the clear_tool_uses trigger, with grok controls keeping OpenAI framing --- .../prompt_templates/chat_template_tools.py | 37 ++ litellm/litellm_core_utils/token_counter.py | 117 ++++-- .../test_chat_template_tool_count_wire.py | 383 ++++++++++++++++++ ...ls_prompt_templates_chat_template_tools.py | 31 ++ .../litellm_core_utils/test_token_counter.py | 189 ++++++++- 5 files changed, 728 insertions(+), 29 deletions(-) create mode 100644 litellm/litellm_core_utils/prompt_templates/chat_template_tools.py create mode 100644 tests/integration/providers/test_chat_template_tool_count_wire.py create mode 100644 tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_chat_template_tools.py diff --git a/litellm/litellm_core_utils/prompt_templates/chat_template_tools.py b/litellm/litellm_core_utils/prompt_templates/chat_template_tools.py new file mode 100644 index 00000000000..3c437ca7fbb --- /dev/null +++ b/litellm/litellm_core_utils/prompt_templates/chat_template_tools.py @@ -0,0 +1,37 @@ +import json +import re +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from typing import Final + + +@dataclass(frozen=True, slots=True) +class ChatTemplateToolSection: + preamble_tokens: int + implicit_system_turn_tokens: int + render_tool_schemas: Callable[[Sequence[Mapping[str, object]]], str] + + +def _deepseek_v4_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str: + return "\n".join(json.dumps(function, ensure_ascii=False) for function in functions) + + +def _qwen3_5_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str: + return "".join( + "\n" + json.dumps({"type": "function", "function": function}, ensure_ascii=False) for function in functions + ) + + +_TOOL_SECTIONS: Final = ( + ( + re.compile(r"deepseek-v4(?![.\d]|p\d)", re.IGNORECASE), + ChatTemplateToolSection(222, 0, _deepseek_v4_tool_schemas), + ), + (re.compile(r"qwen3[._]?5(?!\d)", re.IGNORECASE), ChatTemplateToolSection(196, 5, _qwen3_5_tool_schemas)), +) + + +def chat_template_tool_section(model: str | None) -> ChatTemplateToolSection | None: + if not model: + return None + return next((section for pattern, section in _TOOL_SECTIONS if pattern.search(model)), None) diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py index 7ac3e9c80c5..8aded69aed2 100644 --- a/litellm/litellm_core_utils/token_counter.py +++ b/litellm/litellm_core_utils/token_counter.py @@ -35,6 +35,10 @@ from litellm.constants import ( TOKEN_COUNTER_MAX_EXACT_CHARS, ) from litellm.litellm_core_utils.asyncify import asyncify +from litellm.litellm_core_utils.prompt_templates.chat_template_tools import ( + ChatTemplateToolSection, + chat_template_tool_section, +) from litellm.litellm_core_utils.tokenizer import HuggingFaceTokenizer, OpenAIEncoding if TYPE_CHECKING: @@ -347,6 +351,9 @@ EXTRAPOLATION_SAMPLES: Final = 16 T_ParamSpec: Final = ParamSpec("T_ParamSpec") T_Retval = TypeVar("T_Retval") _COUNT_OFFLOAD_LIMITER: Final = anyio.lowlevel.RunVar[anyio.CapacityLimiter]("litellm_count_offload_limiter") +_ANTHROPIC_TOOL_KEYS_MAPPED_BY_BRIDGE: Final = frozenset( + ("name", "input_schema", "description", "cache_control", "strict", "type", "eager_input_streaming") +) def _count_offload_limiter_for_this_loop() -> anyio.CapacityLimiter: @@ -404,6 +411,7 @@ class _MessageCountParams: self.tokens_per_message = 3 self.tokens_per_name = 1 self.count_function = _get_count_function(model, custom_tokenizer) + self.tool_section = chat_template_tool_section(model) def token_counter( @@ -467,7 +475,9 @@ def token_counter( num_tokens = _count_messages(params, new_messages, use_default_image_token_count, default_token_count) if count_response_tokens is False: includes_system_message: Final = any(message.get("role", None) == "system" for message in new_messages) - num_tokens += _count_extra(params.count_function, tools, tool_choice, includes_system_message) + num_tokens += _count_extra( + params.count_function, tools, tool_choice, includes_system_message, params.tool_section + ) else: raise ValueError("Either text or messages must be provided") @@ -501,7 +511,8 @@ def messages_reach_token_count( _count_messages(params, [message], use_default_image_token_count, None) for message in new_messages ) running_totals: Final = accumulate( - per_message_counts, initial=_count_extra(params.count_function, tools, None, includes_system_message) + per_message_counts, + initial=_count_extra(params.count_function, tools, None, includes_system_message, params.tool_section), ) return any(total >= threshold for total in running_totals) @@ -592,6 +603,7 @@ def _count_extra( tools: list[ChatCompletionToolParam] | None, tool_choice: ChatCompletionNamedToolChoiceParam | None, includes_system_message: bool, + tool_section: ChatTemplateToolSection | None, ) -> int: """Count extra tokens for function definitions and tool choices. Args: @@ -599,26 +611,56 @@ def _count_extra( tools (Optional[List[ChatCompletionToolParam]]): The available tools. tool_choice (Optional[ChatCompletionNamedToolChoiceParam]): The tool choice. includes_system_message (bool): Whether the messages include a system message. + tool_section (Optional[ChatTemplateToolSection]): How the model's chat template renders tools, if known. """ + reply_primer_tokens: Final = 3 + tool_choice_tokens: Final = 0 if tool_section is not None else _count_tool_choice(count_function, tool_choice) + return ( + reply_primer_tokens + + _count_tool_definitions(count_function, tools, includes_system_message, tool_section) + + tool_choice_tokens + ) - num_tokens = 3 # every reply is primed with <|start|>assistant<|message|> - if tools: - num_tokens += count_function(_format_function_definitions(tools)) - num_tokens += 9 # Additional tokens for function definition of tools - # If there's a system message and tools are present, subtract four tokens - if tools and includes_system_message: - num_tokens -= 4 - # If tool_choice is 'none', add one token. - # If it's an object, add 4 + the number of tokens in the function name. - # If it's undefined or 'auto', don't add anything. +def _count_tool_definitions( + count_function: TokenCounterFunction, + tools: Sequence[object] | None, + includes_system_message: bool, + tool_section: ChatTemplateToolSection | None, +) -> int: + if not tools: + return 0 + if tool_section is not None: + return _count_chat_template_tools(count_function, tools, includes_system_message, tool_section) + system_message_discount: Final = 4 if includes_system_message else 0 + return count_function(_format_function_definitions(tools)) + 9 - system_message_discount + + +def _count_tool_choice( + count_function: TokenCounterFunction, + tool_choice: ChatCompletionNamedToolChoiceParam | None, +) -> int: if tool_choice == "none": - num_tokens += 1 - elif isinstance(tool_choice, dict): - num_tokens += 7 - num_tokens += count_function(str(tool_choice["function"]["name"])) + return 1 + if isinstance(tool_choice, dict): + return 7 + count_function(str(tool_choice["function"]["name"])) + return 0 - return num_tokens + +def _count_chat_template_tools( + count_function: TokenCounterFunction, + tools: Sequence[object], + includes_system_message: bool, + tool_section: ChatTemplateToolSection, +) -> int: + functions: Final = tuple( + function for function in _function_definitions(_tools_sent_as_functions(tools)) if function.get("name") + ) + if not functions: + return 0 + system_turn_tokens: Final = 0 if includes_system_message else tool_section.implicit_system_turn_tokens + schemas: Final = tool_section.render_tool_schemas(functions) + return tool_section.preamble_tokens + system_turn_tokens + count_function(schemas) def _get_extrapolating_count_function( @@ -1034,15 +1076,31 @@ def _format_function_definitions(tools: Sequence[object]) -> str: lines: Final = [] lines.append("namespace functions {") lines.append("") - for tool in tools: - if not isinstance(tool, Mapping): - continue - for function in _function_definitions_for_tool(cast(Mapping[str, object], tool)): - lines.extend(_format_single_function_definition(function)) + for function in _function_definitions(tools): + lines.extend(_format_single_function_definition(function)) lines.append("} // namespace functions") return "\n".join(lines) +def _tools_sent_as_functions(tools: Sequence[object]) -> tuple[Mapping[str, object], ...]: + return tuple(tool for tool in _tool_mappings(tools) if not _is_anthropic_web_search_tool(tool)) + + +def _is_anthropic_web_search_tool(tool: Mapping[str, object]) -> bool: + return tool.get("name") == "web_search" and "input_schema" not in tool and tool.get("type") != "function" + + +def _tool_mappings(tools: Sequence[object]) -> Iterable[Mapping[str, object]]: + for tool in tools: + if isinstance(tool, Mapping): + yield cast(Mapping[str, object], tool) + + +def _function_definitions(tools: Sequence[object]) -> Iterable[Mapping[str, object]]: + for tool in _tool_mappings(tools): + yield from _function_definitions_for_tool(tool) + + def _function_definitions_for_tool(tool: Mapping[str, object]) -> Iterable[Mapping[str, object]]: function: Final = tool.get("function") if isinstance(function, Mapping): @@ -1054,12 +1112,21 @@ def _function_definitions_for_tool(tool: Mapping[str, object]) -> Iterable[Mappi if isinstance(declaration, Mapping): yield declaration return - parameters: Final = tool.get("input_schema") or tool.get("parameters") or {} + input_schema: Final = tool.get("input_schema") + parameters: Final = input_schema or tool.get("parameters") or {} normalized_parameters: Final = parameters if isinstance(parameters, Mapping) else {} + unmapped_anthropic_keys: Final = ( + {key: value for key, value in tool.items() if key not in _ANTHROPIC_TOOL_KEYS_MAPPED_BY_BRIDGE} + if input_schema is not None + else {} + ) + description: Final = tool.get("description") + strict: Final = tool.get("strict") yield { "name": tool.get("name"), - "description": tool.get("description"), - "parameters": normalized_parameters, + "parameters": {**normalized_parameters, **unmapped_anthropic_keys}, + **({"description": description} if description is not None else {}), + **({"strict": bool(strict)} if strict is not None else {}), } diff --git a/tests/integration/providers/test_chat_template_tool_count_wire.py b/tests/integration/providers/test_chat_template_tool_count_wire.py new file mode 100644 index 00000000000..d2b85e7168f --- /dev/null +++ b/tests/integration/providers/test_chat_template_tool_count_wire.py @@ -0,0 +1,383 @@ +import json +import re +import uuid +from collections.abc import Iterator, Mapping, Sequence +from contextlib import ExitStack +from dataclasses import dataclass +from itertools import chain +from pathlib import Path +from typing import Final + +import httpx +import pytest +import yaml +from integration._support.client import ( + JSON_OBJECT, + Gateway, + Scenario, + gateway_from_environment, + object_value, + string_value, +) +from integration._support.openai_wire import chat_reply +from integration._support.process import graceful_stop_seconds, owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from litellm.litellm_core_utils.default_encoding import encoding as cl100k_base +from pydantic import JsonValue + +pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120) + +_API_KEY: Final = "scripted-provider-key" +_DEEPSEEK_V4: Final = "azure_ai/DeepSeek-V4-Flash-0731" +_QWEN3_5: Final = "databricks/databricks-qwen35-122b-a10b" +_GROK: Final = "azure_ai/grok-4-1" +_DEEPSEEK_V4_TOOL_PREAMBLE: Final = 222 +_QWEN3_5_TOOL_PREAMBLE: Final = 196 +_QWEN3_5_OPENED_SYSTEM_TURN: Final = 5 +_OPENAI_TOOL_OVERHEAD: Final = 9 +_SCHEMA: Final[dict[str, JsonValue]] = { + "type": "object", + "properties": {"query": {"type": "string"}}, + "required": ["query"], +} +_FUNCTION: Final[dict[str, JsonValue]] = {"name": "lookup", "parameters": _SCHEMA} +_OPENAI_NAMESPACE: Final = "namespace functions {\n\ntype lookup = (_: {\nquery: string,\n}) => any;\n\n} // namespace functions" +_ANTHROPIC_TOOL: Final[dict[str, JsonValue]] = {"name": "lookup", "input_schema": _SCHEMA} +_OPENAI_TOOL: Final[dict[str, JsonValue]] = {"type": "function", "function": _FUNCTION} +_RESPONSES_TOOL: Final[dict[str, JsonValue]] = {"type": "function", **_FUNCTION} +_NAMED_TOOL_CHOICE: Final[dict[str, JsonValue]] = {"type": "function", "function": {"name": "lookup"}} +_WEB_SEARCH_TOOL: Final[dict[str, JsonValue]] = {"type": "web_search_20250305", "name": "web_search", "max_uses": 3} +_PROMPT: Final = "Find record 14." +_MESSAGES: Final[list[JsonValue]] = [{"role": "user", "content": _PROMPT}] +_INPUT_PRICE: Final = 0.001 +_OUTPUT_PRICE: Final = 0.002 +_TINY_BUDGET: Final = 0.000000001 +_ESTIMATED_COST: Final = re.compile(r"Estimated request cost: ([^\s,]+)") + + +def _tokens(text: str) -> int: + return len(cl100k_base.encode(text)) + + +def _deepseek_v4_tool_tokens(function: Mapping[str, JsonValue]) -> int: + return _DEEPSEEK_V4_TOOL_PREAMBLE + _tokens(json.dumps(function, ensure_ascii=False)) + + +def _qwen3_5_tool_tokens(function: Mapping[str, JsonValue], *, with_system: bool) -> int: + opened_system_turn: Final = 0 if with_system else _QWEN3_5_OPENED_SYSTEM_TURN + schema: Final = json.dumps({"type": "function", "function": function}, ensure_ascii=False) + return _QWEN3_5_TOOL_PREAMBLE + opened_system_turn + _tokens("\n" + schema) + + +def _completion(_request: Request) -> Reply: + return chat_reply(f"chatcmpl-{uuid.uuid4().hex}", "scripted-model", "OK", stream=False) + + +def _deployment(scenario: Scenario, group: str, model: str, api_base: str) -> str: + name: Final = f"{group}-{uuid.uuid4().hex}" + created: Final = scenario.gateway.post( + "/model/new", + { + "model_name": name, + "litellm_params": {"model": model, "api_base": api_base, "api_key": _API_KEY}, + "model_info": {}, + }, + ) + scenario.cleanups.callback(scenario.delete_model, string_value(object_value(created["model_info"])["id"])) + return name + + +@dataclass(frozen=True, slots=True) +class _CountRoute: + path: str + field: str + tool: dict[str, JsonValue] + + def body(self, model: str, tools: Sequence[JsonValue]) -> dict[str, JsonValue]: + prompt: Final[dict[str, JsonValue]] = ( + {"input": _PROMPT} if self.path == "/v1/responses/input_tokens" else {"messages": _MESSAGES} + ) + return {"model": model, **prompt, **({"tools": list(tools)} if tools else {})} + + +_COUNT_ROUTES: Final[Mapping[str, _CountRoute]] = { + "messages-count-tokens": _CountRoute("/v1/messages/count_tokens", "input_tokens", _ANTHROPIC_TOOL), + "utils-token-counter": _CountRoute("/utils/token_counter", "total_tokens", _OPENAI_TOOL), + "responses-input-tokens": _CountRoute("/v1/responses/input_tokens", "input_tokens", _RESPONSES_TOOL), +} +_MESSAGES_COUNT: Final = _COUNT_ROUTES["messages-count-tokens"] + + +def _count(gateway: Gateway, route: _CountRoute, body: Mapping[str, JsonValue]) -> int: + response: Final = gateway.request("POST", route.path, body) + assert response.status_code == 200, response.text + counted: Final = JSON_OBJECT.validate_json(response.content)[route.field] + assert isinstance(counted, int), response.text + return counted + + +@pytest.mark.parametrize("route", sorted(_COUNT_ROUTES)) +def test_count_routes_count_deepseek_v4_tools_the_way_its_chat_template_renders_them( + gateway: Gateway, route: str +) -> None: + selected: Final = _COUNT_ROUTES[route] + with wire_server(_completion) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, "count", _DEEPSEEK_V4, wire.url) + without_tools: Final = _count(gateway, selected, selected.body(model, ())) + with_tool: Final = _count(gateway, selected, selected.body(model, (selected.tool,))) + upstream: Final = wire.drain() + assert upstream == () + assert with_tool - without_tools == _deepseek_v4_tool_tokens(_FUNCTION) + + +@pytest.mark.parametrize("with_system", (False, True), ids=("no-system", "system")) +def test_messages_count_tokens_counts_qwen3_5_tools_the_way_its_chat_template_renders_them( + gateway: Gateway, with_system: bool +) -> None: + system: Final[dict[str, JsonValue]] = {"system": "Be brief."} if with_system else {} + with wire_server(_completion) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, "count", _QWEN3_5, wire.url) + without_tools: Final = _count(gateway, _MESSAGES_COUNT, {**_MESSAGES_COUNT.body(model, ()), **system}) + with_tool: Final = _count( + gateway, _MESSAGES_COUNT, {**_MESSAGES_COUNT.body(model, (_ANTHROPIC_TOOL,)), **system} + ) + upstream: Final = wire.drain() + assert upstream == () + assert with_tool - without_tools == _qwen3_5_tool_tokens(_FUNCTION, with_system=with_system) + + +def test_messages_count_tokens_skips_web_search_and_counts_unmapped_tool_keys_for_deepseek_v4( + gateway: Gateway, +) -> None: + deferred_function: Final[dict[str, JsonValue]] = {"name": "lookup", "parameters": {**_SCHEMA, "defer_loading": True}} + with wire_server(_completion) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, "count", _DEEPSEEK_V4, wire.url) + plain: Final = _count(gateway, _MESSAGES_COUNT, _MESSAGES_COUNT.body(model, (_ANTHROPIC_TOOL,))) + with_web_search: Final = _count( + gateway, _MESSAGES_COUNT, _MESSAGES_COUNT.body(model, (_ANTHROPIC_TOOL, _WEB_SEARCH_TOOL)) + ) + deferred: Final = _count( + gateway, _MESSAGES_COUNT, _MESSAGES_COUNT.body(model, ({**_ANTHROPIC_TOOL, "defer_loading": True},)) + ) + upstream: Final = wire.drain() + assert upstream == () + assert with_web_search == plain + assert deferred - plain == _deepseek_v4_tool_tokens(deferred_function) - _deepseek_v4_tool_tokens(_FUNCTION) + + +def test_messages_count_tokens_keeps_openai_tool_framing_for_other_models(gateway: Gateway) -> None: + with wire_server(_completion) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, "count", _GROK, wire.url) + without_tools: Final = _count(gateway, _MESSAGES_COUNT, _MESSAGES_COUNT.body(model, ())) + plain: Final = _count(gateway, _MESSAGES_COUNT, _MESSAGES_COUNT.body(model, (_ANTHROPIC_TOOL,))) + deferred: Final = _count( + gateway, _MESSAGES_COUNT, _MESSAGES_COUNT.body(model, ({**_ANTHROPIC_TOOL, "defer_loading": True},)) + ) + upstream: Final = wire.drain() + assert upstream == () + assert plain - without_tools == _OPENAI_TOOL_OVERHEAD + _tokens(_OPENAI_NAMESPACE) + assert deferred == plain + + +@dataclass(frozen=True, slots=True) +class _Endpoint: + path: str + body: dict[str, JsonValue] + tools: dict[str, JsonValue] + + +_ENDPOINTS: Final[Mapping[str, _Endpoint]] = { + "chat-completions": _Endpoint( + "/v1/chat/completions", + {"messages": _MESSAGES, "max_tokens": 1}, + {"tools": [_OPENAI_TOOL], "tool_choice": _NAMED_TOOL_CHOICE}, + ), + "messages": _Endpoint( + "/v1/messages", + {"messages": _MESSAGES, "max_tokens": 1}, + {"tools": [_ANTHROPIC_TOOL], "tool_choice": {"type": "auto"}}, + ), + "responses": _Endpoint( + "/v1/responses", + {"input": _PROMPT, "max_output_tokens": 16}, + {"tools": [_RESPONSES_TOOL]}, + ), +} + + +def _send(gateway: Gateway, endpoint: _Endpoint, group: str, key: str, *, tools: bool) -> httpx.Response: + return gateway.request( + "POST", + endpoint.path, + {"model": group, **endpoint.body, **(endpoint.tools if tools else {})}, + key=key, + ) + + +@dataclass(frozen=True, slots=True) +class _BudgetRig: + gateway: Gateway + wire: Wire + group: str + + +def _strict_budget_config(directory: Path, group: str, api_base: str) -> Path: + base: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + config: Final = { + **base, + "model_list": [ + { + "model_name": group, + "litellm_params": { + "model": _DEEPSEEK_V4, + "api_base": api_base, + "api_key": _API_KEY, + "input_cost_per_token": _INPUT_PRICE, + "output_cost_per_token": _OUTPUT_PRICE, + }, + } + ], + "general_settings": {**object_value(base["general_settings"]), "fail_closed_budget_enforcement": True}, + } + path: Final = directory / f"chat-template-tool-budget-{uuid.uuid4().hex[:8]}.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +@pytest.fixture(scope="module") +def strict_budget(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_BudgetRig]: + directory: Final = tmp_path_factory.mktemp("chat-template-tool-budget") + group: Final = f"deepseek-v4-flash-{uuid.uuid4().hex}" + with ExitStack() as stack: + rig_gateway: Final = stack.enter_context(gateway_from_environment()) + wire: Final = stack.enter_context(wire_server(_completion)) + owned: Final = stack.enter_context( + owned_proxy_process(rig_gateway, directory, {}, config=_strict_budget_config(directory, group, wire.url)) + ) + yield _BudgetRig(owned.gateway, wire, group) + + +def _estimated_cost(response: httpx.Response) -> float: + assert response.status_code in (400, 422, 429), f"{response.status_code} {response.text}" + error: Final = object_value(JSON_OBJECT.validate_json(response.content)["error"]) + estimate: Final = _ESTIMATED_COST.search(string_value(error["message"])) + assert estimate is not None, response.text + return float(estimate[1]) + + +@pytest.mark.parametrize("endpoint", ("chat-completions", "messages")) +def test_budget_reservation_estimates_deepseek_v4_tools_the_way_its_chat_template_renders_them( + strict_budget: _BudgetRig, endpoint: str +) -> None: + selected: Final = _ENDPOINTS[endpoint] + with strict_budget.gateway.scenario() as scenario: + key: Final = scenario.key(models=[strict_budget.group], max_budget=_TINY_BUDGET) + without_tools: Final = _estimated_cost(_send(strict_budget.gateway, selected, strict_budget.group, key, tools=False)) + with_tools: Final = _estimated_cost(_send(strict_budget.gateway, selected, strict_budget.group, key, tools=True)) + assert strict_budget.wire.drain() == () + assert with_tools - without_tools == pytest.approx(_deepseek_v4_tool_tokens(_FUNCTION) * _INPUT_PRICE) + + +def _project_key(scenario: Scenario, group: str) -> str: + team: Final = scenario.team() + project: Final = scenario.project(team, model_itpm_limit={group: _deepseek_v4_tool_tokens(_FUNCTION)}) + return scenario.key(team_id=team, project_id=project) + + +@pytest.mark.parametrize("endpoint", sorted(_ENDPOINTS)) +def test_project_itpm_reserves_deepseek_v4_tools_the_way_its_chat_template_renders_them( + gateway: Gateway, endpoint: str +) -> None: + with wire_server(_completion) as wire, gateway.scenario() as scenario: + group: Final = _deployment(scenario, "deepseek-v4-flash", _DEEPSEEK_V4, wire.url) + response: Final = _send(gateway, _ENDPOINTS[endpoint], group, _project_key(scenario, group), tools=True) + upstream: Final = wire.drain() + assert response.status_code == 429, response.text + assert "model_per_project_itpm" in response.text, response.text + assert upstream == () + + +def test_project_itpm_keeps_openai_tool_framing_for_other_models(gateway: Gateway) -> None: + chat: Final = _ENDPOINTS["chat-completions"] + with wire_server(_completion) as wire, gateway.scenario() as scenario: + group: Final = _deployment(scenario, "grok-4-1", _GROK, wire.url) + response: Final = gateway.request( + "POST", + chat.path, + {"model": group, **chat.body, "tools": [_OPENAI_TOOL]}, + key=_project_key(scenario, group), + ) + upstream: Final = wire.drain() + assert response.status_code == 200, response.text + assert len(upstream) == 1, upstream + + +_FILES: Final = 3 +_READ_FILE_FUNCTION: Final[dict[str, JsonValue]] = { + "name": "read_file", + "parameters": {"type": "object", "properties": {"path": {"type": "string"}}}, +} +_READ_FILE_TOOL: Final[dict[str, JsonValue]] = { + "name": "read_file", + "input_schema": _READ_FILE_FUNCTION["parameters"], +} + + +def _read_file_round(marker: str, index: int) -> tuple[JsonValue, ...]: + return ( + {"role": "user", "content": f"Read file {index}."}, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": f"toolu_{index}", "name": "read_file", "input": {"path": f"{index}.txt"}} + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": f"toolu_{index}", + "content": f"short file body.", + } + ], + }, + ) + + +def _agent_body(group: str, marker: str) -> dict[str, JsonValue]: + rounds: Final = tuple(chain.from_iterable(_read_file_round(marker, index) for index in range(_FILES))) + return { + "model": group, + "max_tokens": 16, + "tools": [_READ_FILE_TOOL], + "messages": [*rounds, {"role": "user", "content": "Summarize what you read."}], + "context_management": { + "edits": [ + { + "type": "clear_tool_uses_20250919", + "trigger": {"type": "input_tokens", "value": _deepseek_v4_tool_tokens(_READ_FILE_FUNCTION)}, + "keep": {"type": "tool_uses", "value": 1}, + } + ] + }, + } + + +@pytest.mark.parametrize( + ("model", "files_kept"), + ((_DEEPSEEK_V4, (False, False, True)), (_GROK, (True, True, True))), + ids=("deepseek-v4", "grok-4-1"), +) +def test_messages_context_management_trigger_counts_tools_the_way_the_models_chat_template_renders_them( + gateway: Gateway, model: str, files_kept: tuple[bool, ...] +) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_completion) as wire, gateway.scenario() as scenario: + group: Final = _deployment(scenario, "context", model, wire.url) + response: Final = gateway.request("POST", "/v1/messages", _agent_body(group, marker)) + upstream: Final = wire.drain() + assert response.status_code == 200, response.text + assert len(upstream) == 1, upstream + sent: Final = upstream[0].body.decode() + assert tuple(f"" in sent for index in range(_FILES)) == files_kept diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_chat_template_tools.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_chat_template_tools.py new file mode 100644 index 00000000000..51249e01d4d --- /dev/null +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_chat_template_tools.py @@ -0,0 +1,31 @@ +from typing import Final + +import pytest + +from litellm.litellm_core_utils.prompt_templates.chat_template_tools import chat_template_tool_section + +_DESCRIBED_FUNCTION: Final = {"name": "f", "description": "é", "parameters": {}} +_BARE_FUNCTION: Final = {"name": "g", "parameters": {}} + + +# Tool schemas as rendered by render_tools in encoding/encoding_dsv4.py (deepseek-ai/DeepSeek-V4-Flash) and by +# chat_template.jinja (Qwen/Qwen3.5-122B-A10B), both read on Hugging Face on 2026-10-17 +@pytest.mark.parametrize( + ("model", "rendered"), + [ + ( + "DeepSeek-V4-Flash-0731", + '{"name": "f", "description": "é", "parameters": {}}\n{"name": "g", "parameters": {}}', + ), + ( + "databricks-qwen35-122b-a10b", + '\n{"type": "function", "function": {"name": "f", "description": "é", "parameters": {}}}' + '\n{"type": "function", "function": {"name": "g", "parameters": {}}}', + ), + ], +) +def test_chat_template_tool_section_renders_schemas_like_the_models_template(model: str, rendered: str): + section: Final = chat_template_tool_section(model) + + assert section is not None + assert section.render_tool_schemas((_DESCRIBED_FUNCTION, _BARE_FUNCTION)) == rendered diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 84c139e867b..6126d919180 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -9,7 +9,7 @@ import subprocess import sys import threading import time -from collections.abc import Mapping +from collections.abc import Callable, Mapping from concurrent.futures import Future, wait from itertools import accumulate, chain from pathlib import Path @@ -39,6 +39,7 @@ from litellm.litellm_core_utils.token_counter import ( offload_token_count, ) from litellm.litellm_core_utils.token_counter import token_counter as token_counter_new +from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter from tests.large_text import text from tests.unit.litellm_core_utils.event_loop_lag import ( assert_loop_stayed_free, @@ -515,6 +516,183 @@ def test_token_counter_skips_non_mapping_tools(): assert mixed_tokens == valid_tokens +_WEATHER_FUNCTION: Final = { + "name": "get_weather", + "description": "Get the current weather for a city", + "parameters": { + "type": "object", + "properties": {"location": {"type": "string", "description": "City name"}}, + "required": ["location"], + }, +} +_TIME_FUNCTION: Final = { + "name": "get_time", + "description": "Get the local time for a city", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, +} +_WEB_SEARCH_FUNCTION: Final = { + "name": "web_search", + "description": "Search the web", + "parameters": {"type": "object", "properties": {"query": {"type": "string"}}, "required": ["query"]}, +} + + +def _openai_tool(function: Mapping[str, object]) -> dict[str, object]: + return {"type": "function", "function": function} + + +def _anthropic_tool(function: Mapping[str, object]) -> dict[str, object]: + return {"name": function["name"], "description": function["description"], "input_schema": function["parameters"]} + + +_USER_MESSAGES: Final = ({"role": "user", "content": "What is the weather in Paris today?"},) +_SYSTEM_AND_USER_MESSAGES: Final = ({"role": "system", "content": "You are a helpful assistant."}, *_USER_MESSAGES) + + +@pytest.mark.parametrize("tool_shape", [_openai_tool, _anthropic_tool]) +@pytest.mark.parametrize( + ("model", "messages", "functions", "billed_prompt_tokens"), + [ + # usage.prompt_tokens billed for this request by Azure AI Foundry DeepSeek-V4-Flash-0731, + # Databricks databricks-qwen35-122b-a10b and DashScope qwen3.5-plus on 2026-10-17 + ("DeepSeek-V4-Flash-0731", _USER_MESSAGES, (_WEATHER_FUNCTION,), 295), + ("DeepSeek-V4-Flash-0731", _USER_MESSAGES, (_WEATHER_FUNCTION, _TIME_FUNCTION), 348), + ("DeepSeek-V4-Flash-0731", _SYSTEM_AND_USER_MESSAGES, (_WEATHER_FUNCTION,), 300), + ("databricks-qwen35-122b-a10b", _USER_MESSAGES, (_WEATHER_FUNCTION,), 289), + ("databricks-qwen35-122b-a10b", _USER_MESSAGES, (_WEATHER_FUNCTION, _TIME_FUNCTION), 352), + ("databricks-qwen35-122b-a10b", _SYSTEM_AND_USER_MESSAGES, (_WEATHER_FUNCTION,), 296), + ("qwen3.5-plus", _USER_MESSAGES, (_WEATHER_FUNCTION,), 284), + ("qwen3.5-plus", _USER_MESSAGES, (_WEATHER_FUNCTION, _TIME_FUNCTION), 342), + ], +) +def test_token_counter_tools_match_billed_prompt_for_json_tool_chat_templates( + tool_shape: Callable[[Mapping[str, object]], dict[str, object]], + model: str, + messages: tuple[dict[str, str], ...], + functions: tuple[Mapping[str, object], ...], + billed_prompt_tokens: int, +): + counted: Final = token_counter_new( + model=model, + messages=list(messages), + tools=[tool_shape(function) for function in functions], + ) + + assert abs(counted - billed_prompt_tokens) <= billed_prompt_tokens * 0.05 + + +@pytest.mark.parametrize( + ("model", "counted_like"), + [ + ("DeepSeek-V4-Pro", "DeepSeek-V4-Flash-0731"), + ("accounts/fireworks/models/deepseek-v4-flash", "DeepSeek-V4-Flash-0731"), + ("DeepSeek-V4.1-Flash", "deepseek-v3.2"), + ("accounts/fireworks/models/deepseek-v4p1-flash", "deepseek-v3.2"), + ("Qwen/Qwen3.5-122B-A10B", "databricks-qwen35-122b-a10b"), + ("qwen3-235b-a22b", "deepseek-v3.2"), + ], +) +def test_token_counter_counts_tools_by_the_models_chat_template_family(model: str, counted_like: str): + tools: Final = [_openai_tool(_WEATHER_FUNCTION), _openai_tool(_TIME_FUNCTION)] + + assert token_counter_new(model=model, messages=list(_USER_MESSAGES), tools=tools) == token_counter_new( + model=counted_like, messages=list(_USER_MESSAGES), tools=tools + ) + + +@pytest.mark.parametrize("model", ["DeepSeek-V4-Flash-0731", "databricks-qwen35-122b-a10b"]) +@pytest.mark.parametrize( + "builtin_tool", + [ + {"type": "web_search_preview"}, + {"type": "web_search_20250305", "name": "web_search", "max_uses": 5}, + {"name": "web_search"}, + ], +) +def test_token_counter_skips_tools_not_sent_as_functions_for_chat_template_families( + model: str, builtin_tool: dict[str, object] +): + weather_tool: Final = _openai_tool(_WEATHER_FUNCTION) + + assert token_counter_new(model=model, messages=list(_USER_MESSAGES), tools=[builtin_tool]) == token_counter_new( + model=model, messages=list(_USER_MESSAGES) + ) + assert token_counter_new( + model=model, messages=list(_USER_MESSAGES), tools=[builtin_tool, weather_tool] + ) == token_counter_new(model=model, messages=list(_USER_MESSAGES), tools=[weather_tool]) + + +def _responses_tool(function: Mapping[str, object]) -> dict[str, object]: + return {"type": "function", **function} + + +def _gemini_tool(function: Mapping[str, object]) -> dict[str, object]: + return {"function_declarations": [function]} + + +@pytest.mark.parametrize("model", ["DeepSeek-V4-Flash-0731", "databricks-qwen35-122b-a10b"]) +@pytest.mark.parametrize("function", [_WEATHER_FUNCTION, _WEB_SEARCH_FUNCTION]) +@pytest.mark.parametrize("tool_shape", [_responses_tool, _gemini_tool]) +def test_token_counter_counts_function_tools_like_chat_completions_tools_for_chat_template_families( + model: str, function: Mapping[str, object], tool_shape: Callable[[Mapping[str, object]], dict[str, object]] +): + assert token_counter_new( + model=model, messages=list(_USER_MESSAGES), tools=[tool_shape(function)] + ) == token_counter_new(model=model, messages=list(_USER_MESSAGES), tools=[_openai_tool(function)]) + + +# Azure AI Foundry DeepSeek-V4-Flash-0731 billed 295 prompt tokens for the weather tool with each of +# these tool_choice values on 2026-10-17, the same as without one +@pytest.mark.parametrize("tool_choice", ["none", "required", {"type": "function", "function": {"name": "get_weather"}}]) +def test_token_counter_tool_choice_adds_nothing_for_deepseek_v4(tool_choice: object): + tools: Final = [_openai_tool(_WEATHER_FUNCTION)] + + assert token_counter_new( + model="DeepSeek-V4-Flash-0731", messages=list(_USER_MESSAGES), tools=tools, tool_choice=tool_choice + ) == token_counter_new(model="DeepSeek-V4-Flash-0731", messages=list(_USER_MESSAGES), tools=tools) + + +@pytest.mark.parametrize("model", ["DeepSeek-V4-Flash-0731", "databricks-qwen35-122b-a10b"]) +@pytest.mark.parametrize( + "anthropic_tool", + [ + {"name": "get_weather", "input_schema": _WEATHER_FUNCTION["parameters"]}, + _anthropic_tool(_WEATHER_FUNCTION), + {**_anthropic_tool(_WEATHER_FUNCTION), "strict": True}, + {**_anthropic_tool(_WEATHER_FUNCTION), "defer_loading": True, "input_examples": [{"location": "Paris"}]}, + _anthropic_tool(_WEB_SEARCH_FUNCTION), + ], +) +def test_token_counter_counts_anthropic_tools_as_sent_to_chat_template_families( + model: str, anthropic_tool: dict[str, object] +): + sent_tools, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_tools_to_openai([anthropic_tool]) # pyright: ignore[reportArgumentType] # raw Anthropic tool dict + + assert token_counter_new(model=model, messages=list(_USER_MESSAGES), tools=[anthropic_tool]) == token_counter_new( + model=model, messages=list(_USER_MESSAGES), tools=sent_tools + ) + + +# Qwen3.5's chat_template.jinja opens a system turn for the tool schemas when the request has none, 5 tokens that a +# request with a system message already pays, while DeepSeek V4's encoding_dsv4.py appends them to the first turn +# either way (both read on Hugging Face on 2026-10-17) +@pytest.mark.parametrize( + ("model", "implicit_system_turn_tokens"), [("DeepSeek-V4-Flash-0731", 0), ("databricks-qwen35-122b-a10b", 5)] +) +def test_token_counter_counts_the_tools_system_turn_only_without_a_system_message( + model: str, implicit_system_turn_tokens: int +): + tools: Final = [_openai_tool(_WEATHER_FUNCTION)] + system_cost_with_tools: Final = token_counter_new( + model=model, messages=list(_SYSTEM_AND_USER_MESSAGES), tools=tools + ) - token_counter_new(model=model, messages=list(_USER_MESSAGES), tools=tools) + system_cost_without_tools: Final = token_counter_new( + model=model, messages=list(_SYSTEM_AND_USER_MESSAGES) + ) - token_counter_new(model=model, messages=list(_USER_MESSAGES)) + + assert system_cost_with_tools == system_cost_without_tools - implicit_system_turn_tokens + + class NeedsToleranceUpdateError(Exception): """Custom exception to mark tests that have improved""" @@ -1770,14 +1948,17 @@ _THRESHOLD_TEST_TOOLS: Final = [ ] -def test_messages_reach_token_count_agrees_with_token_counter_at_every_threshold() -> None: +@pytest.mark.parametrize( + "model", ["claude-3-5-sonnet-20240620", "DeepSeek-V4-Flash-0731", "databricks-qwen35-122b-a10b"] +) +def test_messages_reach_token_count_agrees_with_token_counter_at_every_threshold(model: str) -> None: """The threshold check is the same arithmetic as token_counter(...) >= threshold, including the tools and system-message adjustments, so the boundary values must agree exactly.""" from litellm.litellm_core_utils.token_counter import messages_reach_token_count messages = _threshold_test_messages(turns=12) total = token_counter_new( - model="claude-3-5-sonnet-20240620", + model=model, messages=messages, tools=_THRESHOLD_TEST_TOOLS, use_default_image_token_count=True, @@ -1785,7 +1966,7 @@ def test_messages_reach_token_count_agrees_with_token_counter_at_every_threshold assert total > 100 for threshold in (0, 1, total - 1, total, total + 1, 10 * total): assert messages_reach_token_count( - model="claude-3-5-sonnet-20240620", + model=model, messages=messages, threshold=threshold, tools=_THRESHOLD_TEST_TOOLS, From 90af729933fe04626d233ed7790752ff1720cc57 Mon Sep 17 00:00:00 2001 From: Jon Varley Date: Mon, 19 Oct 2026 18:37:25 +0100 Subject: [PATCH 07/20] fix(cost-map): enable function calling + tool choice for fireworks deepseek-v4p1-flash-1017 (#47326) Co-authored-by: Cursor --- litellm/model_prices_and_context_window_backup.json | 4 ++-- model_prices_and_context_window.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index e0ace9c4f39..5b68eeb85e3 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -75966,11 +75966,11 @@ "output_cost_per_token": 1.2e-06, "output_cost_per_token_priority": 1.5e-06, "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", - "supports_function_calling": false, + "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": false, + "supports_tool_choice": true, "supports_vision": true }, "github_copilot/claude-sonnet-4.6": { diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e0ace9c4f39..5b68eeb85e3 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -75966,11 +75966,11 @@ "output_cost_per_token": 1.2e-06, "output_cost_per_token_priority": 1.5e-06, "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", - "supports_function_calling": false, + "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, - "supports_tool_choice": false, + "supports_tool_choice": true, "supports_vision": true }, "github_copilot/claude-sonnet-4.6": { From 2f49e7f1c3e77dc676f0aacf501c99e3884e1f2a Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 10:58:49 -0700 Subject: [PATCH 08/20] test(cost): add tier 1 cost tracking cases and wire bridge legs for 2026-10-19 (#47342) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../cost_calculation/cases/gemini.py | 21 +++++--- .../cost_calculation/cases/openai.py | 3 ++ .../cost_calculation/cases/vertex_ai.py | 53 ++++++++++++++++++- 3 files changed, 70 insertions(+), 7 deletions(-) diff --git a/tests/integration/cost_calculation/cases/gemini.py b/tests/integration/cost_calculation/cases/gemini.py index bc6665c423f..8a50e91ec01 100644 --- a/tests/integration/cost_calculation/cases/gemini.py +++ b/tests/integration/cost_calculation/cases/gemini.py @@ -3494,6 +3494,10 @@ GEMINI_GEMINI_4_ARGON_ROUTE_BRIDGES: Final = bridge_pairs( (GEMINI_GEMINI_4_ARGON_INPUT_TEXT, GEMINI_GEMINI_4_ARGON_STREAM), ("/v1/responses", "/v1/messages"), ) +GEMINI_GEMINI_3_8_FLASH_ROUTE_BRIDGES: Final = bridge_pairs( + (GEMINI_GEMINI_3_8_FLASH_INPUT_TEXT, GEMINI_GEMINI_3_8_FLASH_STREAM), + ("/v1/responses", "/v1/messages"), +) GEMINI_GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES: Final = bridge_pairs( (GEMINI_GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT, GEMINI_GEMINI_3_1_PRO_PREVIEW_STREAM), ("/v1/responses", "/v1/messages"), @@ -3572,6 +3576,7 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( GEMINI_GEMINI_3_1_PRO_PREVIEW_TOOL_CALL, GEMINI_GEMINI_3_1_PRO_PREVIEW_SERVICE_TIER_PRIORITY, *(pair.bridged for pair in GEMINI_GEMINI_4_ARGON_ROUTE_BRIDGES), + *(pair.bridged for pair in GEMINI_GEMINI_3_8_FLASH_ROUTE_BRIDGES), *(pair.bridged for pair in GEMINI_GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES), ) @@ -3606,6 +3611,14 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = ( plain=GEMINI_GEMINI_4_ARGON_ROUTE_BRIDGES[1].bridged, streamed=GEMINI_GEMINI_4_ARGON_ROUTE_BRIDGES[3].bridged, ), + StreamParityTestCase( + plain=GEMINI_GEMINI_3_8_FLASH_ROUTE_BRIDGES[0].bridged, + streamed=GEMINI_GEMINI_3_8_FLASH_ROUTE_BRIDGES[2].bridged, + ), + StreamParityTestCase( + plain=GEMINI_GEMINI_3_8_FLASH_ROUTE_BRIDGES[1].bridged, + streamed=GEMINI_GEMINI_3_8_FLASH_ROUTE_BRIDGES[3].bridged, + ), StreamParityTestCase( plain=GEMINI_GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES[0].bridged, streamed=GEMINI_GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES[2].bridged, @@ -3618,14 +3631,10 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = ( BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = ( *bridge_pairs( - ( - GEMINI_GEMINI_3_8_FLASH_INPUT_TEXT, - GEMINI_GEMINI_3_8_FLASH_CACHE_READ, - GEMINI_GEMINI_3_8_FLASH_REASONING, - GEMINI_GEMINI_3_8_FLASH_STREAM, - ), + (GEMINI_GEMINI_3_8_FLASH_CACHE_READ, GEMINI_GEMINI_3_8_FLASH_REASONING), ("/v1/responses", "/v1/messages"), ), + *GEMINI_GEMINI_3_8_FLASH_ROUTE_BRIDGES, *GEMINI_GEMINI_4_ARGON_ROUTE_BRIDGES, *GEMINI_GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES, ) diff --git a/tests/integration/cost_calculation/cases/openai.py b/tests/integration/cost_calculation/cases/openai.py index f610fd9591a..acba66223e5 100644 --- a/tests/integration/cost_calculation/cases/openai.py +++ b/tests/integration/cost_calculation/cases/openai.py @@ -12008,6 +12008,9 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( GPT_6_ASTRA_RESPONSES_NATIVE_STREAM, GPT_5_6_TERRA_RESPONSES_NATIVE_STREAM, GPT_5_6_TERRA_MESSAGES_BRIDGED_JSON, + GPT_6_1_SOL_MESSAGES_BRIDGED_JSON, + GPT_6_LUNA_MESSAGES_BRIDGED_JSON, + GPT_6_ASTRA_MESSAGES_BRIDGED_JSON, GPT_6_1_SOL_MESSAGES_BRIDGED_STREAM, GPT_6_LUNA_MESSAGES_BRIDGED_STREAM, GPT_6_ASTRA_MESSAGES_BRIDGED_STREAM, diff --git a/tests/integration/cost_calculation/cases/vertex_ai.py b/tests/integration/cost_calculation/cases/vertex_ai.py index b512df46d1d..3b825bfe7b2 100644 --- a/tests/integration/cost_calculation/cases/vertex_ai.py +++ b/tests/integration/cost_calculation/cases/vertex_ai.py @@ -3146,6 +3146,41 @@ VERTEX_AI_CLAUDE_HAIKU_5_5_CACHE_READ: Final = _vertex_claude_shape_case( cache_read_cost=0.00351039, ), ) +VERTEX_AI_CLAUDE_HAIKU_5_5_TIERED_INPUT_ABOVE_100K: Final = _vertex_claude_shape_case( + VERTEX_AI_CLAUDE_HAIKU_5_5_INPUT_TEXT, + "vertex_ai/claude-haiku-5-5-tiered_input_above_100k", + "vertex_ai/claude-haiku-5-5", + _vertex_claude_usage(input_tokens=150000, output_tokens=480), + ExactExpected( + spend=0.9906, + input_cost=0.975, + output_cost=0.0156, + prompt_tokens=150000, + completion_tokens=480, + ), + response_content=[ + {"type": "thinking", "thinking": "scripted thought", "signature": "scripted signature"}, + {"type": "text", "text": "scripted answer for cost fixtures"}, + ], +) +VERTEX_AI_CLAUDE_HAIKU_5_5_TIERED_CACHE_READ_ABOVE_100K: Final = _vertex_claude_shape_case( + VERTEX_AI_CLAUDE_HAIKU_5_5_CACHE_READ, + "vertex_ai/claude-haiku-5-5-tiered_cache_read_above_100k", + "vertex_ai/claude-haiku-5-5", + _vertex_claude_usage(input_tokens=4096, output_tokens=480, cache_read_input_tokens=110592), + ExactExpected( + spend=0.1141088, + input_cost=0.0985088, + output_cost=0.0156, + prompt_tokens=114688, + completion_tokens=480, + cache_read_cost=0.0718848, + ), + response_content=[ + {"type": "thinking", "thinking": "scripted thought", "signature": "scripted signature"}, + {"type": "text", "text": "scripted answer for cost fixtures"}, + ], +) VERTEX_AI_CLAUDE_HAIKU_5_5_CACHE_WRITE_5M: Final = _vertex_claude_shape_case( VERTEX_AI_CLAUDE_SONNET_5_5_CACHE_WRITE_5M, "vertex_ai/claude-haiku-5-5-cache_write_5m", @@ -3530,6 +3565,10 @@ GEMINI_4_ARGON_ROUTE_BRIDGES: Final = bridge_pairs( (GEMINI_4_ARGON_INPUT_TEXT, GEMINI_4_ARGON_STREAM), ("/v1/responses", "/v1/messages"), ) +GEMINI_3_8_FLASH_ROUTE_BRIDGES: Final = bridge_pairs( + (GEMINI_3_8_FLASH_INPUT_TEXT, GEMINI_3_8_FLASH_STREAM), + ("/v1/responses", "/v1/messages"), +) GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES: Final = bridge_pairs( (GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT, GEMINI_3_1_PRO_PREVIEW_STREAM), ("/v1/responses", "/v1/messages"), @@ -3604,6 +3643,8 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( VERTEX_AI_CLAUDE_HAIKU_5_5_INPUT_TEXT, VERTEX_AI_CLAUDE_HAIKU_5_5_STREAM, VERTEX_AI_CLAUDE_HAIKU_5_5_CACHE_READ, + VERTEX_AI_CLAUDE_HAIKU_5_5_TIERED_INPUT_ABOVE_100K, + VERTEX_AI_CLAUDE_HAIKU_5_5_TIERED_CACHE_READ_ABOVE_100K, VERTEX_AI_CLAUDE_HAIKU_5_5_CACHE_WRITE_5M, VERTEX_AI_CLAUDE_HAIKU_5_5_CACHE_WRITE_1H, VERTEX_AI_CLAUDE_HAIKU_5_5_THINKING, @@ -3633,6 +3674,7 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( GEMINI_4_ARGON_REASONING, GEMINI_4_ARGON_TOOL_CALL, *(pair.bridged for pair in GEMINI_4_ARGON_ROUTE_BRIDGES), + *(pair.bridged for pair in GEMINI_3_8_FLASH_ROUTE_BRIDGES), *(pair.bridged for pair in GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES), ) @@ -3699,6 +3741,14 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = ( plain=GEMINI_4_ARGON_ROUTE_BRIDGES[1].bridged, streamed=GEMINI_4_ARGON_ROUTE_BRIDGES[3].bridged, ), + StreamParityTestCase( + plain=GEMINI_3_8_FLASH_ROUTE_BRIDGES[0].bridged, + streamed=GEMINI_3_8_FLASH_ROUTE_BRIDGES[2].bridged, + ), + StreamParityTestCase( + plain=GEMINI_3_8_FLASH_ROUTE_BRIDGES[1].bridged, + streamed=GEMINI_3_8_FLASH_ROUTE_BRIDGES[3].bridged, + ), StreamParityTestCase( plain=GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES[0].bridged, streamed=GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES[2].bridged, @@ -3711,9 +3761,10 @@ PARITY: Final[tuple[StreamParityTestCase, ...]] = ( BRIDGE_COST_PARITY: Final[tuple[BridgeCostParityTestCase, ...]] = ( *bridge_pairs( - (GEMINI_3_8_FLASH_INPUT_TEXT, GEMINI_3_8_FLASH_CACHE_READ, GEMINI_3_8_FLASH_STREAM), + (GEMINI_3_8_FLASH_CACHE_READ, GEMINI_3_8_FLASH_REASONING), ("/v1/responses", "/v1/messages"), ), + *GEMINI_3_8_FLASH_ROUTE_BRIDGES, *GEMINI_3_1_PRO_PREVIEW_ROUTE_BRIDGES, *GEMINI_4_ARGON_ROUTE_BRIDGES, *VERTEX_AI_CLAUDE_HAIKU_5_5_BRIDGE_PAIRS, From 0197b961716f2086ea4cb7dfb1dc89a05010db48 Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 11:17:21 -0700 Subject: [PATCH 09/20] chore(pricing): add github_copilot/claude-opus-4.8 deprecation date (#47370) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 1 + model_prices_and_context_window.json | 1 + 2 files changed, 2 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 5b68eeb85e3..6675bb170b8 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -32524,6 +32524,7 @@ "github_copilot/claude-opus-4.8": { "cache_creation_input_token_cost": 6.25e-06, "cache_read_input_token_cost": 5e-07, + "deprecation_date": "2026-11-16", "input_cost_per_token": 5e-06, "litellm_provider": "github_copilot", "max_input_tokens": 200000, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 5b68eeb85e3..6675bb170b8 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -32524,6 +32524,7 @@ "github_copilot/claude-opus-4.8": { "cache_creation_input_token_cost": 6.25e-06, "cache_read_input_token_cost": 5e-07, + "deprecation_date": "2026-11-16", "input_cost_per_token": 5e-06, "litellm_provider": "github_copilot", "max_input_tokens": 200000, From 037e805b970230e4fb5c94784ea1b95e009a2b59 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:19:43 +0000 Subject: [PATCH 10/20] build(deps): bump aiohttp to 3.14.5 to clear osv-scan (#47369) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- uv.lock | 475 +++++++++++++++++++++++++++----------------------------- 1 file changed, 226 insertions(+), 249 deletions(-) diff --git a/uv.lock b/uv.lock index cb77c470885..beb66fe42b2 100644 --- a/uv.lock +++ b/uv.lock @@ -85,7 +85,7 @@ wheels = [ [[package]] name = "aiohttp" -version = "3.14.3" +version = "3.14.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohappyeyeballs" }, @@ -98,126 +98,127 @@ dependencies = [ { name = "typing-extensions", marker = "python_full_version < '3.13'" }, { name = "yarl" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/58/d9/22ce5786ac0c1653ae8b6c23bded02c1686d11f0dbb45b31ce128e0df985/aiohttp-3.14.3.tar.gz", hash = "sha256:9491196535a88924a60afd5b5f434b5b203b6cc616250878dbdb223a8f7844bc", size = 7971213, upload-time = "2026-07-23T01:57:27.037Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6c/4c/bdccd81e9ee225b69c60e7766c9a5b05364f118f4d383713b89a682d772d/aiohttp-3.14.5.tar.gz", hash = "sha256:5558a7f5a05af9ecf744af91e5baefc436f93c9333e656c27ec253f9a6bbe178", size = 8078052, upload-time = "2026-10-11T01:05:12.408Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2d/4d/4a99fb425c5e0cad715eea7bd190aff46f38b959a0a2dadb993705d34b26/aiohttp-3.14.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:eb0495d778817619273c108784292be161a924b9f5ae5cbbc70a2caa6838250b", size = 765848, upload-time = "2026-07-23T01:52:08.217Z" }, - { url = "https://files.pythonhosted.org/packages/74/e8/43b85dc55b8e950dc644babe762add781319ea881b57b33d2cce12017d12/aiohttp-3.14.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c3c200cf9757edd785051dc699c7ecbec22110dbfcb3fefc7a9f9695eda8ea7a", size = 517476, upload-time = "2026-07-23T01:52:10.846Z" }, - { url = "https://files.pythonhosted.org/packages/7f/9e/73b582c4dbbc3c12ef4473822475effaabf1f934b56f14f5b03fe5d3a2af/aiohttp-3.14.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd51ebf9d3a00c074df4ede271023f4d2dba289bcc740b88191872716014e3c5", size = 515334, upload-time = "2026-07-23T01:52:12.636Z" }, - { url = "https://files.pythonhosted.org/packages/79/03/e98c3c9e05a5bdf97defe5ff9169baba4f0ec9a901f2d60e0f060c2f051e/aiohttp-3.14.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:134ac5ddcf61c6fad984b9a5727d83492ada43d63471db20fb73042c13fca62f", size = 1708830, upload-time = "2026-07-23T01:52:14.538Z" }, - { url = "https://files.pythonhosted.org/packages/d7/2c/26e60b694844dfd2176c57f913a22d0cd6a16f9ff202cbda7580d0328b98/aiohttp-3.14.3-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:70c987b27534f9ae1a723f47ae921571d616da21d3208282bf4c52af5164ac43", size = 1674012, upload-time = "2026-07-23T01:52:16.486Z" }, - { url = "https://files.pythonhosted.org/packages/38/65/672df92e3172cd876aacfa97a952ac560877eb169384b2991ac5b273de4c/aiohttp-3.14.3-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1b59533861b70a2185c8f4f350f791f39d64358ef6944ce71c5240c9ec0982c9", size = 1767015, upload-time = "2026-07-23T01:52:18.28Z" }, - { url = "https://files.pythonhosted.org/packages/9e/c5/228dec7bfec1c373cc2217cdeb47d6456dcd7a13a4c55144930a75ae3851/aiohttp-3.14.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1c5281acc88b92396f88c7e1e2748f8466689df22b80170e4f51efa712fb47a8", size = 1858700, upload-time = "2026-07-23T01:52:20.08Z" }, - { url = "https://files.pythonhosted.org/packages/bd/ff/cb36724e8c8d17f90ada567a9ff3efe1d6e9b549fba697a242aece180f21/aiohttp-3.14.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:48d67b87db6279c044760787eb01f6413032c2e6f3ba1cafaa492b1c8e578479", size = 1714075, upload-time = "2026-07-23T01:52:22.071Z" }, - { url = "https://files.pythonhosted.org/packages/9f/3a/296a4135c6366376263aeef54b15caca1f07676c2ae0c525d7832f2f808a/aiohttp-3.14.3-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f53bcd52f585e1ac3e590d61434eb61f9a88c38df041b4ea126d97144344a77b", size = 1588234, upload-time = "2026-07-23T01:52:23.757Z" }, - { url = "https://files.pythonhosted.org/packages/7d/81/9d5d853ef892dc066d1eb6db0e87a47348b920c1c879aa554612fdbd9d79/aiohttp-3.14.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0fdea2281997af69da84c77ffa6f5938a0285f21fb3887c249d67419ca865b3d", size = 1677300, upload-time = "2026-07-23T01:52:25.861Z" }, - { url = "https://files.pythonhosted.org/packages/68/96/021d386ae32d9b26d4b88df2e794546232ff56bb6be952bf6be227c0bbc7/aiohttp-3.14.3-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:cda5fd5c95ad7a125a2e8464acc78b98b94c475a3780d6aa0aa157c93f470f4d", size = 1691501, upload-time = "2026-07-23T01:52:28Z" }, - { url = "https://files.pythonhosted.org/packages/29/9f/af66adce26a14af135c003cbd0f44ccaa68cebd30ff8ac99ca47fb4958f7/aiohttp-3.14.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6debfa7312ff9d4c124dc71d72e9a0a4b9e0879e48ba6fcb42bef5c3300289e2", size = 1735113, upload-time = "2026-07-23T01:52:29.995Z" }, - { url = "https://files.pythonhosted.org/packages/2f/90/28c390d4c9851effe52ac25b5a2e1d92246acd00728b4fc7975dafb67484/aiohttp-3.14.3-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:f4e05329faa0ea1a404b37de4f034fd2c2defcca06a68dc6745e4e56c88e8a48", size = 1577486, upload-time = "2026-07-23T01:52:31.937Z" }, - { url = "https://files.pythonhosted.org/packages/db/c2/00e23a1bf2abb70dd353f6987db7e7f2491d0261f7363997738c71c98f95/aiohttp-3.14.3-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:a3a8296e7ab5c295f53f1041487cb088e1480775aafbf7fe545d93b770a0f96f", size = 1751353, upload-time = "2026-07-23T01:52:33.688Z" }, - { url = "https://files.pythonhosted.org/packages/6e/7d/d51a706a8cbfa57f0611127daf61ab3ae02ab8420b0407412079227d1c65/aiohttp-3.14.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5373dc80ad1aa2fb9ad95c83f24eef418bbda3a61375f128e5b0192e4f3f9b32", size = 1698681, upload-time = "2026-07-23T01:52:38.167Z" }, - { url = "https://files.pythonhosted.org/packages/ec/b0/90bd5cd9fdd9787cb4211d284d1fb8401339a933cb0227a15b71e789232f/aiohttp-3.14.3-cp310-cp310-win32.whl", hash = "sha256:a3e22975f905b89a55a488c2a08f2fdb2186175349e917d48985cc468a3d4c6e", size = 456733, upload-time = "2026-07-23T01:52:41.823Z" }, - { url = "https://files.pythonhosted.org/packages/d8/15/fe5b8f6a71ae112bc677163d0b0701bda5dc15005249582258ede0eb88c7/aiohttp-3.14.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdd0e2834dce1a26c1bbe26464861e16bbe217042cbff619247c11594472518c", size = 480460, upload-time = "2026-07-23T01:52:43.905Z" }, - { url = "https://files.pythonhosted.org/packages/54/00/45e98b6645cd7f00a4b78b749ebd309094b0eaeb2d2e96157eadbc0d0050/aiohttp-3.14.3-cp310-cp310-win_arm64.whl", hash = "sha256:eac645b09bcfdf73df7536331f0678c1086ea250981118ddb5199e17ccef72bb", size = 453479, upload-time = "2026-07-23T01:52:46.075Z" }, - { url = "https://files.pythonhosted.org/packages/f8/5c/b3e4ff8ad43a8afef9602c5e90285936da1beaea8b029016b793891f03c3/aiohttp-3.14.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e568e14940c09955aa51f4e645b6daa18a581c5dcfcd73744dcc86a856e3ced3", size = 764250, upload-time = "2026-07-23T01:52:48.525Z" }, - { url = "https://files.pythonhosted.org/packages/0e/da/f1b384465e51449d844056b75070461da03a9a23e6c1747003695bf4172a/aiohttp-3.14.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:54cfcdee2770dac994417cbb0ee1f3eb0e7cb6b30c79bf44f2c02ff79ec5124a", size = 516281, upload-time = "2026-07-23T01:52:51.047Z" }, - { url = "https://files.pythonhosted.org/packages/b9/3f/01264f820ee2e3712a827892b1cd6ff80f3300c1fcbffbb45714a915d47a/aiohttp-3.14.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:21c016079415ed3fd676963e9793700a566d85dbbd6bfc564b9b2d209147dcc8", size = 514742, upload-time = "2026-07-23T01:52:53.779Z" }, - { url = "https://files.pythonhosted.org/packages/9e/8d/a71c6f2db52ac1ed142b133f7feddaa6b70539c3f4de24d7e226c95b794c/aiohttp-3.14.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d6088ec9894113802bddb3c09e974929aed2c7b3a8c456219b8aab4481f1a239", size = 1780613, upload-time = "2026-07-23T01:52:56.948Z" }, - { url = "https://files.pythonhosted.org/packages/a5/11/3dd9b3fb3a170f6ec9011b5291d876a6fab4086714c9e158600edf01b4fd/aiohttp-3.14.3-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:16ea7e24c309fb7c0bbd505d149abe4fe4dccfb8db911db7dbec0921bc889a6f", size = 1737688, upload-time = "2026-07-23T01:52:59.294Z" }, - { url = "https://files.pythonhosted.org/packages/6d/3e/834c26918be7d88068822b40e0db30fca50b5f4fe79104aa16a93f1d74e6/aiohttp-3.14.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:56f355e79f71aef2a85c80305cc915f894b170dba76de5fe84f6351939b83c06", size = 1845742, upload-time = "2026-07-23T01:53:01.641Z" }, - { url = "https://files.pythonhosted.org/packages/cc/c9/49ab8572df7d66bc13d11e31f781292badb04180dd87ba98733066c6aed7/aiohttp-3.14.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:18c441d0a8fca6de8d1f546849b9f0ab20d435993e2c5b59562b2fae6be2f929", size = 1928412, upload-time = "2026-07-23T01:53:04.018Z" }, - { url = "https://files.pythonhosted.org/packages/a5/b9/2b8f0c0ce09c87a1daf80fd483431b56b1435d3f62789bc86f572e1245de/aiohttp-3.14.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53e7b4ce82b54a8bcc71b3b67a5cbd177ca1d7f592cbc92cd38b7349f73482db", size = 1786220, upload-time = "2026-07-23T01:53:06.481Z" }, - { url = "https://files.pythonhosted.org/packages/85/00/9c45f81de11710460edfa1dc81317b6e882703b160926c879a9d20da9fcc/aiohttp-3.14.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f55119f7bf25f49ed210f6096090715da24f2943c62102448915fde3c62877ce", size = 1637231, upload-time = "2026-07-23T01:53:10.258Z" }, - { url = "https://files.pythonhosted.org/packages/19/ce/967d628e910756f3539c6107cb7844a1b69440dcb3029a5ee7871b09ab63/aiohttp-3.14.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9aa6e61fdf20105c4144e755bd586008ff450791d67b1c8146fdc15959c4d51c", size = 1753161, upload-time = "2026-07-23T01:53:13.817Z" }, - { url = "https://files.pythonhosted.org/packages/11/b2/0c3d4114f0aee4f580f5b3b4eb71b24d7a23b834ea506a4dfebe76513f35/aiohttp-3.14.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ccd4893707b3e2a13e39c90d43cf80edf2e4d0457935bcc103bf2346214c3f15", size = 1756356, upload-time = "2026-07-23T01:53:16.211Z" }, - { url = "https://files.pythonhosted.org/packages/63/5d/99e7d91c82f1399d1ae2a854e080bd1493fbc31e5e959dbc4ec33dac3bec/aiohttp-3.14.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:b2466434105a4e03113c36ec775cc2ebe6676b62eae326fa670bb607ef788c1c", size = 1819846, upload-time = "2026-07-23T01:53:18.289Z" }, - { url = "https://files.pythonhosted.org/packages/ad/05/d5e1cb6480eeffd3f901d40a2c5e2d1e7effdc797837da3b490272699f13/aiohttp-3.14.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ba59d59aba08ac02fc03b0c8983ccd5ee39a199d0552ce9e6d2b4845b34d59ae", size = 1628531, upload-time = "2026-07-23T01:53:23.86Z" }, - { url = "https://files.pythonhosted.org/packages/c9/90/b934682bcaefae18a9e04f3dff5b68522ba810906358ae5029b68110ea3b/aiohttp-3.14.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ed099d105449c4f9e84f24af203cd131349d4761d8813fa7e02c32e7128cd910", size = 1832712, upload-time = "2026-07-23T01:53:27.551Z" }, - { url = "https://files.pythonhosted.org/packages/21/df/6061679faaf81fac746e7307c7adb71e858071a5d34c27583afefc64f543/aiohttp-3.14.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:152516815ef926786a0b6ae2b8f1fd2e0c71582dee0b435636865316fd4891b7", size = 1775014, upload-time = "2026-07-23T01:53:30.223Z" }, - { url = "https://files.pythonhosted.org/packages/8a/1d/f854878bbc69b88faefe924b619a34a6f59ec05fd387c77690667eaa75eb/aiohttp-3.14.3-cp311-cp311-win32.whl", hash = "sha256:a4af35c443e0b1a1bd6a8af3f3485d7fda15c142751a00f3ff8090f0b93346fa", size = 456006, upload-time = "2026-07-23T01:53:34.97Z" }, - { url = "https://files.pythonhosted.org/packages/73/0c/2af9d1674baccd1dbd47282a93d660a22e57ef6167c856deb24b4214fbab/aiohttp-3.14.3-cp311-cp311-win_amd64.whl", hash = "sha256:e1e74298bab6ee0d6e749ed4fd1901c7e604bdda32c03d787a2cc71c46d0433d", size = 481069, upload-time = "2026-07-23T01:53:39.673Z" }, - { url = "https://files.pythonhosted.org/packages/8e/76/88401ff3fc95e85c5fc38d588f36f55e61ecb64343b2bc8d69326f453cc0/aiohttp-3.14.3-cp311-cp311-win_arm64.whl", hash = "sha256:03cd2bde3d7f085b64e549c985f4bb928cad7e8ecf5323bfca320db548d81b39", size = 453021, upload-time = "2026-07-23T01:53:43.749Z" }, - { url = "https://files.pythonhosted.org/packages/18/d4/eb96299230e20acf2efae207cb8d69051f1f68e357e5ea5e479bf6fb097a/aiohttp-3.14.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:39aded8c7f3b935b54aab1d8d73c70ec0ee2d3ec3b943e0e86611bc150ba47f5", size = 754690, upload-time = "2026-07-23T01:53:47.332Z" }, - { url = "https://files.pythonhosted.org/packages/88/11/e7a70a209eb9a067c0d3212b518a0134e3484f5178c7533878b6b514d469/aiohttp-3.14.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5bcb6ff3fdab1258a192679ff1a05d44f59626430aa05cd1a9d2447423599228", size = 509484, upload-time = "2026-07-23T01:53:51.159Z" }, - { url = "https://files.pythonhosted.org/packages/30/07/4bbc222cc8dbe31d4c3e8a5baad2286e4d42026ac0c570027b89afce6344/aiohttp-3.14.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:617105e2c3018ee38d0c8ce5ee3c84f621a6d8b9f723202aacaff28449ca91ee", size = 511949, upload-time = "2026-07-23T01:53:55.083Z" }, - { url = "https://files.pythonhosted.org/packages/54/b9/42e74c46b7b7c794b995bbc1f573fb48950c38b19d8600c62a6804ee2d67/aiohttp-3.14.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f631fe87a6f30df5fbe6d79640b25e4cffb38c31c7fb6f10871517b84b0f8c1a", size = 1765282, upload-time = "2026-07-23T01:53:59.662Z" }, - { url = "https://files.pythonhosted.org/packages/6b/ed/62bc4d74363ad346d518e0720363a949f63e2e23439a79eb5813d4d29bb3/aiohttp-3.14.3-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a94dbaae5ae27bd849c93570669bff91e0510f33a80805738e3de72a7be0447b", size = 1741511, upload-time = "2026-07-23T01:54:04.063Z" }, - { url = "https://files.pythonhosted.org/packages/d0/9f/181e8a8bc79e47d13c7fc4540bd7a3b729d9505609c61f392a8dd2fbfe55/aiohttp-3.14.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8f2f1c4c032c7cedd7d8da6f54c97b70266c6570c3108d3fdffee7188bb70529", size = 1810680, upload-time = "2026-07-23T01:54:09.882Z" }, - { url = "https://files.pythonhosted.org/packages/5c/9a/dec94d6ad694552fe3424e3f1928d7a606a5d9d9433a04e7ecdd9d38ae7f/aiohttp-3.14.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ea05e1f97ceea523942d9b2a7d7c0359d781d683d6b043f5943a602b14da4787", size = 1905646, upload-time = "2026-07-23T01:54:13.475Z" }, - { url = "https://files.pythonhosted.org/packages/52/b7/7cd31f29d6055bd711ae6e669367fba6f5ae9de463910a793e30556a8db7/aiohttp-3.14.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:543906c127fb1d929b95076db19b83fa2d46751006ff1e23b093aa5ac4d8db42", size = 1792122, upload-time = "2026-07-23T01:54:15.752Z" }, - { url = "https://files.pythonhosted.org/packages/66/73/10b1ef93afa61f4963c746257b70ced619cf31a4798671de5fdb2608501d/aiohttp-3.14.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0a5ff2dfbb9ce645fa5b8ef3e02c6c0b9cc3f6030ff863d0c51fffc50cb5541b", size = 1591127, upload-time = "2026-07-23T01:54:19.489Z" }, - { url = "https://files.pythonhosted.org/packages/49/ed/3b203fa6de1b338c14acdc06bf6ca9b043b7944f005966958c2ced932cde/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:041badb8f84396357c4d3ad26de6afd7a32b112f43d3c63045c0c8278cfd2043", size = 1725210, upload-time = "2026-07-23T01:54:24.129Z" }, - { url = "https://files.pythonhosted.org/packages/28/b7/1c2aab8c706436dcc28598452488ac9cd7c409da815237c28c27d58993e6/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:530125ee1163c4219af35dc3aa1206e541e7b31b6efc1a3f93b70a136f65d427", size = 1764848, upload-time = "2026-07-23T01:54:27.973Z" }, - { url = "https://files.pythonhosted.org/packages/54/50/94c28f08b131c4bf10984ea2c7a536c9920608bb2d6e7f95642c30cc87b7/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c8653fd547c93a61aadc612007790f5555cdd18946fa48cf45e26d8ea4ea473d", size = 1777102, upload-time = "2026-07-23T01:54:31.775Z" }, - { url = "https://files.pythonhosted.org/packages/13/d4/e7d09ba7d345fb2d74440fd2fa033c5e079fac05552927705986f41a364f/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:89176250f686cb9853c0fb7ead90e639e915b84a6f43eedc2a4e7ec21f1037f0", size = 1580205, upload-time = "2026-07-23T01:54:34.518Z" }, - { url = "https://files.pythonhosted.org/packages/a3/84/072a91d68e1e1eb587985b54baab94221277f877e8ef274fc213a0ceae28/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3a26434dafe408229ff3403458ca58de24fb51936504decac49ce6755f77e59d", size = 1797219, upload-time = "2026-07-23T01:54:36.995Z" }, - { url = "https://files.pythonhosted.org/packages/e0/eb/aad34e897e668424d6e995da5dff8a4a09af93363d3392488772957a63aa/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d1558173930a5a8d3069cee5c92fc91c87c4dbcb099debbb3622053717145a19", size = 1768629, upload-time = "2026-07-23T01:54:40.103Z" }, - { url = "https://files.pythonhosted.org/packages/b6/2b/6bb88ddba0fecd9122aa3ebcad25996cf6c083a4a7040dbb3a4f97972af6/aiohttp-3.14.3-cp312-cp312-win32.whl", hash = "sha256:16100ad3ab8d649fdfbee87602d9d2dcdca9df0b9eda8a1b5fdc0d41f96da559", size = 451481, upload-time = "2026-07-23T01:54:42.547Z" }, - { url = "https://files.pythonhosted.org/packages/76/9b/f2f8f108da17ecef2cc3efc424e8b7ad3782b1a8360f7b8eae8ced84f6ea/aiohttp-3.14.3-cp312-cp312-win_amd64.whl", hash = "sha256:33a2d7c28d33797a2e99923dffa63f83d908a19b6bf26cfe80fa790aa5e1a75a", size = 476845, upload-time = "2026-07-23T01:54:44.853Z" }, - { url = "https://files.pythonhosted.org/packages/3e/44/28dac80a8941b604f4da10ce21097614ca1bf905ce93dca28d8d7de9c1e7/aiohttp-3.14.3-cp312-cp312-win_arm64.whl", hash = "sha256:362a3fd481769cac1a824514bcd86fda51c65e8fe6e051099e008fddde6db17c", size = 448050, upload-time = "2026-07-23T01:54:47.087Z" }, - { url = "https://files.pythonhosted.org/packages/57/be/5afd201cc0ab139029aadb75392efe85a293403d9dd3a3226161c21ce00c/aiohttp-3.14.3-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:2e9878ae68e4a5f1c0abe4dd497dbc3d51946f5837b56759e2a02e78fa90ef86", size = 506269, upload-time = "2026-07-23T01:54:49.075Z" }, - { url = "https://files.pythonhosted.org/packages/22/09/dec8189d62b45ade009f6792a2264b942a90cb88aeaf181239933cd72c3c/aiohttp-3.14.3-cp313-cp313-android_21_x86_64.whl", hash = "sha256:f3d2669fe7dec7fc359ecdb5984b29b50d85d5d00f8c1cb61de4f4a24ee42627", size = 515166, upload-time = "2026-07-23T01:54:51.894Z" }, - { url = "https://files.pythonhosted.org/packages/28/24/2854869d29ed8a8b19d74f9ec6629515f7e04d02dd329d9d179201e58e47/aiohttp-3.14.3-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:cc7cb243a68167172f48c1fd43cee91ec4b1d40cefd190edd43369d1a6bc9c82", size = 486263, upload-time = "2026-07-23T01:54:54.223Z" }, - { url = "https://files.pythonhosted.org/packages/d4/dd/57187c8be2a35aea65eaee3bd2c3dcbbcf0204f5106c89637e3610380cd1/aiohttp-3.14.3-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:78253b573e6ffab5028924fc98bc281aae05445969982a10864bc360dea2016c", size = 492299, upload-time = "2026-07-23T01:54:56.236Z" }, - { url = "https://files.pythonhosted.org/packages/b9/11/06ae6ed8f0d414edf4068861e233d8fe23ee699bfd4b3ceb8663db948a62/aiohttp-3.14.3-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7041d52c3a7fa20c9e8c182b534704abb19502c8bdcbde7ab23bfda6f642394f", size = 502235, upload-time = "2026-07-23T01:54:58.377Z" }, - { url = "https://files.pythonhosted.org/packages/7e/a3/559639c34a345d2cf7c52dff6838119f2eaf29eb508227b5b83f573af813/aiohttp-3.14.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ac74facc01463f138b0da5580329cfcc82818dea5656e83ddcd11268fc12ff80", size = 750883, upload-time = "2026-07-23T01:55:00.65Z" }, - { url = "https://files.pythonhosted.org/packages/91/cd/41e131f13afd1e7b0172a9d9eda085ef90eb8439f41f0d279db81ed3ae60/aiohttp-3.14.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d6218d92e450824e9b4881f44e8c09f1853b490f9a64130801024a4793b1b3b0", size = 508473, upload-time = "2026-07-23T01:55:02.945Z" }, - { url = "https://files.pythonhosted.org/packages/bc/6b/e7f13410d391c6e55b4c007a8de024355389d7d459e3d64c42b2d33617e5/aiohttp-3.14.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:11fb37ef075669eee52ab1928fbf6e1741fada40409fa309ebde9607a962aebf", size = 509190, upload-time = "2026-07-23T01:55:05.173Z" }, - { url = "https://files.pythonhosted.org/packages/97/21/6464573e53d69672cc1eada3e5c5cb2d2efa82701e8305a0f2047a576967/aiohttp-3.14.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:55bdcc472aafe2de4a253045cc128007a64f1e0264fb675791e132ea5edaa3bd", size = 1761478, upload-time = "2026-07-23T01:55:07.383Z" }, - { url = "https://files.pythonhosted.org/packages/1a/81/d217043a4c17fbce360905e3b2bdd20139ebc9a2de836d035d179c4da006/aiohttp-3.14.3-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c39846c3aad97a8530c89d7a3869a8f8e9e3762c6ac0504481e5c80948f7e807", size = 1735092, upload-time = "2026-07-23T01:55:09.803Z" }, - { url = "https://files.pythonhosted.org/packages/a1/66/e13a02d0eeb1a9a502402a977abb4e4abff9fe4051c26f80558c57a7c975/aiohttp-3.14.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5895ef58c4620afe02fa16044f023dc4dafec08158f9d08874a46a7dbc0341b8", size = 1800546, upload-time = "2026-07-23T01:55:12.012Z" }, - { url = "https://files.pythonhosted.org/packages/26/5e/57d42fca1d18cb5acc1cad945d017fabc5d6ae71d8a08ad66be8dc3ee544/aiohttp-3.14.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa9467a8113aa69d3d7c55a70ef0b7c636010a40993f3df9d9d0d73b3eb7ef24", size = 1895250, upload-time = "2026-07-23T01:55:14.357Z" }, - { url = "https://files.pythonhosted.org/packages/ca/1c/7da8d08e74d56f00070822f9638ff3f1c563f8ad87d1efa996c87bfc8644/aiohttp-3.14.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d7d2deec16eeedf55f2c7cf75b521ea3856a5177e123844f8fd0f114ce252cb5", size = 1789289, upload-time = "2026-07-23T01:55:16.668Z" }, - { url = "https://files.pythonhosted.org/packages/cd/0f/cf16bcf56896981c1a0319f5d5db9337994b5165730c48a8fa07e9b34be6/aiohttp-3.14.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dd54d0e8717de95939766febac482ac0474d8ac3b048115f9f2b1d23a16e7db4", size = 1586706, upload-time = "2026-07-23T01:55:18.913Z" }, - { url = "https://files.pythonhosted.org/packages/fe/6f/76eac12a7f2480e1e304f842efdb07db33256b0d9165b866b6ef0806c202/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:df82f3787c940c94986b34222d59c9e38843fba85139f36e85255a82ad5355a9", size = 1724652, upload-time = "2026-07-23T01:55:21.296Z" }, - { url = "https://files.pythonhosted.org/packages/39/b6/19c8c592baeeb94b75f966547d40c02ac7590902306ec5863d5c027cf506/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:42a67efc36300d052fb4508a53e8b6901b9284b599ae63945c377569c5fcc1e1", size = 1756239, upload-time = "2026-07-23T01:55:23.705Z" }, - { url = "https://files.pythonhosted.org/packages/dc/c9/4e9383150296f97f873b680c4de8fb2cd88608fb9f48c79edcb111611abc/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:7a75aa63cbf9b21cfaf60dc2657e19df2c2867d91707d653fee171ffeedd1371", size = 1769161, upload-time = "2026-07-23T01:55:26.082Z" }, - { url = "https://files.pythonhosted.org/packages/aa/1e/147bdc6cc5de5f3ab011be8bf5d6e786633249f22c20bae06f85e45f5387/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:e92eb8acc45eb6a9f4935071a77edf5b85cc6f8dfad5cd99e97653c26593cdde", size = 1578759, upload-time = "2026-07-23T01:55:28.846Z" }, - { url = "https://files.pythonhosted.org/packages/fd/31/78388a9d6040ece2e11df62ea229a822cf5e52d238374b220ae9975b2623/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b014a6ed7cf912e787149fdc529166d3ceabac23f26efeea3158c9aba2354e7e", size = 1792025, upload-time = "2026-07-23T01:55:31.457Z" }, - { url = "https://files.pythonhosted.org/packages/03/51/a3d29fdf2c25d796746af8ad6fe56a45d6256c38b0a8a2ed752e1160b3a2/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3d4f72af88ac2474bb5bca640030320e3d38a0163a1d7533500e87be458eef71", size = 1768477, upload-time = "2026-07-23T01:55:33.87Z" }, - { url = "https://files.pythonhosted.org/packages/29/a6/442e18b5afeade534d877a2dc3c3e392aff8d49787890b0cf84790410267/aiohttp-3.14.3-cp313-cp313-win32.whl", hash = "sha256:5f08ec777f35ee70720233b8b9811d3bb5d728137f30ac91b7457709c3261ac0", size = 451069, upload-time = "2026-07-23T01:55:36.121Z" }, - { url = "https://files.pythonhosted.org/packages/9d/69/3d876ac02659f271cf7f6769f14a8e3de5b6e888ed8b5a7e998086a4cec8/aiohttp-3.14.3-cp313-cp313-win_amd64.whl", hash = "sha256:dff9461ec275f22135650d5ba4b4931a11f3958df7dfbb8db630000d4dee0883", size = 476518, upload-time = "2026-07-23T01:55:38.303Z" }, - { url = "https://files.pythonhosted.org/packages/b2/0e/50d6e6471cd31edce8b282bdec59375a3a69124d8a989a0b1313355cae52/aiohttp-3.14.3-cp313-cp313-win_arm64.whl", hash = "sha256:ddcac3c6b382e81f1dd0499199d4136b877beb4cb5ef770bbbfba56c4b8f55d2", size = 447676, upload-time = "2026-07-23T01:55:40.451Z" }, - { url = "https://files.pythonhosted.org/packages/c8/20/887fdcf832326571b370ffc347b3e70abe101096f3720126aac161b1d872/aiohttp-3.14.3-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:49f7325beb0f85ef4aef5f48f490269575f83e6e2acad00a1d80b807eb027062", size = 509067, upload-time = "2026-07-23T01:55:42.618Z" }, - { url = "https://files.pythonhosted.org/packages/ad/a3/92cec936f78cc4bf0fa5554ebe593b73459d94e3c62303e1902a4cccb6f7/aiohttp-3.14.3-cp314-cp314-android_24_x86_64.whl", hash = "sha256:e3be98a7c30b8c25d573dafba7171d66dfb05ee6a9070fc46535464ff97700a6", size = 514774, upload-time = "2026-07-23T01:55:44.937Z" }, - { url = "https://files.pythonhosted.org/packages/29/ba/2a0c38df3fc557620b6a5acd98364af050053b6285b4dc7ee74100c63c18/aiohttp-3.14.3-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:614c61d478b83953e261d02bb2df750f17227cd33ef8002945bf5aebbde21919", size = 488134, upload-time = "2026-07-23T01:55:47.135Z" }, - { url = "https://files.pythonhosted.org/packages/48/d6/d51b7d4bf309af3693940d8ffd2b9ed0b682434ef85959b7c9c137f60cf8/aiohttp-3.14.3-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:1caa7b0d05f3e3a36f87788c59e970a7ee1cefcfcbb924a9f138c4a6551c9cb7", size = 494201, upload-time = "2026-07-23T01:55:49.451Z" }, - { url = "https://files.pythonhosted.org/packages/3f/5a/8f624384e5f1efabb5229b94157eb966b021e97bdb188c62860c2ae243c2/aiohttp-3.14.3-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:dfa68deb2a443bdaa3ea5297b0699c1464f08aef3812b486d1348eee61b07dc0", size = 502766, upload-time = "2026-07-23T01:55:51.656Z" }, - { url = "https://files.pythonhosted.org/packages/a6/26/4ff0164370deec18fb19254ee4ab10b7a73304ac0c860b13f5f84663759b/aiohttp-3.14.3-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:e72ee89e28d907a18f46959b4eb0bb06701cc7f8cf4366e00029e2ccfaaf5924", size = 756557, upload-time = "2026-07-23T01:55:53.964Z" }, - { url = "https://files.pythonhosted.org/packages/97/a3/7056b86dc0d9ec709ea9777eae3b0161428f943372f8b98c01c11593b682/aiohttp-3.14.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ad4c8b7488d745d2ca4838ebd8ae5ba9b56341d30b1da43640e4ce87f9f49646", size = 510168, upload-time = "2026-07-23T01:55:56.22Z" }, - { url = "https://files.pythonhosted.org/packages/85/ed/0357a015892fd68058bf2d39d3fd1958e459b997a7db30aaa6aaa434ae96/aiohttp-3.14.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:db332af25642007330fca8be5c4d194caf2bea7a7fc84415aff3497af5dfee6b", size = 512957, upload-time = "2026-07-23T01:55:58.437Z" }, - { url = "https://files.pythonhosted.org/packages/47/d1/8aba53f15ccb2238405f5e9d30e2a8ca44f93878c26e7165ade00d374b1c/aiohttp-3.14.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:25bd2708db6bdf6a6630dd37bdcdfcb47c4434d22ac69c64665b802910140b30", size = 1750149, upload-time = "2026-07-23T01:56:00.856Z" }, - { url = "https://files.pythonhosted.org/packages/49/bd/40c3fee327529284375c6701cbb0fa4600cc2e8432af1378f897e2ef7d3a/aiohttp-3.14.3-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:cef89a58e628c4efcac3275c2d68083f82426dcdc89c1492a6f654f9f7ea6ab9", size = 1707685, upload-time = "2026-07-23T01:56:03.371Z" }, - { url = "https://files.pythonhosted.org/packages/2a/a3/ca0cc6724cca8114b05694abd916060758c79894c3aa5b012cdadc1bc28e/aiohttp-3.14.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c23ec8ee9d5ab2f5421f9c7fffce208435607af27fd46d4a44e031954352838f", size = 1803911, upload-time = "2026-07-23T01:56:05.817Z" }, - { url = "https://files.pythonhosted.org/packages/95/b5/85b099c299c3ffd38ad9b3e43694c8a346934e4a30c88c4fd5a841234f77/aiohttp-3.14.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e2667f0bbe7eb6c74eae5e9691441ad186e5845ca3cff63230fc09c4e7514f5d", size = 1876929, upload-time = "2026-07-23T01:56:08.413Z" }, - { url = "https://files.pythonhosted.org/packages/d5/b7/1da684a04175473fa4cddbf9a2f572e79514c3fd27a74597f43057d4f3da/aiohttp-3.14.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:18cb43369747b2ae007bd2655fb8e63a099c2ff1d207962943636dac989b3147", size = 1761112, upload-time = "2026-07-23T01:56:10.918Z" }, - { url = "https://files.pythonhosted.org/packages/d1/16/bc4b55e3e5cb175fd69c53c90d60d2f47797cb343da5106e23863dc4dba4/aiohttp-3.14.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d77640cc618c1d99fc4f8589c0f24a730adfa54eb1e57ef7bf0c8dfb78da898c", size = 1583500, upload-time = "2026-07-23T01:56:13.613Z" }, - { url = "https://files.pythonhosted.org/packages/2a/e8/13a9d957a1ee40837f46aa30f0f4c657e673ad86a2e6362a9f9be20d26d9/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:53e5179d8abb5710f8e83ba207c41c8d1261fcffd4616500e15ca2b7a33be10a", size = 1713940, upload-time = "2026-07-23T01:56:15.969Z" }, - { url = "https://files.pythonhosted.org/packages/38/05/d33c680c1bcf1c7e130f9cbfc1fc02fe8bb0c4af2a94a53dd5fb56131e5c/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:cd817772b2fcf2b8c0905795318485f9ec16eae60b29feb7f4c77085311637f0", size = 1724413, upload-time = "2026-07-23T01:56:18.591Z" }, - { url = "https://files.pythonhosted.org/packages/85/1d/af798d306f7a74b6a632dbcabcf62a4c91391b7582d2a8c6d7712e2cc54e/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4e3ac92d90e92773b2362d506068e9a948192bd553e743c5b2429e28527c8661", size = 1770748, upload-time = "2026-07-23T01:56:21.074Z" }, - { url = "https://files.pythonhosted.org/packages/a8/92/ad720d472556a995049206867765e9410969684f86ee09423ff9969044c1/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:3f42e9b78301f11c8f861746175d8b9c1ccef713fcad9eab396e2f6db8ed4a22", size = 1577564, upload-time = "2026-07-23T01:56:23.475Z" }, - { url = "https://files.pythonhosted.org/packages/60/ad/0ed7586cbef7a884e23a752fa2bb987a122e6a5dd50dab109258d0a95193/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:9d9edccfe496b476db5f398d97b865e9a6752bcf8aec4eef8390ce20fb64bb41", size = 1782080, upload-time = "2026-07-23T01:56:25.994Z" }, - { url = "https://files.pythonhosted.org/packages/97/ea/dbaed0d73e8a69aad653b045dab451c67c2454bb731a37b45a86593e9422/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:1c5ec8fb1bcc31a8466f74aaf26c345d5c386fa4bd08a3f0eb9c7a4a3fe8b5bf", size = 1745813, upload-time = "2026-07-23T01:56:28.604Z" }, - { url = "https://files.pythonhosted.org/packages/81/1b/6893d4bc57e434fc93a6c9217c637d967a0b651d989f6e3265179375754a/aiohttp-3.14.3-cp314-cp314-win32.whl", hash = "sha256:38901a84da3ce22249f6e860bf8f90d141bcab7da090cc398f8bb58c0e44b7da", size = 455872, upload-time = "2026-07-23T01:56:31.031Z" }, - { url = "https://files.pythonhosted.org/packages/f5/8b/c7baa1ba1eda4db6989baefe5de6d99834921b84ebd7918624febcb9f290/aiohttp-3.14.3-cp314-cp314-win_amd64.whl", hash = "sha256:8b3b60de05f3dcb6f6a00f818bb2ec781cee4de0645f59ccaf99b1d1823b6100", size = 481030, upload-time = "2026-07-23T01:56:33.365Z" }, - { url = "https://files.pythonhosted.org/packages/22/8c/c29d067df825a2df88ca432db848aa2fe8199598359cc06c12b09320cac9/aiohttp-3.14.3-cp314-cp314-win_arm64.whl", hash = "sha256:1576145bdceeb92382d899751e12743a3a5b8e460a841e3e50543859e54864dc", size = 453669, upload-time = "2026-07-23T01:56:35.731Z" }, - { url = "https://files.pythonhosted.org/packages/6a/a4/9c033beb355d39b6147980597ec9645e4729243f686ee4dc73945de72030/aiohttp-3.14.3-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:8800c996b01c2772a783e3e46f3e1abd5823029adca0df54231960de9bfefa5b", size = 791403, upload-time = "2026-07-23T01:56:37.972Z" }, - { url = "https://files.pythonhosted.org/packages/80/ca/87c32a0a7704583cfc49660bd817889bae5b830bf53b5dcb4e92145ac2da/aiohttp-3.14.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ebe8e504f058fe91223351cecd2d9d6946c9d241bb0250d898ffbdf584cc72b0", size = 526413, upload-time = "2026-07-23T01:56:40.523Z" }, - { url = "https://files.pythonhosted.org/packages/9e/d8/8ec0e471248c500acdce2be3f46db8fb62b5eb60efef072529cc85ee1d26/aiohttp-3.14.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:30402d03a7c0ff52bce290b57e564e9079fd9d0cb545c8aba73f86a103162d2e", size = 532135, upload-time = "2026-07-23T01:56:42.876Z" }, - { url = "https://files.pythonhosted.org/packages/fe/45/f8919fd936e8b79fcd9bda7b6d8e62613462a713f4f17987fd7c34399142/aiohttp-3.14.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9fc7b5bfec6573f3ae844f457fdde5adeb713f8b8e4a81ad64fc207b49383716", size = 1922742, upload-time = "2026-07-23T01:56:45.528Z" }, - { url = "https://files.pythonhosted.org/packages/f6/ec/9ca76b28a27525b0cc53e20842e0228b022f301ce1f436b7d814b4aaf2df/aiohttp-3.14.3-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8a5fd34f7f7410d1730d5c2ba873cacb2eed3fede366feb268a70ba22581ed8f", size = 1787371, upload-time = "2026-07-23T01:56:48.045Z" }, - { url = "https://files.pythonhosted.org/packages/b1/04/6acdbf17315f7b55f1937e3387acb89a3cddeb4995689553d064af8e92ab/aiohttp-3.14.3-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:270d3dace9ca2f10f0da5d8ebe519b7a310fc6112ed916e32df5866df0888553", size = 1912623, upload-time = "2026-07-23T01:56:50.605Z" }, - { url = "https://files.pythonhosted.org/packages/86/e6/438b0c79ca6f45eb9fd9817dd4c01a91919a38c0de5ee9e05e2b4dc0ece7/aiohttp-3.14.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3ae5b3a59436d089b5395d910121a390feed4d00578eb95a0fd1a329fe963100", size = 2005515, upload-time = "2026-07-23T01:56:53.153Z" }, - { url = "https://files.pythonhosted.org/packages/bb/6b/62cbd6577758699525f5c712d1ddef57d9875fbab0ae8d5f5a202fd598f8/aiohttp-3.14.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2498f0fe69ead802f9675beca44a7c21c62fdaa4ec5145ea1c3ad6edbee29f85", size = 1879906, upload-time = "2026-07-23T01:56:55.818Z" }, - { url = "https://files.pythonhosted.org/packages/00/95/18bcbf830a21dc3aae24d8f6b6feaf3db1d2090242d00a7868db2ffb0b67/aiohttp-3.14.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a0dc483c00da8b673abbb367eb6f8d8f4bcec30eb58529ea13cb42e7fd2dfa33", size = 1675849, upload-time = "2026-07-23T01:56:58.861Z" }, - { url = "https://files.pythonhosted.org/packages/a9/19/47f4968659c5e23606c3790c80fc624e691c153d036148449ee84d31b287/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c7d3a97c678d34fc5b59da671ee9cd630096ddc643e7b5a30d54a2a6f3574d3f", size = 1843496, upload-time = "2026-07-23T01:57:01.591Z" }, - { url = "https://files.pythonhosted.org/packages/64/af/38c33c4dd82fddcb4e56c4653b6f1072a8edbc6b7fa15809f14932c41e2d/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:f8fb78a83c9e5f741ca3a68cfb455c1f5bb83b4e7249a3848b3cd78d0a8563b0", size = 1827746, upload-time = "2026-07-23T01:57:05.131Z" }, - { url = "https://files.pythonhosted.org/packages/a1/9d/0537cda4885ac8f5b7053d164dd06312f4c483a4edcb8ee5b8aaf2a989bf/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:74ab5b6a9fb13e873e5a90946588baecaf488745e1db1a4a5c433f971f035098", size = 1853810, upload-time = "2026-07-23T01:57:08.043Z" }, - { url = "https://files.pythonhosted.org/packages/19/fe/26f9c5e6458385aa86497836b0dea6fb2f027827d63f37c7856cce9286ee/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:bd52f811e65f6fb634b1047159657c98f52b407f8efec907bcfc09da9a4c0a25", size = 1668895, upload-time = "2026-07-23T01:57:10.837Z" }, - { url = "https://files.pythonhosted.org/packages/ec/4c/618b1db9b9ba079b8875d2cdf78e7c4a3bf72903bd5850fee7dd9544600a/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f0f177d1b195b9e06376cfd7d308d8a1b920909a609d03ac82a8c73bbb16d3b9", size = 1883833, upload-time = "2026-07-23T01:57:13.672Z" }, - { url = "https://files.pythonhosted.org/packages/94/c6/bd959bd1e4771f9fd944e9e436224c48c77b018b73b519b5aad346335bcc/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:498c6c623134f8e09a3c4e60bcd607a0b4590dd7dbf08dd40851b27cbb520ccb", size = 1844251, upload-time = "2026-07-23T01:57:16.593Z" }, - { url = "https://files.pythonhosted.org/packages/5e/19/08d41839658bdd44a0ed2480f3891705ecb487ce28c0dde62c9040c997e0/aiohttp-3.14.3-cp314-cp314t-win32.whl", hash = "sha256:b304db572b4368edd8dda8a2274f73156fe15558fca4a917cb8a09fc47af5963", size = 474180, upload-time = "2026-07-23T01:57:19.306Z" }, - { url = "https://files.pythonhosted.org/packages/99/5d/3cd6ef0a2b2851f7ab913b5b079334781bd50ff56a323e4454063377a080/aiohttp-3.14.3-cp314-cp314t-win_amd64.whl", hash = "sha256:b20032766aedf6261c7a566585a40867d092ac03a0d81592d5370ef9b054f99b", size = 500528, upload-time = "2026-07-23T01:57:21.762Z" }, - { url = "https://files.pythonhosted.org/packages/a4/37/cfd1ed540a4d318da025590d96b728e63713c09e9377950fc655dadeb856/aiohttp-3.14.3-cp314-cp314t-win_arm64.whl", hash = "sha256:2e1161602f45a54de2ce0905243a95f58cb42dcd378402f3697f5e0b21e9d2e7", size = 469280, upload-time = "2026-07-23T01:57:24.241Z" }, + { url = "https://files.pythonhosted.org/packages/ca/82/f7f0652e3a845e095d8505a2807a0fe5782abb71f99b2f516c93ff0be492/aiohttp-3.14.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ef692a24087a699c0a4a26af45e746e0c1eae2116f6d8a5ff91d8aae2b867b45", size = 834351, upload-time = "2026-10-11T00:59:07.668Z" }, + { url = "https://files.pythonhosted.org/packages/95/c5/10cb0a5195bc3ec05a74ee494f8431a15e559800f2e869f3526ad4a16f04/aiohttp-3.14.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1220353657ad49493551f089ce02f1a348fd57ffd585bfec77f2f3c4fe3a7346", size = 560663, upload-time = "2026-10-11T00:59:09.853Z" }, + { url = "https://files.pythonhosted.org/packages/32/dc/b7fefdd7dd64d080da2d69d609344191cc94b0e2eba70200c371014c9b36/aiohttp-3.14.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:330900acd0dc4cb8b27f9c127fbaad770964845338493e7906ae3822e82dbf8d", size = 553263, upload-time = "2026-10-11T00:59:11.5Z" }, + { url = "https://files.pythonhosted.org/packages/61/5f/3271cad1b34347e8bf27abee20d501ebdc56cec97e8ae59cd3fabb23d917/aiohttp-3.14.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0684952aeae1f5dbfe02d46039338513b94009baecd15d8e4098a357c4c4a2a6", size = 1843813, upload-time = "2026-10-11T00:59:13.143Z" }, + { url = "https://files.pythonhosted.org/packages/67/af/2011b30fe61889af112ad380e6fd44a93cca4b51296f612cac666c177882/aiohttp-3.14.5-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d94e44be379e569758fee8a9a58431cfc3c2598c708b92b1cfe96c66b4c94aef", size = 1811243, upload-time = "2026-10-11T00:59:15.091Z" }, + { url = "https://files.pythonhosted.org/packages/e3/ba/72f3ac523ed48d38cdfb169f1b2a2e93ce06d9fcf4f46d0f5692e9539bcc/aiohttp-3.14.5-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5af42135fdfebdadbc2bcd9c0842a48ccf0d62794c36a260b21dc4b94d1e0119", size = 1898815, upload-time = "2026-10-11T00:59:17.224Z" }, + { url = "https://files.pythonhosted.org/packages/e6/69/bbf24f3b555ea1befa7db1e2510c728c8b0e80e61ce360beb7d94bc562b7/aiohttp-3.14.5-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4f07fe3ac408d8b3f768be471dc3f56d43843c47d97c66120534467a15ead197", size = 2022390, upload-time = "2026-10-11T00:59:19.244Z" }, + { url = "https://files.pythonhosted.org/packages/5d/38/01f5fc732d5373d6c099db0343279cd4e6607287d336bab6ff6ae9085a0d/aiohttp-3.14.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f375db73a39f5cf83696d500e21a67f418dc9a988955756f254be8f03b7b3651", size = 1856329, upload-time = "2026-10-11T00:59:21.19Z" }, + { url = "https://files.pythonhosted.org/packages/15/85/1f86fbd8fe34ad447c63e679404a1d5be435fe256a9d281b791fc1ed0d35/aiohttp-3.14.5-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8df7d481654ac96fe1ba9a02a9f67770fdd367823e0d5ef01b922725c4bd2cfa", size = 1712763, upload-time = "2026-10-11T00:59:22.982Z" }, + { url = "https://files.pythonhosted.org/packages/39/1e/ced51cf47e427e7aa6142c3bf6f33022472703bc9061289329532d3c40c0/aiohttp-3.14.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e95c8def4b81c5d68d5cf1f54c07acd7c0d2577af244e5b6da802120825737c6", size = 1812125, upload-time = "2026-10-11T00:59:25.032Z" }, + { url = "https://files.pythonhosted.org/packages/6a/f1/9610b4e263b554ca4191501ba72b952efe06556b647da74887abc5d41049/aiohttp-3.14.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f2ebb54b3f932210503072f09974b4fb574d823e497a944adfdcd140a6a00255", size = 1829550, upload-time = "2026-10-11T00:59:27.078Z" }, + { url = "https://files.pythonhosted.org/packages/ac/8a/1d1fb27cf9fa99b13133de33044c4d10d7705be0d4173d6d56abad1b67f9/aiohttp-3.14.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:038c2c7e8caa26b6c8423779b5eaf1893904048a512c19b32fe841ffa5592b50", size = 1868179, upload-time = "2026-10-11T00:59:29.315Z" }, + { url = "https://files.pythonhosted.org/packages/24/9f/3cfafa86ff025623dcca0d30335db7b8a6f3fa348148441208147e827e04/aiohttp-3.14.5-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b7806e804889231b0e06469fd4a5c06313d1c0a3377322b6d9237fa5e0fe4167", size = 1697721, upload-time = "2026-10-11T00:59:31.161Z" }, + { url = "https://files.pythonhosted.org/packages/62/1d/86439a23bf32296e715d8c75535f0cd701821d36719324baef65bcd6af9e/aiohttp-3.14.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:9bab2045550c4fe0f7baf89574db1b455c195750702ba96fef1f16972b146617", size = 1887000, upload-time = "2026-10-11T00:59:33.329Z" }, + { url = "https://files.pythonhosted.org/packages/1d/0f/6044f2d02b3874966d84cd119f9b8eda46aa2991230078aea4dd17750cc9/aiohttp-3.14.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:96a2e584f0b9ed8f1fa33211397dcf67bb7069866402cb405d191c2f0defb9a3", size = 1842020, upload-time = "2026-10-11T00:59:35.384Z" }, + { url = "https://files.pythonhosted.org/packages/18/64/b49d2a34428ff937631b07dde74d5879a7a2c8559aab39331d40769e8cb6/aiohttp-3.14.5-cp310-cp310-win32.whl", hash = "sha256:602c1e9b718a3275c580149f947e7fac65044c0a20e599553fb12e9700da9eca", size = 489282, upload-time = "2026-10-11T00:59:37.061Z" }, + { url = "https://files.pythonhosted.org/packages/a6/7c/1114e723f1c4a65597820b605a0ddcd7f3b28fdfde3ac8d520d517ca582c/aiohttp-3.14.5-cp310-cp310-win_amd64.whl", hash = "sha256:bea559ad70218d230663e4210875735076a9bfea5994cef34a55a25faeaf2544", size = 512021, upload-time = "2026-10-11T00:59:38.7Z" }, + { url = "https://files.pythonhosted.org/packages/48/97/e08c646f4bad63d77876b24591d013d7f33f02fda7c931eafe3224f38467/aiohttp-3.14.5-cp310-cp310-win_arm64.whl", hash = "sha256:dca3fa8d8a0a26679862eccb0b1a9151b2b9f1cd2c212e7a6335778faaff5833", size = 493016, upload-time = "2026-10-11T00:59:40.525Z" }, + { url = "https://files.pythonhosted.org/packages/d8/f3/8997f18890a92c79f77fcfbb4f78ca17c2d4ab109e9eb6d31b4e9de194a0/aiohttp-3.14.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d51db97c96384fbfcaf8f4c65922183a68b94f891c3c10c862ef5f6df2adbb1f", size = 832358, upload-time = "2026-10-11T00:59:42.195Z" }, + { url = "https://files.pythonhosted.org/packages/8e/42/084651e9efb5cadb99265f786df60f2a7b353cead7bfd2c87003cb4867ad/aiohttp-3.14.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ae53924aa853a7a2ca20ed4142c7c6b56338e4d4cd999e2980075b9efc2e257a", size = 559636, upload-time = "2026-10-11T00:59:43.677Z" }, + { url = "https://files.pythonhosted.org/packages/3d/fe/92838944e601f0fe195fd3f3ada37e29fbee3d19bfd323fe15937ec79d36/aiohttp-3.14.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a2c473a355f9239efcb72c92d5abfd8fcdb0cc78c8e9af607e72ca12dbb36593", size = 552588, upload-time = "2026-10-11T00:59:45.592Z" }, + { url = "https://files.pythonhosted.org/packages/8f/b8/dd9b95c5c20ceae1b47738e656bd79e9883e3117a9cc8b143901eb604e9a/aiohttp-3.14.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:32e8fa6644e541fcd7e02430588c7fc93b602c1778ea0bc345505db76b61cfb4", size = 1928224, upload-time = "2026-10-11T00:59:47.294Z" }, + { url = "https://files.pythonhosted.org/packages/c6/47/70010cd2ba8746968d53d433ff328c33f06a66c99da5f3de759c64a6b2eb/aiohttp-3.14.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:579f97d5120f2971876d2ddca2968135f6944d00c44c3a6590ad7d86ca9b403f", size = 1877674, upload-time = "2026-10-11T00:59:49.472Z" }, + { url = "https://files.pythonhosted.org/packages/c5/a1/b026f071dbd87f0bdba86b92d47e46f3899e3ad7143b6e0fe7e7887090b4/aiohttp-3.14.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:24409db442e2fb6e766bc7f3943851a8381dec3098140e43bb2e843b79e31b12", size = 1985850, upload-time = "2026-10-11T00:59:51.296Z" }, + { url = "https://files.pythonhosted.org/packages/36/d1/f7b6f6f8c3cb5a71b53baeddc2e70c224c28b12a23a55c42994c504019b7/aiohttp-3.14.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5e8f97c0488ffda3082766ac0f2c8150a9a58c4d05788330e479cfd449b37939", size = 2105471, upload-time = "2026-10-11T00:59:53.191Z" }, + { url = "https://files.pythonhosted.org/packages/42/74/2a2b22953de15c6c8a80debf26db3047e4e8a5a5db6dd7c2c6c01eec0605/aiohttp-3.14.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:50a195903119008fe9cc68710535eb37f556ffffd6a7759afe70a2c145587045", size = 1929915, upload-time = "2026-10-11T00:59:55.609Z" }, + { url = "https://files.pythonhosted.org/packages/33/ad/f80e8d33933d0eacd0217efa3b7fb32c9f48ed480ed0db53cf9948bf474f/aiohttp-3.14.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0133c3c3b54a0bf1e71fa5c1ad95c93f07fd54e24ef1fe182f5122e1573d2bf1", size = 1773832, upload-time = "2026-10-11T00:59:57.611Z" }, + { url = "https://files.pythonhosted.org/packages/04/a4/0273d239f3e3bb69438f209c64c82e1f98dc60d5db264795e41f7dd05dcb/aiohttp-3.14.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c172db893e516e1358e65a95ee20b7ce7173963eefe318b6ab2a2220688b999e", size = 1901001, upload-time = "2026-10-11T01:00:00.605Z" }, + { url = "https://files.pythonhosted.org/packages/16/ed/7dd439d26c654645bc34ca3c9a822fc11c1c51801fed1cefdfe1cc41c1f6/aiohttp-3.14.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:f2a7966bda23dd85051f1661ce0ace38d6890e05ec6c357ecae9d2479cba377e", size = 1907218, upload-time = "2026-10-11T01:00:02.699Z" }, + { url = "https://files.pythonhosted.org/packages/32/dd/86249c3b8248562a17fd3e22ee0c164378d65766f01aceef62cf52783710/aiohttp-3.14.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:4887d130a7bbfed3a85493bb5a25e5b5b558d40c1d986dd16970d2bb26d63793", size = 1958650, upload-time = "2026-10-11T01:00:04.851Z" }, + { url = "https://files.pythonhosted.org/packages/bc/0b/ca4d53d68f683ce195807fd0aeb063bf6d1c6b39343fff234c9524e8d5a9/aiohttp-3.14.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:225c579c23b68b343cccea27a7e06e3bd8ec23a09c30b427eb3f1e4ca6239b20", size = 1760057, upload-time = "2026-10-11T01:00:06.978Z" }, + { url = "https://files.pythonhosted.org/packages/6b/42/005437ffd7fa56c2ce3347add40404b654a32754ce91f2e2a14b9e5ab2bc/aiohttp-3.14.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ab52d8f1fc1b64821c1fbad64a647ed6203627004059a6d1ed4f0858a1499703", size = 1978081, upload-time = "2026-10-11T01:00:08.868Z" }, + { url = "https://files.pythonhosted.org/packages/03/71/3a5b66fe1b7b23d3c6524350817ed55de5feb7fbb4cccba7cbe044d118dd/aiohttp-3.14.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:cb131d775a1573c1aee66656bd78b023577bbdb6cb8349a07773bd4f73e68a6e", size = 1922954, upload-time = "2026-10-11T01:00:11.035Z" }, + { url = "https://files.pythonhosted.org/packages/69/84/d08a554f281d42fd7c6b2b6dd7738f3e3a7a9f657ddd87e11ca2091cdfe3/aiohttp-3.14.5-cp311-cp311-win32.whl", hash = "sha256:e87046c8ff77a8decdb6a41d8ab25824b47531b2da933aeab0c1e21c7acff329", size = 488101, upload-time = "2026-10-11T01:00:12.933Z" }, + { url = "https://files.pythonhosted.org/packages/0f/15/52b5e65f02b33686ae49f1518548ae4f5a7adaf31d9d3c54fa12a143137b/aiohttp-3.14.5-cp311-cp311-win_amd64.whl", hash = "sha256:6f275c11d1aa6d4c458e05a68be084efe3c55a113d99e3f46a318098e52948fc", size = 512575, upload-time = "2026-10-11T01:00:14.743Z" }, + { url = "https://files.pythonhosted.org/packages/b0/b9/bb75012635f3defb0f7cbb7c4ae391e36bb9cb3fbdca3b9944e2ebebf743/aiohttp-3.14.5-cp311-cp311-win_arm64.whl", hash = "sha256:b032a0023eb41d768ce77d83210ab2a3c389bc0b09313273c7e1eca48c10a755", size = 493179, upload-time = "2026-10-11T01:00:16.961Z" }, + { url = "https://files.pythonhosted.org/packages/d5/94/6ba86efddcb616c811b40e6a0dfdd862738f647e4e3860a961075d5e9ed8/aiohttp-3.14.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:df37b620684e19b5e25724412518ccafc3b1a49cdac706fdbd2f983fad943450", size = 817499, upload-time = "2026-10-11T01:00:18.901Z" }, + { url = "https://files.pythonhosted.org/packages/5e/e1/7bca6d84dabd228aa8eb4b7f9feac2586aaa9be5505d7d65bf287a615c43/aiohttp-3.14.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ef60869969180ec2464f1349aff07138ae35ca2200f0946cb3552e49e8f301a8", size = 549467, upload-time = "2026-10-11T01:00:20.854Z" }, + { url = "https://files.pythonhosted.org/packages/64/91/11b89f45ca486252dd67dd5f3231fec04bf5518da39a95cb3997619f17fb/aiohttp-3.14.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d079c0a0135c36e7beb6f1c88087c8f108dc5891cdd0b5eafa778421bda70ed2", size = 548067, upload-time = "2026-10-11T01:00:22.659Z" }, + { url = "https://files.pythonhosted.org/packages/22/ff/c6615806c14aab34f82b9424ccde8ce6e417315fd57ce1c5b4d4747888e1/aiohttp-3.14.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:abfda5cb094a829f7bc25216a32f7db2e85cc65bd59910f8e7b40b3d9b224764", size = 1896245, upload-time = "2026-10-11T01:00:24.638Z" }, + { url = "https://files.pythonhosted.org/packages/39/b2/25a8c971ae6a92c8394d77e42422d7f38989e05cf41a5ceb92d73d67ab7e/aiohttp-3.14.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9cc882cf8619109583c906b4d4a85d6a111a98afa34b7a450d1e08118d016820", size = 1872180, upload-time = "2026-10-11T01:00:26.838Z" }, + { url = "https://files.pythonhosted.org/packages/2f/d5/99f93ea36cc5205e47c1e5a803e087f2ad21b5430b5db2e942cb6e988a37/aiohttp-3.14.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7457580535e019e1247ea35d6a02bf081ad30c26d0cbc210c93f6c3ab67a0835", size = 1937261, upload-time = "2026-10-11T01:00:28.74Z" }, + { url = "https://files.pythonhosted.org/packages/40/a6/9ac9c9e6695040bd73d2584a1b59a9388f6433c76d5294a7bf591e21ffe5/aiohttp-3.14.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c5ed596aedb9c42afd3fe0aae3117725378ac73d2cc5ddc735056fbdb96c5d02", size = 2064994, upload-time = "2026-10-11T01:00:30.623Z" }, + { url = "https://files.pythonhosted.org/packages/da/e4/aa172eb534b7f02f1f8ff1c3213347eaf3cf218db91a727c6863c22f1035/aiohttp-3.14.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:20f085697d7e911f1f73c43ed03fafbed1e7121797e2eb5428efa80398060584", size = 1919630, upload-time = "2026-10-11T01:00:32.548Z" }, + { url = "https://files.pythonhosted.org/packages/06/7d/4eedafc5bababa8932636c141e346806966eade12c0b7e5946d43bf8218b/aiohttp-3.14.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:74b0a9c8270f9b0a11410e124ff8d4f18bfc1f1837440ec84da5ae7b50927b5d", size = 1710016, upload-time = "2026-10-11T01:00:34.471Z" }, + { url = "https://files.pythonhosted.org/packages/b2/94/eee018537ba19da0ceb2ac79cab83faed4ac49568e08376e2799043f2538/aiohttp-3.14.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:19e2ba471507c34f8252402ab50f5ab512398b9ea8c8f1cb26beb3f75793ba30", size = 1858112, upload-time = "2026-10-11T01:00:36.581Z" }, + { url = "https://files.pythonhosted.org/packages/68/76/354653a306547238f3427283905972d796ba7c292ba9977abec9f2b6f260/aiohttp-3.14.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:d418ce2af40c6bb685b3f663e9e8de27cb0a22431d8e88a167348d7f01878073", size = 1896371, upload-time = "2026-10-11T01:00:38.478Z" }, + { url = "https://files.pythonhosted.org/packages/f2/ec/63e8c7136b570e356345ad3174e3820fdc973ea10712cb6c649bf875755a/aiohttp-3.14.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:70cb4008ac2ed1e0ca9e824deb4b53d3aa0d939109698ebf1e723a84337bd794", size = 1905836, upload-time = "2026-10-11T01:00:40.527Z" }, + { url = "https://files.pythonhosted.org/packages/57/d8/11365bda144b127928cd42533d0eff78a55613c9e28f81941bd6630ea887/aiohttp-3.14.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:a23fe35d776bc03cb495938b9594450d047e3bc08c5255315a82323e9cb7d2dd", size = 1696534, upload-time = "2026-10-11T01:00:42.686Z" }, + { url = "https://files.pythonhosted.org/packages/2f/3d/82df0461b18e00b2998f205c03e0d3010222478c43640aceb8e03dcd7e8f/aiohttp-3.14.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3e0eb43bed3c6801a6cee315195377789e90b2a72c2277a475b578535312488d", size = 1933970, upload-time = "2026-10-11T01:00:44.71Z" }, + { url = "https://files.pythonhosted.org/packages/03/ad/6ddfe0aacd931c17b53533336d97e9d11a98b96d6ae815a9da0b19f82ccf/aiohttp-3.14.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3be7dd397d64ca3e1869626fa9318aaebb54b7bf93bc72d7a205448d83e4f748", size = 1899501, upload-time = "2026-10-11T01:00:46.629Z" }, + { url = "https://files.pythonhosted.org/packages/9e/8e/189bdd9ae4793059bb09f6dc880f211a6c6c7859cebcbf33912dbfab7dd7/aiohttp-3.14.5-cp312-cp312-win32.whl", hash = "sha256:eb324e2009fb54db30a071dad7caf6998ee2879c4704007efb244514dad1fec1", size = 480370, upload-time = "2026-10-11T01:00:48.468Z" }, + { url = "https://files.pythonhosted.org/packages/ae/ce/1f08114679d49655b30a6e0a29858375c94b82c1de1a0bd0a20c2fee8b02/aiohttp-3.14.5-cp312-cp312-win_amd64.whl", hash = "sha256:2cc38a4f2b516bef1714e690df87a0e043faf1a7693c82d860091684453d5111", size = 506141, upload-time = "2026-10-11T01:00:50.272Z" }, + { url = "https://files.pythonhosted.org/packages/79/d4/c7b4f60b16a1b7e43249fa9031ae05e7e8c341ba4b7d1866f914dafeaa0e/aiohttp-3.14.5-cp312-cp312-win_arm64.whl", hash = "sha256:a63afd1f757de949028387e65a7127b61ad0f775432dbb0e62816ae619fe69ac", size = 488108, upload-time = "2026-10-11T01:00:52.321Z" }, + { url = "https://files.pythonhosted.org/packages/d3/e1/2841e020ebb7aefae5513586193e011e06313d9a6bdbd296622afbbce204/aiohttp-3.14.5-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:9ad7e6aa38c20da1be697874349c4c273c8a03b7887169665081706398d0439a", size = 538932, upload-time = "2026-10-11T01:00:54.3Z" }, + { url = "https://files.pythonhosted.org/packages/f7/a6/7fb8ea8fe96bcc7b7c7a36d10f021d99d01a8dc8a4b3f0ddacecfad9a80e/aiohttp-3.14.5-cp313-cp313-android_24_x86_64.whl", hash = "sha256:f59c7673465908cbe506117176156c127f29f917677afceada34957179221d91", size = 553329, upload-time = "2026-10-11T01:00:56.442Z" }, + { url = "https://files.pythonhosted.org/packages/de/64/d056e3c27647dc25af1a592cf356245382ea7c808171b9dac7677afedfc8/aiohttp-3.14.5-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:1b5416552740edf07234cc9437d0706f2acb67b93c198670b1a68e1b2b587dec", size = 521367, upload-time = "2026-10-11T01:00:58.345Z" }, + { url = "https://files.pythonhosted.org/packages/3e/e4/95226147e11d4db916fd1d495dcf85af8e3816e38333e42718241196e848/aiohttp-3.14.5-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:43351bdb5e4c3cb7d1772368e988534e869a74db7778079a83782c11c69535c7", size = 528322, upload-time = "2026-10-11T01:01:00.211Z" }, + { url = "https://files.pythonhosted.org/packages/17/cd/1d3c9192cafdb51cad62b2d3ded96cff9cc8af51893210aadd448a325389/aiohttp-3.14.5-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:c8c4478bef6d57fcfda15dae461ea3c9f06aa7b257c58df3f2300174ccbb185a", size = 539874, upload-time = "2026-10-11T01:01:02.06Z" }, + { url = "https://files.pythonhosted.org/packages/f6/0c/dfa33aecc7d4d1dc75e05248f5eac5a0edf4d09e7b44d93ab62529b0c1db/aiohttp-3.14.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:c2c30484dd1417ef98b51021ffa2cc0d7f3c78918adaaaab7e70817335ab3e02", size = 813836, upload-time = "2026-10-11T01:01:04.01Z" }, + { url = "https://files.pythonhosted.org/packages/33/17/4a63738052d20567d55529d6daa1b9480d906fd52930fbcf6d3fbed618f0/aiohttp-3.14.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:dab9ac5a67c8d1f070c00fa8fccb7cbd1b8dcc1a8d6b42f37540df9b3d4cc603", size = 548070, upload-time = "2026-10-11T01:01:06.035Z" }, + { url = "https://files.pythonhosted.org/packages/15/e5/b57e58695a757fd4c02497c033fced96a69c631b13866c43d336530c9670/aiohttp-3.14.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e1cc2bfaee8c214f06080a7c7d5772419b8a1108e8e5349236189811823fb02a", size = 545786, upload-time = "2026-10-11T01:01:07.817Z" }, + { url = "https://files.pythonhosted.org/packages/9a/68/8c2c67a3aedf46e00f3c42f04fbc6983de80d4ed5786151e33681ba45883/aiohttp-3.14.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:74efb69332b85675b1eabd760a8cfc2e2cf42c60607c66f88014c1bdfb40942d", size = 1886492, upload-time = "2026-10-11T01:01:09.834Z" }, + { url = "https://files.pythonhosted.org/packages/ab/4b/74aab5e8d28c62e8f795b4fe8f38cf5586fd264a9a27bd2141ef6490333d/aiohttp-3.14.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:42b5e616946dbaf505e2bff18c9af2cd4ef9e7ef300ee58a6e951a5b7cf147ae", size = 1870057, upload-time = "2026-10-11T01:01:12.045Z" }, + { url = "https://files.pythonhosted.org/packages/a8/f7/eafc3b1988302b1815d9fd4a21071be5c360d616c0a430d02fd92dc97688/aiohttp-3.14.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f9033b43f511f27547c557dcaba0177649e10a3725336ccd2cce0fdc1dc4850d", size = 1925119, upload-time = "2026-10-11T01:01:14.09Z" }, + { url = "https://files.pythonhosted.org/packages/a1/04/78d8f294f74dd570f3898ff20402349fce524176045df98ba727d6846a68/aiohttp-3.14.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:15a310d3c71398e3d7bfc93a1a73fbe664315cd9e9016b8efc1cff85eeab7155", size = 2053137, upload-time = "2026-10-11T01:01:16.344Z" }, + { url = "https://files.pythonhosted.org/packages/32/51/395d225ef36f5a50d8e548dcd3141bfdbcd31fb6eed859022c573d2c4d66/aiohttp-3.14.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1ffa3a523a36d8628f98c06492ae16a31a23d14c0b4ec721757b477319f656d6", size = 1911038, upload-time = "2026-10-11T01:01:18.653Z" }, + { url = "https://files.pythonhosted.org/packages/ff/a4/2aec1aa06d82e8a244843b5dae31d78061e5e76744270a86dd0ee051c889/aiohttp-3.14.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5fb6a6e919bfb703227bc1ce6579281b84b1a2ba57deb9794dfdbec7dcd1e40c", size = 1702205, upload-time = "2026-10-11T01:01:20.904Z" }, + { url = "https://files.pythonhosted.org/packages/16/27/6051bfde7b6f418f70edd60d655fa426abb3355fa0981764739d87ecf160/aiohttp-3.14.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:43e1b7994a8b038125f722bff07492ef501110722c2727c408995d9fb864c421", size = 1848588, upload-time = "2026-10-11T01:01:22.918Z" }, + { url = "https://files.pythonhosted.org/packages/9e/44/55efc06fc26c4e6e1c095f231b4c222bf2d86d64a8eebbc24b2bb5958ea8/aiohttp-3.14.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:5f3e96071686755d9cd3600c3880183eb94b012178e92746d68101800f0ed8a3", size = 1887540, upload-time = "2026-10-11T01:01:25.278Z" }, + { url = "https://files.pythonhosted.org/packages/48/dc/1502bfdc2a65760d386ac6a00090b0addaa8a3c9c60b3f8127fad3a9afb2/aiohttp-3.14.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:cd88b01f3d37b7a2a34f91d98f14720206f1ea3d540843fab2d649dd5fb91fec", size = 1895030, upload-time = "2026-10-11T01:01:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/93/7e/44174bb6288264418c9eec07a5e35180969c0d5a796c7af544db3cb8a33a/aiohttp-3.14.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:f2ed8b64dc0c651c0f5a9c926777719770021251b8f336d97c4b80b660836ce1", size = 1693244, upload-time = "2026-10-11T01:01:29.39Z" }, + { url = "https://files.pythonhosted.org/packages/47/dd/b507d64e50db23888582fff08eda13998b12f9dea70c072edaac19218380/aiohttp-3.14.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a9918e58faf62ba2c7147927d06057aec78f42475aff5048047ec47e7265a600", size = 1926308, upload-time = "2026-10-11T01:01:31.634Z" }, + { url = "https://files.pythonhosted.org/packages/98/4b/5b51b4f63e3f2793151f4aea49c48fe1e00baeb7cec9c7a206de499f8de0/aiohttp-3.14.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:42f320d4a5b00b9af0bddcfec5407dc6f2d9816f006b2f79ebbaa31f16895df3", size = 1888406, upload-time = "2026-10-11T01:01:33.715Z" }, + { url = "https://files.pythonhosted.org/packages/ff/13/d5e818a5eaba9f822016727299f41c0e1075799f82a6433ec508a93ab867/aiohttp-3.14.5-cp313-cp313-win32.whl", hash = "sha256:3ae800a20947e2c2e53088047d021e6bf7d51560cc49f6a0737a1f79d2e3a13c", size = 479970, upload-time = "2026-10-11T01:01:35.677Z" }, + { url = "https://files.pythonhosted.org/packages/8d/d0/8eca2c65aa467320990d78fb2005f38ed3588944280c39ff9deb6423fef1/aiohttp-3.14.5-cp313-cp313-win_amd64.whl", hash = "sha256:d05e94cdfe0d15d0206f970722d2554780ce562787b21b218b275447f8751319", size = 505985, upload-time = "2026-10-11T01:01:37.574Z" }, + { url = "https://files.pythonhosted.org/packages/7a/f8/4cdd65305d2fca14b886bea9ed2abb1fe726287872524e56e3d26692b47d/aiohttp-3.14.5-cp313-cp313-win_arm64.whl", hash = "sha256:f001b571ead90ca1770f1e616db255351a1703317f20374c361ef22f12c06d09", size = 487629, upload-time = "2026-10-11T01:01:39.477Z" }, + { url = "https://files.pythonhosted.org/packages/43/be/3184a1d34a8be665569eadb7e9e764b4629e4f3413e241cb2e4d6fecf3b3/aiohttp-3.14.5-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:939042d5cda21d41a6f512e7cc8b8e33a2aebff863352251da495fbd91b673b5", size = 540673, upload-time = "2026-10-11T01:01:41.354Z" }, + { url = "https://files.pythonhosted.org/packages/60/2a/d35f3ba4cf157b072e3b674bf9983047ca5ea5173c995d32d877e1191d36/aiohttp-3.14.5-cp314-cp314-android_24_x86_64.whl", hash = "sha256:6da32b5ff3fd78d244e37300463434c7145162bfd2b6e9e915ab164da37f7343", size = 554087, upload-time = "2026-10-11T01:01:43.719Z" }, + { url = "https://files.pythonhosted.org/packages/fc/d2/61a33880ca4eaca95a9c60ca3f6beed15555af1028652dfaac601627787b/aiohttp-3.14.5-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:1b438b73c38111818d0c9d6a5c2bfed8584c8e503a49ef085d70e874ec846738", size = 522613, upload-time = "2026-10-11T01:01:46.154Z" }, + { url = "https://files.pythonhosted.org/packages/31/1d/de579b299d2225dc2c6fd99d579d91f16c02a913fb5af9a3cf2fe9bd88ba/aiohttp-3.14.5-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:755933b107ea7a6a9ac916f635a70595a5b1a32fac10a8ff0b9f2ab88555550c", size = 529682, upload-time = "2026-10-11T01:01:48.477Z" }, + { url = "https://files.pythonhosted.org/packages/f4/4a/ddb923564e15e053b6e060b0036e1694dcadcb13aa476c5a87dcad20e336/aiohttp-3.14.5-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:b3cc509327c7b27f6f4727a8830f4004f6df7766e179f2f4b8e54e65c0bec5d3", size = 540120, upload-time = "2026-10-11T01:01:50.474Z" }, + { url = "https://files.pythonhosted.org/packages/3d/36/a640fbecaa53727a5900b892bdbe17b5f3e8cc88903e22864fe41b654def/aiohttp-3.14.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:7bd8ac754ebd6733a3e2a0dd1674c4d8ab086196803fd8dcd776f07b4e2607d9", size = 818795, upload-time = "2026-10-11T01:01:52.665Z" }, + { url = "https://files.pythonhosted.org/packages/ef/b6/d52ca608859e271b5fa7944074802dc45f60e52c318a4ddc34edbf73586e/aiohttp-3.14.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:e724a7b6091f0b1ac064f9d1b15ff9ec52e6033a86cdae649e5f086e32a3c0db", size = 549667, upload-time = "2026-10-11T01:01:54.65Z" }, + { url = "https://files.pythonhosted.org/packages/ce/b5/05b8ac39a76ff4bca89f42a4c2471560c71f894ff6e4bc16158c951874e3/aiohttp-3.14.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c32e26310cc10e547f53cd13d39a369034f69dcb7d749d5cb0e5f67bc196b6ba", size = 548922, upload-time = "2026-10-11T01:01:56.547Z" }, + { url = "https://files.pythonhosted.org/packages/19/b0/5aa186d56ce2334dabe29b70bd99dc8ae926ee44184c0de64a53d415a4f3/aiohttp-3.14.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1eb8167961ec4dfcc8cb9dd50bd0ee72519f7ef496be95203e49e27b01618382", size = 1882422, upload-time = "2026-10-11T01:01:59.258Z" }, + { url = "https://files.pythonhosted.org/packages/db/f7/7d5c91bb9620db300c8ddb05337a9014301acb626223faff3abcb8ea47d7/aiohttp-3.14.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c1d60eafd9c7e8e74abd03a5b00df44e7febfe6d9b89b559c0a6551eef0699d4", size = 1843621, upload-time = "2026-10-11T01:02:01.417Z" }, + { url = "https://files.pythonhosted.org/packages/30/0a/b208953b96d8f24b75f6da704f508e6c5cf3022f52b60c61933df082e89c/aiohttp-3.14.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:137351bf20bbed9a65e839f4a4452ac377389bdb2f2857d2acffef38f5e9f2d1", size = 1927915, upload-time = "2026-10-11T01:02:03.697Z" }, + { url = "https://files.pythonhosted.org/packages/f6/79/90ebcccb55e2d1e11a1fed581d83bb966e38fb35fb4b2577fdc980f8707a/aiohttp-3.14.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fba47bc2c3d7303c3d027c6cf4d07626c37b1314ac81f5820c31032e0ca1f677", size = 2036339, upload-time = "2026-10-11T01:02:06.046Z" }, + { url = "https://files.pythonhosted.org/packages/a6/66/55a8904b3a129fafdf94f9cc0a2e4ca09a3c914650be52355db7ad0bbdb6/aiohttp-3.14.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:94684b879ac1d71e4238850c99b62dc1b28d9086b156a2555f082010b85a865c", size = 1888041, upload-time = "2026-10-11T01:02:08.384Z" }, + { url = "https://files.pythonhosted.org/packages/0b/b8/96b25da7329a52e42c812b1e8b076386039ec4fc312afa043d173d8147fc/aiohttp-3.14.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:56572c42e3ecd636de8d2c3dd54cf5fc939cb5c32eb56297f176a0d366fac622", size = 1703849, upload-time = "2026-10-11T01:02:10.903Z" }, + { url = "https://files.pythonhosted.org/packages/1b/43/fbf976e3ae4c038d6f5c84945ab2150298c2d71201674d4e53d158063e75/aiohttp-3.14.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a95529a92a446db351675f4aab518feaf5e99842f63f5dd17160c2b74f382db3", size = 1849703, upload-time = "2026-10-11T01:02:13.15Z" }, + { url = "https://files.pythonhosted.org/packages/8b/7d/218e912f4c1d89bde7ac551409be57ad2f6121638e56942d395a7cb1fa58/aiohttp-3.14.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:3edbece0379b8b4aaa67619b8aa2399bb66fce372cd5911098a434ea77220aa0", size = 1852695, upload-time = "2026-10-11T01:02:15.295Z" }, + { url = "https://files.pythonhosted.org/packages/5a/42/252a1b9287e3b6e393a1c3bd1776f36af30f5f25f071bbf2b0cb7eba9116/aiohttp-3.14.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:56d9828f204331a5ca8850fcfe2bcce95a149f1f223f60cc7216e5524978e480", size = 1893493, upload-time = "2026-10-11T01:02:17.93Z" }, + { url = "https://files.pythonhosted.org/packages/78/97/71cae83d5100556fad1521684f7cd1e3e578432644f850245ed3bd969310/aiohttp-3.14.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:20064a177a070d789ee64a50b01a9161d3468e989baacfc6c714aa685c4b332f", size = 1693468, upload-time = "2026-10-11T01:02:20.666Z" }, + { url = "https://files.pythonhosted.org/packages/1f/69/73d88e97a8b5f0ca7a946d0011c0de99fb188b1687c7948ecd0553dc5bf0/aiohttp-3.14.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:81c2b3dfd56c62bee6108e4852d5970b4cf9086390b6983f52b666e878c1f115", size = 1915398, upload-time = "2026-10-11T01:02:23.011Z" }, + { url = "https://files.pythonhosted.org/packages/05/f0/881644bcb15d4b258daea9b720a0af9dc4330496cc8d6ade9090cdd0cffc/aiohttp-3.14.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:09ec102b4b8c9a920275733bbc11fdbb615efe6f9231a06007c0218d336fb77a", size = 1871470, upload-time = "2026-10-11T01:02:25.278Z" }, + { url = "https://files.pythonhosted.org/packages/7f/de/19d9ebbcce5aedaa3242d8a99ff8816a60bdb7629fb0084bf4a45bfd1f62/aiohttp-3.14.5-cp314-cp314-win32.whl", hash = "sha256:9c428eb2bd8817588d16a0ab898aa4eb5d141f896aa2b394cc79a4cf61d9a8e2", size = 485517, upload-time = "2026-10-11T01:02:27.579Z" }, + { url = "https://files.pythonhosted.org/packages/a9/74/8cdaf0e58c2588371670d5a9a8215bbb971d36940b6e5d051967dd05c07d/aiohttp-3.14.5-cp314-cp314-win_amd64.whl", hash = "sha256:6f967dde489ca6a8c02d093ab245d2cbf50ccb5c36adf0188b17b0ca39d24b67", size = 511755, upload-time = "2026-10-11T01:02:29.685Z" }, + { url = "https://files.pythonhosted.org/packages/1a/6b/e0100e25502430a531c7cf1482a378d0b65bf728ab60c01ee270e56bc469/aiohttp-3.14.5-cp314-cp314-win_arm64.whl", hash = "sha256:1d2d981b53dd09a319e3570ef8cc3bbc3ef86f5a7abef0f6b2bff3867db3a9e7", size = 495151, upload-time = "2026-10-11T01:02:32.163Z" }, + { url = "https://files.pythonhosted.org/packages/9d/c2/ca2ead7b655688c53c03aeeb6e96e6851c9ff08d6be7b13802f53a6ae8fd/aiohttp-3.14.5-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:9ce66feae6ac65327379460380549bf1b8df8e17c4e25df2a2bcf168272e3bed", size = 855396, upload-time = "2026-10-11T01:02:34.443Z" }, + { url = "https://files.pythonhosted.org/packages/a7/70/22206fea409255a240c926ce11de48de354ae2bb90ca44f709c05497585d/aiohttp-3.14.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:27c2322e03f66101acb09869ce1cf1efc04994ee95e1735b69827bf8c8b9d781", size = 567734, upload-time = "2026-10-11T01:02:36.644Z" }, + { url = "https://files.pythonhosted.org/packages/ce/e5/79a36c118308b56f8667d67e05d2fb6dc638ab45985704cdb199631bedaa/aiohttp-3.14.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ff75a7537413a86e7cafe98e0e1d6e3dc4b15c6349896e7d5c6b881bfdb6d550", size = 567379, upload-time = "2026-10-11T01:02:38.757Z" }, + { url = "https://files.pythonhosted.org/packages/63/a3/2ebec7dece3b1f02c30d2e484647f6f7b13952b1bb40a4cb285b849e8432/aiohttp-3.14.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c061aa954daaf57d2a4b8374f9fca621ef0e1b603584431c220c22458c59b6d", size = 2050686, upload-time = "2026-10-11T01:02:41.169Z" }, + { url = "https://files.pythonhosted.org/packages/3a/d2/7e4d093db2f4450482652e7ef19a9e19919028f5135aa52bc4078c3beb80/aiohttp-3.14.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1612fa5857b37bf32e5c1eaeefb96e3b01e9c70679eec81f0934e8a600080863", size = 1915077, upload-time = "2026-10-11T01:02:43.563Z" }, + { url = "https://files.pythonhosted.org/packages/a6/88/bd40d09958442a0a1df67da81de496361d6e2afc04f9db2950d835da750b/aiohttp-3.14.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:adbeee7d6fd4cf5fe0aece2fb3edc4243615d3180430ba8149d01a90670cac99", size = 2027204, upload-time = "2026-10-11T01:02:46.185Z" }, + { url = "https://files.pythonhosted.org/packages/63/eb/3a601c1f8d3103c1a60ea981f20924855da9f575d2007fc38f8898b792fb/aiohttp-3.14.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b2966998927d7bed9db12c0a4647b0c7b179755878fc9c357fe1ffd3e3b0c1a5", size = 2149520, upload-time = "2026-10-11T01:02:48.812Z" }, + { url = "https://files.pythonhosted.org/packages/22/d0/4e41bfe1b1ce1cb6f6d2e59fa7a88ef5cf92c402b2d07e9018778ba9edbc/aiohttp-3.14.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e317e0fb6b16212c881d2205a7d87414c29acd69320b3aa6dce9d9c7b86fe4f", size = 2002039, upload-time = "2026-10-11T01:02:51.293Z" }, + { url = "https://files.pythonhosted.org/packages/6f/5e/72067019545c502b881b031153c437752ecef48d7d213bc0218ccebb4bfb/aiohttp-3.14.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:50343c1757b4b6f6708eeaf24534b32f19dfb99fb1b762c00420867a62fc81e0", size = 1786164, upload-time = "2026-10-11T01:02:53.533Z" }, + { url = "https://files.pythonhosted.org/packages/d9/fe/7741efd6119bfb7a00827fe6f7b84b4409de58d888ae21adc5a9a6824992/aiohttp-3.14.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:083673c7a94c3ea035caaa5ca04288bdb44887abfe1f5ba23294e6a4b03efd2d", size = 1970689, upload-time = "2026-10-11T01:02:55.888Z" }, + { url = "https://files.pythonhosted.org/packages/43/e7/342a13bf67f34d269bf2f7e870ecd72b99c832cc6f0a271a9210c0ebfb84/aiohttp-3.14.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2528cb4c6b92008c76ac9ac6298624069bb2db91ff4929905512d1d84485f658", size = 1951967, upload-time = "2026-10-11T01:02:58.467Z" }, + { url = "https://files.pythonhosted.org/packages/e7/d4/fdb3b27340617e5e64df18a70fa89097778652ea5c7c7e2f79def76999c3/aiohttp-3.14.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b1b8ece1e71132d2afba4dbc0c3d62c766e25165990b25db1196c04969eb3d84", size = 1968511, upload-time = "2026-10-11T01:03:00.883Z" }, + { url = "https://files.pythonhosted.org/packages/83/b2/e8f88298de78d1a951f36f9f966d38ec6ed1d4721303ba02064a545e8aa6/aiohttp-3.14.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:fce9523df31cea6284f3e2c479876750d7687cf671d7b25d32b19effc0e86441", size = 1778497, upload-time = "2026-10-11T01:03:03.206Z" }, + { url = "https://files.pythonhosted.org/packages/22/68/9ccdb93d664345c546be7f34480b921774d8c0d98f47e70d7e03b115d475/aiohttp-3.14.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:09e0eb18c7e0c8777e2f9149de63799195b9b3ca1b5c81ba6f32f2c6b8628210", size = 1999676, upload-time = "2026-10-11T01:03:05.829Z" }, + { url = "https://files.pythonhosted.org/packages/57/4a/a33cfa6dcb00e94194ae4fe710432ca4ed111e016356b4ba4d2f4c3a124c/aiohttp-3.14.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6774814fd5c338e72ee0da5cbb9432816df450e69c019f72b5d29bdec2a1792d", size = 1963437, upload-time = "2026-10-11T01:03:08.196Z" }, + { url = "https://files.pythonhosted.org/packages/f4/20/eacbecfea3b5c3dcbfc9b023e3a5460f43e5dda16d06a2877ebe7184c3f3/aiohttp-3.14.5-cp314-cp314t-win32.whl", hash = "sha256:33f706574e32c6e694f352a856e05caf18f7f2c871b3e87b41c55ea452b409ab", size = 503973, upload-time = "2026-10-11T01:03:10.481Z" }, + { url = "https://files.pythonhosted.org/packages/ff/78/18eec294f6c8c5dc845dcf6d730a0147d8d0f17e86138a7bdb85e43a30fa/aiohttp-3.14.5-cp314-cp314t-win_amd64.whl", hash = "sha256:5ba14a839fbe87cf7c12a6b5661c05f324a296eb8363141edb3944ba63d4c9d3", size = 531190, upload-time = "2026-10-11T01:03:12.716Z" }, + { url = "https://files.pythonhosted.org/packages/9d/39/e53f8169acc85271ebd12b5b32ad7f1541b35639ccbe0f49034c64785d10/aiohttp-3.14.5-cp314-cp314t-win_arm64.whl", hash = "sha256:1061b364556e8172e8d46b0b183adeeb73e8c42d30ebc745591e1bd89acad52e", size = 510360, upload-time = "2026-10-11T01:03:15.08Z" }, + { url = "https://files.pythonhosted.org/packages/68/30/173960c42b05a6c59f7558e4b12a4b0d9ba376cf6aa9bde7f9e08a30ca8d/aiohttp-3.14.5-py3-none-any.whl", hash = "sha256:efc21a454892828368b11c2c780de0ff8bc991f73f6b99c6b66e56205470929b", size = 279517, upload-time = "2026-10-11T01:05:08.523Z" }, ] [[package]] @@ -10792,142 +10793,118 @@ wheels = [ [[package]] name = "yarl" -version = "1.23.0" +version = "1.25.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "idna" }, { name = "multidict" }, { name = "propcache" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/23/6e/beb1beec874a72f23815c1434518bfc4ed2175065173fb138c3705f658d4/yarl-1.23.0.tar.gz", hash = "sha256:53b1ea6ca88ebd4420379c330aea57e258408dd0df9af0992e5de2078dc9f5d5", size = 194676, upload-time = "2026-03-01T22:07:53.373Z" } +sdist = { url = "https://files.pythonhosted.org/packages/75/16/e8be8e2fb175bbf41a0680381a319f1199fae256588241a2ac8677eafb49/yarl-1.25.1.tar.gz", hash = "sha256:03dd38de09bc213e9a8b29761eec33ee1d5318dac0e49d8af36e4d27830e23a7", size = 246245, upload-time = "2026-09-15T19:35:02.264Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8b/0d/9cc638702f6fc3c7a3685bcc8cf2a9ed7d6206e932a49f5242658047ef51/yarl-1.23.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cff6d44cb13d39db2663a22b22305d10855efa0fa8015ddeacc40bc59b9d8107", size = 123764, upload-time = "2026-03-01T22:04:09.7Z" }, - { url = "https://files.pythonhosted.org/packages/7a/35/5a553687c5793df5429cd1db45909d4f3af7eee90014888c208d086a44f0/yarl-1.23.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e4c53f8347cd4200f0d70a48ad059cabaf24f5adc6ba08622a23423bc7efa10d", size = 86282, upload-time = "2026-03-01T22:04:11.892Z" }, - { url = "https://files.pythonhosted.org/packages/68/2e/c5a2234238f8ce37a8312b52801ee74117f576b1539eec8404a480434acc/yarl-1.23.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2a6940a074fb3c48356ed0158a3ca5699c955ee4185b4d7d619be3c327143e05", size = 86053, upload-time = "2026-03-01T22:04:13.292Z" }, - { url = "https://files.pythonhosted.org/packages/74/3f/bbd8ff36fb038622797ffbaf7db314918bb4d76f1cc8a4f9ca7a55fe5195/yarl-1.23.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ed5f69ce7be7902e5c70ea19eb72d20abf7d725ab5d49777d696e32d4fc1811d", size = 99395, upload-time = "2026-03-01T22:04:15.133Z" }, - { url = "https://files.pythonhosted.org/packages/77/04/9516bc4e269d2a3ec9c6779fcdeac51ce5b3a9b0156f06ac7152e5bba864/yarl-1.23.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:389871e65468400d6283c0308e791a640b5ab5c83bcee02a2f51295f95e09748", size = 92143, upload-time = "2026-03-01T22:04:16.829Z" }, - { url = "https://files.pythonhosted.org/packages/c7/63/88802d1f6b1cb1fc67d67a58cd0cf8a1790de4ce7946e434240f1d60ab4a/yarl-1.23.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:dda608c88cf709b1d406bdfcd84d8d63cff7c9e577a403c6108ce8ce9dcc8764", size = 107643, upload-time = "2026-03-01T22:04:18.519Z" }, - { url = "https://files.pythonhosted.org/packages/8e/db/4f9b838f4d8bdd6f0f385aed8bbf21c71ed11a0b9983305c302cbd557815/yarl-1.23.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8c4fe09e0780c6c3bf2b7d4af02ee2394439d11a523bbcf095cf4747c2932007", size = 108700, upload-time = "2026-03-01T22:04:20.373Z" }, - { url = "https://files.pythonhosted.org/packages/50/12/95a1d33f04a79c402664070d43b8b9f72dc18914e135b345b611b0b1f8cc/yarl-1.23.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:31c9921eb8bd12633b41ad27686bbb0b1a2a9b8452bfdf221e34f311e9942ed4", size = 102769, upload-time = "2026-03-01T22:04:23.055Z" }, - { url = "https://files.pythonhosted.org/packages/86/65/91a0285f51321369fd1a8308aa19207520c5f0587772cfc2e03fc2467e90/yarl-1.23.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5f10fd85e4b75967468af655228fbfd212bdf66db1c0d135065ce288982eda26", size = 101114, upload-time = "2026-03-01T22:04:25.031Z" }, - { url = "https://files.pythonhosted.org/packages/58/80/c7c8244fc3e5bc483dc71a09560f43b619fab29301a0f0a8f936e42865c7/yarl-1.23.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:dbf507e9ef5688bada447a24d68b4b58dd389ba93b7afc065a2ba892bea54769", size = 98883, upload-time = "2026-03-01T22:04:27.281Z" }, - { url = "https://files.pythonhosted.org/packages/86/e7/71ca9cc9ca79c0b7d491216177d1aed559d632947b8ffb0ee60f7d8b23e3/yarl-1.23.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:85e9beda1f591bc73e77ea1c51965c68e98dafd0fec72cdd745f77d727466716", size = 94172, upload-time = "2026-03-01T22:04:28.554Z" }, - { url = "https://files.pythonhosted.org/packages/6a/3f/6c6c8a0fe29c26fb2db2e8d32195bb84ec1bfb8f1d32e7f73b787fcf349b/yarl-1.23.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:0e1fdaa14ef51366d7757b45bde294e95f6c8c049194e793eedb8387c86d5993", size = 107010, upload-time = "2026-03-01T22:04:30.385Z" }, - { url = "https://files.pythonhosted.org/packages/56/38/12730c05e5ad40a76374d440ed8b0899729a96c250516d91c620a6e38fc2/yarl-1.23.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:75e3026ab649bf48f9a10c0134512638725b521340293f202a69b567518d94e0", size = 100285, upload-time = "2026-03-01T22:04:31.752Z" }, - { url = "https://files.pythonhosted.org/packages/34/92/6a7be9239f2347234e027284e7a5f74b1140cc86575e7b469d13fba1ebfe/yarl-1.23.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:80e6d33a3d42a7549b409f199857b4fb54e2103fc44fb87605b6663b7a7ff750", size = 108230, upload-time = "2026-03-01T22:04:33.844Z" }, - { url = "https://files.pythonhosted.org/packages/5e/81/4aebccfa9376bd98b9d8bfad20621a57d3e8cfc5b8631c1fa5f62cdd03f4/yarl-1.23.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5ec2f42d41ccbd5df0270d7df31618a8ee267bfa50997f5d720ddba86c4a83a6", size = 103008, upload-time = "2026-03-01T22:04:35.856Z" }, - { url = "https://files.pythonhosted.org/packages/38/0f/0b4e3edcec794a86b853b0c6396c0a888d72dfce19b2d88c02ac289fb6c1/yarl-1.23.0-cp310-cp310-win32.whl", hash = "sha256:debe9c4f41c32990771be5c22b56f810659f9ddf3d63f67abfdcaa2c6c9c5c1d", size = 83073, upload-time = "2026-03-01T22:04:38.268Z" }, - { url = "https://files.pythonhosted.org/packages/a0/71/ad95c33da18897e4c636528bbc24a1dd23fe16797de8bc4ec667b8db0ba4/yarl-1.23.0-cp310-cp310-win_amd64.whl", hash = "sha256:ab5f043cb8a2d71c981c09c510da013bc79fd661f5c60139f00dd3c3cc4f2ffb", size = 87328, upload-time = "2026-03-01T22:04:39.558Z" }, - { url = "https://files.pythonhosted.org/packages/e2/14/dfa369523c79bccf9c9c746b0a63eb31f65db9418ac01275f7950962e504/yarl-1.23.0-cp310-cp310-win_arm64.whl", hash = "sha256:263cd4f47159c09b8b685890af949195b51d1aa82ba451c5847ca9bc6413c220", size = 82463, upload-time = "2026-03-01T22:04:41.454Z" }, - { url = "https://files.pythonhosted.org/packages/a2/aa/60da938b8f0997ba3a911263c40d82b6f645a67902a490b46f3355e10fae/yarl-1.23.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b35d13d549077713e4414f927cdc388d62e543987c572baee613bf82f11a4b99", size = 123641, upload-time = "2026-03-01T22:04:42.841Z" }, - { url = "https://files.pythonhosted.org/packages/24/84/e237607faf4e099dbb8a4f511cfd5efcb5f75918baad200ff7380635631b/yarl-1.23.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cbb0fef01f0c6b38cb0f39b1f78fc90b807e0e3c86a7ff3ce74ad77ce5c7880c", size = 86248, upload-time = "2026-03-01T22:04:44.757Z" }, - { url = "https://files.pythonhosted.org/packages/b2/0d/71ceabc14c146ba8ee3804ca7b3d42b1664c8440439de5214d366fec7d3a/yarl-1.23.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:dc52310451fc7c629e13c4e061cbe2dd01684d91f2f8ee2821b083c58bd72432", size = 85988, upload-time = "2026-03-01T22:04:46.365Z" }, - { url = "https://files.pythonhosted.org/packages/8c/6c/4a90d59c572e46b270ca132aca66954f1175abd691f74c1ef4c6711828e2/yarl-1.23.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b2c6b50c7b0464165472b56b42d4c76a7b864597007d9c085e8b63e185cf4a7a", size = 100566, upload-time = "2026-03-01T22:04:47.639Z" }, - { url = "https://files.pythonhosted.org/packages/49/fb/c438fb5108047e629f6282a371e6e91cf3f97ee087c4fb748a1f32ceef55/yarl-1.23.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:aafe5dcfda86c8af00386d7781d4c2181b5011b7be3f2add5e99899ea925df05", size = 92079, upload-time = "2026-03-01T22:04:48.925Z" }, - { url = "https://files.pythonhosted.org/packages/d9/13/d269aa1aed3e4f50a5a103f96327210cc5fa5dd2d50882778f13c7a14606/yarl-1.23.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9ee33b875f0b390564c1fb7bc528abf18c8ee6073b201c6ae8524aca778e2d83", size = 108741, upload-time = "2026-03-01T22:04:50.838Z" }, - { url = "https://files.pythonhosted.org/packages/85/fb/115b16f22c37ea4437d323e472945bea97301c8ec6089868fa560abab590/yarl-1.23.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4c41e021bc6d7affb3364dc1e1e5fa9582b470f283748784bd6ea0558f87f42c", size = 108099, upload-time = "2026-03-01T22:04:52.499Z" }, - { url = "https://files.pythonhosted.org/packages/9a/64/c53487d9f4968045b8afa51aed7ca44f58b2589e772f32745f3744476c82/yarl-1.23.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:99c8a9ed30f4164bc4c14b37a90208836cbf50d4ce2a57c71d0f52c7fb4f7598", size = 102678, upload-time = "2026-03-01T22:04:55.176Z" }, - { url = "https://files.pythonhosted.org/packages/85/59/cd98e556fbb2bf8fab29c1a722f67ad45c5f3447cac798ab85620d1e70af/yarl-1.23.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f2af5c81a1f124609d5f33507082fc3f739959d4719b56877ab1ee7e7b3d602b", size = 100803, upload-time = "2026-03-01T22:04:56.588Z" }, - { url = "https://files.pythonhosted.org/packages/9e/c0/b39770b56d4a9f0bb5f77e2f1763cd2d75cc2f6c0131e3b4c360348fcd65/yarl-1.23.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6b41389c19b07c760c7e427a3462e8ab83c4bb087d127f0e854c706ce1b9215c", size = 100163, upload-time = "2026-03-01T22:04:58.492Z" }, - { url = "https://files.pythonhosted.org/packages/e7/64/6980f99ab00e1f0ff67cb84766c93d595b067eed07439cfccfc8fb28c1a6/yarl-1.23.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:1dc702e42d0684f42d6519c8d581e49c96cefaaab16691f03566d30658ee8788", size = 93859, upload-time = "2026-03-01T22:05:00.268Z" }, - { url = "https://files.pythonhosted.org/packages/38/69/912e6c5e146793e5d4b5fe39ff5b00f4d22463dfd5a162bec565ac757673/yarl-1.23.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:0e40111274f340d32ebcc0a5668d54d2b552a6cca84c9475859d364b380e3222", size = 108202, upload-time = "2026-03-01T22:05:02.273Z" }, - { url = "https://files.pythonhosted.org/packages/59/97/35ca6767524687ad64e5f5c31ad54bc76d585585a9fcb40f649e7e82ffed/yarl-1.23.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:4764a6a7588561a9aef92f65bda2c4fb58fe7c675c0883862e6df97559de0bfb", size = 99866, upload-time = "2026-03-01T22:05:03.597Z" }, - { url = "https://files.pythonhosted.org/packages/d3/1c/1a3387ee6d73589f6f2a220ae06f2984f6c20b40c734989b0a44f5987308/yarl-1.23.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:03214408cfa590df47728b84c679ae4ef00be2428e11630277be0727eba2d7cc", size = 107852, upload-time = "2026-03-01T22:05:04.986Z" }, - { url = "https://files.pythonhosted.org/packages/a4/b8/35c0750fcd5a3f781058bfd954515dd4b1eab45e218cbb85cf11132215f1/yarl-1.23.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:170e26584b060879e29fac213e4228ef063f39128723807a312e5c7fec28eff2", size = 102919, upload-time = "2026-03-01T22:05:06.397Z" }, - { url = "https://files.pythonhosted.org/packages/e5/1c/9a1979aec4a81896d597bcb2177827f2dbee3f5b7cc48b2d0dadb644b41d/yarl-1.23.0-cp311-cp311-win32.whl", hash = "sha256:51430653db848d258336cfa0244427b17d12db63d42603a55f0d4546f50f25b5", size = 82602, upload-time = "2026-03-01T22:05:08.444Z" }, - { url = "https://files.pythonhosted.org/packages/93/22/b85eca6fa2ad9491af48c973e4c8cf6b103a73dbb271fe3346949449fca0/yarl-1.23.0-cp311-cp311-win_amd64.whl", hash = "sha256:bf49a3ae946a87083ef3a34c8f677ae4243f5b824bfc4c69672e72b3d6719d46", size = 87461, upload-time = "2026-03-01T22:05:10.145Z" }, - { url = "https://files.pythonhosted.org/packages/93/95/07e3553fe6f113e6864a20bdc53a78113cda3b9ced8784ee52a52c9f80d8/yarl-1.23.0-cp311-cp311-win_arm64.whl", hash = "sha256:b39cb32a6582750b6cc77bfb3c49c0f8760dc18dc96ec9fb55fbb0f04e08b928", size = 82336, upload-time = "2026-03-01T22:05:11.554Z" }, - { url = "https://files.pythonhosted.org/packages/88/8a/94615bc31022f711add374097ad4144d569e95ff3c38d39215d07ac153a0/yarl-1.23.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:1932b6b8bba8d0160a9d1078aae5838a66039e8832d41d2992daa9a3a08f7860", size = 124737, upload-time = "2026-03-01T22:05:12.897Z" }, - { url = "https://files.pythonhosted.org/packages/e3/6f/c6554045d59d64052698add01226bc867b52fe4a12373415d7991fdca95d/yarl-1.23.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:411225bae281f114067578891bc75534cfb3d92a3b4dfef7a6ca78ba354e6069", size = 87029, upload-time = "2026-03-01T22:05:14.376Z" }, - { url = "https://files.pythonhosted.org/packages/19/2a/725ecc166d53438bc88f76822ed4b1e3b10756e790bafd7b523fe97c322d/yarl-1.23.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:13a563739ae600a631c36ce096615fe307f131344588b0bc0daec108cdb47b25", size = 86310, upload-time = "2026-03-01T22:05:15.71Z" }, - { url = "https://files.pythonhosted.org/packages/99/30/58260ed98e6ff7f90ba84442c1ddd758c9170d70327394a6227b310cd60f/yarl-1.23.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9cbf44c5cb4a7633d078788e1b56387e3d3cf2b8139a3be38040b22d6c3221c8", size = 97587, upload-time = "2026-03-01T22:05:17.384Z" }, - { url = "https://files.pythonhosted.org/packages/76/0a/8b08aac08b50682e65759f7f8dde98ae8168f72487e7357a5d684c581ef9/yarl-1.23.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:53ad387048f6f09a8969631e4de3f1bf70c50e93545d64af4f751b2498755072", size = 92528, upload-time = "2026-03-01T22:05:18.804Z" }, - { url = "https://files.pythonhosted.org/packages/52/07/0b7179101fe5f8385ec6c6bb5d0cb9f76bd9fb4a769591ab6fb5cdbfc69a/yarl-1.23.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4a59ba56f340334766f3a4442e0efd0af895fae9e2b204741ef885c446b3a1a8", size = 105339, upload-time = "2026-03-01T22:05:20.235Z" }, - { url = "https://files.pythonhosted.org/packages/d3/8a/36d82869ab5ec829ca8574dfcb92b51286fcfb1e9c7a73659616362dc880/yarl-1.23.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:803a3c3ce4acc62eaf01eaca1208dcf0783025ef27572c3336502b9c232005e7", size = 105061, upload-time = "2026-03-01T22:05:22.268Z" }, - { url = "https://files.pythonhosted.org/packages/66/3e/868e5c3364b6cee19ff3e1a122194fa4ce51def02c61023970442162859e/yarl-1.23.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a3d2bff8f37f8d0f96c7ec554d16945050d54462d6e95414babaa18bfafc7f51", size = 100132, upload-time = "2026-03-01T22:05:23.638Z" }, - { url = "https://files.pythonhosted.org/packages/cf/26/9c89acf82f08a52cb52d6d39454f8d18af15f9d386a23795389d1d423823/yarl-1.23.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c75eb09e8d55bceb4367e83496ff8ef2bc7ea6960efb38e978e8073ea59ecb67", size = 99289, upload-time = "2026-03-01T22:05:25.749Z" }, - { url = "https://files.pythonhosted.org/packages/6f/54/5b0db00d2cb056922356104468019c0a132e89c8d3ab67d8ede9f4483d2a/yarl-1.23.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:877b0738624280e34c55680d6054a307aa94f7d52fa0e3034a9cc6e790871da7", size = 96950, upload-time = "2026-03-01T22:05:27.318Z" }, - { url = "https://files.pythonhosted.org/packages/f6/40/10fa93811fd439341fad7e0718a86aca0de9548023bbb403668d6555acab/yarl-1.23.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:b5405bb8f0e783a988172993cfc627e4d9d00432d6bbac65a923041edacf997d", size = 93960, upload-time = "2026-03-01T22:05:28.738Z" }, - { url = "https://files.pythonhosted.org/packages/bc/d2/8ae2e6cd77d0805f4526e30ec43b6f9a3dfc542d401ac4990d178e4bf0cf/yarl-1.23.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c3a3598a832590c5a3ce56ab5576361b5688c12cb1d39429cf5dba30b510760", size = 104703, upload-time = "2026-03-01T22:05:30.438Z" }, - { url = "https://files.pythonhosted.org/packages/2f/0c/b3ceacf82c3fe21183ce35fa2acf5320af003d52bc1fcf5915077681142e/yarl-1.23.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:8419ebd326430d1cbb7efb5292330a2cf39114e82df5cc3d83c9a0d5ebeaf2f2", size = 98325, upload-time = "2026-03-01T22:05:31.835Z" }, - { url = "https://files.pythonhosted.org/packages/9d/e0/12900edd28bdab91a69bd2554b85ad7b151f64e8b521fe16f9ad2f56477a/yarl-1.23.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:be61f6fff406ca40e3b1d84716fde398fc08bc63dd96d15f3a14230a0973ed86", size = 105067, upload-time = "2026-03-01T22:05:33.358Z" }, - { url = "https://files.pythonhosted.org/packages/15/61/74bb1182cf79c9bbe4eb6b1f14a57a22d7a0be5e9cedf8e2d5c2086474c3/yarl-1.23.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3ceb13c5c858d01321b5d9bb65e4cf37a92169ea470b70fec6f236b2c9dd7e34", size = 100285, upload-time = "2026-03-01T22:05:35.4Z" }, - { url = "https://files.pythonhosted.org/packages/69/7f/cd5ef733f2550de6241bd8bd8c3febc78158b9d75f197d9c7baa113436af/yarl-1.23.0-cp312-cp312-win32.whl", hash = "sha256:fffc45637bcd6538de8b85f51e3df3223e4ad89bccbfca0481c08c7fc8b7ed7d", size = 82359, upload-time = "2026-03-01T22:05:36.811Z" }, - { url = "https://files.pythonhosted.org/packages/f5/be/25216a49daeeb7af2bec0db22d5e7df08ed1d7c9f65d78b14f3b74fd72fc/yarl-1.23.0-cp312-cp312-win_amd64.whl", hash = "sha256:f69f57305656a4852f2a7203efc661d8c042e6cc67f7acd97d8667fb448a426e", size = 87674, upload-time = "2026-03-01T22:05:38.171Z" }, - { url = "https://files.pythonhosted.org/packages/d2/35/aeab955d6c425b227d5b7247eafb24f2653fedc32f95373a001af5dfeb9e/yarl-1.23.0-cp312-cp312-win_arm64.whl", hash = "sha256:6e87a6e8735b44816e7db0b2fbc9686932df473c826b0d9743148432e10bb9b9", size = 81879, upload-time = "2026-03-01T22:05:40.006Z" }, - { url = "https://files.pythonhosted.org/packages/9a/4b/a0a6e5d0ee8a2f3a373ddef8a4097d74ac901ac363eea1440464ccbe0898/yarl-1.23.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:16c6994ac35c3e74fb0ae93323bf8b9c2a9088d55946109489667c510a7d010e", size = 123796, upload-time = "2026-03-01T22:05:41.412Z" }, - { url = "https://files.pythonhosted.org/packages/67/b6/8925d68af039b835ae876db5838e82e76ec87b9782ecc97e192b809c4831/yarl-1.23.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4a42e651629dafb64fd5b0286a3580613702b5809ad3f24934ea87595804f2c5", size = 86547, upload-time = "2026-03-01T22:05:42.841Z" }, - { url = "https://files.pythonhosted.org/packages/ae/50/06d511cc4b8e0360d3c94af051a768e84b755c5eb031b12adaaab6dec6e5/yarl-1.23.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7c6b9461a2a8b47c65eef63bb1c76a4f1c119618ffa99ea79bc5bb1e46c5821b", size = 85854, upload-time = "2026-03-01T22:05:44.85Z" }, - { url = "https://files.pythonhosted.org/packages/c4/f4/4e30b250927ffdab4db70da08b9b8d2194d7c7b400167b8fbeca1e4701ca/yarl-1.23.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2569b67d616eab450d262ca7cb9f9e19d2f718c70a8b88712859359d0ab17035", size = 98351, upload-time = "2026-03-01T22:05:46.836Z" }, - { url = "https://files.pythonhosted.org/packages/86/fc/4118c5671ea948208bdb1492d8b76bdf1453d3e73df051f939f563e7dcc5/yarl-1.23.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e9d9a4d06d3481eab79803beb4d9bd6f6a8e781ec078ac70d7ef2dcc29d1bea5", size = 92711, upload-time = "2026-03-01T22:05:48.316Z" }, - { url = "https://files.pythonhosted.org/packages/56/11/1ed91d42bd9e73c13dc9e7eb0dd92298d75e7ac4dd7f046ad0c472e231cd/yarl-1.23.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f514f6474e04179d3d33175ed3f3e31434d3130d42ec153540d5b157deefd735", size = 106014, upload-time = "2026-03-01T22:05:50.028Z" }, - { url = "https://files.pythonhosted.org/packages/ce/c9/74e44e056a23fbc33aca71779ef450ca648a5bc472bdad7a82339918f818/yarl-1.23.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fda207c815b253e34f7e1909840fd14299567b1c0eb4908f8c2ce01a41265401", size = 105557, upload-time = "2026-03-01T22:05:51.416Z" }, - { url = "https://files.pythonhosted.org/packages/66/fe/b1e10b08d287f518994f1e2ff9b6d26f0adeecd8dd7d533b01bab29a3eda/yarl-1.23.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:34b6cf500e61c90f305094911f9acc9c86da1a05a7a3f5be9f68817043f486e4", size = 101559, upload-time = "2026-03-01T22:05:52.872Z" }, - { url = "https://files.pythonhosted.org/packages/72/59/c5b8d94b14e3d3c2a9c20cb100119fd534ab5a14b93673ab4cc4a4141ea5/yarl-1.23.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d7504f2b476d21653e4d143f44a175f7f751cd41233525312696c76aa3dbb23f", size = 100502, upload-time = "2026-03-01T22:05:54.954Z" }, - { url = "https://files.pythonhosted.org/packages/77/4f/96976cb54cbfc5c9fd73ed4c51804f92f209481d1fb190981c0f8a07a1d7/yarl-1.23.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:578110dd426f0d209d1509244e6d4a3f1a3e9077655d98c5f22583d63252a08a", size = 98027, upload-time = "2026-03-01T22:05:56.409Z" }, - { url = "https://files.pythonhosted.org/packages/63/6e/904c4f476471afdbad6b7e5b70362fb5810e35cd7466529a97322b6f5556/yarl-1.23.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:609d3614d78d74ebe35f54953c5bbd2ac647a7ddb9c30a5d877580f5e86b22f2", size = 95369, upload-time = "2026-03-01T22:05:58.141Z" }, - { url = "https://files.pythonhosted.org/packages/9d/40/acfcdb3b5f9d68ef499e39e04d25e141fe90661f9d54114556cf83be8353/yarl-1.23.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4966242ec68afc74c122f8459abd597afd7d8a60dc93d695c1334c5fd25f762f", size = 105565, upload-time = "2026-03-01T22:06:00.286Z" }, - { url = "https://files.pythonhosted.org/packages/5e/c6/31e28f3a6ba2869c43d124f37ea5260cac9c9281df803c354b31f4dd1f3c/yarl-1.23.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:e0fd068364a6759bc794459f0a735ab151d11304346332489c7972bacbe9e72b", size = 99813, upload-time = "2026-03-01T22:06:01.712Z" }, - { url = "https://files.pythonhosted.org/packages/08/1f/6f65f59e72d54aa467119b63fc0b0b1762eff0232db1f4720cd89e2f4a17/yarl-1.23.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:39004f0ad156da43e86aa71f44e033de68a44e5a31fc53507b36dd253970054a", size = 105632, upload-time = "2026-03-01T22:06:03.188Z" }, - { url = "https://files.pythonhosted.org/packages/a3/c4/18b178a69935f9e7a338127d5b77d868fdc0f0e49becd286d51b3a18c61d/yarl-1.23.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e5723c01a56c5028c807c701aa66722916d2747ad737a046853f6c46f4875543", size = 101895, upload-time = "2026-03-01T22:06:04.651Z" }, - { url = "https://files.pythonhosted.org/packages/8f/54/f5b870b5505663911dba950a8e4776a0dbd51c9c54c0ae88e823e4b874a0/yarl-1.23.0-cp313-cp313-win32.whl", hash = "sha256:1b6b572edd95b4fa8df75de10b04bc81acc87c1c7d16bcdd2035b09d30acc957", size = 82356, upload-time = "2026-03-01T22:06:06.04Z" }, - { url = "https://files.pythonhosted.org/packages/7a/84/266e8da36879c6edcd37b02b547e2d9ecdfea776be49598e75696e3316e1/yarl-1.23.0-cp313-cp313-win_amd64.whl", hash = "sha256:baaf55442359053c7d62f6f8413a62adba3205119bcb6f49594894d8be47e5e3", size = 87515, upload-time = "2026-03-01T22:06:08.107Z" }, - { url = "https://files.pythonhosted.org/packages/00/fd/7e1c66efad35e1649114fa13f17485f62881ad58edeeb7f49f8c5e748bf9/yarl-1.23.0-cp313-cp313-win_arm64.whl", hash = "sha256:fb4948814a2a98e3912505f09c9e7493b1506226afb1f881825368d6fb776ee3", size = 81785, upload-time = "2026-03-01T22:06:10.181Z" }, - { url = "https://files.pythonhosted.org/packages/9c/fc/119dd07004f17ea43bb91e3ece6587759edd7519d6b086d16bfbd3319982/yarl-1.23.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:aecfed0b41aa72b7881712c65cf764e39ce2ec352324f5e0837c7048d9e6daaa", size = 130719, upload-time = "2026-03-01T22:06:11.708Z" }, - { url = "https://files.pythonhosted.org/packages/e6/0d/9f2348502fbb3af409e8f47730282cd6bc80dec6630c1e06374d882d6eb2/yarl-1.23.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:a41bcf68efd19073376eb8cf948b8d9be0af26256403e512bb18f3966f1f9120", size = 89690, upload-time = "2026-03-01T22:06:13.429Z" }, - { url = "https://files.pythonhosted.org/packages/50/93/e88f3c80971b42cfc83f50a51b9d165a1dbf154b97005f2994a79f212a07/yarl-1.23.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:cde9a2ecd91668bcb7f077c4966d8ceddb60af01b52e6e3e2680e4cf00ad1a59", size = 89851, upload-time = "2026-03-01T22:06:15.53Z" }, - { url = "https://files.pythonhosted.org/packages/1c/07/61c9dd8ba8f86473263b4036f70fb594c09e99c0d9737a799dfd8bc85651/yarl-1.23.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5023346c4ee7992febc0068e7593de5fa2bf611848c08404b35ebbb76b1b0512", size = 95874, upload-time = "2026-03-01T22:06:17.553Z" }, - { url = "https://files.pythonhosted.org/packages/9e/e9/f9ff8ceefba599eac6abddcfb0b3bee9b9e636e96dbf54342a8577252379/yarl-1.23.0-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d1009abedb49ae95b136a8904a3f71b342f849ffeced2d3747bf29caeda218c4", size = 88710, upload-time = "2026-03-01T22:06:19.004Z" }, - { url = "https://files.pythonhosted.org/packages/eb/78/0231bfcc5d4c8eec220bc2f9ef82cb4566192ea867a7c5b4148f44f6cbcd/yarl-1.23.0-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a8d00f29b42f534cc8aa3931cfe773b13b23e561e10d2b26f27a8d309b0e82a1", size = 101033, upload-time = "2026-03-01T22:06:21.203Z" }, - { url = "https://files.pythonhosted.org/packages/cd/9b/30ea5239a61786f18fd25797151a17fbb3be176977187a48d541b5447dd4/yarl-1.23.0-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:95451e6ce06c3e104556d73b559f5da6c34a069b6b62946d3ad66afcd51642ea", size = 100817, upload-time = "2026-03-01T22:06:22.738Z" }, - { url = "https://files.pythonhosted.org/packages/62/e2/a4980481071791bc83bce2b7a1a1f7adcabfa366007518b4b845e92eeee3/yarl-1.23.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:531ef597132086b6cf96faa7c6c1dcd0361dd5f1694e5cc30375907b9b7d3ea9", size = 97482, upload-time = "2026-03-01T22:06:24.21Z" }, - { url = "https://files.pythonhosted.org/packages/e5/1e/304a00cf5f6100414c4b5a01fc7ff9ee724b62158a08df2f8170dfc72a2d/yarl-1.23.0-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:88f9fb0116fbfcefcab70f85cf4b74a2b6ce5d199c41345296f49d974ddb4123", size = 95949, upload-time = "2026-03-01T22:06:25.697Z" }, - { url = "https://files.pythonhosted.org/packages/68/03/093f4055ed4cae649ac53bca3d180bd37102e9e11d048588e9ab0c0108d0/yarl-1.23.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:e7b0460976dc75cb87ad9cc1f9899a4b97751e7d4e77ab840fc9b6d377b8fd24", size = 95839, upload-time = "2026-03-01T22:06:27.309Z" }, - { url = "https://files.pythonhosted.org/packages/b9/28/4c75ebb108f322aa8f917ae10a8ffa4f07cae10a8a627b64e578617df6a0/yarl-1.23.0-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:115136c4a426f9da976187d238e84139ff6b51a20839aa6e3720cd1026d768de", size = 90696, upload-time = "2026-03-01T22:06:29.048Z" }, - { url = "https://files.pythonhosted.org/packages/23/9c/42c2e2dd91c1a570402f51bdf066bfdb1241c2240ba001967bad778e77b7/yarl-1.23.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:ead11956716a940c1abc816b7df3fa2b84d06eaed8832ca32f5c5e058c65506b", size = 100865, upload-time = "2026-03-01T22:06:30.525Z" }, - { url = "https://files.pythonhosted.org/packages/74/05/1bcd60a8a0a914d462c305137246b6f9d167628d73568505fce3f1cb2e65/yarl-1.23.0-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:fe8f8f5e70e6dbdfca9882cd9deaac058729bcf323cf7a58660901e55c9c94f6", size = 96234, upload-time = "2026-03-01T22:06:32.692Z" }, - { url = "https://files.pythonhosted.org/packages/90/b2/f52381aac396d6778ce516b7bc149c79e65bfc068b5de2857ab69eeea3b7/yarl-1.23.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:a0e317df055958a0c1e79e5d2aa5a5eaa4a6d05a20d4b0c9c3f48918139c9fc6", size = 100295, upload-time = "2026-03-01T22:06:34.268Z" }, - { url = "https://files.pythonhosted.org/packages/e5/e8/638bae5bbf1113a659b2435d8895474598afe38b4a837103764f603aba56/yarl-1.23.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6f0fd84de0c957b2d280143522c4f91a73aada1923caee763e24a2b3fda9f8a5", size = 97784, upload-time = "2026-03-01T22:06:35.864Z" }, - { url = "https://files.pythonhosted.org/packages/80/25/a3892b46182c586c202629fc2159aa13975d3741d52ebd7347fd501d48d5/yarl-1.23.0-cp313-cp313t-win32.whl", hash = "sha256:93a784271881035ab4406a172edb0faecb6e7d00f4b53dc2f55919d6c9688595", size = 88313, upload-time = "2026-03-01T22:06:37.39Z" }, - { url = "https://files.pythonhosted.org/packages/43/68/8c5b36aa5178900b37387937bc2c2fe0e9505537f713495472dcf6f6fccc/yarl-1.23.0-cp313-cp313t-win_amd64.whl", hash = "sha256:dd00607bffbf30250fe108065f07453ec124dbf223420f57f5e749b04295e090", size = 94932, upload-time = "2026-03-01T22:06:39.579Z" }, - { url = "https://files.pythonhosted.org/packages/c6/cc/d79ba8292f51f81f4dc533a8ccfb9fc6992cabf0998ed3245de7589dc07c/yarl-1.23.0-cp313-cp313t-win_arm64.whl", hash = "sha256:ac09d42f48f80c9ee1635b2fcaa819496a44502737660d3c0f2ade7526d29144", size = 84786, upload-time = "2026-03-01T22:06:41.988Z" }, - { url = "https://files.pythonhosted.org/packages/90/98/b85a038d65d1b92c3903ab89444f48d3cee490a883477b716d7a24b1a78c/yarl-1.23.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:21d1b7305a71a15b4794b5ff22e8eef96ff4a6d7f9657155e5aa419444b28912", size = 124455, upload-time = "2026-03-01T22:06:43.615Z" }, - { url = "https://files.pythonhosted.org/packages/39/54/bc2b45559f86543d163b6e294417a107bb87557609007c007ad889afec18/yarl-1.23.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:85610b4f27f69984932a7abbe52703688de3724d9f72bceb1cca667deff27474", size = 86752, upload-time = "2026-03-01T22:06:45.425Z" }, - { url = "https://files.pythonhosted.org/packages/24/f9/e8242b68362bffe6fb536c8db5076861466fc780f0f1b479fc4ffbebb128/yarl-1.23.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23f371bd662cf44a7630d4d113101eafc0cfa7518a2760d20760b26021454719", size = 86291, upload-time = "2026-03-01T22:06:46.974Z" }, - { url = "https://files.pythonhosted.org/packages/ea/d8/d1cb2378c81dd729e98c716582b1ccb08357e8488e4c24714658cc6630e8/yarl-1.23.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c4a80f77dc1acaaa61f0934176fccca7096d9b1ff08c8ba9cddf5ae034a24319", size = 99026, upload-time = "2026-03-01T22:06:48.459Z" }, - { url = "https://files.pythonhosted.org/packages/0a/ff/7196790538f31debe3341283b5b0707e7feb947620fc5e8236ef28d44f72/yarl-1.23.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:bd654fad46d8d9e823afbb4f87c79160b5a374ed1ff5bde24e542e6ba8f41434", size = 92355, upload-time = "2026-03-01T22:06:50.306Z" }, - { url = "https://files.pythonhosted.org/packages/c1/56/25d58c3eddde825890a5fe6aa1866228377354a3c39262235234ab5f616b/yarl-1.23.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:682bae25f0a0dd23a056739f23a134db9f52a63e2afd6bfb37ddc76292bbd723", size = 106417, upload-time = "2026-03-01T22:06:52.1Z" }, - { url = "https://files.pythonhosted.org/packages/51/8a/882c0e7bc8277eb895b31bce0138f51a1ba551fc2e1ec6753ffc1e7c1377/yarl-1.23.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a82836cab5f197a0514235aaf7ffccdc886ccdaa2324bc0aafdd4ae898103039", size = 106422, upload-time = "2026-03-01T22:06:54.424Z" }, - { url = "https://files.pythonhosted.org/packages/42/2b/fef67d616931055bf3d6764885990a3ac647d68734a2d6a9e1d13de437a2/yarl-1.23.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1c57676bdedc94cd3bc37724cf6f8cd2779f02f6aba48de45feca073e714fe52", size = 101915, upload-time = "2026-03-01T22:06:55.895Z" }, - { url = "https://files.pythonhosted.org/packages/18/6a/530e16aebce27c5937920f3431c628a29a4b6b430fab3fd1c117b26ff3f6/yarl-1.23.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c7f8dc16c498ff06497c015642333219871effba93e4a2e8604a06264aca5c5c", size = 100690, upload-time = "2026-03-01T22:06:58.21Z" }, - { url = "https://files.pythonhosted.org/packages/88/08/93749219179a45e27b036e03260fda05190b911de8e18225c294ac95bbc9/yarl-1.23.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:5ee586fb17ff8f90c91cf73c6108a434b02d69925f44f5f8e0d7f2f260607eae", size = 98750, upload-time = "2026-03-01T22:06:59.794Z" }, - { url = "https://files.pythonhosted.org/packages/d9/cf/ea424a004969f5d81a362110a6ac1496d79efdc6d50c2c4b2e3ea0fc2519/yarl-1.23.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:17235362f580149742739cc3828b80e24029d08cbb9c4bda0242c7b5bc610a8e", size = 94685, upload-time = "2026-03-01T22:07:01.375Z" }, - { url = "https://files.pythonhosted.org/packages/e2/b7/14341481fe568e2b0408bcf1484c652accafe06a0ade9387b5d3fd9df446/yarl-1.23.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:0793e2bd0cf14234983bbb371591e6bea9e876ddf6896cdcc93450996b0b5c85", size = 106009, upload-time = "2026-03-01T22:07:03.151Z" }, - { url = "https://files.pythonhosted.org/packages/0a/e6/5c744a9b54f4e8007ad35bce96fbc9218338e84812d36f3390cea616881a/yarl-1.23.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:3650dc2480f94f7116c364096bc84b1d602f44224ef7d5c7208425915c0475dd", size = 100033, upload-time = "2026-03-01T22:07:04.701Z" }, - { url = "https://files.pythonhosted.org/packages/0c/23/e3bfc188d0b400f025bc49d99793d02c9abe15752138dcc27e4eaf0c4a9e/yarl-1.23.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f40e782d49630ad384db66d4d8b73ff4f1b8955dc12e26b09a3e3af064b3b9d6", size = 106483, upload-time = "2026-03-01T22:07:06.231Z" }, - { url = "https://files.pythonhosted.org/packages/72/42/f0505f949a90b3f8b7a363d6cbdf398f6e6c58946d85c6d3a3bc70595b26/yarl-1.23.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:94f8575fbdf81749008d980c17796097e645574a3b8c28ee313931068dad14fe", size = 102175, upload-time = "2026-03-01T22:07:08.4Z" }, - { url = "https://files.pythonhosted.org/packages/aa/65/b39290f1d892a9dd671d1c722014ca062a9c35d60885d57e5375db0404b5/yarl-1.23.0-cp314-cp314-win32.whl", hash = "sha256:c8aa34a5c864db1087d911a0b902d60d203ea3607d91f615acd3f3108ac32169", size = 83871, upload-time = "2026-03-01T22:07:09.968Z" }, - { url = "https://files.pythonhosted.org/packages/a9/5b/9b92f54c784c26e2a422e55a8d2607ab15b7ea3349e28359282f84f01d43/yarl-1.23.0-cp314-cp314-win_amd64.whl", hash = "sha256:63e92247f383c85ab00dd0091e8c3fa331a96e865459f5ee80353c70a4a42d70", size = 89093, upload-time = "2026-03-01T22:07:11.501Z" }, - { url = "https://files.pythonhosted.org/packages/e0/7d/8a84dc9381fd4412d5e7ff04926f9865f6372b4c2fd91e10092e65d29eb8/yarl-1.23.0-cp314-cp314-win_arm64.whl", hash = "sha256:70efd20be968c76ece7baa8dafe04c5be06abc57f754d6f36f3741f7aa7a208e", size = 83384, upload-time = "2026-03-01T22:07:13.069Z" }, - { url = "https://files.pythonhosted.org/packages/dd/8d/d2fad34b1c08aa161b74394183daa7d800141aaaee207317e82c790b418d/yarl-1.23.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:9a18d6f9359e45722c064c97464ec883eb0e0366d33eda61cb19a244bf222679", size = 131019, upload-time = "2026-03-01T22:07:14.903Z" }, - { url = "https://files.pythonhosted.org/packages/19/ff/33009a39d3ccf4b94d7d7880dfe17fb5816c5a4fe0096d9b56abceea9ac7/yarl-1.23.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:2803ed8b21ca47a43da80a6fd1ed3019d30061f7061daa35ac54f63933409412", size = 89894, upload-time = "2026-03-01T22:07:17.372Z" }, - { url = "https://files.pythonhosted.org/packages/0c/f1/dab7ac5e7306fb79c0190766a3c00b4cb8d09a1f390ded68c85a5934faf5/yarl-1.23.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:394906945aa8b19fc14a61cf69743a868bb8c465efe85eee687109cc540b98f4", size = 89979, upload-time = "2026-03-01T22:07:19.361Z" }, - { url = "https://files.pythonhosted.org/packages/aa/b1/08e95f3caee1fad6e65017b9f26c1d79877b502622d60e517de01e72f95d/yarl-1.23.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:71d006bee8397a4a89f469b8deb22469fe7508132d3c17fa6ed871e79832691c", size = 95943, upload-time = "2026-03-01T22:07:21.266Z" }, - { url = "https://files.pythonhosted.org/packages/c0/cc/6409f9018864a6aa186c61175b977131f373f1988e198e031236916e87e4/yarl-1.23.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:62694e275c93d54f7ccedcfef57d42761b2aad5234b6be1f3e3026cae4001cd4", size = 88786, upload-time = "2026-03-01T22:07:23.129Z" }, - { url = "https://files.pythonhosted.org/packages/76/40/cc22d1d7714b717fde2006fad2ced5efe5580606cb059ae42117542122f3/yarl-1.23.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a31de1613658308efdb21ada98cbc86a97c181aa050ba22a808120bb5be3ab94", size = 101307, upload-time = "2026-03-01T22:07:24.689Z" }, - { url = "https://files.pythonhosted.org/packages/8f/0d/476c38e85ddb4c6ec6b20b815bdd779aa386a013f3d8b85516feee55c8dc/yarl-1.23.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fb1e8b8d66c278b21d13b0a7ca22c41dd757a7c209c6b12c313e445c31dd3b28", size = 100904, upload-time = "2026-03-01T22:07:26.287Z" }, - { url = "https://files.pythonhosted.org/packages/72/32/0abe4a76d59adf2081dcb0397168553ece4616ada1c54d1c49d8936c74f8/yarl-1.23.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:50f9d8d531dfb767c565f348f33dd5139a6c43f5cbdf3f67da40d54241df93f6", size = 97728, upload-time = "2026-03-01T22:07:27.906Z" }, - { url = "https://files.pythonhosted.org/packages/b7/35/7b30f4810fba112f60f5a43237545867504e15b1c7647a785fbaf588fac2/yarl-1.23.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:575aa4405a656e61a540f4a80eaa5260f2a38fff7bfdc4b5f611840d76e9e277", size = 95964, upload-time = "2026-03-01T22:07:30.198Z" }, - { url = "https://files.pythonhosted.org/packages/2d/86/ed7a73ab85ef00e8bb70b0cb5421d8a2a625b81a333941a469a6f4022828/yarl-1.23.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:041b1a4cefacf65840b4e295c6985f334ba83c30607441ae3cf206a0eed1a2e4", size = 95882, upload-time = "2026-03-01T22:07:32.132Z" }, - { url = "https://files.pythonhosted.org/packages/19/90/d56967f61a29d8498efb7afb651e0b2b422a1e9b47b0ab5f4e40a19b699b/yarl-1.23.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:d38c1e8231722c4ce40d7593f28d92b5fc72f3e9774fe73d7e800ec32299f63a", size = 90797, upload-time = "2026-03-01T22:07:34.404Z" }, - { url = "https://files.pythonhosted.org/packages/72/00/8b8f76909259f56647adb1011d7ed8b321bcf97e464515c65016a47ecdf0/yarl-1.23.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:d53834e23c015ee83a99377db6e5e37d8484f333edb03bd15b4bc312cc7254fb", size = 101023, upload-time = "2026-03-01T22:07:35.953Z" }, - { url = "https://files.pythonhosted.org/packages/ac/e2/cab11b126fb7d440281b7df8e9ddbe4851e70a4dde47a202b6642586b8d9/yarl-1.23.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:2e27c8841126e017dd2a054a95771569e6070b9ee1b133366d8b31beb5018a41", size = 96227, upload-time = "2026-03-01T22:07:37.594Z" }, - { url = "https://files.pythonhosted.org/packages/c2/9b/2c893e16bfc50e6b2edf76c1a9eb6cb0c744346197e74c65e99ad8d634d0/yarl-1.23.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:76855800ac56f878847a09ce6dba727c93ca2d89c9e9d63002d26b916810b0a2", size = 100302, upload-time = "2026-03-01T22:07:39.334Z" }, - { url = "https://files.pythonhosted.org/packages/28/ec/5498c4e3a6d5f1003beb23405671c2eb9cdbf3067d1c80f15eeafe301010/yarl-1.23.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e09fd068c2e169a7070d83d3bde728a4d48de0549f975290be3c108c02e499b4", size = 98202, upload-time = "2026-03-01T22:07:41.717Z" }, - { url = "https://files.pythonhosted.org/packages/fe/c3/cd737e2d45e70717907f83e146f6949f20cc23cd4bf7b2688727763aa458/yarl-1.23.0-cp314-cp314t-win32.whl", hash = "sha256:73309162a6a571d4cbd3b6a1dcc703c7311843ae0d1578df6f09be4e98df38d4", size = 90558, upload-time = "2026-03-01T22:07:43.433Z" }, - { url = "https://files.pythonhosted.org/packages/e1/19/3774d162f6732d1cfb0b47b4140a942a35ca82bb19b6db1f80e9e7bdc8f8/yarl-1.23.0-cp314-cp314t-win_amd64.whl", hash = "sha256:4503053d296bc6e4cbd1fad61cf3b6e33b939886c4f249ba7c78b602214fabe2", size = 97610, upload-time = "2026-03-01T22:07:45.773Z" }, - { url = "https://files.pythonhosted.org/packages/51/47/3fa2286c3cb162c71cdb34c4224d5745a1ceceb391b2bd9b19b668a8d724/yarl-1.23.0-cp314-cp314t-win_arm64.whl", hash = "sha256:44bb7bef4ea409384e3f8bc36c063d77ea1b8d4a5b2706956c0d6695f07dcc25", size = 86041, upload-time = "2026-03-01T22:07:49.026Z" }, - { url = "https://files.pythonhosted.org/packages/69/68/c8739671f5699c7dc470580a4f821ef37c32c4cb0b047ce223a7f115757f/yarl-1.23.0-py3-none-any.whl", hash = "sha256:a2df6afe50dea8ae15fa34c9f824a3ee958d785fd5d089063d960bae1daa0a3f", size = 48288, upload-time = "2026-03-01T22:07:51.388Z" }, + { url = "https://files.pythonhosted.org/packages/85/92/55fa9ee84cb8ec9930a910b5753936a926f918d8cc8965bdac571479c095/yarl-1.25.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:142c06c4d6a35ee3ec5da08499805e879cb3ca7c1fbfbecb0140fe72403818d6", size = 144695, upload-time = "2026-09-15T19:29:53.114Z" }, + { url = "https://files.pythonhosted.org/packages/16/b4/9edc8e605b16b2eb5bd0c2ccc73e2b15aab583862460767fb43f91f11839/yarl-1.25.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:24ce942011a61953e7d313438038f4d32ff21387b775f58a957f7a07dd55ef95", size = 104332, upload-time = "2026-09-15T19:29:55.443Z" }, + { url = "https://files.pythonhosted.org/packages/ae/de/4204e6646e278b1cd4a9cf73ebe8b7cfcf16693f1249b324eb273d67602a/yarl-1.25.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9e23c82b63cd7652fc24d33ed6cc17099d607aa3b4fc4ddc75e95062f3d82df4", size = 104449, upload-time = "2026-09-15T19:29:57.239Z" }, + { url = "https://files.pythonhosted.org/packages/1d/fc/21d74789198ad68a23a209ab0d73fe5a52d39ebbd9945041e72827acdd77/yarl-1.25.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8ee202350cf57abf0e9502a41601841019c25d3db7ff52d980aaf31446254059", size = 116828, upload-time = "2026-09-15T19:29:59.031Z" }, + { url = "https://files.pythonhosted.org/packages/8f/31/90891ddb61848c067ebd1fb505ab5902a6e7d4707c2f3823f67ff37883d3/yarl-1.25.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5df89f769cc8ff94c3d7e7603386fba309d25ce5240132d26c15baa8d0e96c4c", size = 107150, upload-time = "2026-09-15T19:30:00.868Z" }, + { url = "https://files.pythonhosted.org/packages/a1/15/f3d18743d3688b5f01aa83034b6c00665692ab20e1d0c89a29000d4979ea/yarl-1.25.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:83e9f4a25085bd4b7214701a0794ff1f50fc633ffb8bdfebf07abdd81c2db126", size = 124704, upload-time = "2026-09-15T19:30:03.128Z" }, + { url = "https://files.pythonhosted.org/packages/99/37/718789d8004775d6a2db86b2afc72b1d156e0590b88a1b8634ddd4bdb8ee/yarl-1.25.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e636b64d24fd9c38053c5e389a1174c66361fa49dcfd220f4dd35b4abde7cb89", size = 129246, upload-time = "2026-09-15T19:30:05.014Z" }, + { url = "https://files.pythonhosted.org/packages/28/b9/7818ac6dec7fbcd16be2d19495807c01e2c38834e57d41c63356a75e786b/yarl-1.25.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e5637ca8d0bd7fb72648a6c7934af4baaccb697657f7438c9d264fc2abb8b0b1", size = 118071, upload-time = "2026-09-15T19:30:06.977Z" }, + { url = "https://files.pythonhosted.org/packages/b2/e3/8d098cafb30a64df7283b5a5dd0d32d22a71b18a1b9ec071d0776ffe5c2a/yarl-1.25.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:683e362b8ba453080f7489c66f4ea794e751c35b72e7eab3575ef784c2fbc7fb", size = 116180, upload-time = "2026-09-15T19:30:08.836Z" }, + { url = "https://files.pythonhosted.org/packages/20/9c/fbe9432478d87e00eef80242e15179639df162cb3d5d8d978fc56dcb6c51/yarl-1.25.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:df23df54b5114a17c2d0ef192433e2e5a9f0c5178c32375e90b7cfc965f349d0", size = 116578, upload-time = "2026-09-15T19:30:10.782Z" }, + { url = "https://files.pythonhosted.org/packages/b1/8e/b3dfd03732236b86b0bcf06a72a37de412ff6f82e719a947c3c651d4dece/yarl-1.25.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f53dcd26694f148f738edc052b5a69234833e739f10f4c3287bdfd8ec0f7b326", size = 108886, upload-time = "2026-09-15T19:30:12.951Z" }, + { url = "https://files.pythonhosted.org/packages/83/e1/94dacb650d4963d5f844bc3df7ba70288cb7b68ed5a4d070da71acb19b14/yarl-1.25.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:b8075fe90bc08e40b8b8a1874fab42ee4c7b56af05c5886e9cc841397f916908", size = 124098, upload-time = "2026-09-15T19:30:14.963Z" }, + { url = "https://files.pythonhosted.org/packages/d6/3f/c93f76a218258c7bfc60dd27fd56e9102bdbc625517f4a6ee673295fe4e4/yarl-1.25.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:a8c2b841478068440d8b733005d13a5ef535b9928cbc05f17182d410f32ba449", size = 115768, upload-time = "2026-09-15T19:30:16.736Z" }, + { url = "https://files.pythonhosted.org/packages/a8/79/4d93f13c3b05cda3c962805dec28cbc255c50239b3457808abc5633a00c2/yarl-1.25.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:ca32926d7d77bcc8838425c4c95e040a3ace1cb7dfdae599013458dcda2607ca", size = 122461, upload-time = "2026-09-15T19:30:18.901Z" }, + { url = "https://files.pythonhosted.org/packages/c5/8e/e2ba83b3a9bfc1d3b882ad35fa8abc04a7af4d9356f2e329a23d0c8d889d/yarl-1.25.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:192a866877a49993949ef1975864ad8728bea28ee810f6abe1a0729c2b500426", size = 118295, upload-time = "2026-09-15T19:30:21.07Z" }, + { url = "https://files.pythonhosted.org/packages/42/67/cb5ea1baa0c0ac60bda44ce59f601133154af2a2e67722d3517f8793855d/yarl-1.25.1-cp310-cp310-win_amd64.whl", hash = "sha256:3f4d48a6112712973e676bd792121fee470e432d749177162d9949d5c9460a1b", size = 102929, upload-time = "2026-09-15T19:30:23.122Z" }, + { url = "https://files.pythonhosted.org/packages/2f/85/9a1e98de10fc0e738d517efd2a9984c3e7db6cf35fa72b2b373988a9e9b7/yarl-1.25.1-cp310-cp310-win_arm64.whl", hash = "sha256:48796ea00a303961507dc6c8437c4b325a6fc3f95f7c36c71b91ea9a8150963c", size = 98854, upload-time = "2026-09-15T19:30:25.102Z" }, + { url = "https://files.pythonhosted.org/packages/83/b3/2cea721d495ca57f8f414aa4867ee263f486b274e07463158cf52f02ba7f/yarl-1.25.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9d693bf4bf534e9ba3ae2780cfd577f5135629f7b5ac653490859d0b77864865", size = 143794, upload-time = "2026-09-15T19:30:26.946Z" }, + { url = "https://files.pythonhosted.org/packages/55/e6/cd145cff8e5cf60b8b3c41fbecfa2a45028a8dec3fbc52bec03595ff3d3b/yarl-1.25.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ab2054c5531af2a9ba7b69b8ec91e4f884420e83a8c5e579b013084cb57e5e5d", size = 103993, upload-time = "2026-09-15T19:30:28.661Z" }, + { url = "https://files.pythonhosted.org/packages/ce/7c/fbb40fe2d53747c40aa36a9e2bd2178a202f942bda0b670f3306d4aefbce/yarl-1.25.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:564fdc7085d2245ab84f88882fdb1d6ac0723124bff6ded35bfb1c00f812630d", size = 104010, upload-time = "2026-09-15T19:30:31.069Z" }, + { url = "https://files.pythonhosted.org/packages/f0/0b/5a516f70641092283f57cf3670bdb75e7327bcc0dcb5038697e4dfbfd569/yarl-1.25.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:acae6b45d1ace09b6ba3876da43b88366ef368f73b988c7f57e14231753d4420", size = 116444, upload-time = "2026-09-15T19:30:32.894Z" }, + { url = "https://files.pythonhosted.org/packages/b6/8a/d1a627f827b0a404ae0f5647cab0534959081cde13b0756a783469fb3b5e/yarl-1.25.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1fb2a01ba8cd9c5d2c5dc1ec35e0fc951d04b4f037541d4ac090c993ce58b3d7", size = 107565, upload-time = "2026-09-15T19:30:35.188Z" }, + { url = "https://files.pythonhosted.org/packages/aa/cf/c8e0aaec886840a6c4480fe44eaec7cd4319f79a563b79321473be79c56f/yarl-1.25.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e92b6bcc741b86d67606c40d3cb9c7cc8e6c737f81e31f4a94efc204456c92e3", size = 125006, upload-time = "2026-09-15T19:30:37.1Z" }, + { url = "https://files.pythonhosted.org/packages/50/26/0cce366d54a93cdc8342965dc7663385e4db161625cc1e8b18e786753d24/yarl-1.25.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:72c34ac7ad4314c19362d5ce27626dcc8429bd30bbf8c179f4234078851f9492", size = 128717, upload-time = "2026-09-15T19:30:38.904Z" }, + { url = "https://files.pythonhosted.org/packages/95/0c/a71501bbc1a674ff72c4d6c2b75f4d9a5af819f5244c3a7558080a8802c5/yarl-1.25.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d5add7b4ca7afeea91d52e4d4e4db3b1fe9885b71f07054560d8c4296b7441a2", size = 117728, upload-time = "2026-09-15T19:30:40.809Z" }, + { url = "https://files.pythonhosted.org/packages/e7/6f/c3267ca01defeed9ed9c4ff9b17bd54915432c405945233265b707475d1c/yarl-1.25.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:def538065f9e4d4cf1ae164bd59aba00dfa84f03923e0de4c3788f252d6bcd17", size = 116224, upload-time = "2026-09-15T19:30:42.818Z" }, + { url = "https://files.pythonhosted.org/packages/8f/69/fad57ee52d648431718ee0f1f68966a99c1352c3924688ffbdcc9d3fe51a/yarl-1.25.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0a191bfdb30a79b98e5d175d75285f9fcb78bf0e46ba5efda042e1c72071a0de", size = 116299, upload-time = "2026-09-15T19:30:44.966Z" }, + { url = "https://files.pythonhosted.org/packages/2a/d4/6a8c1e29f33338687ca278cba0a8fbf6525a322c2c02a9a500ccbe041152/yarl-1.25.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:71f42c5b9a948c113bbdebfa544598321431d064ff959d32e99b1feb61d68345", size = 108625, upload-time = "2026-09-15T19:30:46.874Z" }, + { url = "https://files.pythonhosted.org/packages/e8/d2/3a35ae791c9cb6522c106923ff25c3230d999091e5e65511bca23bbd9914/yarl-1.25.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:72849d892954be4d09e569b8b831ac39ce58417fedc767d4308a0fe542018a40", size = 124515, upload-time = "2026-09-15T19:30:49.082Z" }, + { url = "https://files.pythonhosted.org/packages/be/fd/2b022109a6b4af0f7dc371cf7500af380b0d4f034010243e1b0ce218dc93/yarl-1.25.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:efb01a106f971cb3752856bca2318bbdf7f01bd8823779c461586cbe5ffd5258", size = 115711, upload-time = "2026-09-15T19:30:51.208Z" }, + { url = "https://files.pythonhosted.org/packages/18/59/f7586271136c3ddb0126bbfe661844699369b76b555fe38c4efe86870b2e/yarl-1.25.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:a1daf47cd95a7c3a63456336bc5aaa8c86dd3a47d07ed3d0e76132ae4666a5a1", size = 122751, upload-time = "2026-09-15T19:30:53.535Z" }, + { url = "https://files.pythonhosted.org/packages/a0/0b/07f7a2d881f7e16c385b47fc1753382600847cad305dcc7fa0c25828acf8/yarl-1.25.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9489e6abf47ba37f332075a91444c7cfedb03e6ce99fbb2f116bfe1ce810da3b", size = 117983, upload-time = "2026-09-15T19:30:55.277Z" }, + { url = "https://files.pythonhosted.org/packages/aa/9d/8cdceec66a9b940700cb45931741403f045b162afd79bcb93c41cadd0972/yarl-1.25.1-cp311-cp311-win_amd64.whl", hash = "sha256:d7306dee25b8a0e737363f347362b875094b4dc4e367311470656ae420fdbf8e", size = 102894, upload-time = "2026-09-15T19:30:57.606Z" }, + { url = "https://files.pythonhosted.org/packages/17/f1/7ec357db1d3ad2863542d71e8fe64a126bbec17b134cd7d30951196809a5/yarl-1.25.1-cp311-cp311-win_arm64.whl", hash = "sha256:abb1384477f5901d436b5d2e5465954de46ea6098f59163d243660b5c4461d35", size = 98609, upload-time = "2026-09-15T19:30:59.705Z" }, + { url = "https://files.pythonhosted.org/packages/75/b3/cd32ac66ae622b854c2df0ac52106dda220d361b65a64fde7d5b3684aa3f/yarl-1.25.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:94d7aa6debf92a1dd14cb5280b083a764169a13cfb23a452111160274ed989f4", size = 144798, upload-time = "2026-09-15T19:31:01.821Z" }, + { url = "https://files.pythonhosted.org/packages/61/fb/a2c52a8007c2051ba74662afb112ecf3d00346af4c25e33df9d80fd14fb8/yarl-1.25.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:83d4a37e4b95da4d8bda930d6d35b75b4cdadbacbb4980cae290ea3100b5d51d", size = 104583, upload-time = "2026-09-15T19:31:04.05Z" }, + { url = "https://files.pythonhosted.org/packages/be/dd/ee38aec8e09fdf957e50d4085453fbe202f56c6c3b4cf07b81cdb4f09ee9/yarl-1.25.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e029648f9c951db30e98a7d7ec90835db88ec4b32820efe2a9bdc2287e032eb6", size = 104325, upload-time = "2026-09-15T19:31:06.338Z" }, + { url = "https://files.pythonhosted.org/packages/1e/b3/058dbfb1857b484c9cf9cc135659f50b85ce66e03c99e44dc2f7b6161f55/yarl-1.25.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d781294bb815ecb5ea57ff6bbf8038e0a31a95fdf3e1788f66e0dc100d64b58", size = 115358, upload-time = "2026-09-15T19:31:08.593Z" }, + { url = "https://files.pythonhosted.org/packages/db/39/29693446cf0cf6b15a0e2f75a5d40f93c56819b05b0622196f45e95b5cc0/yarl-1.25.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e12c538e00e7c1b286a07061046b90e8124e6a9793efae2c70db6a4aad07faad", size = 107658, upload-time = "2026-09-15T19:31:10.802Z" }, + { url = "https://files.pythonhosted.org/packages/86/b3/3c4dd7e1af43b931fba95e0a722737f2ea94a6d199c802585282831d7abd/yarl-1.25.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7e4de3ac4adbad3d0bc7c6f4360a7dbff5de2f15e3b723be3198074e17fd9c40", size = 122660, upload-time = "2026-09-15T19:31:12.84Z" }, + { url = "https://files.pythonhosted.org/packages/bd/b5/1b60dbc3cfc9c5712b15148c206748f2bc93953ffdbe25ea75b63dfc89c9/yarl-1.25.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:419f392a1da624877975709e3864dfe833af6cc7671b39318086d456e288380c", size = 126506, upload-time = "2026-09-15T19:31:15.088Z" }, + { url = "https://files.pythonhosted.org/packages/bc/7b/ca212cbe170ac8b96e45317ecbcf9c3c3ecf0cdec98d5b088a9c4088929b/yarl-1.25.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c6f117789d22dce188e5754e8bc65b7e6ebf8cb73963b9fa761f672a5883769d", size = 117050, upload-time = "2026-09-15T19:31:17.241Z" }, + { url = "https://files.pythonhosted.org/packages/cb/c3/72b4938cdbe619ad71ac156182faef4908846b84dc3ca4dbb4c4e6f84014/yarl-1.25.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:80e47012e730da131c9f059c80936783f9659aae22dc31c03c0595590d11ed54", size = 114174, upload-time = "2026-09-15T19:31:19.294Z" }, + { url = "https://files.pythonhosted.org/packages/e8/43/268717870f9ba0cc9701a95181587f6dc8c5f387aab4aeecc83158f38a79/yarl-1.25.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e80f557716fd765439577131e526b8942ffc2c07bdbc5e39fa62f660ba1e963f", size = 114944, upload-time = "2026-09-15T19:31:21.414Z" }, + { url = "https://files.pythonhosted.org/packages/da/84/baa5bf504d51fe062c4bcaf62936da97fffb43285978d0b39984824231fd/yarl-1.25.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:f61964f235a43738bfac50da46fc4254943a7eea3051aeb0b6fc7c992c29fadc", size = 108263, upload-time = "2026-09-15T19:31:23.388Z" }, + { url = "https://files.pythonhosted.org/packages/a4/28/779a2ed9e0152a601a27039bed9aead3f0b79797a67e2c44bfa444622dd8/yarl-1.25.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e546fe1d4a93ebc2910f0d768baff19faa09843ab3f2036a67ed6e69fae4419d", size = 122184, upload-time = "2026-09-15T19:31:25.343Z" }, + { url = "https://files.pythonhosted.org/packages/f8/1f/118e9e5b8f07694d63fd3222e801d7782270003f1a222aa798df3f8d5933/yarl-1.25.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:cce0727fd5ac04d372fa9bbfde9febc2bcf209aadfcf0468e45dec72719895d1", size = 114001, upload-time = "2026-09-15T19:31:27.465Z" }, + { url = "https://files.pythonhosted.org/packages/0f/ae/a4cf1cf372313734b17996d4007f9f73596e7a178b9485802e5494ecf484/yarl-1.25.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:af4ea5b37403ef4e30f3927eaed540db942bde01d8d3ff083527c0704d1c9c68", size = 120565, upload-time = "2026-09-15T19:31:29.47Z" }, + { url = "https://files.pythonhosted.org/packages/05/79/ad94f93ca731bc9e44d321833ab96b82a4f9f5f63cf773f81a4aeea5ecc1/yarl-1.25.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:68782fdb4027b8d1eee25ec35e9a6db05e863b899eb0310b3a33b6c3fef55707", size = 117060, upload-time = "2026-09-15T19:31:31.367Z" }, + { url = "https://files.pythonhosted.org/packages/bb/cc/51a7b4abf4ac593b8e7eb3794b28e5a35ae26eed8bc04787628d215af82f/yarl-1.25.1-cp312-cp312-win_amd64.whl", hash = "sha256:7d575b54cb3863ef9bc290ea4b009999d55dc237326131e4853cf33e888fee03", size = 102593, upload-time = "2026-09-15T19:31:33.329Z" }, + { url = "https://files.pythonhosted.org/packages/9d/21/0941a6b93a58b59a1ec75e5333bf06929b671309c43c0cd201c172d9c39f/yarl-1.25.1-cp312-cp312-win_arm64.whl", hash = "sha256:bc3ac7bf569f6b64dad04dd7808c7872dae8a97df657856eac05e9b7e3614a85", size = 97697, upload-time = "2026-09-15T19:31:35.855Z" }, + { url = "https://files.pythonhosted.org/packages/7b/ed/2f3129bbcc9a5c8ba12cc2b29d8060a3bab9c8043c456cfd4b5ca3188890/yarl-1.25.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:25868beca8b6765f8f7d0e11fe6dd7c66dd4b0793b9500286d20cc92352126a5", size = 143623, upload-time = "2026-09-15T19:31:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/17/e1/f1bc3390fdca352826676b531d0712736f156919090206700421d46b2c37/yarl-1.25.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:10b2fd95332f0d716d5eee3c9fb2ce8eada19082de7fee83d32e37992fd75c26", size = 104011, upload-time = "2026-09-15T19:31:40.25Z" }, + { url = "https://files.pythonhosted.org/packages/a8/aa/50acc5c3e5da04172ae3c281c75405af4d2ca911e16120ab0563f4dffb66/yarl-1.25.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0f12afda4eea8c8994a76d4df1875c765194f5fbe8a9d197929ea303caee29ec", size = 103677, upload-time = "2026-09-15T19:31:42.46Z" }, + { url = "https://files.pythonhosted.org/packages/30/d2/7d1e0ab9f8390e1fbcede5a6dbf70d23c96ad09b8c5567f3a514d1ddb0e2/yarl-1.25.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:14b79a30a93a3ce2e8832603fd0ab780ada281b0ba5110b519a634f2d7d7d1fc", size = 115392, upload-time = "2026-09-15T19:31:44.371Z" }, + { url = "https://files.pythonhosted.org/packages/71/e1/5ba1e3a2a22139213655e760919038e8ed7e2d4a99826d0bbddb3beb96e5/yarl-1.25.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4bd6340d20ae2c7ca719b87b426e808e90743b676d05d4c26c4fb5ca71f41184", size = 107493, upload-time = "2026-09-15T19:31:46.273Z" }, + { url = "https://files.pythonhosted.org/packages/f5/53/780653d5e0f73831f467cf13548912e5eec97f21dc49fc8daf21da027df4/yarl-1.25.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:126a2533570c554719ca40a1288fdee1700b6bc82e7131aa69fa85252d92e651", size = 122537, upload-time = "2026-09-15T19:31:48.654Z" }, + { url = "https://files.pythonhosted.org/packages/03/92/d54fa70236c6036271c9c9c09fd978df5cbe3ef49ef6c46e9b833476d215/yarl-1.25.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a3faadac7d812ddac258feb57b9846b60c1b437c4f4b9ad42595c6f6fe4390df", size = 126170, upload-time = "2026-09-15T19:31:50.872Z" }, + { url = "https://files.pythonhosted.org/packages/0e/b7/a82a49bf88340b837ef6972b508a1604ae377b9e6904b46b10cf5f1cf925/yarl-1.25.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:be80550d9bfe83d9b62398a37081a90434e6df2d978ec345c3d2820de6beddab", size = 117012, upload-time = "2026-09-15T19:31:53.189Z" }, + { url = "https://files.pythonhosted.org/packages/ef/78/5d684b411e3f3602464ee9b538db48205038f8605872985f61efb809ced0/yarl-1.25.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e07595c7d6f4db270ceede356a1bd1c07a34f1c26f958d1ed0cd7b48e0d2bba3", size = 114950, upload-time = "2026-09-15T19:31:55.694Z" }, + { url = "https://files.pythonhosted.org/packages/2f/11/51d82b852c64f7fad0fc7a7ff3031517204887e874c722bbca839c0b23ac/yarl-1.25.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:eb96ed1ae6c7d072d60840c0434aef07a2df611812810807fbc54263a6053e9a", size = 115428, upload-time = "2026-09-15T19:31:57.966Z" }, + { url = "https://files.pythonhosted.org/packages/e4/49/9d1978049bf646b9ea918313926453c6901b71c92f097467777d47d36a88/yarl-1.25.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3feb99222553a8cbedfa52c2f59dd84c3f50d5b582c728d522caf8d72769a54b", size = 108428, upload-time = "2026-09-15T19:32:00.048Z" }, + { url = "https://files.pythonhosted.org/packages/43/35/7b8f1ebb45d7ec3dda7d1909bf44f458de41ef91e2937f107733582a5166/yarl-1.25.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:a2ed0ba415ccdf08f14bf544cb78346d0f76086707ffee24921a2c84dbf1305a", size = 121961, upload-time = "2026-09-15T19:32:02.436Z" }, + { url = "https://files.pythonhosted.org/packages/63/d6/d8b689ab7ca26edeb85f6ff28812aac7a25376eefc1780e303a7bfbaceff/yarl-1.25.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:2b49375d22299b0a834c2bca72f39aaecc270d96fb24c30424899676f487b22a", size = 114961, upload-time = "2026-09-15T19:32:04.456Z" }, + { url = "https://files.pythonhosted.org/packages/cf/d5/1a1798ea4dc6b7ee3260010a27907ebc697c95dae99817d817ed446d24aa/yarl-1.25.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ef74070ac553c59eb4f04258722066d6c6135b7baa03b2e9f2da65c096e96d98", size = 120036, upload-time = "2026-09-15T19:32:06.5Z" }, + { url = "https://files.pythonhosted.org/packages/91/8d/b1b35ed7903da6669b1d367cb2c09436acd4ff508029b4f39a0c0c2058fc/yarl-1.25.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0a66db89ea473abeac4b70523cafd94db3772380e565f9d28af7a179b7af71fa", size = 117276, upload-time = "2026-09-15T19:32:09.401Z" }, + { url = "https://files.pythonhosted.org/packages/3a/8f/4db01cef62caff0d7a4593ed694fb8a41a27a11158cab80d290221f13e57/yarl-1.25.1-cp313-cp313-win_amd64.whl", hash = "sha256:1f51020b2eb8a003c84925638ec63c21a750a4bddd3a22ec8eac6a742dadf1b9", size = 101945, upload-time = "2026-09-15T19:32:11.545Z" }, + { url = "https://files.pythonhosted.org/packages/c0/5e/3ce00497c5c0babb74d4130c10c3828ccd215b4819d12020c42429f991ac/yarl-1.25.1-cp313-cp313-win_arm64.whl", hash = "sha256:b10dd0557ba422715b5206b3743192135a6022acca8baec51aa127d0a75db8fe", size = 97270, upload-time = "2026-09-15T19:32:14.127Z" }, + { url = "https://files.pythonhosted.org/packages/80/cf/54023edfab7aa773b860503db0c56e962ccab0922803ee97988c176ea090/yarl-1.25.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:a9ca696eb02e5c02a8afd872ada510eba9b7fe6e68b9572c2e9a9b1941e31e2e", size = 143975, upload-time = "2026-09-15T19:32:16.416Z" }, + { url = "https://files.pythonhosted.org/packages/d7/a8/e6c1be0e6761d0f2d10bbf33a3e1e02b99dc83874d92945d7b461a72481e/yarl-1.25.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a5877f2255aab518ebe528289037699201d5dc5f045f2396cb30aa02db22f57f", size = 104018, upload-time = "2026-09-15T19:32:18.364Z" }, + { url = "https://files.pythonhosted.org/packages/6e/bb/dda344765ffd3430afe1a1c66c866a57fae67786537d4f14607df6505ac1/yarl-1.25.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7a5c3115595995779ee21f2567035793911c3802a43c74f3fbb0314929ec67ac", size = 104156, upload-time = "2026-09-15T19:32:20.459Z" }, + { url = "https://files.pythonhosted.org/packages/e5/5f/ed1538bcd06009fe990d6d283dd7667f639e62a81e35c6d8c6ef6c08fb3c/yarl-1.25.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:77e5099b99b37f3cf79c246998ca9f7313a78054cd1809ec46bc1afad47e1c4c", size = 116025, upload-time = "2026-09-15T19:32:22.766Z" }, + { url = "https://files.pythonhosted.org/packages/a2/af/2185daf56b99830d3356ecfada46faaa49945de6626e842b7728088d4980/yarl-1.25.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6efaf45df6a849cef613a03a94c845647456662f85438c886bb67a9c027c8c2c", size = 106985, upload-time = "2026-09-15T19:32:24.749Z" }, + { url = "https://files.pythonhosted.org/packages/c1/65/bc1ae564fb4b04a30b6a8f250e787772581c57e4c3d5cf07ac3359de3103/yarl-1.25.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d5f90e44653c4e0f78501ed9bb7d3fce835a8d62b7c6ed0cb16557534087e743", size = 123030, upload-time = "2026-09-15T19:32:27.084Z" }, + { url = "https://files.pythonhosted.org/packages/6a/3e/e2afcde10d74e53b3fa889960991efb3019beda2b1682a01de720a302056/yarl-1.25.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:632da579b2d879f6bad20f2cfa35ded1efe2f4f77f8abb26a6234a5b236acd2f", size = 126765, upload-time = "2026-09-15T19:32:29.332Z" }, + { url = "https://files.pythonhosted.org/packages/a2/be/415b00c0fe5a0615b062a456b26623d7ec91c2bee20faea1a14045aa0469/yarl-1.25.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:30eec96e8a91bd588ce897c9543f6d5d8d34b28fbcba28a4dedf20ebeae9fe57", size = 117199, upload-time = "2026-09-15T19:32:31.49Z" }, + { url = "https://files.pythonhosted.org/packages/97/27/3d8c63ddd3e8bcfd033748ab93876678ce59bacd66e4cb1ed851c9c5b37e/yarl-1.25.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:12b6bc4906e11f5e1a1cdcb12296e7afbd366c783cc8073403cd2fb74334e453", size = 115187, upload-time = "2026-09-15T19:32:34.137Z" }, + { url = "https://files.pythonhosted.org/packages/39/b7/7a81d0be1a502a26a0d4326c6f2ecb736c824f570ea1c6529f2b0b227b50/yarl-1.25.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9d6ed3d17bccce4c05343e1ca8da13bc5c02c812a4e7282ddd05e8769322d3fc", size = 116085, upload-time = "2026-09-15T19:32:36.438Z" }, + { url = "https://files.pythonhosted.org/packages/f0/69/39fff459916aa0fab42215dc47b759586fd80f94aa56dfc4a7c15ba6e0dc/yarl-1.25.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:f38a70074041d3b7e138e452799f5174198bae5bd5ab2000917badf403908c5f", size = 107996, upload-time = "2026-09-15T19:32:38.959Z" }, + { url = "https://files.pythonhosted.org/packages/c0/39/80b9a55a3335590451d9ecf3eb593a8c635351f4c905ef056d7e8a8fd9e7/yarl-1.25.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4ca89e4e21854ed27ec753297dde84b16c9f8e53b14a4866fb44457d643c19f8", size = 122549, upload-time = "2026-09-15T19:32:41.151Z" }, + { url = "https://files.pythonhosted.org/packages/42/7d/a179c6757818bb59372a4adafd09f7f26a3b4a0f04c3ae404b544c0b0c82/yarl-1.25.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:1ab7618921a93767387a4b83776f751588f5b5ae9bb5bc96620e2e2e00bca868", size = 115107, upload-time = "2026-09-15T19:32:43.072Z" }, + { url = "https://files.pythonhosted.org/packages/32/2b/a773ac867e4ab53a98ed98e5cefe3bae31e6f550252ca9d1de266f1a40c5/yarl-1.25.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0ae12ff2b805fa02c4dab838005caef735e39986322698c48588d3beacb65c62", size = 120666, upload-time = "2026-09-15T19:32:45.061Z" }, + { url = "https://files.pythonhosted.org/packages/bc/41/52be6505e85b0f76b4f85b01b5de7e06a0512201abc2c95e14e099549174/yarl-1.25.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:90c30ed53546da833c700115c0064c22120d1b1560f474699fd31f22dd668233", size = 117505, upload-time = "2026-09-15T19:32:47.177Z" }, + { url = "https://files.pythonhosted.org/packages/f5/01/349c0386caedbbe488d519f252df54efac8a1459282d466c474bdd84a620/yarl-1.25.1-cp314-cp314-win_amd64.whl", hash = "sha256:acfa7e22aa6c6e7a5996a41d275bfa01efa7ea56ab890590280e9063e2cf5c1b", size = 103446, upload-time = "2026-09-15T19:32:49.615Z" }, + { url = "https://files.pythonhosted.org/packages/5c/f0/8ec63180f77912f0dc4e5a42760cb8c08d20da1d5ace3578a01b84d1f3d8/yarl-1.25.1-cp314-cp314-win_arm64.whl", hash = "sha256:8e7d98cdbb6d71e726f7d525952867096053d1f290dd4e3c50d7d313a136f414", size = 99159, upload-time = "2026-09-15T19:32:51.686Z" }, + { url = "https://files.pythonhosted.org/packages/47/7d/92d2220d6886b70ab1ed8579533ac2af2dfac716d5d929001daff7986df9/yarl-1.25.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:d21f0fa80a02d05299207eeaafef345d812ace96d5306e4ef265e1d419a615fa", size = 150071, upload-time = "2026-09-15T19:32:53.911Z" }, + { url = "https://files.pythonhosted.org/packages/64/fc/b245e448124bcda9340df38e3553fa222b50260fca027a84095e9bd8642d/yarl-1.25.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:17c9877a89fb6e2bca6f9087eb24cd7fb434653946ef5075e470d23d49b52287", size = 106780, upload-time = "2026-09-15T19:32:56.443Z" }, + { url = "https://files.pythonhosted.org/packages/51/e2/9a6ce2e334ebf218a30335ae76fb1696459430d42f733b8cb0d7d65b84d3/yarl-1.25.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:29273edf1530e397bd07cb784db1fbe0d2590b77569f2e24679a9c0a2d763b94", size = 107361, upload-time = "2026-09-15T19:32:58.827Z" }, + { url = "https://files.pythonhosted.org/packages/ed/70/66e8c76b569b450d16e190f15071c916c3df70b0e33927e415ac497cf0c2/yarl-1.25.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b7abffdf37af1cec6a2ad69b827aa84320db5894791bc8ed932dc93fb274b7e9", size = 114396, upload-time = "2026-09-15T19:33:02.24Z" }, + { url = "https://files.pythonhosted.org/packages/73/23/0d82838a05c57fdc05bc8b66e8c92dcc0df15e27463a5f163142d521c682/yarl-1.25.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2239a02249d9326655419e0168a28ca9008938eaab31dc29fc875c217927a6c0", size = 104882, upload-time = "2026-09-15T19:33:04.494Z" }, + { url = "https://files.pythonhosted.org/packages/86/d4/ea08615c4edaa6049a13a2f1128944d068d1893abda7d708d4d7ea01599a/yarl-1.25.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:664ec6a520b74a1df2810666eb67695fcb77fa663e6ea0a25aaf2e529cb24dfa", size = 119485, upload-time = "2026-09-15T19:33:06.583Z" }, + { url = "https://files.pythonhosted.org/packages/1a/82/0898bdce9b1ae403b308b9c733d0d24af4a3464270c2c081f457b16c3e0d/yarl-1.25.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4f1c91f5a5980a937ff8e238e98e6897e1ad74a4b1e2c0d68c73b5ffbb3f5c0b", size = 122490, upload-time = "2026-09-15T19:33:08.653Z" }, + { url = "https://files.pythonhosted.org/packages/d1/38/97d79b81c342b78246cfedb74809e68841f3198d21653e10d3232bd9c622/yarl-1.25.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7c88edaec8c349ad4c5ad4c486a3defcc4b80ceb2f074436ffa0a87caf5e76a6", size = 115336, upload-time = "2026-09-15T19:33:11.056Z" }, + { url = "https://files.pythonhosted.org/packages/8e/9d/2577896554cd310dc470adb6da0b7dd0b435cb63e2565204a7ac240e504c/yarl-1.25.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:35dcbea443fafb3eece757ad4e514560ddeb6c34cfae1582c620d7b293d7feee", size = 111825, upload-time = "2026-09-15T19:33:13.204Z" }, + { url = "https://files.pythonhosted.org/packages/29/6b/7ac49d8ba84a5c4bd73415a4c949d22c749cb3762579b3d50e48019a78aa/yarl-1.25.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:882569ff613758cac762a457a5d72d6e211b28d4bcfea89d1d71ea942b02eac0", size = 114655, upload-time = "2026-09-15T19:33:15.553Z" }, + { url = "https://files.pythonhosted.org/packages/e5/18/e5942a16723f5b72f9b1297fd5a85a54f6300cd15c0dcb5005b90cd89156/yarl-1.25.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:d0f1489233a254bb3643d2f05de7d59019254d81daeca6b9162fe9edef57e0c7", size = 106395, upload-time = "2026-09-15T19:33:17.599Z" }, + { url = "https://files.pythonhosted.org/packages/7b/2d/549fa46240781513ebc47ae7eb418df428a163a2a3d644cc9cbb3ecb7846/yarl-1.25.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:f41753a76f4f63927d03a0d8ba8f5ce0f2083bec29a8cfaccc55371b1564b96b", size = 119277, upload-time = "2026-09-15T19:33:19.973Z" }, + { url = "https://files.pythonhosted.org/packages/76/16/4763f78dcdc0b3b9fb3842b04afe72b9320857c6a69300c62a0eab03d119/yarl-1.25.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:8fb0eb4955adf0579001581f2f71a126e8781ba61bcd120f127b0401163c6c2d", size = 112504, upload-time = "2026-09-15T19:33:22.464Z" }, + { url = "https://files.pythonhosted.org/packages/ae/b4/974e3edfe0d188393ce1cb9de400111c63fe61f4eb3b772a500d84c970d1/yarl-1.25.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a1e32763e641a1566507d90a8d3b19bfc3cc04a9d4e5ae3e32189874ed4b58a3", size = 116243, upload-time = "2026-09-15T19:33:24.788Z" }, + { url = "https://files.pythonhosted.org/packages/b0/aa/157b940428da80c104ca09666a740e51c94963df65d5b112e06b52e4d7a8/yarl-1.25.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:65b5b2066651b7432d389e9799d979c703bcc6ef44266bb8153ef54e91e4aab3", size = 115822, upload-time = "2026-09-15T19:33:26.886Z" }, + { url = "https://files.pythonhosted.org/packages/7e/af/19fbdce41412e1b96825544cc52cd7029d3724655d0988237972f078bd29/yarl-1.25.1-cp314-cp314t-win_amd64.whl", hash = "sha256:734f6e5400352ac4254456003d462866c684703570929cff7a7bde015d0cb371", size = 107386, upload-time = "2026-09-15T19:33:29.009Z" }, + { url = "https://files.pythonhosted.org/packages/2a/99/f6431c8968e89be608d74b28ae2d024521b2953f27dd44e0dece5e04f67a/yarl-1.25.1-cp314-cp314t-win_arm64.whl", hash = "sha256:287e99ff5aa4dc1c7630bfc683ded6f106d756c99dec432a2d7f197a784f51c6", size = 102094, upload-time = "2026-09-15T19:33:31.151Z" }, + { url = "https://files.pythonhosted.org/packages/54/22/318c7980066769c6bcd9221ed2248294f5698811da099013098c670565ed/yarl-1.25.1-py3-none-any.whl", hash = "sha256:681c758b0490f9e96b78e5fa8e8dc6e648e9185bb6eaebe73183c33ea0c445f3", size = 63617, upload-time = "2026-09-15T19:34:59.616Z" }, ] [[package]] From 714d84acb609d873b4f86de49ef89d48b49b2c1d Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 11:20:46 -0700 Subject: [PATCH 11/20] fix(cost-map): add fireworks minimax m3.1 flash standard prices (#47371) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 5 ++++- model_prices_and_context_window.json | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 6675bb170b8..de76b6e1bdf 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -76118,15 +76118,18 @@ ] }, "fireworks_ai/accounts/fireworks/models/minimax-m3p1-flash": { + "cache_read_input_token_cost": 1.5e-08, "cache_read_input_token_cost_priority": 2.25e-08, + "input_cost_per_token": 1.5e-07, "input_cost_per_token_priority": 2.25e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 1048576, "max_tokens": 1048576, "mode": "chat", + "output_cost_per_token": 6e-07, "output_cost_per_token_priority": 9e-07, - "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", + "source": "https://platform.minimax.io/docs/guides/pricing", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 6675bb170b8..de76b6e1bdf 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -76118,15 +76118,18 @@ ] }, "fireworks_ai/accounts/fireworks/models/minimax-m3p1-flash": { + "cache_read_input_token_cost": 1.5e-08, "cache_read_input_token_cost_priority": 2.25e-08, + "input_cost_per_token": 1.5e-07, "input_cost_per_token_priority": 2.25e-07, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "max_output_tokens": 1048576, "max_tokens": 1048576, "mode": "chat", + "output_cost_per_token": 6e-07, "output_cost_per_token_priority": 9e-07, - "source": "https://api.fireworks.ai/v1/serverless/models?format=nested", + "source": "https://platform.minimax.io/docs/guides/pricing", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, From 40e4ade6a6d5824a072e90a391cf8b2d32675a48 Mon Sep 17 00:00:00 2001 From: Justin Israel Date: Tue, 20 Oct 2026 07:40:38 +1300 Subject: [PATCH 12/20] feat(proxy): opt-in self-serve budget edits for key owners on /key/update (#44763) * feat(proxy): opt-in self-serve budget edits for key owners on /key/update Adds general_settings.self_serve_budget_policy (disabled by default) so a non-admin can change max_budget or budget_limits on a key they created and still own. lower_only allows adding or lowering limits. ceiling also allows raising or removing them when the owner could create that key with the new budget via /key/generate (same member permissions and delegation ceiling). soft_budget and spend stay admin-only * fix(proxy): sync dashboard API types and cover the UI session cap on update * test(proxy): cover team role restriction for owner budget raises Drop two unreachable branches from the non-raising key generation check. * fix(proxy): close self-serve budget bypasses for linked budgets, null caps and retained limits Keys with a linked budget_id need an admin to change max_budget, owners cannot leave a key uncapped when an upperbound max_budget is configured, and the ceiling check now runs on the budget the key would carry after the update. * fix(proxy): keep open key budget windows and the admin path on self-serve budget edits A resubmitted budget_limits window now keeps its stored reset_at while that period is still running, so adding or editing one window no longer restarts the others. Team and org admins answer False on a ceiling loosen so they fall through to _check_key_admin_access as before. The self-serve tests feed the team through the cache and prisma mocks instead of patching get_team_object, and every helper is typed * perf(proxy): group key budget windows by duration in one pass The per-duration minimum used to rescan the whole window list once per distinct duration, so a request with thousands of windows pinned the event loop before any permission check ran. Sort once and take each group's minimum instead * refactor(proxy): take budget windows as a Sequence in the delegation ceiling checks * test(proxy): assert allowed self-serve budget edits return no denial Also drop the unreachable assert_never arm from the exhaustive budget change match. --------- Co-authored-by: ryan-crabbe-berri --- litellm/proxy/_types.py | 4 + .../management_endpoints/key_budget_change.py | 135 +++++ .../key_management_endpoints.py | 280 ++++++++--- .../team_member_permission_checks.py | 17 +- .../test_key_budget_change.py | 467 ++++++++++++++++++ .../test_key_management_endpoints.py | 45 ++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 5 + 7 files changed, 874 insertions(+), 79 deletions(-) create mode 100644 litellm/proxy/management_endpoints/key_budget_change.py create mode 100644 tests/unit/proxy/management_endpoints/test_key_budget_change.py diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index bc32978d477..e4f77ac0005 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2793,6 +2793,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): None, description="Fine-grained control over which object types to load from the database when store_model_in_db is True. Available types: 'models', 'mcp', 'guardrails', 'vector_stores', 'pass_through_endpoints', 'prompts', 'model_cost_map', 'tools', 'config_overrides'. If not set, all objects are loaded (default behavior).", ) + self_serve_budget_policy: Literal["disabled", "lower_only", "ceiling"] | None = Field( + None, + description="Lets a non-admin change max_budget or budget_limits on a key they created and still own via /key/update. 'disabled' (default) requires an admin for any budget change. 'lower_only' lets the owner add or lower limits. 'ceiling' also lets the owner raise or remove limits when they could create that key with the new budget via /key/generate.", + ) user_mcp_management_mode: UserMCPManagementMode | None = Field( None, description="Controls how non-admin users interact with MCP servers in the dashboard. 'restricted' shows only accessible servers, 'view_all' lists every server in read-only mode.", diff --git a/litellm/proxy/management_endpoints/key_budget_change.py b/litellm/proxy/management_endpoints/key_budget_change.py new file mode 100644 index 00000000000..3143ef53d76 --- /dev/null +++ b/litellm/proxy/management_endpoints/key_budget_change.py @@ -0,0 +1,135 @@ +"""Classifies a /key/update request's budget edits as tightening or loosening the key's stored limits.""" + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from itertools import groupby +from types import MappingProxyType +from typing import Final, Literal, TypeAlias + +from pydantic import TypeAdapter, ValidationError + +from litellm._logging import verbose_proxy_logger +from litellm.models.team import BudgetLimitEntry +from litellm.models.verification_token import LiteLLM_VerificationToken +from litellm.proxy._types import UpdateKeyRequest + + +@dataclass(frozen=True, slots=True) +class KeyBudgetUnchanged: + kind: Literal["unchanged"] = "unchanged" + + +@dataclass(frozen=True, slots=True) +class KeyBudgetTightened: + kind: Literal["tightened"] = "tightened" + + +@dataclass(frozen=True, slots=True) +class KeyBudgetLoosened: + kind: Literal["loosened"] = "loosened" + + +@dataclass(frozen=True, slots=True) +class KeyBudgetAdminOnly: + kind: Literal["admin_only"] = "admin_only" + + +KeyBudgetChange: TypeAlias = KeyBudgetUnchanged | KeyBudgetTightened | KeyBudgetLoosened | KeyBudgetAdminOnly + +_FieldChange: TypeAlias = Literal["unchanged", "tightened", "loosened"] + +SELF_SERVE_BUDGET_POLICY_SETTING: Final = "self_serve_budget_policy" +SelfServeBudgetPolicy: TypeAlias = Literal["disabled", "lower_only", "ceiling"] + +_STORED_WINDOWS: Final = TypeAdapter(list[BudgetLimitEntry]) +_POLICY: Final = TypeAdapter(SelfServeBudgetPolicy) + + +def resolve_self_serve_budget_policy(general_settings: Mapping[str, object]) -> SelfServeBudgetPolicy: + configured: Final = general_settings.get(SELF_SERVE_BUDGET_POLICY_SETTING, "disabled") + try: + return _POLICY.validate_python(configured) + except ValidationError: + verbose_proxy_logger.warning( + "general_settings.%s=%r is not one of disabled, lower_only, ceiling; using disabled", + SELF_SERVE_BUDGET_POLICY_SETTING, + configured, + ) + return "disabled" + + +def _max_budget_change(data: UpdateKeyRequest, existing: LiteLLM_VerificationToken) -> _FieldChange: + if "max_budget" not in data.model_fields_set or data.max_budget == existing.max_budget: + return "unchanged" + if data.max_budget is None: + return "loosened" + if existing.max_budget is None or data.max_budget < existing.max_budget: + return "tightened" + return "loosened" + + +def _duration(window: BudgetLimitEntry) -> str: + return window.budget_duration + + +def _caps_by_duration(windows: Sequence[BudgetLimitEntry]) -> Mapping[str, float]: + grouped: Final = groupby(sorted(windows, key=_duration), key=_duration) + return MappingProxyType({duration: min(w.max_budget for w in group) for duration, group in grouped}) + + +def _stored_caps(existing: LiteLLM_VerificationToken) -> Mapping[str, float] | None: + try: + return _caps_by_duration(_STORED_WINDOWS.validate_python(existing.budget_limits or [])) + except ValidationError: + return None + + +def _budget_limits_change(data: UpdateKeyRequest, existing: LiteLLM_VerificationToken) -> _FieldChange: + """ + Window spend counters are keyed by the exact `budget_duration` string, so a window only + counts as kept when the same string survives with a cap no higher than before. + """ + if "budget_limits" not in data.model_fields_set: + return "unchanged" + stored: Final = _stored_caps(existing) + if stored is None: + return "loosened" + requested: Final = _caps_by_duration(data.budget_limits or ()) + kept_or_lowered: Final = all(d in requested and requested[d] <= cap for d, cap in stored.items()) + return "tightened" if kept_or_lowered else "loosened" + + +@dataclass(frozen=True, slots=True) +class EffectiveKeyBudget: + max_budget: float | None + budget_limits: tuple[BudgetLimitEntry, ...] + + +def stored_budget_windows(existing: LiteLLM_VerificationToken) -> tuple[BudgetLimitEntry, ...]: + """The key's stored windows, or none when the column holds something the window model cannot read.""" + try: + return tuple(_STORED_WINDOWS.validate_python(existing.budget_limits or [])) + except ValidationError: + return () + + +def effective_key_budget(data: UpdateKeyRequest, existing: LiteLLM_VerificationToken) -> EffectiveKeyBudget: + """The budget the key would carry after the update, with omitted fields keeping their stored values.""" + max_budget: Final = data.max_budget if "max_budget" in data.model_fields_set else existing.max_budget + if "budget_limits" in data.model_fields_set: + return EffectiveKeyBudget(max_budget=max_budget, budget_limits=tuple(data.budget_limits or ())) + return EffectiveKeyBudget(max_budget=max_budget, budget_limits=stored_budget_windows(existing)) + + +def classify_key_budget_change(data: UpdateKeyRequest, existing: LiteLLM_VerificationToken) -> KeyBudgetChange: + overrides_linked_budget: Final = ( + existing.budget_id is not None and _max_budget_change(data, existing) != "unchanged" + ) + if data.spend is not None or "soft_budget" in data.model_fields_set or overrides_linked_budget: + return KeyBudgetAdminOnly() + changes: Final = frozenset((_max_budget_change(data, existing), _budget_limits_change(data, existing))) + if "loosened" in changes: + return KeyBudgetLoosened() + if "tightened" in changes: + return KeyBudgetTightened() + return KeyBudgetUnchanged() diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index e87ca70035c..022eba499a4 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -106,6 +106,17 @@ from litellm.proxy.management_endpoints.common_utils import ( # noqa: F401 # l validate_budget_duration, validate_finite_spend, ) +from litellm.proxy.management_endpoints.key_budget_change import ( + KeyBudgetAdminOnly, + KeyBudgetChange, + KeyBudgetLoosened, + KeyBudgetTightened, + KeyBudgetUnchanged, + classify_key_budget_change, + effective_key_budget, + resolve_self_serve_budget_policy, + stored_budget_windows, +) from litellm.proxy.management_endpoints.model_management_endpoints import ( # noqa: F401 # legacy module exports _add_model_to_db, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export add_model_to_db, @@ -190,6 +201,7 @@ from litellm.types.router import Deployment from litellm.types.utils import ( BudgetConfig, PersonalUIKeyGenerationConfig, + StandardKeyGenerationConfig, TeamUIKeyGenerationConfig, ) @@ -848,6 +860,33 @@ def key_generation_check( return _personal_key_generation_check(user_api_key_dict=user_api_key_dict, data=data) +def _caller_may_generate_key( + team_table: LiteLLM_TeamTableCachedObj | None, + user_api_key_dict: UserAPIKeyAuth, +) -> bool: + """Non-raising mirror of the role and member-permission gates in `key_generation_check`.""" + settings: Final = litellm.key_generation_settings or StandardKeyGenerationConfig() + if team_table is None: + personal: Final = settings.get("personal_key_generation") + return ( + personal is None + or "allowed_user_roles" not in personal + or user_api_key_dict.user_role in personal["allowed_user_roles"] + ) + team_settings: Final = settings.get( + "team_key_generation", TeamUIKeyGenerationConfig(allowed_team_member_roles=["admin", "user"]) + ) + caller_team_role: Final = _get_caller_team_role(team_table=team_table, user_api_key_dict=user_api_key_dict) + if caller_team_role is None or ( + "allowed_team_member_roles" in team_settings + and caller_team_role not in team_settings["allowed_team_member_roles"] + ): + return False + return caller_team_role == "admin" or TeamMemberPermissionChecks.is_route_granted_to_team_members( + team_table=team_table, route=KeyManagementRoutes.KEY_GENERATE.value + ) + + def raise_on_invalid_key_logging_config(metadata: Mapping[str, object] | None) -> None: """Key-level logging writes go through key metadata, not /team/callback. @@ -1072,7 +1111,7 @@ _check_permissions_caller_permission: Final = check_permissions_caller_permissio def _check_budget_limits_delegation_ceiling( - budget_limits: list[BudgetLimitEntry] | None, + budget_limits: Sequence[BudgetLimitEntry] | None, delegation_ceiling: float | None, user_api_key_dict: UserAPIKeyAuth, is_ui_session_team_key: bool, @@ -1122,6 +1161,67 @@ def _check_budget_limits_delegation_ceiling( ) +def _enforce_budget_delegation_ceiling( + requested_max_budget: float | None, + requested_team_id: str | None, + budget_limits: Sequence[BudgetLimitEntry] | None, + team_table: LiteLLM_TeamTableCachedObj | None, + user_api_key_dict: UserAPIKeyAuth, +) -> None: + # Delegated-authority ceiling (GHSA-q775-qw9r-2r4g): a non-admin caller + # cannot grant a key a higher budget than their own authority. + # UI session personal keys are capped by user_max_budget when it is available. + is_ui_session_token: Final = user_api_key_dict.team_id == UI_SESSION_TOKEN_TEAM_ID + is_ui_session_team_key: Final = is_ui_session_token and requested_team_id is not None + if ( + user_api_key_dict.is_session_token + and user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value + and not is_ui_session_team_key + and requested_max_budget is not None + and team_table is None + ): + raise HTTPException( + status_code=400, + detail={ + "error": ( + f"max_budget ({requested_max_budget}) cannot be set without " + "specifying team_id when using a CLI session token." + ) + }, + ) + delegation_ceiling: Final = ( + user_api_key_dict.user_max_budget + if is_ui_session_token and user_api_key_dict.user_max_budget is not None + else user_api_key_dict.max_budget + if user_api_key_dict.max_budget is not None + else (team_table.max_budget if user_api_key_dict.is_session_token and team_table is not None else None) + ) + if ( + user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value + and not is_ui_session_team_key + and requested_max_budget is not None + and delegation_ceiling is not None + and requested_max_budget > delegation_ceiling + ): + raise HTTPException( + status_code=400, + detail={ + "error": ( + f"max_budget ({requested_max_budget}) cannot exceed the caller's " + f"own max_budget ({delegation_ceiling})." + ) + }, + ) + + _check_budget_limits_delegation_ceiling( + budget_limits=budget_limits, + delegation_ceiling=delegation_ceiling, + user_api_key_dict=user_api_key_dict, + is_ui_session_team_key=is_ui_session_team_key, + team_table=team_table, + ) + + async def validate_team_id_used_in_service_account_request( team_id: str | None, prisma_client: PrismaClient | None, @@ -1364,57 +1464,12 @@ async def _common_key_generation_helper( # check if user set upperbound key/generate params on config.yaml _enforce_upperbound_key_params(data, fill_defaults=True) - # Delegated-authority ceiling (GHSA-q775-qw9r-2r4g): a non-admin caller - # cannot grant a key a higher budget than their own authority. - # UI session personal keys are capped by user_max_budget when it is available. - is_ui_session_token: Final = user_api_key_dict.team_id == UI_SESSION_TOKEN_TEAM_ID - is_ui_session_team_key = is_ui_session_token and _requested_team_id is not None - if ( - user_api_key_dict.is_session_token - and user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value - and not is_ui_session_team_key - and _requested_max_budget is not None - and team_table is None - ): - raise HTTPException( - status_code=400, - detail={ - "error": ( - f"max_budget ({_requested_max_budget}) cannot be set without " - "specifying team_id when using a CLI session token." - ) - }, - ) - delegation_ceiling: Final = ( - user_api_key_dict.user_max_budget - if is_ui_session_token and user_api_key_dict.user_max_budget is not None - else user_api_key_dict.max_budget - if user_api_key_dict.max_budget is not None - else (team_table.max_budget if user_api_key_dict.is_session_token and team_table is not None else None) - ) - if ( - user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value - and not is_ui_session_team_key - and _requested_max_budget is not None - and delegation_ceiling is not None - and _requested_max_budget > delegation_ceiling - ): - raise HTTPException( - status_code=400, - detail={ - "error": ( - f"max_budget ({_requested_max_budget}) cannot exceed the caller's " - f"own max_budget ({delegation_ceiling})." - ) - }, - ) - - _check_budget_limits_delegation_ceiling( + _enforce_budget_delegation_ceiling( + requested_max_budget=_requested_max_budget, + requested_team_id=_requested_team_id, budget_limits=data.budget_limits, - delegation_ceiling=delegation_ceiling, - user_api_key_dict=user_api_key_dict, - is_ui_session_team_key=is_ui_session_team_key, team_table=team_table, + user_api_key_dict=user_api_key_dict, ) check_permissions_caller_permission( data=data, @@ -2750,16 +2805,12 @@ async def prepare_key_update_data( non_default_values["budget_duration"] = budget_duration if "budget_limits" in non_default_values: - raw_windows: Final = non_default_values["budget_limits"] - if raw_windows: - from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time - - initialized_windows: Final = [] - for window in raw_windows: - w = window if isinstance(window, dict) else window.model_dump() - w["reset_at"] = get_budget_reset_time(budget_duration=w["budget_duration"]).isoformat() - initialized_windows.append(w) - non_default_values["budget_limits"] = json.dumps(initialized_windows) + requested_windows: Final = _budget_limit_windows(data.budget_limits) + if requested_windows: + stored_windows: Final = stored_budget_windows(existing_key_row) + non_default_values["budget_limits"] = json.dumps( + [_window_with_reset_at(window, stored_windows) for window in requested_windows] + ) else: # [] / None clears the field; prisma-client-py has no DbNull # sentinel for Json? columns, so store the JSON literal null @@ -3327,6 +3378,62 @@ async def _acting_as_team_admin_for_key_update( return True +async def _owner_may_change_key_budget( + budget_change: KeyBudgetChange, + data: UpdateKeyRequest, + existing_key_row: LiteLLM_VerificationToken, + user_api_key_dict: UserAPIKeyAuth, + prisma_client: PrismaClient, + user_api_key_cache: UserApiKeyCache, +) -> bool: + """ + Whether the key's owner may make this budget change without admin rights, per + general_settings.self_serve_budget_policy. `ceiling` lets the owner loosen only when they + could have created the key with the new budget anyway (same generate permissions and + delegation ceiling as /key/generate). Team and org admins answer False on a loosen so they + keep the admin path, which has no ceiling. + """ + policy: Final = resolve_self_serve_budget_policy(_general_settings()) + if policy == "disabled": + return False + match budget_change: + case KeyBudgetUnchanged() | KeyBudgetTightened(): + return True + case KeyBudgetAdminOnly(): + return False + case KeyBudgetLoosened(): + if policy == "lower_only": + return False + team_id: Final = data.team_id if "team_id" in data.model_fields_set else existing_key_row.team_id + team_table: Final = ( + None + if team_id is None + else await get_team_object( + team_id=team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + ) + ) + if team_table is not None and await get_team_access().allows( + user_api_key_dict, team_table, TEAM_OR_ORG_ADMIN + ): + return False + if not _caller_may_generate_key(team_table=team_table, user_api_key_dict=user_api_key_dict): + return False + effective: Final = effective_key_budget(data=data, existing=existing_key_row) + upperbound: Final = litellm.upperbound_key_generate_params + if effective.max_budget is None and upperbound is not None and upperbound.max_budget is not None: + return False + _enforce_budget_delegation_ceiling( + requested_max_budget=effective.max_budget, + requested_team_id=team_id, + budget_limits=effective.budget_limits, + team_table=team_table, + user_api_key_dict=user_api_key_dict, + ) + return True + + async def _validate_update_key_data( data: UpdateKeyRequest, existing_key_row: LiteLLM_VerificationToken, @@ -3420,22 +3527,18 @@ async def _validate_update_key_data( # - Anyone else (non-PROXY_ADMIN, not the owner, not a team member # on a team key): must pass _check_key_admin_access (PROXY_ADMIN # / key-owner / team-admin / org-admin of the key). - # - max_budget / spend / budget_limits: always require the admin - # check, even for the key owner or a team member (matches the - # existing admin-only budget semantics). budget_limits uses - # model_fields_set because an explicit null/[] clears the field - # and must gate the same as setting or changing it. + # - max_budget / budget_limits: admin check unless + # general_settings.self_serve_budget_policy lets the key owner make the + # change (_owner_may_change_key_budget). + # - soft_budget: admin check, since it writes the key's budget row, + # which can be a shared /budget/new tier. # - spend gates on presence alone (not a value diff): the DB spend # lags the live cross-pod counter, so letting an "unchanged" spend # through the non-admin path would let a key owner / team member # overwrite the live counter below real usage and silently weaken # enforcement. - _is_budget_change: Final = ( - (data.max_budget is not None and data.max_budget != existing_key_row.max_budget) - or data.spend is not None - or "budget_limits" in data.model_fields_set - or "soft_budget" in data.model_fields_set - ) + budget_change: Final = classify_key_budget_change(data=data, existing=existing_key_row) + _is_budget_change: Final = not isinstance(budget_change, KeyBudgetUnchanged) _existing_metadata: Final = getattr(existing_key_row, "metadata", None) _existing_throttle: Final = ( @@ -3484,8 +3587,19 @@ async def _validate_update_key_data( # non-budget change means the caller was authorized — skip the redundant # _check_key_admin_access that would otherwise require team/org admin status. _key_is_team_key: Final = getattr(existing_key_row, "team_id", None) is not None - can_skip_admin_check: Final = (caller_is_creator or _key_is_team_key) and not ( - _is_budget_change or is_project_change + budget_change_allowed: Final = not _is_budget_change or ( + caller_is_creator + and await _owner_may_change_key_budget( + budget_change=budget_change, + data=data, + existing_key_row=existing_key_row, + user_api_key_dict=user_api_key_dict, + prisma_client=checked_prisma_client, + user_api_key_cache=user_api_key_cache, + ) + ) + can_skip_admin_check: Final = ( + (caller_is_creator or _key_is_team_key) and budget_change_allowed and not is_project_change ) if (not _is_proxy_admin) and not can_skip_admin_check: hashed_key: Final = existing_key_row.token @@ -6459,6 +6573,26 @@ def _budget_limit_windows(budget_limits: Sequence[object] | str | None) -> tuple return tuple(raw_window if isinstance(raw_window, dict) else raw_window.model_dump() for raw_window in raw_windows) +def _window_with_reset_at( + window: Mapping[str, object], stored_windows: Sequence[BudgetLimitEntry] +) -> Mapping[str, object]: + """A resubmitted window keeps its running period, so editing one window never restarts the others.""" + duration: Final = str(window["budget_duration"]) + now: Final = datetime.now(timezone.utc) + open_reset_at: Final = next( + (w.reset_at for w in stored_windows if w.budget_duration == duration and _is_open_reset_at(w.reset_at, now)), + None, + ) + reset_at: Final = open_reset_at if open_reset_at is not None else get_budget_reset_time(budget_duration=duration) + return {**window, "reset_at": reset_at.isoformat()} + + +def _is_open_reset_at(reset_at: datetime | None, now: datetime) -> bool: + if reset_at is None: + return False + return (reset_at if reset_at.tzinfo is not None else reset_at.replace(tzinfo=timezone.utc)) > now + + def _advance_one_key_budget_window(window: Mapping[str, object]) -> Mapping[str, object]: """Restart one budget window from now, by advancing its `reset_at`. diff --git a/litellm/proxy/management_helpers/team_member_permission_checks.py b/litellm/proxy/management_helpers/team_member_permission_checks.py index 3e6d0e47265..834b032b04f 100644 --- a/litellm/proxy/management_helpers/team_member_permission_checks.py +++ b/litellm/proxy/management_helpers/team_member_permission_checks.py @@ -101,6 +101,13 @@ class TeamMemberPermissionChecks: code=401, ) + @staticmethod + def is_route_granted_to_team_members(team_table: LiteLLM_TeamTableCachedObj, route: str) -> bool: + granted_routes: Final = TeamMemberPermissionChecks._get_list_of_route_enum_as_str( + TeamMemberPermissionChecks.get_permissions_for_team_member(team_table=team_table) + ) + return RouteChecks.check_route_access(route=route, allowed_routes=granted_routes) + @staticmethod def does_team_member_have_permissions_for_endpoint( team_member_role: Literal["admin", "user"] | None, @@ -118,12 +125,10 @@ class TeamMemberPermissionChecks: if team_member_role == "admin": return True - _team_member_permissions: Final = TeamMemberPermissionChecks.get_permissions_for_team_member( - team_table=team_table, - ) - team_member_permissions = TeamMemberPermissionChecks._get_list_of_route_enum_as_str(_team_member_permissions) - - if not RouteChecks.check_route_access(route=route, allowed_routes=team_member_permissions): + if not TeamMemberPermissionChecks.is_route_granted_to_team_members(team_table=team_table, route=route): + team_member_permissions: Final = TeamMemberPermissionChecks._get_list_of_route_enum_as_str( + TeamMemberPermissionChecks.get_permissions_for_team_member(team_table=team_table) + ) raise ProxyException( message=f"Team member does not have permissions for endpoint: {route}. You only have access to the following endpoints: {team_member_permissions} for team {team_table.team_id}. To create keys for this team, please ask your proxy admin to check the team member permission settings and update the settings to allow team member users to create keys.", type=ProxyErrorTypes.team_member_permission_error, diff --git a/tests/unit/proxy/management_endpoints/test_key_budget_change.py b/tests/unit/proxy/management_endpoints/test_key_budget_change.py new file mode 100644 index 00000000000..baf8d870117 --- /dev/null +++ b/tests/unit/proxy/management_endpoints/test_key_budget_change.py @@ -0,0 +1,467 @@ +from collections.abc import Mapping, Sequence +from types import MappingProxyType +from typing import Final +from unittest.mock import AsyncMock, MagicMock + +import pytest +from fastapi import HTTPException + +from litellm.constants import UI_SESSION_TOKEN_TEAM_ID +from litellm.models.verification_token import LiteLLM_VerificationToken +from litellm.proxy._types import ( + LiteLLM_TeamTableCachedObj, + LitellmUserRoles, + Member, + ProxyException, + UpdateKeyRequest, + UserAPIKeyAuth, +) +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.management_endpoints.key_budget_change import ( + KeyBudgetAdminOnly, + KeyBudgetChange, + KeyBudgetLoosened, + KeyBudgetTightened, + KeyBudgetUnchanged, + classify_key_budget_change, + resolve_self_serve_budget_policy, +) +from litellm.proxy.management_endpoints.key_management_endpoints import _validate_update_key_data +from litellm.types.proxy.management_endpoints.ui_sso import LiteLLM_UpperboundKeyGenerateParams + +_OWNER: Final = "owner-1" +_TEAM_ADMIN: Final = "team-admin" +_TEAM: Final = "team-1" +_WEEKLY: Final[Mapping[str, object]] = MappingProxyType( + {"budget_duration": "7d", "max_budget": 50.0, "reset_at": "2026-10-08T00:00:00+00:00"} +) + + +def _key( + *, + user_id: str = _OWNER, + created_by: str = _OWNER, + team_id: str | None = None, + max_budget: float | None = None, + budget_limits: Sequence[Mapping[str, object]] | None = None, + budget_id: str | None = None, +) -> LiteLLM_VerificationToken: + return LiteLLM_VerificationToken( + token="hashed-owner-key", + user_id=user_id, + created_by=created_by, + team_id=team_id, + max_budget=max_budget, + budget_limits=None if budget_limits is None else [dict(window) for window in budget_limits], + budget_id=budget_id, + metadata={}, + ) + + +def _window(duration: str, cap: float) -> Mapping[str, object]: + return MappingProxyType({"budget_duration": duration, "max_budget": cap}) + + +@pytest.mark.parametrize( + ("update", "stored", "expected"), + [ + (UpdateKeyRequest(key="sk-1", key_alias="renamed"), _key(max_budget=10.0), KeyBudgetUnchanged()), + (UpdateKeyRequest(key="sk-1", max_budget=10.0), _key(max_budget=10.0), KeyBudgetUnchanged()), + (UpdateKeyRequest(key="sk-1", max_budget=5.0), _key(max_budget=10.0), KeyBudgetTightened()), + (UpdateKeyRequest(key="sk-1", max_budget=5.0), _key(), KeyBudgetTightened()), + (UpdateKeyRequest(key="sk-1", max_budget=20.0), _key(max_budget=10.0), KeyBudgetLoosened()), + (UpdateKeyRequest(key="sk-1", max_budget=None), _key(max_budget=10.0), KeyBudgetLoosened()), + (UpdateKeyRequest(key="sk-1", budget_limits=[_window("24h", 5.0)]), _key(), KeyBudgetTightened()), + ( + UpdateKeyRequest(key="sk-1", budget_limits=[_WEEKLY, _window("24h", 5.0)]), + _key(budget_limits=[_WEEKLY]), + KeyBudgetTightened(), + ), + ( + UpdateKeyRequest(key="sk-1", budget_limits=[_window("7d", 50.0)]), + _key(budget_limits=[_WEEKLY]), + KeyBudgetTightened(), + ), + ( + UpdateKeyRequest(key="sk-1", budget_limits=[_window("7d", 40.0)]), + _key(budget_limits=[_WEEKLY]), + KeyBudgetTightened(), + ), + ( + UpdateKeyRequest(key="sk-1", budget_limits=[_window("7d", 60.0)]), + _key(budget_limits=[_WEEKLY]), + KeyBudgetLoosened(), + ), + ( + UpdateKeyRequest(key="sk-1", budget_limits=[_window("24h", 5.0)]), + _key(budget_limits=[_WEEKLY]), + KeyBudgetLoosened(), + ), + (UpdateKeyRequest(key="sk-1", budget_limits=[]), _key(budget_limits=[_WEEKLY]), KeyBudgetLoosened()), + (UpdateKeyRequest(key="sk-1", budget_limits=None), _key(budget_limits=[_WEEKLY]), KeyBudgetLoosened()), + ( + UpdateKeyRequest(key="sk-1", budget_limits=[_window("1d", 5.0)]), + _key(budget_limits=[_window("24h", 5.0)]), + KeyBudgetLoosened(), + ), + ( + UpdateKeyRequest(key="sk-1", budget_limits=[_window("7d", 40.0)]), + _key(budget_limits=[_WEEKLY, _window("7d", 30.0)]), + KeyBudgetLoosened(), + ), + ( + UpdateKeyRequest(key="sk-1", budget_limits=[_window("7d", 40.0), _window("24h", 5.0)]), + _key(budget_limits=[_window("7d", 30.0), _window("24h", 5.0), _window("7d", 50.0)]), + KeyBudgetLoosened(), + ), + ( + UpdateKeyRequest(key="sk-1", budget_limits=[_window("24h", 5.0)]), + _key(budget_limits=[{"budget_duration": "24h"}]), + KeyBudgetLoosened(), + ), + ( + UpdateKeyRequest(key="sk-1", max_budget=5.0, budget_limits=[]), + _key(max_budget=10.0, budget_limits=[_WEEKLY]), + KeyBudgetLoosened(), + ), + (UpdateKeyRequest(key="sk-1", spend=0.0), _key(), KeyBudgetAdminOnly()), + (UpdateKeyRequest(key="sk-1", soft_budget=1.0), _key(), KeyBudgetAdminOnly()), + (UpdateKeyRequest(key="sk-1", soft_budget=None, max_budget=5.0), _key(max_budget=10.0), KeyBudgetAdminOnly()), + ], +) +def test_classify_key_budget_change( + update: UpdateKeyRequest, stored: LiteLLM_VerificationToken, expected: KeyBudgetChange +) -> None: + assert classify_key_budget_change(update, stored) == expected + + +@pytest.mark.parametrize( + ("settings", "expected"), + [ + ({}, "disabled"), + ({"self_serve_budget_policy": "lower_only"}, "lower_only"), + ({"self_serve_budget_policy": "ceiling"}, "ceiling"), + ({"self_serve_budget_policy": "everything"}, "disabled"), + ({"self_serve_budget_policy": True}, "disabled"), + ], +) +def test_resolve_self_serve_budget_policy(settings: Mapping[str, object], expected: str) -> None: + assert resolve_self_serve_budget_policy(settings) == expected + + +def _team(member_permissions: Sequence[str]) -> LiteLLM_TeamTableCachedObj: + return LiteLLM_TeamTableCachedObj( + team_id=_TEAM, + members_with_roles=[Member(user_id=_TEAM_ADMIN, role="admin"), Member(user_id=_OWNER, role="user")], + team_member_permissions=list(member_permissions), + ) + + +def _caller( + *, + user_id: str = _OWNER, + max_budget: float | None = None, + team_id: str | None = None, + user_max_budget: float | None = None, +) -> UserAPIKeyAuth: + return UserAPIKeyAuth( + user_id=user_id, + user_role=LitellmUserRoles.INTERNAL_USER, + max_budget=max_budget, + team_id=team_id, + user_max_budget=user_max_budget, + ) + + +async def _update( + monkeypatch: pytest.MonkeyPatch, + *, + policy: str | None, + data: UpdateKeyRequest, + existing: LiteLLM_VerificationToken, + team: LiteLLM_TeamTableCachedObj | None = None, + caller: UserAPIKeyAuth | None = None, +) -> None: + monkeypatch.setattr( + "litellm.proxy.proxy_server.general_settings", + {} if policy is None else {"self_serve_budget_policy": policy}, + ) + prisma: Final = MagicMock() + prisma.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=existing) + prisma.db.litellm_teamtable.find_unique = AsyncMock(return_value=team) + prisma.writer_db.litellm_teamtable.find_unique = AsyncMock(return_value=team) + await _validate_update_key_data( + data=data, + existing_key_row=existing, + user_api_key_dict=caller or _caller(), + llm_router=None, + premium_user=False, + prisma_client=prisma, + user_api_key_cache=UserApiKeyCache(), + ) + + +async def _denial( + monkeypatch: pytest.MonkeyPatch, + *, + policy: str | None, + data: UpdateKeyRequest, + existing: LiteLLM_VerificationToken, + team: LiteLLM_TeamTableCachedObj | None = None, + caller: UserAPIKeyAuth | None = None, +) -> HTTPException | None: + try: + await _update(monkeypatch, policy=policy, data=data, existing=existing, team=team, caller=caller) + except HTTPException as exc: + return exc + return None + + +_ADD_DAILY_WINDOW: Final = UpdateKeyRequest(key="sk-1", budget_limits=[_WEEKLY, _window("24h", 5.0)]) +_RAISE_WEEKLY_WINDOW: Final = UpdateKeyRequest(key="sk-1", budget_limits=[_window("7d", 80.0)]) + + +def _team_key_with_weekly(*, user_id: str = _OWNER, created_by: str = _OWNER) -> LiteLLM_VerificationToken: + return _key(user_id=user_id, created_by=created_by, team_id=_TEAM, budget_limits=[_WEEKLY]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("policy", "data", "member_permissions"), + [ + ("lower_only", _ADD_DAILY_WINDOW, ["/key/update"]), + ("ceiling", _ADD_DAILY_WINDOW, ["/key/update"]), + ("ceiling", _RAISE_WEEKLY_WINDOW, ["/key/update", "/key/generate"]), + ], +) +async def test_owner_changes_team_key_budget_when_policy_allows( + monkeypatch: pytest.MonkeyPatch, policy: str, data: UpdateKeyRequest, member_permissions: Sequence[str] +) -> None: + assert ( + await _denial( + monkeypatch, + policy=policy, + data=data, + existing=_team_key_with_weekly(), + team=_team(member_permissions), + ) + is None + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("policy", "data", "member_permissions"), + [ + (None, _ADD_DAILY_WINDOW, ["/key/update", "/key/generate"]), + ("disabled", _ADD_DAILY_WINDOW, ["/key/update", "/key/generate"]), + ("lower_only", _RAISE_WEEKLY_WINDOW, ["/key/update", "/key/generate"]), + ("ceiling", _RAISE_WEEKLY_WINDOW, ["/key/update"]), + ("ceiling", UpdateKeyRequest(key="sk-1", soft_budget=1.0), ["/key/update", "/key/generate"]), + ("ceiling", UpdateKeyRequest(key="sk-1", spend=0.0), ["/key/update", "/key/generate"]), + ], +) +async def test_owner_team_key_budget_change_needs_admin( + monkeypatch: pytest.MonkeyPatch, policy: str | None, data: UpdateKeyRequest, member_permissions: Sequence[str] +) -> None: + with pytest.raises(HTTPException) as exc: + await _update( + monkeypatch, + policy=policy, + data=data, + existing=_team_key_with_weekly(), + team=_team(member_permissions), + ) + assert exc.value.status_code == 403 + + +@pytest.mark.asyncio +async def test_key_created_by_someone_else_needs_admin_even_to_tighten(monkeypatch: pytest.MonkeyPatch) -> None: + with pytest.raises(HTTPException) as exc: + await _update( + monkeypatch, + policy="ceiling", + data=_ADD_DAILY_WINDOW, + existing=_team_key_with_weekly(created_by=_TEAM_ADMIN), + team=_team(["/key/update", "/key/generate"]), + ) + assert exc.value.status_code == 403 + + +@pytest.mark.asyncio +async def test_team_member_without_update_permission_is_still_rejected(monkeypatch: pytest.MonkeyPatch) -> None: + with pytest.raises(ProxyException) as exc: + await _update( + monkeypatch, + policy="ceiling", + data=_ADD_DAILY_WINDOW, + existing=_team_key_with_weekly(), + team=_team(["/key/generate"]), + ) + assert str(exc.value.code) == "401" + + +@pytest.mark.asyncio +async def test_ceiling_lets_owner_raise_personal_key_budget(monkeypatch: pytest.MonkeyPatch) -> None: + assert ( + await _denial( + monkeypatch, + policy="ceiling", + data=UpdateKeyRequest(key="sk-1", max_budget=30.0), + existing=_key(max_budget=10.0), + ) + is None + ) + + +@pytest.mark.asyncio +async def test_ceiling_caps_owner_raise_at_callers_own_budget(monkeypatch: pytest.MonkeyPatch) -> None: + with pytest.raises(HTTPException) as exc: + await _update( + monkeypatch, + policy="ceiling", + data=UpdateKeyRequest(key="sk-1", max_budget=30.0), + existing=_key(max_budget=10.0), + caller=_caller(max_budget=20.0), + ) + assert exc.value.status_code == 400 + assert "cannot exceed the caller's own max_budget (20.0)" in str(exc.value.detail) + + +@pytest.mark.asyncio +async def test_ceiling_does_not_cap_a_team_admin_raising_their_own_team_key(monkeypatch: pytest.MonkeyPatch) -> None: + assert ( + await _denial( + monkeypatch, + policy="ceiling", + data=UpdateKeyRequest(key="sk-1", max_budget=30.0), + existing=_key(user_id=_TEAM_ADMIN, created_by=_TEAM_ADMIN, team_id=_TEAM, max_budget=10.0), + team=_team(["/key/update"]), + caller=_caller(user_id=_TEAM_ADMIN, max_budget=20.0), + ) + is None + ) + + +@pytest.mark.asyncio +async def test_ceiling_caps_ui_session_owner_raise_at_user_budget(monkeypatch: pytest.MonkeyPatch) -> None: + with pytest.raises(HTTPException) as exc: + await _update( + monkeypatch, + policy="ceiling", + data=UpdateKeyRequest(key="sk-1", max_budget=30.0), + existing=_key(max_budget=10.0), + caller=_caller(team_id=UI_SESSION_TOKEN_TEAM_ID, user_max_budget=20.0), + ) + assert exc.value.status_code == 400 + assert "cannot exceed the caller's own max_budget (20.0)" in str(exc.value.detail) + + +@pytest.mark.asyncio +async def test_ceiling_respects_personal_key_generation_role_restriction(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + "litellm.key_generation_settings", + {"personal_key_generation": {"allowed_user_roles": [LitellmUserRoles.PROXY_ADMIN.value]}}, + ) + with pytest.raises(HTTPException) as exc: + await _update( + monkeypatch, + policy="ceiling", + data=UpdateKeyRequest(key="sk-1", max_budget=None), + existing=_key(max_budget=10.0), + ) + assert exc.value.status_code == 403 + + +@pytest.mark.asyncio +async def test_lower_only_lets_owner_lower_personal_key_budget_without_generate_rights( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "litellm.key_generation_settings", + {"personal_key_generation": {"allowed_user_roles": [LitellmUserRoles.PROXY_ADMIN.value]}}, + ) + assert ( + await _denial( + monkeypatch, + policy="lower_only", + data=UpdateKeyRequest(key="sk-1", max_budget=5.0), + existing=_key(max_budget=10.0), + ) + is None + ) + + +@pytest.mark.asyncio +async def test_ceiling_respects_team_key_generation_role_restriction(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + "litellm.key_generation_settings", + {"team_key_generation": {"allowed_team_member_roles": ["admin"]}}, + ) + with pytest.raises(HTTPException) as exc: + await _update( + monkeypatch, + policy="ceiling", + data=_RAISE_WEEKLY_WINDOW, + existing=_team_key_with_weekly(), + team=_team(["/key/update", "/key/generate"]), + ) + assert exc.value.status_code == 403 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("policy", ["lower_only", "ceiling"]) +async def test_owner_cannot_override_a_cap_inherited_from_a_linked_budget( + monkeypatch: pytest.MonkeyPatch, policy: str +) -> None: + with pytest.raises(HTTPException) as exc: + await _update( + monkeypatch, + policy=policy, + data=UpdateKeyRequest(key="sk-1", max_budget=100.0), + existing=_key(max_budget=None, budget_id="shared-tier-10"), + ) + assert exc.value.status_code == 403 + + +@pytest.mark.asyncio +async def test_ceiling_cannot_leave_a_key_uncapped_when_an_upperbound_is_configured( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr("litellm.upperbound_key_generate_params", LiteLLM_UpperboundKeyGenerateParams(max_budget=20.0)) + with pytest.raises(HTTPException) as exc: + await _update( + monkeypatch, + policy="ceiling", + data=UpdateKeyRequest(key="sk-1", max_budget=None), + existing=_key(max_budget=10.0), + ) + assert exc.value.status_code == 403 + + +@pytest.mark.asyncio +async def test_ceiling_checks_the_retained_max_budget_when_only_windows_change( + monkeypatch: pytest.MonkeyPatch, +) -> None: + with pytest.raises(HTTPException) as exc: + await _update( + monkeypatch, + policy="ceiling", + data=UpdateKeyRequest(key="sk-1", budget_limits=[]), + existing=_key(max_budget=100.0, budget_limits=[_WEEKLY]), + caller=_caller(max_budget=20.0), + ) + assert exc.value.status_code == 400 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("policy", [None, "disabled"]) +async def test_owner_cannot_clear_max_budget_by_default(monkeypatch: pytest.MonkeyPatch, policy: str | None) -> None: + with pytest.raises(HTTPException) as exc: + await _update( + monkeypatch, + policy=policy, + data=UpdateKeyRequest(key="sk-1", max_budget=None), + existing=_key(max_budget=10.0), + ) + assert exc.value.status_code == 403 diff --git a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py index dffd9b9b513..e451c38a251 100644 --- a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py @@ -2513,6 +2513,51 @@ async def test_prepare_key_update_data_budget_limits_serializes_windows(): assert windows[0]["reset_at"] is not None +def _key_with_weekly_window(reset_at: datetime) -> LiteLLM_VerificationToken: + return LiteLLM_VerificationToken( + token="test-token", + user_id="test-user", + metadata={}, + budget_limits=[{"budget_duration": "7d", "max_budget": 50.0, "reset_at": reset_at.isoformat()}], + ) + + +async def _updated_windows_by_duration( + existing_key: LiteLLM_VerificationToken, budget_limits: list[dict[str, object]] +) -> dict[str, dict[str, object]]: + result: Final = await prepare_key_update_data( + data=UpdateKeyRequest.model_validate({"key": "test-token", "budget_limits": budget_limits}), + existing_key_row=existing_key, + ) + return {window["budget_duration"]: window for window in json.loads(result["budget_limits"])} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("weekly_cap", [50.0, 40.0, 60.0]) +async def test_prepare_key_update_data_keeps_a_resubmitted_window_mid_period(weekly_cap: float) -> None: + open_reset_at: Final = datetime.now(timezone.utc) + timedelta(days=3) + + windows: Final = await _updated_windows_by_duration( + _key_with_weekly_window(open_reset_at), + [{"budget_duration": "7d", "max_budget": weekly_cap}, {"budget_duration": "1d", "max_budget": 5.0}], + ) + + assert datetime.fromisoformat(windows["7d"]["reset_at"]) == open_reset_at + assert windows["7d"]["max_budget"] == weekly_cap + assert datetime.fromisoformat(windows["1d"]["reset_at"]) > datetime.now(timezone.utc) + + +@pytest.mark.asyncio +async def test_prepare_key_update_data_restarts_a_resubmitted_window_whose_period_ended() -> None: + stale_reset_at: Final = datetime.now(timezone.utc) - timedelta(days=1) + + windows: Final = await _updated_windows_by_duration( + _key_with_weekly_window(stale_reset_at), [{"budget_duration": "7d", "max_budget": 50.0}] + ) + + assert datetime.fromisoformat(windows["7d"]["reset_at"]) > datetime.now(timezone.utc) + + @pytest.mark.asyncio async def test_prepare_key_update_data_disable_global_guardrails_false_no_premium( monkeypatch, diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index d697ccac98d..daffc3c0c41 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -30213,6 +30213,11 @@ export interface components { * @default false */ search_tool_deny_by_default: boolean; + /** + * Self Serve Budget Policy + * @description Lets a non-admin change max_budget or budget_limits on a key they created and still own via /key/update. 'disabled' (default) requires an admin for any budget change. 'lower_only' lets the owner add or lower limits. 'ceiling' also lets the owner raise or remove limits when they could create that key with the new budget via /key/generate. + */ + self_serve_budget_policy?: ("disabled" | "lower_only" | "ceiling") | null; /** @description Daily check of the spend LiteLLM captured against the provider's own bill (OpenAI via OPENAI_ADMIN_KEY). Publishes litellm_spend_capture_rate per provider and alerts when the ratio over the lookback window falls under the threshold (default 0.9). Off unless set. */ spend_capture_rate_check?: components["schemas"]["SpendCaptureRateCheckSettings"] | null; /** @description Which keys of LiteLLM_SpendLogs.metadata are written to the database. Set exactly one of 'include' (write only these keys) or 'exclude' (drop these keys). 'status' and 'cold_storage_object_key' are always written. Daily spend tables, budgets and logging callbacks still see every key. Unset writes every key */ From d7620c77a1ebaf30bb55f31030418b9da2f8cd21 Mon Sep 17 00:00:00 2001 From: Paul El Khoury <82139421+pOpO1-9@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:40:56 +0200 Subject: [PATCH 13/20] fix(openai): drop unsupported reasoning effort for gpt-6.1-sol (#43935) * fix(openai): drop unsupported reasoning effort for gpt-6.1-sol gpt-6.1-sol accepts low, medium, high, xhigh, and max. none and minimal were still forwarded, because chat completions ignored an explicit supports_none_reasoning_effort=false and the Responses mapper never applied that contract. Co-authored-by: Cursor * fix(openai): mark the stripped reasoning object as outgoing JSON Co-authored-by: Cursor * Fix Azure responses reading the OpenAI effort map gpt-6-astra on Azure accepts reasoning.effort none. The responses gate was checking the OpenAI entry, which rejects it. Signed-off-by: Paul El Khoury <82139421+pOpO1-9@users.noreply.github.com> Co-authored-by: Cursor * fix(openai): gate every reasoning_effort level from the cost map on chat and Responses One gate reads the model's cost-map row for whatever effort level a request asks for, on Chat Completions and the Responses API, for OpenAI and Azure. xhigh stays opt-in on chat (Azure also keeps none opt-in); every other level is refused only when its supports__reasoning_effort flag is false. A refused level is dropped under drop_params or answered with a local 400, and Responses keeps the rest of the reasoning object when it drops effort. The guards that follow judge temperature by the level the provider will see. Tests move into the mapped test files and derive the expected outcome for each level from the map row, so a new GPT-6 model needs only its row. * test(openai): pin gpt-6.1-sol's refused effort levels and prove the row flags alone drive the gate * fix(cost-map): azure gpt-6-astra refuses none and honors max on Responses, as live calls show * test(reasoning_effort): pin the astra max and none flips through the adapter and Foundry paths * fix(reasoning_effort): keep max out of the route gates and correct the gpt-5.3-codex rows The same model honors max on Responses and refuses it on Chat Completions, so one row flag cannot gate it on either route. The gates now skip max, the Azure gpt-6-astra rows keep max off so the /v1/messages adapter degrades to xhigh and /model_group/info stops short of it as on main, and chat still forwards max for the Responses bridge. The gpt-5.3-codex rows said none and xhigh off and minimal on; the live model answers the other way round (2026-10-01), and an opt-out gate would have dropped the levels Codex CLI sends. * fix(reasoning_effort): bridge function tools to Responses when the chat gate drops none * docs(reasoning_effort): correct the max advertisement note * fix(azure_ai): read the Foundry row on the Responses effort gate and turn none off for Azure gpt-6.1-sol * fix(azure): gate a prefixed Foundry deployment's Responses effort off its row litellm.responses(model="azure_ai/", api_base=) is re-routed to the azure provider with the azure_ai/ prefix intact, so the Azure Responses config saw a prefixed name, skipped it as another provider's pass-through, and forwarded none and minimal to Azure even under drop_params. The Azure gpt-5 resolver also read no row at all for a prefixed name. The Azure Responses gate now detects the family the way the Azure chat gate does, and an azure_ai/ name on the Azure OpenAI configs reads its azure_ai/ row when the map has one, else the azure/ twin, the rule the Foundry chat config already applies to the same deployment. * fix(cost-map): turn none off on azure/eu/gpt-6-astra like its sibling rows * docs(reasoning_effort): say how each gate reads none and shorten the module docstring * fix(azure_ai): call the public effort-level check from the Foundry Responses gate * test(reasoning_effort): fold the Foundry Responses tests into the mapped file and type the row helpers --------- Signed-off-by: Paul El Khoury <82139421+pOpO1-9@users.noreply.github.com> Co-authored-by: Cursor Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: ryan-crabbe-berri --- .../llms/azure/chat/gpt_5_transformation.py | 59 +++----- .../llms/azure/responses/transformation.py | 8 + .../llms/azure_ai/responses/transformation.py | 13 ++ .../llms/openai/chat/gpt_5_transformation.py | 102 ++++++++----- .../llms/openai/responses/transformation.py | 39 ++++- ...odel_prices_and_context_window_backup.json | 22 +-- litellm/responses/bridge_check.py | 26 +++- .../reasoning_effort_capability.py | 25 ++- model_prices_and_context_window.json | 22 +-- ..._handler_reasoning_effort_normalization.py | 26 +++- .../chat/test_azure_gpt5_transformation.py | 143 ++++++++++++++++-- .../response/test_azure_transformation.py | 125 ++++++++++++++- .../chat/test_azure_ai_transformation.py | 17 ++- .../test_azure_ai_responses_transformation.py | 74 +++++++++ .../test_openai_responses_transformation.py | 109 +++++++++++++ .../llms/openai/test_gpt5_transformation.py | 101 +++++++++++++ .../test_reasoning_effort_capability.py | 15 +- tests/unit/test_main.py | 49 +++++- 18 files changed, 822 insertions(+), 153 deletions(-) diff --git a/litellm/llms/azure/chat/gpt_5_transformation.py b/litellm/llms/azure/chat/gpt_5_transformation.py index a23fffb5744..a9b6c27a0df 100644 --- a/litellm/llms/azure/chat/gpt_5_transformation.py +++ b/litellm/llms/azure/chat/gpt_5_transformation.py @@ -1,12 +1,10 @@ """Support for Azure OpenAI gpt-5 model family.""" -from typing import Final +from typing import ClassVar, Final import litellm -from litellm.exceptions import UnsupportedParamsError from litellm.llms.openai.chat.gpt_5_transformation import ( OpenAIGPT5Config, - get_effort_level, is_gpt_reasoning_series_name, ) from litellm.types.llms.openai import AllMessageValues @@ -15,9 +13,16 @@ from .gpt_transformation import AzureOpenAIConfig class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): - """Azure specific handling for gpt-5 models.""" + """Azure specific handling for gpt-5 models. + + reasoning_effort none joins xhigh as opt-in here: Azure rejected it on every gpt-5 + deployment before gpt-5.1 (https://github.com/BerriAI/litellm/issues/16704), and a deployment + name the map never heard of must not forward it. See + https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/reasoning + """ GPT5_SERIES_ROUTE = "gpt5_series/" + OPT_IN_REASONING_EFFORTS: ClassVar[frozenset[str]] = frozenset(("xhigh", "none")) @classmethod def _model_map_lookup_name(cls, model: str) -> str: @@ -27,11 +32,18 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): model_prices_and_context_window.json; ``azure/gpt-5.1`` is. Overriding the shared resolver rather than one lookup means the supports, explicitly-disabled and default-effort answers all read the same entry. + + An ``azure_ai/`` name reaches this config when ``get_llm_provider`` re-routes a Foundry + deployment on an OpenAI-v1 host to the azure provider with the prefix intact. It is the + same product the azure_ai config reads ``azure_ai/`` for, so that row answers when + the map carries it, and the azure/ twin when it does not. """ if model.startswith(cls.GPT5_SERIES_ROUTE): return "azure/" + model[len(cls.GPT5_SERIES_ROUTE) :] if model.startswith("azure/"): return model + if model.startswith("azure_ai/"): + return model if model in litellm.model_cost else "azure/" + model.removeprefix("azure_ai/") return "azure/" + model @classmethod @@ -75,33 +87,7 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): drop_params: bool, api_version: str = "", ) -> dict: - reasoning_effort_value = non_default_params.get("reasoning_effort") or optional_params.get("reasoning_effort") - effective_effort: Final = get_effort_level(reasoning_effort_value) - - # gpt-5.1/5.2/5.4 support reasoning_effort='none', but other gpt-5 models don't - # See: https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/reasoning - supports_none: Final = self._supports_reasoning_effort_level(model, "none") - - if effective_effort == "none" and not supports_none: - if litellm.drop_params is True or (drop_params is not None and drop_params is True): - non_default_params = non_default_params.copy() - optional_params = optional_params.copy() - if get_effort_level(non_default_params.get("reasoning_effort")) == "none": - non_default_params.pop("reasoning_effort") - if get_effort_level(optional_params.get("reasoning_effort")) == "none": - optional_params.pop("reasoning_effort") - else: - raise UnsupportedParamsError( - status_code=400, - message=( - "Azure OpenAI does not support reasoning_effort='none' for this model. " - "To drop this parameter, set `litellm.drop_params=True` or for proxy:\n\n" - "`litellm_settings:\n drop_params: true`\n" - "Issue: https://github.com/BerriAI/litellm/issues/16704" - ), - ) - - result: Final = OpenAIGPT5Config.map_openai_params( + return OpenAIGPT5Config.map_openai_params( self, non_default_params=non_default_params, optional_params=optional_params, @@ -109,17 +95,6 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): drop_params=drop_params, ) - # Only drop reasoning_effort='none' for models that don't support it - result_effort: Final = get_effort_level(result.get("reasoning_effort")) - if result_effort == "none" and not supports_none: - result.pop("reasoning_effort") - - # Azure gpt-5.4+ with tools + reasoning_effort is now routed to the - # Responses API bridge (same as OpenAI), so we no longer need to drop - # reasoning_effort here. See: responses_api_bridge_check() in main.py. - - return result - def transform_request( self, model: str, diff --git a/litellm/llms/azure/responses/transformation.py b/litellm/llms/azure/responses/transformation.py index 75714d5253b..74e65572809 100644 --- a/litellm/llms/azure/responses/transformation.py +++ b/litellm/llms/azure/responses/transformation.py @@ -65,10 +65,18 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): def custom_llm_provider(self) -> LlmProviders: return LlmProviders.AZURE + @staticmethod + def _is_gpt_5_model(model: str) -> bool: + return AzureOpenAIGPT5Config.is_model_gpt_5_model(model) + @staticmethod def _supports_reasoning_effort_none(model: str) -> bool: return AzureOpenAIGPT5Config.supports_reasoning_effort_level(model, "none") + @staticmethod + def _effort_level_is_disabled(model: str, level: str) -> bool: + return AzureOpenAIGPT5Config.row_disables_reasoning_effort_level(model, level) + @staticmethod def _effort_resolves_to_none(model: str, effort: str | None) -> bool: return AzureOpenAIGPT5Config.effort_resolves_to_none(model, effort) diff --git a/litellm/llms/azure_ai/responses/transformation.py b/litellm/llms/azure_ai/responses/transformation.py index 2721f26e0fb..d23de05e148 100644 --- a/litellm/llms/azure_ai/responses/transformation.py +++ b/litellm/llms/azure_ai/responses/transformation.py @@ -3,6 +3,7 @@ from typing import Final import httpx from litellm.llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig +from litellm.llms.azure_ai.chat.transformation import AzureAIGPT5Config from litellm.llms.azure_ai.common_utils import ( AzureFoundryModelInfo, api_key_header_for_base, @@ -27,6 +28,18 @@ class AzureAIResponsesAPIConfig(AzureOpenAIResponsesAPIConfig): def custom_llm_provider(self) -> LlmProviders: return LlmProviders.AZURE_AI + @staticmethod + def _supports_reasoning_effort_none(model: str) -> bool: + return AzureAIGPT5Config.supports_reasoning_effort_level(model, "none") + + @staticmethod + def _effort_level_is_disabled(model: str, level: str) -> bool: + return AzureAIGPT5Config.row_disables_reasoning_effort_level(model, level) + + @staticmethod + def _effort_resolves_to_none(model: str, effort: str | None) -> bool: + return AzureAIGPT5Config.effort_resolves_to_none(model, effort) + def validate_environment(self, headers: dict, model: str, litellm_params: GenericLiteLLMParams | None) -> dict: params: Final = litellm_params or GenericLiteLLMParams() auth_headers: Final = get_azure_ai_auth_headers( diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py index 5bf19844287..8d68e7bbad5 100644 --- a/litellm/llms/openai/chat/gpt_5_transformation.py +++ b/litellm/llms/openai/chat/gpt_5_transformation.py @@ -1,7 +1,7 @@ """Support for OpenAI gpt-5 model family.""" import re -from typing import Final +from typing import ClassVar, Final import litellm from litellm.utils import ( @@ -71,6 +71,10 @@ _get_effort_level = get_effort_level GPT_REASONING_SERIES_MARKERS: Final = ("gpt-5", "gpt-6") +def unsupported_reasoning_effort_message(param: str, level: str, model: str) -> str: + return f"{param}={level} is not supported for {model}. To drop unsupported params set `litellm.drop_params = True`" + + def is_gpt_reasoning_series_name(model: str) -> bool: normalized: Final = model.split("/")[-1] return any(marker in model for marker in GPT_REASONING_SERIES_MARKERS) and "gpt-5-chat" not in normalized @@ -86,6 +90,9 @@ class OpenAIGPT5Config(OpenAIGPTConfig): - Support for GPT-5-Codex models optimized for code generation. """ + OPT_IN_REASONING_EFFORTS: ClassVar[frozenset[str]] = frozenset(("xhigh",)) + ROUTE_SPECIFIC_REASONING_EFFORTS: ClassVar[frozenset[str]] = frozenset(("max",)) + @classmethod def is_model_gpt_5_model(cls, model: str) -> bool: return is_gpt_reasoning_series_name(model) @@ -205,15 +212,10 @@ class OpenAIGPT5Config(OpenAIGPTConfig): return False @classmethod - def _is_reasoning_effort_level_explicitly_disabled(cls, model: str, level: str) -> bool: + def is_reasoning_effort_level_explicitly_disabled(cls, model: str, level: str) -> bool: """Return True only when the model map explicitly sets the capability to False. - Unlike ``_supports_reasoning_effort_level`` (which requires an explicit True), - this method returns True only when ``supports_{level}_reasoning_effort`` is - explicitly set to ``False`` in the model map. A missing key is treated as - supported (i.e. this method returns False = not disabled). - - Use this for opt-out checks where unknown models should be allowed through. + A missing key is not a disable. Unknown models pass through. """ return is_explicitly_disabled_factory( model=cls._model_map_lookup_name(model), @@ -221,6 +223,39 @@ class OpenAIGPT5Config(OpenAIGPTConfig): key=f"supports_{level}_reasoning_effort", ) + @classmethod + def _is_reasoning_effort_level_explicitly_disabled(cls, model: str, level: str) -> bool: + return cls.is_reasoning_effort_level_explicitly_disabled(model, level) + + @classmethod + def refuses_reasoning_effort_level(cls, model: str, level: str) -> bool: + """The chat gate for any requested effort level, read off the model's map row alone. + + An opt-in level (xhigh here; Azure adds none) is refused unless the row says true, since + the providers rejected it on most of the family when it arrived. Every other level is + refused only when the row says false: a model the map never heard of, or a row without the + flag, keeps forwarding what it was sent, and a level with no flag at all (medium, high) is + never refused. So a new model in the family needs its map row and no code change, the + row's supports__reasoning_effort flags being the whole contract. + + A route-specific level (max) is never refused by either gate: the same model honors it + on Responses and refuses it on Chat Completions, so one row flag cannot gate it on + either route. Chat forwards it for the Responses bridge and a plain chat call gets the + provider's own 400, exactly as before the gate read every level. + """ + if level in cls.ROUTE_SPECIFIC_REASONING_EFFORTS: + return False + if level in cls.OPT_IN_REASONING_EFFORTS: + return not cls._supports_reasoning_effort_level(model, level) + return cls.is_reasoning_effort_level_explicitly_disabled(model, level) + + @classmethod + def row_disables_reasoning_effort_level(cls, model: str, level: str) -> bool: + """The Responses gate: opt-out for every level except the route-specific ones.""" + if level in cls.ROUTE_SPECIFIC_REASONING_EFFORTS: + return False + return cls.is_reasoning_effort_level_explicitly_disabled(model, level) + def get_supported_openai_params(self, model: str) -> list: if self.is_model_gpt_5_search_model(model): return [ @@ -281,12 +316,12 @@ class OpenAIGPT5Config(OpenAIGPTConfig): drop_params=drop_params, ) - # Get raw reasoning_effort and effective effort level for all guards. - # Use effective_effort (extracted string) for xhigh validation, "none" checks, and - # tool/sampling guards — dict inputs like {"effort": "none", "summary": "detailed"} - # must be treated as effort="none" to avoid incorrect tool-drop or sampling errors. - raw_reasoning_effort = non_default_params.get("reasoning_effort") or optional_params.get("reasoning_effort") - effective_effort: Final = get_effort_level(raw_reasoning_effort) + # Dict inputs like {"effort": "none", "summary": "detailed"} are judged by their effort + # string so the gate, the sampling guard, and the temperature guard all see one level. + raw_reasoning_effort: Final = non_default_params.get("reasoning_effort") or optional_params.get( + "reasoning_effort" + ) + requested_effort: Final = get_effort_level(raw_reasoning_effort) # Normalize dict reasoning_effort to string for Chat Completions API. # Example: {"effort": "high", "summary": "detailed"} -> "high" @@ -298,31 +333,20 @@ class OpenAIGPT5Config(OpenAIGPTConfig): if "reasoning_effort" in optional_params: optional_params["reasoning_effort"] = normalized - if effective_effort == "xhigh": - # xhigh is an opt-in capability: only allow if model explicitly supports it. - if not self._supports_reasoning_effort_level(model, effective_effort): - if litellm.drop_params or drop_params: - non_default_params.pop("reasoning_effort", None) - optional_params.pop("reasoning_effort", None) - else: - raise litellm.utils.UnsupportedParamsError( - message=(f"reasoning_effort={effective_effort} is not supported for this model."), - status_code=400, - ) - elif effective_effort in ("minimal", "low"): - # minimal/low are opt-out: unknown models pass through; only block when - # the model map explicitly sets supports_{level}_reasoning_effort=false. - # Example: gpt-5.5-pro only accepts {medium, high, xhigh}, so it sets - # supports_low_reasoning_effort=false (and supports_minimal=false). - if self._is_reasoning_effort_level_explicitly_disabled(model, effective_effort): - if litellm.drop_params or drop_params: - non_default_params.pop("reasoning_effort", None) - optional_params.pop("reasoning_effort", None) - else: - raise litellm.utils.UnsupportedParamsError( - message=(f"reasoning_effort={effective_effort} is not supported for this model."), - status_code=400, - ) + if requested_effort is not None and self.refuses_reasoning_effort_level(model, requested_effort): + if litellm.drop_params or drop_params: + non_default_params.pop("reasoning_effort", None) + optional_params.pop("reasoning_effort", None) + else: + raise litellm.utils.UnsupportedParamsError( + message=unsupported_reasoning_effort_message("reasoning_effort", requested_effort, model), + status_code=400, + ) + + # The level the provider will see: None once dropped, so the guards below judge by the model's default effort + effective_effort: Final = get_effort_level( + non_default_params.get("reasoning_effort") or optional_params.get("reasoning_effort") + ) ################################################################ # max_tokens is not supported for gpt-5 models on OpenAI API diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 768d89b9a97..464dc799c16 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -25,7 +25,10 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig -from litellm.llms.openai.chat.gpt_5_transformation import is_gpt_reasoning_series_name +from litellm.llms.openai.chat.gpt_5_transformation import ( + is_gpt_reasoning_series_name, + unsupported_reasoning_effort_message, +) from litellm.responses.litellm_completion_transformation.custom_tools import TOOL_CALL_ITEM_ID_PREFIX_BY_TYPE from litellm.responses.litellm_completion_transformation.reasoning_items import is_litellm_minted_reasoning_item from litellm.responses.litellm_completion_transformation.tool_search import ( @@ -278,6 +281,20 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): return supports_none_reasoning_effort(model=model, custom_llm_provider=None) + @staticmethod + def _effort_level_is_disabled(model: str, level: str) -> bool: + """Whether the model's cost-map row turns this effort level off. + + The same row the chat gate reads, so a level the row refuses is refused on both wires + and a new model needs its row, not code. This surface stays opt-out for every level, + xhigh included, since it never refused a level the row omits; max is the exception, a + level the row cannot gate because the same model takes it here and refuses it on chat. + Azure overrides this so a bare deployment name reads the azure/ entry. + """ + from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config + + return OpenAIGPT5Config.row_disables_reasoning_effort_level(model, level) + @staticmethod def _effort_resolves_to_none(model: str, effort: str | None) -> bool: """Whether this request's reasoning effort ends up as "none", the one condition @@ -384,7 +401,25 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): lookup_name: Final = self._model_map_lookup_name(model) if self._is_gpt_5_model(model=lookup_name): reasoning: Final = params.get("reasoning") or {} - effort: Final = reasoning.get("effort") if isinstance(reasoning, dict) else None + requested_effort: Final = reasoning.get("effort") if isinstance(reasoning, dict) else None + if ( + isinstance(reasoning, dict) + and isinstance(requested_effort, str) + and self._effort_level_is_disabled(lookup_name, requested_effort) + ): + if not (drop_params or litellm.drop_params): + raise litellm.UnsupportedParamsError( + message=unsupported_reasoning_effort_message("reasoning.effort", requested_effort, model), + status_code=400, + ) + remaining: Final = reasoning.copy() + remaining.pop("effort", None) + if remaining: + params["reasoning"] = remaining + else: + params.pop("reasoning", None) + forwarded_reasoning: Final = params.get("reasoning") + effort: Final = forwarded_reasoning.get("effort") if isinstance(forwarded_reasoning, dict) else None supports_none: Final = self._supports_reasoning_effort_none(model=lookup_name) effort_is_none: Final = supports_none and self._effort_resolves_to_none(lookup_name, effort) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index de76b6e1bdf..6e573402427 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -3895,7 +3895,7 @@ "supports_max_reasoning_effort": false, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_cache_breakpoint": true, @@ -8409,7 +8409,7 @@ "supports_max_reasoning_effort": false, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -8458,7 +8458,7 @@ "supports_max_reasoning_effort": false, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -9252,7 +9252,7 @@ "supports_max_reasoning_effort": false, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -33352,9 +33352,9 @@ "supports_response_schema": true, "supports_vision": true, "supports_reasoning": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true + "supports_none_reasoning_effort": true, + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": false }, "chatgpt/gpt-5.3-codex-spark": { "litellm_provider": "chatgpt", @@ -37807,9 +37807,9 @@ "supports_tool_search": true, "supports_vision": true, "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true, + "supports_none_reasoning_effort": true, + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, "deprecation_date": "2027-04-01" }, "gpt-5.3-chat-latest": { @@ -69624,7 +69624,7 @@ "supports_max_reasoning_effort": false, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, diff --git a/litellm/responses/bridge_check.py b/litellm/responses/bridge_check.py index 120a0d7c828..60318d6b184 100644 --- a/litellm/responses/bridge_check.py +++ b/litellm/responses/bridge_check.py @@ -5,6 +5,7 @@ from urllib.parse import urlsplit import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.llms.azure_ai.chat.transformation import AzureAIGPT5Config from litellm.llms.azure_ai.common_utils import ( azure_ai_supports_native_responses, foundry_chat_rejects_function_tools_while_reasoning, @@ -133,6 +134,16 @@ def chat_target_accepts_reasoning_summary(model: str, custom_llm_provider: str | return chat_config is not None and chat_config.accepts_reasoning_summary_in_reasoning_effort +def _chat_gate_refuses_reasoning_effort_none(model: str, custom_llm_provider: str) -> bool: + match custom_llm_provider: + case "azure": + return litellm.AzureOpenAIGPT5Config.refuses_reasoning_effort_level(model, "none") + case "azure_ai": + return AzureAIGPT5Config.refuses_reasoning_effort_level(model, "none") + case _: + return OpenAIGPT5Config.refuses_reasoning_effort_level(model, "none") + + def responses_api_bridge_check( model: str, custom_llm_provider: str, @@ -183,7 +194,9 @@ def responses_api_bridge_check( # server-side), and Chat Completions rejects function tools whenever reasoning is # on ("Function tools with reasoning_effort are not supported ... use # /v1/responses or set reasoning_effort to 'none'"), so only an explicit - # ``"none"`` keeps the request chat-servable. Custom (grammar) tools are served + # ``"none"`` the chat gate forwards keeps the request chat-servable: a ``none`` + # the model's map row refuses is dropped (or refused) before the call, which + # leaves the provider's default effort in force. Custom (grammar) tools are served # natively by Chat Completions with reasoning on, so custom-only requests stay on # chat and keep their native custom tool_call response shape. # - The UNSET-effort arm only fires against endpoints known to enforce that @@ -213,10 +226,13 @@ def responses_api_bridge_check( (tool.get("type") if isinstance(tool, dict) else getattr(tool, "type", None)) == "tool_search" for tool in (tools or ()) ) - if isinstance(reasoning_effort, dict): - reasoning_active = reasoning_effort.get("effort") != "none" or reasoning_effort.get("summary") is not None - else: - reasoning_active = reasoning_effort != "none" + requested_effort: Final = reasoning_effort.get("effort") if isinstance(reasoning_effort, dict) else reasoning_effort + requested_summary: Final = reasoning_effort.get("summary") if isinstance(reasoning_effort, dict) else None + reasoning_active: Final = ( + requested_effort != "none" + or requested_summary is not None + or _chat_gate_refuses_reasoning_effort_none(model, custom_llm_provider) + ) # The reasoning+tools constraint is enforced by the real OpenAI backend behind any api.openai.com # host (the default URL or a PrivateLink hostname such as .privatelink.api.openai.com) and # by Azure OpenAI through the azure provider. Resolve the effective OpenAI base arg>global>env>default diff --git a/litellm/router_utils/reasoning_effort_capability.py b/litellm/router_utils/reasoning_effort_capability.py index 200de29b664..d660adf05a6 100644 --- a/litellm/router_utils/reasoning_effort_capability.py +++ b/litellm/router_utils/reasoning_effort_capability.py @@ -1,20 +1,17 @@ """Resolve which reasoning_effort values a deployment, and by intersection a model group, accepts. -An entry that states its levels outright in reasoning_effort_levels is read first and wins -whole, for a model whose set the per-level flags cannot express: Kimi K3 takes low, high and max, -and no flag can drop medium because medium has none. Every other entry answers through the -supports_*_reasoning_effort flags below, whose polarity mirrors how a request path reads that same -flag. medium and high are unconditional for a reasoning model. minimal and low are opt-out: -openai/chat/gpt_5_transformation.py refuses them only when the map says false. xhigh and max are -opt-in. none is opt-out everywhere except the azure gpt-5 family, whose config raises -UnsupportedParamsError without an explicit true. +An entry that states its levels outright in reasoning_effort_levels wins whole, for a model whose set +the per-level flags cannot express (Kimi K3 takes low, high and max, and no flag can drop medium). +Every other entry answers through its supports_*_reasoning_effort flags, read with the polarity the +chat gate uses: medium and high are unconditional, minimal and low are opt-out (refused only on an +explicit false), xhigh is opt-in, and none is opt-out except on the azure gpt-5 family, whose chat +config refuses it without an explicit true. -xhigh is gated on the request path by the openai and azure gpt-5 configs. max is not gated there at -all: outside the gpt-6-astra rows every entry carrying supports_max_reasoning_effort is Claude-family, -and anthropic/chat/transformation.py gates max on the output_config path while its reasoning_effort -path maps any level to a thinking budget. Making max opt-in is a deliberate trade, then, since an -explicit flag is the only signal that the tier is a real one rather than litellm rounding the level -to a budget, and a missing flag costs advisory metadata rather than a rejected request. +The Responses gate reads none, minimal, low and xhigh as opt-out. Neither gate judges max, which +Chat Completions refuses and Responses honors on the gpt-6 family, so its flag is advisory and +advertised opt-in: outside the gpt-6 rows every entry carrying supports_max_reasoning_effort is +Claude-family, where anthropic/chat/transformation.py gates max on the output_config path, so an +explicit flag is the only signal that the tier is real. A deployment the map describes with no effort flags at all resolves to None rather than to the opt-out defaults. 689 of the map's 854 reasoning entries carry no flag, and the o-series, xai and diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index de76b6e1bdf..6e573402427 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -3895,7 +3895,7 @@ "supports_max_reasoning_effort": false, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_cache_breakpoint": true, @@ -8409,7 +8409,7 @@ "supports_max_reasoning_effort": false, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -8458,7 +8458,7 @@ "supports_max_reasoning_effort": false, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -9252,7 +9252,7 @@ "supports_max_reasoning_effort": false, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -33352,9 +33352,9 @@ "supports_response_schema": true, "supports_vision": true, "supports_reasoning": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true + "supports_none_reasoning_effort": true, + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": false }, "chatgpt/gpt-5.3-codex-spark": { "litellm_provider": "chatgpt", @@ -37807,9 +37807,9 @@ "supports_tool_search": true, "supports_vision": true, "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false, - "supports_minimal_reasoning_effort": true, + "supports_none_reasoning_effort": true, + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": false, "deprecation_date": "2027-04-01" }, "gpt-5.3-chat-latest": { @@ -69624,7 +69624,7 @@ "supports_max_reasoning_effort": false, "supports_minimal_reasoning_effort": false, "supports_native_streaming": true, - "supports_none_reasoning_effort": true, + "supports_none_reasoning_effort": false, "supports_parallel_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_reasoning_effort_normalization.py b/tests/unit/llms/anthropic/pass_through/adapters/test_handler_reasoning_effort_normalization.py index 389398dcbf4..7a4b5badb3a 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_reasoning_effort_normalization.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_handler_reasoning_effort_normalization.py @@ -321,6 +321,23 @@ class TestTheNormalizedTierIsTheTierSent: ): assert _reasoning_effort_sent(model, provider, effort) == expected + @pytest.mark.parametrize( + "model, provider", + [ + ("gpt-5.5", "azure_ai"), + ("azure_ai/gpt-5.5", "azure_ai"), + ("gpt-5.6", "azure"), + ("us/gpt-5.6", "azure"), + ], + ) + def test_an_azure_hosted_deployment_without_the_flag_drops_to_the_tier_it_accepts( + self, local_model_cost_map, model, provider + ): + """Chat Completions answers ``max`` on gpt-5.6 with a 400 naming ``none`` through ``xhigh`` + (OpenAI, 2026-10-01) and none of these Azure rows declares the level, so the adapter sends + the tier below instead of the rejected one.""" + assert _reasoning_effort_sent(model, provider, "max") == "xhigh" + @pytest.mark.parametrize( "model, provider", [ @@ -328,13 +345,16 @@ class TestTheNormalizedTierIsTheTierSent: ("azure_ai/gpt-6-astra", "azure_ai"), ("gpt-6-astra", "azure"), ("us/gpt-6-astra", "azure"), + ("eu/gpt-6-astra", "azure"), ], ) - def test_an_azure_hosted_astra_deployment_drops_to_the_tier_it_accepts( + def test_an_azure_hosted_astra_deployment_drops_to_the_tier_its_chat_route_accepts( self, local_model_cost_map: None, model: str, provider: str ) -> None: - """The deployment answers ``max`` with a 400 naming ``none`` through ``xhigh``, so the rows - say so and the adapter sends the tier below instead of the rejected one.""" + """A Foundry gpt-6-astra answers a Chat Completions ``max`` with a 400 naming low through + xhigh while honoring the same level on the Responses route (2026-10-01). The row's flag is + route-blind, so it keeps max off and the adapter, which cannot know which route the call + takes, sends the tier below instead of the level chat rejects.""" assert _reasoning_effort_sent(model, provider, "max") == "xhigh" def test_the_openai_hosted_twin_still_sends_max(self, local_model_cost_map): diff --git a/tests/unit/llms/azure/chat/test_azure_gpt5_transformation.py b/tests/unit/llms/azure/chat/test_azure_gpt5_transformation.py index 5fcd1f41789..80328022f61 100644 --- a/tests/unit/llms/azure/chat/test_azure_gpt5_transformation.py +++ b/tests/unit/llms/azure/chat/test_azure_gpt5_transformation.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from typing import Final import pytest @@ -369,17 +370,22 @@ def test_azure_gpt_6_astra_takes_the_reasoning_series_request_shape(): model="gpt-6-astra", custom_llm_provider="azure", max_tokens=100, - reasoning_effort="max", + reasoning_effort="xhigh", ) assert params["max_completion_tokens"] == 100 assert "max_tokens" not in params - assert params["reasoning_effort"] == "max" + assert params["reasoning_effort"] == "xhigh" -@pytest.mark.parametrize("model", ["azure/gpt-6-astra", "azure/us/gpt-6-astra"]) -def test_azure_gpt6_astra_reasoning_effort_none_unlocks_temperature(config: AzureOpenAIGPT5Config, model: str): - """Foundry's gpt-6-astra accepts reasoning_effort='none' and, only then, a non-default - temperature (verified live against a Foundry deployment), unlike OpenAI's gpt-6-astra.""" +@pytest.mark.parametrize("model", ["azure/gpt-6-sol", "azure/us/gpt-6-sol"]) +def test_azure_reasoning_effort_none_unlocks_temperature_off_the_azure_row( + config: AzureOpenAIGPT5Config, monkeypatch: pytest.MonkeyPatch, model: str +): + """A deployment whose azure/ row takes none accepts a non-default temperature with it, read off + that row even when OpenAI's twin row says the model refuses none.""" + monkeypatch.setitem( + litellm.model_cost, "gpt-6-sol", {**litellm.model_cost["gpt-6-sol"], "supports_none_reasoning_effort": False} + ) params = config.map_openai_params( non_default_params={"temperature": 0.2, "reasoning_effort": "none"}, optional_params={}, @@ -391,14 +397,127 @@ def test_azure_gpt6_astra_reasoning_effort_none_unlocks_temperature(config: Azur assert params["reasoning_effort"] == "none" -@pytest.mark.parametrize("model", ["azure/gpt-6-astra", "azure/us/gpt-6-astra"]) -def test_azure_gpt6_astra_rejects_reasoning_effort_minimal(config: AzureOpenAIGPT5Config, model: str): - """Foundry's gpt-6-astra lists none, low, medium, high, xhigh and max but not minimal.""" - with pytest.raises(litellm.utils.UnsupportedParamsError): - config.map_openai_params( - non_default_params={"reasoning_effort": "minimal"}, +def _azure_chat_row_forwards(row: Mapping[str, object], level: str) -> bool: + """Azure chat polarity: max is never gated, xhigh and none need an explicit true, every other + level only has to not be false.""" + if level == "max": + return True + flag: Final = row.get(f"supports_{level}_reasoning_effort") + return flag is True if level in ("xhigh", "none") else flag is not False + + +@pytest.mark.parametrize("model", ["azure/gpt-6-astra", "azure/us/gpt-6-astra", "azure/eu/gpt-6-astra"]) +@pytest.mark.parametrize("level", ["none", "minimal", "low", "medium", "high", "xhigh", "max"]) +def test_azure_gpt6_astra_forwards_exactly_the_effort_levels_its_map_row_allows( + config: AzureOpenAIGPT5Config, model: str, level: str +): + """The azure/ row is the whole contract for a Foundry deployment. A live Foundry gpt-6-astra + (model gpt-6-astra-2026-09-03) answered reasoning_effort none with a 400 naming low, medium, + high and xhigh on 2026-10-01, so its row turns none and minimal off. The row keeps max off too, + since chat refuses it, but no gate reads that flag: the Responses route honors max, so chat + forwards it for the bridge and a plain chat call gets the provider's own 400.""" + forwarded: Final = _azure_chat_row_forwards(litellm.model_cost[model], level) + dropped: Final = config.map_openai_params( + non_default_params={"reasoning_effort": level}, + optional_params={}, + model=model, + drop_params=True, + api_version="2025-04-01-preview", + ) + assert ("reasoning_effort" in dropped) is forwarded + if forwarded: + kept: Final = config.map_openai_params( + non_default_params={"reasoning_effort": level}, optional_params={}, model=model, drop_params=False, api_version="2025-04-01-preview", ) + assert kept["reasoning_effort"] == level + else: + with pytest.raises(litellm.utils.UnsupportedParamsError): + config.map_openai_params( + non_default_params={"reasoning_effort": level}, + optional_params={}, + model=model, + drop_params=False, + api_version="2025-04-01-preview", + ) + + +@pytest.mark.parametrize("model", ["azure/gpt-6-astra", "azure/us/gpt-6-astra", "azure/eu/gpt-6-astra"]) +def test_azure_gpt6_astra_row_turns_none_and_minimal_off(model: str): + """Pinned on purpose so the row-derived test above cannot go vacuous: a Foundry gpt-6-astra + deployment refused none on chat and on Responses on 2026-10-01 (400 unsupported_value naming + low, medium, high, xhigh), and minimal the same way; max it refused on chat while honoring it + on Responses, so the route-blind flag stays off. When Azure adds a level, flip the row and + this assertion together.""" + row: Final = litellm.model_cost[model] + assert row["supports_none_reasoning_effort"] is False + assert row["supports_minimal_reasoning_effort"] is False + assert row["supports_max_reasoning_effort"] is False + + +def test_azure_chat_gate_reads_the_foundry_row_when_azure_ai_prefix_survives_provider_remap( + config: AzureOpenAIGPT5Config, monkeypatch: pytest.MonkeyPatch +): + """A Foundry deployment on an OpenAI-v1 host is re-routed to the azure provider with its + azure_ai/ prefix intact, and this config then gates it. The rows are made to disagree on low, + a level neither real row flags, so the prefixed name has to read azure_ai/gpt-6-sol while the + bare deployment name keeps reading azure/gpt-6-sol.""" + monkeypatch.setitem( + litellm.model_cost, + "azure_ai/gpt-6-sol", + {**litellm.model_cost["azure_ai/gpt-6-sol"], "supports_low_reasoning_effort": False}, + ) + monkeypatch.setitem( + litellm.model_cost, + "azure/gpt-6-sol", + {**litellm.model_cost["azure/gpt-6-sol"], "supports_low_reasoning_effort": True}, + ) + prefixed: Final = config.map_openai_params( + non_default_params={"reasoning_effort": "low"}, + optional_params={}, + model="azure_ai/gpt-6-sol", + drop_params=True, + api_version="2025-04-01-preview", + ) + assert "reasoning_effort" not in prefixed + bare: Final = config.map_openai_params( + non_default_params={"reasoning_effort": "low"}, + optional_params={}, + model="gpt-6-sol", + drop_params=True, + api_version="2025-04-01-preview", + ) + assert bare["reasoning_effort"] == "low" + + +def test_azure_chat_gate_falls_back_to_the_azure_row_for_an_azure_ai_name_the_map_lacks( + config: AzureOpenAIGPT5Config, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.delitem(litellm.model_cost, "azure_ai/gpt-6-sol") + monkeypatch.setitem( + litellm.model_cost, + "azure/gpt-6-sol", + {**litellm.model_cost["azure/gpt-6-sol"], "supports_low_reasoning_effort": False}, + ) + dropped: Final = config.map_openai_params( + non_default_params={"reasoning_effort": "low"}, + optional_params={}, + model="azure_ai/gpt-6-sol", + drop_params=True, + api_version="2025-04-01-preview", + ) + assert "reasoning_effort" not in dropped + + +@pytest.mark.parametrize("key", ["azure/gpt-6.1-sol", "azure/gpt-6.1-sol-2026-09-29", "azure_ai/gpt-6.1-sol"]) +def test_azure_gpt61_sol_rows_turn_none_off(key: str): + """Pinned on purpose, as the astra row above: a Foundry gpt-6.1-sol deployment (model + gpt-6.1-sol-2026-09-29) refused none on chat and on Responses on 2026-10-02 (400 unsupported_value + naming low, medium, high, xhigh, and max on Responses), and minimal the same way, while these rows + still said none was on. When Azure adds a level, flip the row and this assertion together.""" + row: Final = litellm.model_cost[key] + assert row["supports_none_reasoning_effort"] is False + assert row["supports_minimal_reasoning_effort"] is False diff --git a/tests/unit/llms/azure/response/test_azure_transformation.py b/tests/unit/llms/azure/response/test_azure_transformation.py index 9b4c618c78b..7ed98f6c705 100644 --- a/tests/unit/llms/azure/response/test_azure_transformation.py +++ b/tests/unit/llms/azure/response/test_azure_transformation.py @@ -727,16 +727,21 @@ def local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None: litellm.add_known_models(model_cost_map=litellm.model_cost) -def test_azure_responses_gpt6_astra_reasoning_effort_none_unlocks_temperature(local_model_cost_map: None): - """Foundry's gpt-6-astra accepts reasoning.effort='none' with a non-default temperature - while OpenAI's gpt-6-astra does not, so the gate must read the azure/ cost-map entry - for the bare deployment name rather than OpenAI's.""" +def test_azure_responses_reasoning_effort_none_unlocks_temperature_off_the_azure_row( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +): + """The gate reads the azure/ cost-map entry for the bare deployment name rather than OpenAI's + twin: with OpenAI's gpt-6-sol row turned off for none, azure/gpt-6-sol's row still lets none + through and, only then, a non-default temperature.""" + monkeypatch.setitem( + litellm.model_cost, "gpt-6-sol", {**litellm.model_cost["gpt-6-sol"], "supports_none_reasoning_effort": False} + ) params = AzureOpenAIResponsesAPIConfig().map_openai_params( response_api_optional_params=ResponsesAPIOptionalRequestParams( temperature=0.2, reasoning={"effort": "none"}, ), - model="gpt-6-astra", + model="gpt-6-sol", drop_params=False, ) assert params["temperature"] == 0.2 @@ -755,6 +760,116 @@ def test_azure_responses_gpt6_astra_rejects_temperature_while_reasoning(local_mo ) +@pytest.mark.parametrize("model", ["gpt-6-astra", "gpt-6.1-sol"]) +@pytest.mark.parametrize("level", ["none", "minimal", "low", "medium", "high", "xhigh", "max"]) +def test_azure_responses_gpt6_effort_levels_follow_the_azure_row(local_model_cost_map: None, model: str, level: str): + """A bare deployment name reads the azure/ row, opt-out for every level but max, so the none + and minimal Foundry refuses on this route (live 400s on 2026-10-01 for gpt-6-astra and 2026-10-02 + for gpt-6.1-sol) are dropped or refused here while the max it honors goes through whatever the + route-blind flag says.""" + forwarded = ( + level == "max" or litellm.model_cost[f"azure/{model}"].get(f"supports_{level}_reasoning_effort") is not False + ) + dropped = AzureOpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params=ResponsesAPIOptionalRequestParams(reasoning={"effort": level}), + model=model, + drop_params=True, + ) + assert ("reasoning" in dropped) is forwarded + if forwarded: + kept = AzureOpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params=ResponsesAPIOptionalRequestParams(reasoning={"effort": level}), + model=model, + drop_params=False, + ) + assert kept["reasoning"] == {"effort": level} + else: + with pytest.raises(litellm.UnsupportedParamsError): + AzureOpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params=ResponsesAPIOptionalRequestParams(reasoning={"effort": level}), + model=model, + drop_params=False, + ) + + +def test_azure_gpt6_astra_keeps_max_on_this_route_whatever_its_row_says(local_model_cost_map: None): + """Pinned on purpose: the row-derived test above exempts max by hand, so this one has to fail + if the gate ever reads the flag. A Foundry gpt-6-astra deployment answered reasoning.effort max + through /v1/responses with 200, the effort echoed back and reasoning tokens billed, on + 2026-10-01, while chat refuses the same level, as OpenAI's chat does. The row keeps max off for + the adapter and the picker, and this gate still forwards it even under drop_params.""" + assert litellm.model_cost["azure/gpt-6-astra"]["supports_max_reasoning_effort"] is False + kept = AzureOpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params=ResponsesAPIOptionalRequestParams(reasoning={"effort": "max"}), + model="gpt-6-astra", + drop_params=True, + ) + assert kept["reasoning"] == {"effort": "max"} + + +@pytest.mark.parametrize("level", ["none", "minimal", "low", "medium", "high", "xhigh", "max"]) +def test_azure_responses_effort_gate_fires_when_azure_ai_prefix_survives_provider_remap( + local_model_cost_map: None, level: str +): + """litellm.responses(model="azure_ai/gpt-6.1-sol", api_base=) is re-routed to the + azure provider with the prefix intact, so this config sees the prefixed name. The gate used to + skip every prefixed name as another provider's pass-through, and a live Foundry gpt-6.1-sol + answered the forwarded none and minimal with 400 on 2026-10-02 even under drop_params. The + prefixed name reads the azure_ai/ row and drops or refuses exactly what that row turns off.""" + forwarded = ( + level == "max" + or litellm.model_cost["azure_ai/gpt-6.1-sol"].get(f"supports_{level}_reasoning_effort") is not False + ) + dropped = AzureOpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params=ResponsesAPIOptionalRequestParams( + max_output_tokens=60, reasoning={"effort": level} + ), + model="azure_ai/gpt-6.1-sol", + drop_params=True, + ) + assert ("reasoning" in dropped) is forwarded + if forwarded: + return + with pytest.raises(litellm.UnsupportedParamsError): + AzureOpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params=ResponsesAPIOptionalRequestParams( + max_output_tokens=60, reasoning={"effort": level} + ), + model="azure_ai/gpt-6.1-sol", + drop_params=False, + ) + + +def test_azure_responses_prefixed_name_reads_the_foundry_row_not_the_azure_twin( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +): + """The rows are made to disagree on low, a level neither real row flags, so the prefixed name + proving it reads azure_ai/gpt-6-sol cannot pass by reading azure/gpt-6-sol, which the bare + deployment name keeps reading.""" + monkeypatch.setitem( + litellm.model_cost, + "azure_ai/gpt-6-sol", + {**litellm.model_cost["azure_ai/gpt-6-sol"], "supports_low_reasoning_effort": False}, + ) + monkeypatch.setitem( + litellm.model_cost, + "azure/gpt-6-sol", + {**litellm.model_cost["azure/gpt-6-sol"], "supports_low_reasoning_effort": True}, + ) + prefixed = AzureOpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params=ResponsesAPIOptionalRequestParams(reasoning={"effort": "low"}), + model="azure_ai/gpt-6-sol", + drop_params=True, + ) + assert "reasoning" not in prefixed + bare = AzureOpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params=ResponsesAPIOptionalRequestParams(reasoning={"effort": "low"}), + model="gpt-6-sol", + drop_params=True, + ) + assert bare["reasoning"] == {"effort": "low"} + + _COMPLETED_RESPONSE: Final = { "id": "resp_dated_chat_deployment", "object": "response", diff --git a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py index 634228da1bf..5186541a62b 100644 --- a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py +++ b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py @@ -306,17 +306,30 @@ def _local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "model_cost", get_model_cost_map(url=litellm.model_cost_map_url)) -def test_foundry_gpt_6_astra_keeps_sampling_params_when_reasoning_effort_is_none(_local_model_cost_map): +def test_foundry_gpt_6_sol_keeps_sampling_params_when_reasoning_effort_is_none(_local_model_cost_map): + """gpt-6-sol's Foundry row still says none, so the sampling params that ride on it stay. gpt-6-astra + used to be the model here until its row turned none off (a live Foundry deployment answered none + with a 400 naming low through xhigh on 2026-10-01); the test below pins that flip.""" optional_params = AzureAIStudioConfig().map_openai_params( non_default_params={"reasoning_effort": "none", "temperature": 0.2, "top_p": 0.9}, optional_params={}, - model="gpt-6-astra", + model="gpt-6-sol", drop_params=False, ) assert optional_params == {"reasoning_effort": "none", "temperature": 0.2, "top_p": 0.9} +def test_foundry_gpt_6_astra_refuses_reasoning_effort_none_before_the_call(_local_model_cost_map): + with pytest.raises(litellm.utils.UnsupportedParamsError): + AzureAIStudioConfig().map_openai_params( + non_default_params={"reasoning_effort": "none"}, + optional_params={}, + model="gpt-6-astra", + drop_params=False, + ) + + def test_azure_ai_grok_stop_parameter_handling(): diff --git a/tests/unit/llms/azure_ai/responses/test_azure_ai_responses_transformation.py b/tests/unit/llms/azure_ai/responses/test_azure_ai_responses_transformation.py index bae956eb061..26076f8bb34 100644 --- a/tests/unit/llms/azure_ai/responses/test_azure_ai_responses_transformation.py +++ b/tests/unit/llms/azure_ai/responses/test_azure_ai_responses_transformation.py @@ -1,12 +1,15 @@ import json +from typing import Final import httpx import pytest import respx import litellm +from litellm.llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig from litellm.llms.azure_ai.responses.transformation import AzureAIResponsesAPIConfig from litellm.responses.main import _will_bridge_to_chat_completions +from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams from litellm.types.router import GenericLiteLLMParams from litellm.utils import ProviderConfigManager @@ -309,3 +312,74 @@ async def test_aresponses_serverless_host_stays_on_chat_bridge(): assert chat_route.called assert not responses_route.called assert chat_route.calls.last.request.headers["Authorization"] == "Bearer fake-key" + + +def _responses_params(**fields: object) -> ResponsesAPIOptionalRequestParams: + return ResponsesAPIOptionalRequestParams(**fields) # pyright: ignore[reportArgumentType] # TypedDict kwargs from a test helper + + +def test_foundry_responses_effort_gate_reads_the_azure_ai_row( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +): + """A Foundry deployment and its Azure OpenAI namesake are different products, so the Foundry + Responses route answers from the azure_ai/ row the Foundry chat route already reads, never from + the azure/ twin. The rows are made to disagree on low, a level neither real row flags, so a + leak from the wrong row cannot hide behind rows that happen to agree.""" + monkeypatch.setitem( + litellm.model_cost, + "azure_ai/gpt-6-sol", + {**litellm.model_cost["azure_ai/gpt-6-sol"], "supports_low_reasoning_effort": False}, + ) + monkeypatch.setitem( + litellm.model_cost, + "azure/gpt-6-sol", + {**litellm.model_cost["azure/gpt-6-sol"], "supports_low_reasoning_effort": True}, + ) + + dropped: Final = AzureAIResponsesAPIConfig().map_openai_params( + response_api_optional_params=_responses_params(reasoning={"effort": "low"}), + model="gpt-6-sol", + drop_params=True, + ) + assert "reasoning" not in dropped + + with pytest.raises(litellm.UnsupportedParamsError): + AzureAIResponsesAPIConfig().map_openai_params( + response_api_optional_params=_responses_params(reasoning={"effort": "low"}), + model="gpt-6-sol", + drop_params=False, + ) + + kept_by_azure_openai: Final = AzureOpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params=_responses_params(reasoning={"effort": "low"}), + model="gpt-6-sol", + drop_params=True, + ) + assert kept_by_azure_openai["reasoning"] == {"effort": "low"} + + +def test_foundry_responses_none_unlocks_temperature_off_the_azure_ai_row( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +): + """The sampling-param unlock that rides on none reads the same azure_ai/ row as the effort gate: + with the azure/ twin turned off for none, the Foundry row still lets none through and, only + then, a non-default temperature.""" + monkeypatch.setitem( + litellm.model_cost, + "azure_ai/gpt-6-sol", + {**litellm.model_cost["azure_ai/gpt-6-sol"], "supports_none_reasoning_effort": True}, + ) + monkeypatch.setitem( + litellm.model_cost, + "azure/gpt-6-sol", + {**litellm.model_cost["azure/gpt-6-sol"], "supports_none_reasoning_effort": False}, + ) + + params: Final = AzureAIResponsesAPIConfig().map_openai_params( + response_api_optional_params=_responses_params(temperature=0.2, reasoning={"effort": "none"}), + model="gpt-6-sol", + drop_params=False, + ) + + assert params["temperature"] == 0.2 + assert params["reasoning"] == {"effort": "none"} diff --git a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py index f85e4db447b..020f491dcfd 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py @@ -2303,6 +2303,115 @@ class TestResponsesSurfaceSharesTheEffortRule: ) assert ("top_p" in mapped) is top_p_survives + @pytest.mark.parametrize("level", ["none", "minimal", "low", "medium", "high", "xhigh", "max"]) + def test_effort_levels_the_map_row_turns_off_are_dropped_or_refused(self, local_model_cost_map, level): + """Same row as the chat gate, read opt-out for every level but max. gpt-6.1-sol's row turns + none and minimal off: OpenAI lists low, medium, high, xhigh and max for it + (https://developers.openai.com/api/docs/models/gpt-6.1-sol, via #43910), and a local proxy + got OpenAI's 400 for none and minimal through /v1/responses on 2026-09-30.""" + forwarded = ( + level == "max" or litellm.model_cost["gpt-6.1-sol"].get(f"supports_{level}_reasoning_effort") is not False + ) + dropped = OpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params={"reasoning": {"effort": level}}, + model="gpt-6.1-sol", + drop_params=True, + ) + assert ("reasoning" in dropped) is forwarded + if forwarded: + kept = OpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params={"reasoning": {"effort": level}}, + model="gpt-6.1-sol", + drop_params=False, + ) + assert kept["reasoning"] == {"effort": level} + else: + with pytest.raises(litellm.UnsupportedParamsError): + OpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params={"reasoning": {"effort": level}}, + model="gpt-6.1-sol", + drop_params=False, + ) + + @pytest.mark.parametrize( + ("flags", "level", "forwarded"), + [ + ({}, "none", True), + ({"supports_none_reasoning_effort": False}, "none", False), + ({"supports_none_reasoning_effort": True}, "none", True), + ({"supports_minimal_reasoning_effort": False}, "minimal", False), + ({"supports_max_reasoning_effort": False}, "max", True), + ({}, "xhigh", True), + ({"supports_xhigh_reasoning_effort": False}, "xhigh", False), + ({"supports_none_reasoning_effort": False}, "medium", True), + ], + ) + def test_the_row_flags_alone_decide_whether_a_level_is_forwarded( + self, local_model_cost_map, monkeypatch, flags, level, forwarded + ): + """Same model name, rewritten row: this surface reads every level opt-out, xhigh included, + so the gate flips with the flags alone. max is the one level no flag can turn off here, + since the same model honors it on this route and refuses it on chat. The chat test module + pins the real row's facts.""" + real_row = litellm.model_cost["gpt-6.1-sol"] + without_flags = { + field: value + for field, value in real_row.items() + if not (field.startswith("supports_") and field.endswith("_reasoning_effort")) + } + monkeypatch.setitem(litellm.model_cost, "gpt-6.1-sol", {**without_flags, **flags}) + dropped = OpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params={"reasoning": {"effort": level}}, + model="gpt-6.1-sol", + drop_params=True, + ) + assert ("reasoning" in dropped) is forwarded + if not forwarded: + with pytest.raises(litellm.UnsupportedParamsError): + OpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params={"reasoning": {"effort": level}}, + model="gpt-6.1-sol", + drop_params=False, + ) + + def test_dropping_a_refused_effort_keeps_the_rest_of_the_reasoning_object(self, local_model_cost_map): + refused = next( + level + for level in ("none", "minimal", "low") + if litellm.model_cost["gpt-6.1-sol"].get(f"supports_{level}_reasoning_effort") is False + ) + mapped = OpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params={"reasoning": {"effort": refused, "summary": "auto"}}, + model="gpt-6.1-sol", + drop_params=True, + ) + assert mapped["reasoning"] == {"summary": "auto"} + + @pytest.mark.parametrize("level", ["none", "minimal", "low", "medium", "high", "xhigh", "max"]) + def test_gpt5_3_codex_forwards_exactly_the_effort_levels_its_map_row_allows(self, local_model_cost_map, level): + """The gate turns a stale false into a local refusal, so the Responses-only codex row has to + say what the route does. On 2026-10-01 /v1/responses answered gpt-5.3-codex none and xhigh + with 200 (effort echoed back) and minimal and max with 400 unsupported_value naming none, + low, medium, high and xhigh, so the row takes none and xhigh and turns minimal off.""" + forwarded = ( + level == "max" or litellm.model_cost["gpt-5.3-codex"].get(f"supports_{level}_reasoning_effort") is not False + ) + mapped = OpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params={"reasoning": {"effort": level}}, + model="gpt-5.3-codex", + drop_params=True, + ) + assert ("reasoning" in mapped) is forwarded + + def test_gpt5_3_codex_row_takes_none_and_xhigh(self, local_model_cost_map): + """Pinned on purpose so the row-derived test above cannot go vacuous: the live answers of + 2026-10-01 listed above. When OpenAI changes the set, flip the row and this assertion + together.""" + row = litellm.model_cost["gpt-5.3-codex"] + assert row["supports_none_reasoning_effort"] is True + assert row["supports_xhigh_reasoning_effort"] is True + assert row["supports_minimal_reasoning_effort"] is False + def test_top_p_raises_without_drop_params(self, local_model_cost_map): with pytest.raises(litellm.UnsupportedParamsError): OpenAIResponsesAPIConfig().map_openai_params( diff --git a/tests/unit/llms/openai/test_gpt5_transformation.py b/tests/unit/llms/openai/test_gpt5_transformation.py index 41f5816600f..7b10411ba20 100644 --- a/tests/unit/llms/openai/test_gpt5_transformation.py +++ b/tests/unit/llms/openai/test_gpt5_transformation.py @@ -1,4 +1,6 @@ import re +from collections.abc import Mapping +from typing import Final import pytest @@ -431,6 +433,105 @@ def test_gpt5_drops_reasoning_effort_minimal_when_requested(config: OpenAIConfig assert "reasoning_effort" not in params +def _chat_row_forwards(row: Mapping[str, object], level: str) -> bool: + """Chat polarity: max is never gated, xhigh needs an explicit true, every other level only has + to not be false.""" + if level == "max": + return True + flag: Final = row.get(f"supports_{level}_reasoning_effort") + return flag is True if level == "xhigh" else flag is not False + + +@pytest.mark.parametrize("level", ["none", "minimal", "low", "medium", "high", "xhigh", "max"]) +def test_gpt6_1_sol_forwards_exactly_the_effort_levels_its_map_row_allows(config: OpenAIConfig, level: str): + """The map row is the whole contract, so a new GPT-6 model is gated by its row with no code + change. gpt-6.1-sol's row turns none and minimal off: OpenAI lists low, medium, high, xhigh + and max for it (https://developers.openai.com/api/docs/models/gpt-6.1-sol, via #43910), and a + local proxy got OpenAI's 400 for none and minimal on both endpoints on 2026-09-30.""" + forwarded: Final = _chat_row_forwards(litellm.model_cost["gpt-6.1-sol"], level) + dropped: Final = config.map_openai_params( + non_default_params={"reasoning_effort": level}, + optional_params={}, + model="gpt-6.1-sol", + drop_params=True, + ) + assert ("reasoning_effort" in dropped) is forwarded + if forwarded: + kept: Final = config.map_openai_params( + non_default_params={"reasoning_effort": level}, + optional_params={}, + model="gpt-6.1-sol", + drop_params=False, + ) + assert kept["reasoning_effort"] == level + else: + with pytest.raises(litellm.utils.UnsupportedParamsError): + config.map_openai_params( + non_default_params={"reasoning_effort": level}, + optional_params={}, + model="gpt-6.1-sol", + drop_params=False, + ) + + +def test_gpt6_1_sol_row_turns_none_and_minimal_off(): + """Pinned on purpose so the row-derived test above cannot go vacuous: OpenAI lists low, medium, + high, xhigh and max for gpt-6.1-sol and neither none nor minimal + (https://developers.openai.com/api/docs/models/gpt-6.1-sol, read 2026-09-30 via #43910), and a + local proxy got OpenAI's 400 for both on 2026-09-30. When OpenAI adds a level, flip the row and + this assertion together.""" + row: Final = litellm.model_cost["gpt-6.1-sol"] + assert row["supports_none_reasoning_effort"] is False + assert row["supports_minimal_reasoning_effort"] is False + + +def _row_without_effort_flags(row: Mapping[str, object]) -> dict[str, object]: + return { + field: value + for field, value in row.items() + if not (field.startswith("supports_") and field.endswith("_reasoning_effort")) + } + + +@pytest.mark.parametrize( + ("flags", "level", "forwarded"), + [ + ({}, "none", True), + ({"supports_none_reasoning_effort": False}, "none", False), + ({"supports_none_reasoning_effort": True}, "none", True), + ({"supports_minimal_reasoning_effort": False}, "minimal", False), + ({"supports_max_reasoning_effort": False}, "max", True), + ({}, "max", True), + ({}, "xhigh", False), + ({"supports_xhigh_reasoning_effort": True}, "xhigh", True), + ({"supports_none_reasoning_effort": False}, "medium", True), + ], +) +def test_the_row_flags_alone_decide_whether_a_chat_level_is_forwarded( + config: OpenAIConfig, monkeypatch: pytest.MonkeyPatch, flags: dict[str, bool], level: str, forwarded: bool +): + """Same model name, rewritten row: the gate flips with the flags and nothing else, so breaking the + gate fails here while the pinned test above catches the real row losing a flag.""" + monkeypatch.setitem( + litellm.model_cost, "gpt-6.1-sol", {**_row_without_effort_flags(litellm.model_cost["gpt-6.1-sol"]), **flags} + ) + dropped: Final = config.map_openai_params( + non_default_params={"reasoning_effort": level}, + optional_params={}, + model="gpt-6.1-sol", + drop_params=True, + ) + assert ("reasoning_effort" in dropped) is forwarded + if not forwarded: + with pytest.raises(litellm.utils.UnsupportedParamsError): + config.map_openai_params( + non_default_params={"reasoning_effort": level}, + optional_params={}, + model="gpt-6.1-sol", + drop_params=False, + ) + + def test_gpt5_minimal_dict_triggers_validation(config: OpenAIConfig): """Dict with effort='minimal' triggers minimal model-support validation.""" with pytest.raises(litellm.utils.UnsupportedParamsError): diff --git a/tests/unit/router_utils/test_reasoning_effort_capability.py b/tests/unit/router_utils/test_reasoning_effort_capability.py index 6a0c274f149..2226b1f39df 100644 --- a/tests/unit/router_utils/test_reasoning_effort_capability.py +++ b/tests/unit/router_utils/test_reasoning_effort_capability.py @@ -395,22 +395,25 @@ class TestGpt6AstraAdvertisesItsDocumentedLevels: [ ("azure/gpt-6-astra", "azure"), ("azure/us/gpt-6-astra", "azure"), + ("azure/eu/gpt-6-astra", "azure"), ("azure_ai/gpt-6-astra", "azure_ai"), ], ) - def test_an_azure_hosted_deployment_advertises_none_but_not_max( + def test_an_azure_hosted_deployment_advertises_the_levels_both_its_routes_accept( self, local_model_cost_map, model, custom_llm_provider ): - """Microsoft hosts the same model with a different level set than OpenAI does. Verified live - on both Azure routes: none returns 200 with zero reasoning tokens and unlocks temperature, - which OpenAI's API rejects, while max returns 400 unsupported_value naming none through - xhigh as the levels it does take.""" + """Verified live on 2026-10-01 against a Foundry gpt-6-astra deployment (model + gpt-6-astra-2026-09-03): none returns 400 unsupported_value on chat and on Responses, naming + low, medium, high and xhigh as the levels it takes; Responses also honors max with reasoning + tokens billed while chat refuses it, as OpenAI's chat does. The flag is route-blind, so the + row keeps max off and the advertised set stops at the level every route accepts. + Earlier evidence (2026-09-05, commit e79f3ec5205) had none accepted on chat; the deployment + no longer behaves that way.""" from litellm.utils import get_model_info_helper model_info = dict(get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)) assert resolve_supported_reasoning_efforts(model_info, deployment_is_mapped=True) == ( - "none", "low", "medium", "high", diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 7c292ea8f08..63f440d4aba 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -1305,6 +1305,50 @@ def test_responses_api_bridge_check_gpt_5_4_tools_with_reasoning_none_stays_chat assert model_info.get("mode") != "responses" +@pytest.mark.parametrize( + "model, custom_llm_provider, reasoning_effort", + [ + ("gpt-6.1-sol", "openai", "none"), + ("gpt-6.1-sol", "openai", {"effort": "none"}), + ("gpt-6-astra", "azure", "none"), + ], +) +def test_responses_api_bridge_check_tools_with_a_none_the_row_refuses_routes_to_responses( + local_model_cost_map: None, model: str, custom_llm_provider: str, reasoning_effort: object +) -> None: + """The ``none`` escape hatch only holds when the chat gate forwards it. These rows refuse + ``none``, so the gate drops it (or refuses the call) and the provider runs its default effort, + under which Chat Completions rejects function tools; the bridge has to fire the way it does + for an unset effort. Row-derived: the test follows the flag, never the vendor's answer.""" + from litellm.main import responses_api_bridge_check + + row_key: Final = model if custom_llm_provider == "openai" else f"{custom_llm_provider}/{model}" + assert litellm.model_cost[row_key]["supports_none_reasoning_effort"] is False + + model_info, _ = responses_api_bridge_check( + model=model, + custom_llm_provider=custom_llm_provider, + tools=[{"type": "function", "function": {"name": "get_capital"}}], + reasoning_effort=reasoning_effort, + ) + + assert model_info.get("mode") == "responses" + + +def test_responses_api_bridge_check_a_none_the_row_refuses_stays_chat_without_tools( + local_model_cost_map: None, +) -> None: + """Without function tools nothing forces the Responses route: the dropped ``none`` leaves a + plain chat call at the default effort, as for any other dropped level.""" + from litellm.main import responses_api_bridge_check + + model_info, _ = responses_api_bridge_check( + model="gpt-6.1-sol", custom_llm_provider="openai", reasoning_effort="none" + ) + + assert model_info.get("mode") != "responses" + + def test_responses_api_bridge_check_reasoning_none_with_summary_still_routes_to_responses(): """A reasoning summary is Responses-only regardless of effort value.""" from litellm.main import responses_api_bridge_check @@ -1712,11 +1756,15 @@ _FOUNDRY_FUNCTION_TOOL: Final = ({"type": "function", "function": {"name": "get_ pytest.param("gpt-6-astra", "https://myresource.openai.azure.com", None, id="gpt-6-azure-openai-host"), pytest.param("gpt-5.6-sol", _FOUNDRY_API_BASE, "low", id="gpt-5.6-explicit-effort"), pytest.param("gpt-5.6-sol", _FOUNDRY_API_BASE, {"effort": "high"}, id="gpt-5.6-explicit-effort-dict"), + pytest.param("gpt-6-astra", _FOUNDRY_API_BASE, "none", id="gpt-6-row-refused-none"), ], ) def test_responses_api_bridge_check_azure_ai_foundry_rejected_tools_route_to_responses( model_name, api_base, reasoning_effort ): + """The explicit ``none`` only keeps a Foundry gpt-6 request on chat while the deployment takes + it; the ``azure_ai/gpt-6-astra`` row says it does not, so the gate drops it and the call runs at + the default effort, under which Foundry rejects function tools from gpt-6 on.""" from litellm.main import responses_api_bridge_check model_info, model = responses_api_bridge_check( @@ -1734,7 +1782,6 @@ def test_responses_api_bridge_check_azure_ai_foundry_rejected_tools_route_to_res @pytest.mark.parametrize( "model_name, api_base, reasoning_effort", [ - pytest.param("gpt-6-astra", _FOUNDRY_API_BASE, "none", id="explicit-none-stays-chat"), pytest.param("gpt-5.6-sol", _FOUNDRY_API_BASE, None, id="gpt-5.6-unset-effort-stays-chat"), pytest.param("gpt-5.6-sol", _FOUNDRY_API_BASE, "none", id="gpt-5.6-explicit-none-stays-chat"), pytest.param("gpt-5.5", _FOUNDRY_API_BASE, "high", id="gpt-5.5-explicit-effort-stays-chat"), From f07596a5fdf9cefb9e49212477b08ebc1d68444d Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Mon, 19 Oct 2026 12:21:18 -0700 Subject: [PATCH 14/20] fix(helm): stop litellm-helm from injecting placeholder models into every install (#47056) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- helm/litellm-helm/README.md | 13 +++++++--- helm/litellm-helm/tests/deployment_tests.yaml | 26 +++++++++++++++++++ helm/litellm-helm/values.yaml | 12 +-------- 3 files changed, 37 insertions(+), 14 deletions(-) diff --git a/helm/litellm-helm/README.md b/helm/litellm-helm/README.md index ff432c451e5..e059d45e037 100644 --- a/helm/litellm-helm/README.md +++ b/helm/litellm-helm/README.md @@ -70,11 +70,18 @@ proxyConfigMap: proxy_config: general_settings: master_key: os.environ/PROXY_MASTER_KEY + model_list: [] +``` + +The chart ships no models, so add them under `proxy_config.model_list` or from the Admin UI with `store_model_in_db: true` + +``` +proxy_config: model_list: - - model_name: gpt-3.5-turbo + - model_name: gpt-5.6 litellm_params: - model: gpt-3.5-turbo - api_key: eXaMpLeOnLy + model: openai/gpt-5.6 + api_key: os.environ/OPENAI_API_KEY ``` #### Example using existing `proxyConfigMap` instead of creating it: diff --git a/helm/litellm-helm/tests/deployment_tests.yaml b/helm/litellm-helm/tests/deployment_tests.yaml index ee946038202..292eeadfd13 100644 --- a/helm/litellm-helm/tests/deployment_tests.yaml +++ b/helm/litellm-helm/tests/deployment_tests.yaml @@ -270,6 +270,32 @@ tests: - matchRegex: path: data["config.yaml"] pattern: '(?m)^\s*health_check_details:\s*false$' + - it: should not render any placeholder models when the user sets proxy_config without model_list + template: configmap-litellm.yaml + set: + proxy_config.general_settings.store_model_in_db: true + asserts: + - notMatchRegex: + path: data["config.yaml"] + pattern: 'model_name:' + - matchRegex: + path: data["config.yaml"] + pattern: 'store_model_in_db: true' + - matchRegex: + path: data["config.yaml"] + pattern: 'master_key: os.environ/PROXY_MASTER_KEY' + - it: should render a user-supplied model_list as the only models + template: configmap-litellm.yaml + set: + proxy_config.model_list: + - model_name: my-model + litellm_params: + model: openai/my-model + api_key: os.environ/OPENAI_API_KEY + asserts: + - matchRegex: + path: data["config.yaml"] + pattern: 'model_list:\n- litellm_params:\n api_key: os.environ/OPENAI_API_KEY\n model: openai/my-model\n model_name: my-model\n$' - it: should allow overriding liveness, readiness, and startup probes template: deployment.yaml set: diff --git a/helm/litellm-helm/values.yaml b/helm/litellm-helm/values.yaml index 199c4595149..8450c5f2840 100644 --- a/helm/litellm-helm/values.yaml +++ b/helm/litellm-helm/values.yaml @@ -180,17 +180,7 @@ proxyConfigMap: # Examples: https://github.com/BerriAI/litellm/tree/main/litellm/proxy/example_config_yaml # Reference: https://docs.litellm.ai/docs/proxy/configs proxy_config: - model_list: - # At least one model must exist for the proxy to start. - - model_name: gpt-3.5-turbo - litellm_params: - model: gpt-3.5-turbo - api_key: eXaMpLeOnLy - - model_name: fake-openai-endpoint - litellm_params: - model: openai/fake - api_key: fake-key - api_base: https://exampleopenaiendpoint-production.up.railway.app/ + model_list: [] general_settings: master_key: os.environ/PROXY_MASTER_KEY From 2e427c6247903b133bd8e3db9076bf5d70d3f9f0 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Mon, 19 Oct 2026 12:37:47 -0700 Subject: [PATCH 15/20] chore(deps): require anyio>=4.14.0 to rule out the lock-waiter deadlock (#47378) anyio below 4.14.0 can leave a cancelled waiter queued on an asyncio Lock or Semaphore, so the next task to acquire waits forever on a lock nobody holds (agronholm/anyio#1145, fixed in 4.14.0). httpcore takes these locks for every httpx request, including the prisma query-engine client litellm imports anyio directly but never declared it, so pip and uv kept an already installed anyio 4.13.0 when installing litellm and accepted an explicit anyio==4.13.0 next to it. uv.lock already resolves 4.14.2, so Docker and CI installs do not change. Both the litellm and litellm-core manifests get the floor, and the lock only records the new direct dependency Co-authored-by: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> --- packaging/litellm-core/pyproject.toml | 1 + pyproject.toml | 1 + tests/unit/test_package_metadata.py | 17 +++++++++++++++++ uv.lock | 2 ++ 4 files changed, 21 insertions(+) diff --git a/packaging/litellm-core/pyproject.toml b/packaging/litellm-core/pyproject.toml index cf87cc6b936..2add6868bd9 100644 --- a/packaging/litellm-core/pyproject.toml +++ b/packaging/litellm-core/pyproject.toml @@ -24,6 +24,7 @@ dependencies = [ "click>=8.0.0,<9.0", "jinja2>=3.1.6,<4.0", "aiohttp>=3.14.2,<4.0", + "anyio>=4.14.0,<5.0", "async-timeout>=4.0.3,<6.0; python_version < '3.11'", "pydantic>=2.11.0,<3.0.0; python_version < '3.14'", "pydantic>=2.12.0,<3.0.0; python_version >= '3.14'", diff --git a/pyproject.toml b/pyproject.toml index f6697a1b57a..6c71140e6b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ dependencies = [ "click>=8.0.0,<9.0", "jinja2>=3.1.6,<4.0", "aiohttp>=3.14.2,<4.0", + "anyio>=4.14.0,<5.0", "async-timeout>=4.0.3,<6.0; python_version < '3.11'", "pydantic>=2.11.0,<3.0.0; python_version < '3.14'", "pydantic>=2.12.0,<3.0.0; python_version >= '3.14'", diff --git a/tests/unit/test_package_metadata.py b/tests/unit/test_package_metadata.py index fb6069291d8..dc6e3dca64e 100644 --- a/tests/unit/test_package_metadata.py +++ b/tests/unit/test_package_metadata.py @@ -5,6 +5,7 @@ from typing import Final from packaging.requirements import Requirement from packaging.specifiers import SpecifierSet from packaging.utils import NormalizedName, canonicalize_name +from packaging.version import Version if sys.version_info >= (3, 11): import tomllib @@ -18,6 +19,8 @@ OPTIONAL_DEPENDENCIES: Final = PYPROJECT["project"]["optional-dependencies"] AIOHTTP_POOL_POISONING_RANGE: Final = SpecifierSet(">=3.14.0,<3.14.2") AIOHTTP_POOL_POISONING_RELEASES: Final = ("3.14.0", "3.14.1") AIOHTTP_POOL_POISONING_SOURCE: Final = "aio-libs/aiohttp#12953, fixed in aiohttp 3.14.2" +ANYIO_STRANDED_LOCK_WAITER_FIX: Final = Version("4.14.0") +ANYIO_STRANDED_LOCK_WAITER_SOURCE: Final = "agronholm/anyio#1145, fixed in anyio 4.14.0 released 2026-06-15" CLI_RUNTIME_DEPENDENCIES: Final = frozenset({"rich", "pyyaml", "requests"}) PROXY_SERVER_ONLY_DEPENDENCIES: Final = frozenset( { @@ -92,3 +95,17 @@ def test_locked_aiohttp_version_is_not_pool_poisoning() -> None: locked: Final = _locked_package("aiohttp")["version"] assert isinstance(locked, str) assert not AIOHTTP_POOL_POISONING_RANGE.contains(locked), AIOHTTP_POOL_POISONING_SOURCE + + +def test_declared_anyio_floor_is_at_or_above_stranded_lock_waiter_fix() -> None: + declared: Final = tuple( + Requirement(requirement) + for requirement in PYPROJECT["project"]["dependencies"] + if canonicalize_name(Requirement(requirement).name) == "anyio" + ) + assert len(declared) == 1, ANYIO_STRANDED_LOCK_WAITER_SOURCE + assert declared[0].marker is None, ANYIO_STRANDED_LOCK_WAITER_SOURCE + assert any( + specifier.operator == ">=" and Version(specifier.version) >= ANYIO_STRANDED_LOCK_WAITER_FIX + for specifier in declared[0].specifier + ), ANYIO_STRANDED_LOCK_WAITER_SOURCE diff --git a/uv.lock b/uv.lock index beb66fe42b2..69d9f777687 100644 --- a/uv.lock +++ b/uv.lock @@ -4508,6 +4508,7 @@ version = "1.107.0" source = { editable = "." } dependencies = [ { name = "aiohttp" }, + { name = "anyio" }, { name = "async-timeout", marker = "python_full_version < '3.11'" }, { name = "boto3" }, { name = "click" }, @@ -4761,6 +4762,7 @@ requires-dist = [ { name = "a2a-sdk", marker = "extra == 'extra-proxy'", specifier = ">=1.1.0,<2.0" }, { name = "aiohttp", specifier = ">=3.14.2,<4.0" }, { name = "anthropic", extras = ["vertex"], marker = "extra == 'proxy-runtime'", specifier = ">=0.84.0,<1.0" }, + { name = "anyio", specifier = ">=4.14.0,<5.0" }, { name = "apscheduler", marker = "extra == 'proxy'", specifier = ">=3.11.2,<4.0" }, { name = "async-timeout", marker = "python_full_version < '3.11'", specifier = ">=4.0.3,<6.0" }, { name = "audioread", marker = "extra == 'stt-nvidia-riva'", specifier = ">=3.0.1" }, From 3e18a4b55d5dcab3a5514c554a890ed0f9441392 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Mon, 19 Oct 2026 13:18:20 -0700 Subject: [PATCH 16/20] fix(router): record cooled deployments on the all-cooled failure row (#47120) * fix(router): record cooled deployments on the all-cooled failure row A RouterRateLimitError now carries the cooled deployments of the route the request resolves to for the caller's team (model group, alias, wildcard or team route), each with its masked api_base. get_error_information copies them into error_information.error_cooldown_deployments so the SpendLogs failure row and every logging callback can tell which deployments were cooling * fix(router): name cooled deployments from the router's own resolved route Use the deployments the router resolved for the request, before health and cooldown filtering, instead of re-deriving the route. Alias chains, the provider-model fallback, wildcards and team filtering now match routing by construction. A routing-strategy refusal records null. * fix(router): name only the cooled deployments tag and region routing allow The refusal's cooled-deployment list now leaves out deployments the request's tags or the end user's allowed region keep it from, so a tenant's refusal row no longer shows another tenant's api_base * fix(router): keep failure logging working for refusal subclasses that skip init A RouterRateLimitError subclass that never calls its parent init now logs a null cooled-deployment list instead of breaking the failure payload * test(router): call the refusal routing helpers directly The router coverage check needs each router function called by name from a unit test --------- Co-authored-by: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> --- litellm/litellm_core_utils/litellm_logging.py | 3 + litellm/router.py | 104 +++++ litellm/router_utils/cooldown_handlers.py | 24 +- litellm/router_utils/handle_error.py | 12 +- litellm/types/router.py | 4 + litellm/types/utils.py | 6 + .../spend/test_router_cooldown_failure_row.py | 358 +++++++++++++++++ .../test_litellm_logging.py | 44 +++ .../router_utils/test_cooldown_handlers.py | 361 +++++++++++++++++- 9 files changed, 913 insertions(+), 3 deletions(-) create mode 100644 tests/integration/spend/test_router_cooldown_failure_row.py diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 0ac216c2029..3a1dafa331c 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -151,6 +151,7 @@ from litellm.types.llms.openai import ( from litellm.types.mcp import MCPPostCallResponseObject from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.rerank import RerankResponse +from litellm.types.router import RouterRateLimitError from litellm.types.utils import ( DEPLOYMENT_SCOPED_PRICING_FIELDS, CachingDetails, @@ -6692,6 +6693,7 @@ class StandardLoggingPayloadSetup: rate_limit_category: Final = validate_rate_limit_category(getattr(original_exception, "category", None)) rate_limit_type: Final = validate_rate_limit_type(getattr(original_exception, "rate_limit_type", None)) budget_error: Final = original_exception if isinstance(original_exception, BudgetExceededError) else None + router_error: Final = original_exception if isinstance(original_exception, RouterRateLimitError) else None provider_request_id: Final = _get_provider_request_id(original_exception) if original_exception else None return StandardLoggingPayloadErrorInformation( @@ -6707,6 +6709,7 @@ class StandardLoggingPayloadSetup: error_budget_entity_id=budget_error.entity_id if budget_error else None, error_budget_limit=budget_error.max_budget if budget_error else None, error_budget_spend=budget_error.current_cost if budget_error else None, + error_cooldown_deployments=router_error.cooldown_deployments if router_error else None, normalized_error=normalize_error(original_exception, error_status, error_message), ) diff --git a/litellm/router.py b/litellm/router.py index 113565fecaa..af79fa763c4 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -222,6 +222,7 @@ from litellm.router_utils.cooldown_handlers import ( _first_present, # pyright: ignore[reportPrivateUsage] - shared internal helper across router_utils submodules, matching the other cooldown_handlers imports on this line async_get_cooldown_deployments, async_get_cooldown_deployments_with_debug_info, + get_cooldown_deployment_info, get_cooldown_deployments, is_advisor_orchestration_failure, is_background_response_poll_not_found, @@ -12598,6 +12599,80 @@ class Router: if isinstance(model_info, Mapping) and model_info.get("id") is not None ) + def _region_routable_deployments( + self, + deployments: Sequence[DeploymentTypedDict], + request_kwargs: Mapping[str, object] | None, + has_request_input: bool, + ) -> tuple[Mapping[str, Any], ...]: + """ + `deployments` without the ones `_pre_call_checks` keeps this request away from for being outside its allowed + region, typed for the tag routing filter + """ + allowed_model_region: Final = None if request_kwargs is None else request_kwargs.get("allowed_model_region") + if allowed_model_region is None or not self.enable_pre_call_checks or not has_request_input: + return tuple(deployments) + if not isinstance(allowed_model_region, str): + return () + return tuple( + deployment + for deployment in deployments + if is_region_allowed( + litellm_params=LiteLLM_Params.model_validate(deployment["litellm_params"]), + allowed_model_region=allowed_model_region, + ) + ) + + async def _async_routable_cooled_deployment_ids( + self, + model: str, + request_deployments: Sequence[DeploymentTypedDict], + cooldown_deployments: Iterable[str], + request_kwargs: Mapping[str, object], + has_request_input: bool, + ) -> frozenset[str]: + """ + Ids of the cooling deployments in `request_deployments` that region and tag routing would send this request + to, each tag routing pick counted along with the picks it falls back to once that one is unavailable, judged + without recording a tag match on the request + """ + cooled_ids: Final = frozenset(cooldown_deployments) + metadata_variable_name: Final = get_metadata_variable_name_from_kwargs(request_kwargs) + metadata: Final = request_kwargs.get(metadata_variable_name) + if metadata_variable_name in request_kwargs and not isinstance(metadata, Mapping): + return frozenset() + tag_request: Final[Mapping[str, Any]] = ( + {**request_kwargs, metadata_variable_name: {**metadata}} + if isinstance(metadata, Mapping) + else request_kwargs + ) + routable_ids = frozenset[str]() # rebind-ok: gains each pass's tag routing picks + remaining = tuple( # rebind-ok: each pass drops what tag routing just picked + deployment + for deployment in self._region_routable_deployments(request_deployments, request_kwargs, has_request_input) + if self._deployment_ids((deployment,)) <= cooled_ids + ) + while remaining: + try: + picked = await get_deployments_for_tag( + llm_router_instance=self, + model=model, + healthy_deployments=remaining, + request_kwargs=tag_request, + metadata_variable_name=metadata_variable_name, + ) + except ValueError: # tag routing sends this request to none of the rest + break + picked_identities = frozenset(id(deployment) for deployment in picked) + rest = tuple(deployment for deployment in remaining if id(deployment) not in picked_identities) + if len(rest) == len(remaining): + break + routable_ids |= self._deployment_ids( + tuple(deployment for deployment in remaining if id(deployment) in picked_identities) + ) + remaining = rest + return routable_ids + def has_model_id(self, candidate_id: str) -> bool: """ O(1) membership check for a deployment ID without allocating large lists. @@ -14457,6 +14532,13 @@ class Router: litellm_router_instance=self, model=model, parent_otel_span=parent_otel_span, + route_model_ids=await self._async_routable_cooled_deployment_ids( + model=model, + request_deployments=request_deployments, + cooldown_deployments=cooldown_deployments, + request_kwargs=cast(Mapping[str, object], request_kwargs), # cast-ok: str-keyed request kwargs + has_request_input=messages is not None or input is not None, + ), ) raise exception @@ -15656,6 +15738,17 @@ class Router: enable_pre_call_checks=self.enable_pre_call_checks, cooldown_list=_cooldown_list, model_ids=model_ids, + cooldown_deployments=get_cooldown_deployment_info( + self, + self._deployment_ids( + self._region_routable_deployments( + request_deployments, + cast(Mapping[str, object] | None, request_kwargs), # cast-ok: str-keyed request kwargs + messages is not None or input is not None, + ) + ), + _cooldown_list, + ), ) if strategy == "simple-shuffle": @@ -15830,6 +15923,17 @@ class Router: enable_pre_call_checks=self.enable_pre_call_checks, cooldown_list=_cooldown_list, model_ids=pass_through_model_ids, + cooldown_deployments=get_cooldown_deployment_info( + self, + self._deployment_ids( + self._region_routable_deployments( + request_deployments, + cast(Mapping[str, object] | None, request_kwargs), # cast-ok: str-keyed request kwargs + messages is not None or input is not None, + ) + ), + _cooldown_list, + ), ) # 6. Apply load balancing strategy diff --git a/litellm/router_utils/cooldown_handlers.py b/litellm/router_utils/cooldown_handlers.py index bbd895320bd..7ed4864ebc3 100644 --- a/litellm/router_utils/cooldown_handlers.py +++ b/litellm/router_utils/cooldown_handlers.py @@ -8,7 +8,7 @@ Router cooldown handlers import asyncio import math -from collections.abc import Mapping +from collections.abc import Collection, Mapping, Sequence from datetime import datetime from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final @@ -28,12 +28,14 @@ from litellm.constants import ( ) from litellm.exceptions import MidStreamFallbackError from litellm.litellm_core_utils.credential_accessor import CredentialAccessor +from litellm.litellm_core_utils.litellm_logging import mask_api_base_credentials from litellm.router_utils.cooldown_cache import ROUTER_COOLDOWNS_TARGET, CooldownCacheValue from litellm.router_utils.cooldown_callbacks import router_cooldown_event_callback from litellm.types.router import Deployment from litellm.types.utils import ( BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, BACKGROUND_RESPONSE_STATUS_POLL_CALL_ORIGIN, + CooldownDeploymentInfo, InternalCallOrigin, ) @@ -644,6 +646,26 @@ def get_cooldown_deployments(litellm_router_instance: LitellmRouter, parent_otel _get_cooldown_deployments = get_cooldown_deployments +def _deployment_api_base(litellm_router_instance: LitellmRouter, model_id: str) -> str | None: + deployment: Final = litellm_router_instance.get_deployment(model_id=model_id) + api_base: Final = deployment.litellm_params.api_base if deployment else None + return None if api_base is None else mask_api_base_credentials(api_base) + + +def get_cooldown_deployment_info( + litellm_router_instance: LitellmRouter, route_model_ids: Collection[str], cooldown_list: Sequence[str] +) -> tuple[CooldownDeploymentInfo, ...]: + """ + The deployments the router resolved for this request (`route_model_ids`) that are in `cooldown_list`, each + with its masked api_base + """ + return tuple( + CooldownDeploymentInfo(model_id=model_id, api_base=_deployment_api_base(litellm_router_instance, model_id)) + for model_id in cooldown_list + if model_id in route_model_ids + ) + + def should_cooldown_based_on_allowed_fails_policy( litellm_router_instance: LitellmRouter, deployment: str, diff --git a/litellm/router_utils/handle_error.py b/litellm/router_utils/handle_error.py index 14ed6ed2adc..38d9ef05706 100644 --- a/litellm/router_utils/handle_error.py +++ b/litellm/router_utils/handle_error.py @@ -1,9 +1,11 @@ +from collections.abc import Collection from typing import TYPE_CHECKING, Any, Final from litellm._logging import redact_secrets, verbose_router_logger from litellm.constants import MAX_EXCEPTION_MESSAGE_LENGTH from litellm.router_utils.cooldown_handlers import ( async_get_cooldown_deployments_with_debug_info, + get_cooldown_deployment_info, ) from litellm.types.integrations.slack_alerting import AlertType from litellm.types.router import RouterRateLimitError @@ -69,7 +71,10 @@ async def send_llm_exception_alert( async def async_raise_no_deployment_exception( - litellm_router_instance: LitellmRouter, model: str, parent_otel_span: Span | None + litellm_router_instance: LitellmRouter, + model: str, + parent_otel_span: Span | None, + route_model_ids: Collection[str] | None = None, ): """ Raises a RouterRateLimitError if no deployment is found for the given model. @@ -94,4 +99,9 @@ async def async_raise_no_deployment_exception( enable_pre_call_checks=litellm_router_instance.enable_pre_call_checks, cooldown_list=cooldown_list_ids, model_ids=model_ids, + cooldown_deployments=( + None + if route_model_ids is None + else get_cooldown_deployment_info(litellm_router_instance, route_model_ids, cooldown_list_ids) + ), ) diff --git a/litellm/types/router.py b/litellm/types/router.py index 68dbc701c9e..46617e1b753 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -43,6 +43,7 @@ from .llms.bedrock import AwsSessionTag from .llms.openai import OpenAIFileObject from .search import SearchProvider from .utils import ( + CooldownDeploymentInfo, CustomPricingLiteLLMParams, MirroredPricingParams, ModelResponse, @@ -1106,6 +1107,7 @@ class RouterErrorTypes(str, enum.Enum): class RouterRateLimitError(ValueError): status_code: Final = 429 + cooldown_deployments: tuple[CooldownDeploymentInfo, ...] | None = None def __init__( self, @@ -1114,11 +1116,13 @@ class RouterRateLimitError(ValueError): enable_pre_call_checks: bool, cooldown_list: list, model_ids: Sequence[str] = (), + cooldown_deployments: Sequence[CooldownDeploymentInfo] | None = None, ) -> None: self.model = model self.cooldown_time = cooldown_time self.enable_pre_call_checks = enable_pre_call_checks self.cooldown_list = cooldown_list + self.cooldown_deployments = None if cooldown_deployments is None else tuple(cooldown_deployments) self.all_deployments_in_cooldown = bool(model_ids) and frozenset(model_ids) <= frozenset(cooldown_list) self.type = ( RouterErrorTypes.all_deployments_in_cooldown.value diff --git a/litellm/types/utils.py b/litellm/types/utils.py index fd1d58adafb..2e05107f82d 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3386,6 +3386,11 @@ class StandardLoggingZeroCostDiagnostic(TypedDict): missing_pricing_keys: ReadOnly[tuple[str, ...]] +class CooldownDeploymentInfo(TypedDict): + model_id: ReadOnly[str] + api_base: ReadOnly[str | None] + + class StandardLoggingPayloadErrorInformation(TypedDict, total=False): error_code: str | None error_class: str | None @@ -3414,6 +3419,7 @@ class StandardLoggingPayloadErrorInformation(TypedDict, total=False): error_budget_entity_id: str | None error_budget_limit: float | None error_budget_spend: float | None + error_cooldown_deployments: ReadOnly[tuple[CooldownDeploymentInfo, ...] | None] normalized_error: ReadOnly[str | None] diff --git a/tests/integration/spend/test_router_cooldown_failure_row.py b/tests/integration/spend/test_router_cooldown_failure_row.py new file mode 100644 index 00000000000..04e36785823 --- /dev/null +++ b/tests/integration/spend/test_router_cooldown_failure_row.py @@ -0,0 +1,358 @@ +import asyncio +import json +import uuid +from collections.abc import Callable, Iterator, Mapping +from concurrent.futures import ThreadPoolExecutor +from contextlib import ExitStack +from dataclasses import dataclass +from itertools import product +from pathlib import Path +from typing import Final + +import anthropic +import httpx +import openai +import pytest +import yaml +from integration._support.client import GATEWAY_LIMITS, Gateway, eventually, gateway_from_environment, object_value +from integration._support.database import read_rows +from integration._support.process import owned_proxy_process +from integration._support.redis_process import OwnedRedis, owned_redis +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue +from redis import Redis + +pytestmark: Final = pytest.mark.timeout(300) + +COOLDOWN_SECONDS: Final = 120 +NO_DEPLOYMENTS: Final = "No deployments available" +ROW_SECONDS: Final = 70 +CHAT: Final = "/v1/chat/completions" +MESSAGES: Final = "/v1/messages" +RESPONSES: Final = "/v1/responses" +FAILURE_ROW: Final = ( + "SELECT status, model_group, model_id, api_base, metadata::jsonb -> 'error_information' AS error " + 'FROM "LiteLLM_SpendLogs" WHERE request_id = %s' +) +ENDPOINT_CELLS: Final = tuple((path, stream) for path in (CHAT, MESSAGES, RESPONSES) for stream in (False, True)) +ENDPOINT_GROUPS: Final = tuple( + f"{path.rsplit('/', 1)[-1]}-{'stream' if stream else 'json'}" for path, stream in ENDPOINT_CELLS +) +SDK_CELLS: Final = ("openai-sync", "openai-async-stream", "anthropic-sync") +TAGGED: Final = "tagged" +REGIONAL: Final = "regional" +GROUPS: Final = (*ENDPOINT_GROUPS, *SDK_CELLS, "neighbour", "scoped", "burst", TAGGED, REGIONAL) +LANES: Final = ("a", "b") +LANE_PARAMS: Final[Mapping[tuple[str, str], Mapping[str, JsonValue]]] = { + (TAGGED, "a"): {"tags": ["tenant-a"]}, + (TAGGED, "b"): {"tags": ["default"]}, + (REGIONAL, "a"): {"region_name": "eu"}, + (REGIONAL, "b"): {"region_name": "us"}, +} +TENANT_A: Final[Mapping[str, JsonValue]] = {"metadata": {"tags": ["tenant-a"]}} +UNTAGGED: Final[Mapping[str, JsonValue]] = {} + + +@dataclass(frozen=True, slots=True) +class Rig: + gateway: Gateway + cache: OwnedRedis + prefix: str + wire: Wire + + def group(self, name: str) -> str: + return f"{self.prefix}-{name}" + + def deployments(self, name: str) -> Mapping[str, str]: + return {f"{self.group(name)}-{lane}": f"{self.wire.url}/{self.group(name)}/{lane}/v1" for lane in LANES} + + +@dataclass(frozen=True, slots=True) +class Outcome: + status: int + detail: str + call_id: str + + +def is_model_probe(request: Request) -> bool: + return (request.method, request.target) == ("GET", "/v1/models") + + +def rate_limited(request: Request) -> Reply: + if is_model_probe(request): + return Reply(body=b'{"object": "list", "data": []}') + body: Final = {"error": {"message": "scripted 429", "type": "rate_limit_error", "code": "rate_limit_exceeded"}} + return Reply(status=429, body=json.dumps(body).encode()) + + +def cooldown_config(directory: Path, cache: OwnedRedis, prefix: str, wire: Wire) -> Path: + base: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + assert isinstance(base, dict) + model_list: Final = [ + { + "model_name": f"{prefix}-{name}", + "litellm_params": { + "model": "openai/gpt-5.4", + "api_base": f"{wire.url}/{prefix}-{name}/{lane}/v1", + "api_key": "synthetic-openai-key", + **LANE_PARAMS.get((name, lane), {}), + }, + "model_info": { + "id": f"{prefix}-{name}-{lane}", + **({"enable_tag_filtering": True} if name == TAGGED else {}), + }, + } + for name, lane in product(GROUPS, LANES) + ] + config: Final = { + **base, + "model_list": model_list, + "router_settings": { + "num_retries": 0, + "allowed_fails": 0, + "cooldown_time": COOLDOWN_SECONDS, + "enable_pre_call_checks": True, + "redis_host": cache.host, + "redis_port": cache.port, + }, + } + path: Final = directory / f"cooldown-{prefix}.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + directory: Final = tmp_path_factory.mktemp("router-cooldown-failure-row") + prefix: Final = f"cooldown-{uuid.uuid4().hex[:8]}" + with ExitStack() as stack: + upstream: Final = stack.enter_context(gateway_from_environment()) + cache: Final = stack.enter_context(owned_redis(directory)) + wire: Final = stack.enter_context(wire_server(rate_limited)) + config: Final = cooldown_config(directory, cache, prefix, wire) + owned: Final = stack.enter_context(owned_proxy_process(upstream, directory, {}, config=config, workers=2)) + yield Rig(owned.gateway, cache, prefix, wire) + + +def cooled(cache: OwnedRedis, deployment_id: str) -> bool: + with Redis(host=cache.host, port=cache.port) as client: + return int(client.exists(f"deployment:{deployment_id}:cooldown")) == 1 + + +def request_body(path: str, group: str, stream: bool) -> dict[str, JsonValue]: + content: Final = f"cooldown row {uuid.uuid4().hex}" + match path: + case "/v1/messages": + return { + "model": group, + "max_tokens": 8, + "messages": [{"role": "user", "content": content}], + "stream": stream, + } + case "/v1/responses": + return {"model": group, "input": content, "stream": stream} + case _: + return {"model": group, "messages": [{"role": "user", "content": content}], "stream": stream} + + +def send(gateway: Gateway, path: str, group: str, stream: bool = False) -> Outcome: + response: Final = gateway.request("POST", path, request_body(path, group, stream), headers={"Connection": "close"}) + return Outcome(response.status_code, response.text, response.headers["x-litellm-call-id"]) + + +def send_chat(rig: Rig, name: str, extra: Mapping[str, JsonValue]) -> Outcome: + body: Final = {**request_body(CHAT, rig.group(name), False), **extra} + response: Final = rig.gateway.request("POST", CHAT, body, headers={"Connection": "close"}) + return Outcome(response.status_code, response.text, response.headers["x-litellm-call-id"]) + + +def failure_row(call_id: str) -> dict[str, JsonValue]: + rows: Final = eventually( + lambda: read_rows(FAILURE_ROW, (call_id,)), lambda found: len(found) == 1, seconds=ROW_SECONDS + ) + assert rows[0]["status"] == "failure", rows + return rows[0] + + +def until_router_refusal(attempt: Callable[[], Outcome], tries: int = 10) -> tuple[Outcome, ...]: + def walk() -> Iterator[Outcome]: + for _ in range(tries): + outcome: Final = attempt() + yield outcome + if NO_DEPLOYMENTS in outcome.detail: + return + + history: Final = tuple(walk()) + assert NO_DEPLOYMENTS in history[-1].detail, [(o.status, o.detail[:200]) for o in history] + return history + + +def cool_group(rig: Rig, name: str) -> tuple[Outcome, ...]: + history: Final = until_router_refusal(lambda: send(rig.gateway, CHAT, rig.group(name))) + assert all(cooled(rig.cache, identity) for identity in rig.deployments(name)), history + return history + + +def expected_cooldown(deployments: Mapping[str, str]) -> list[JsonValue]: + return [{"model_id": identity, "api_base": deployments[identity]} for identity in sorted(deployments)] + + +def cooldown_entries(error: Mapping[str, JsonValue]) -> list[JsonValue] | None: + entries: Final = error.get("error_cooldown_deployments") + if not isinstance(entries, list): + return None + return sorted(entries, key=lambda entry: str(object_value(entry)["model_id"])) + + +def assert_refusal_row(call_id: str, deployments: Mapping[str, str]) -> Mapping[str, JsonValue]: + row: Final = failure_row(call_id) + error: Final = object_value(row["error"]) + assert (error["error_class"], error["error_code"]) == ("RouterRateLimitError", "429"), row + assert cooldown_entries(error) == expected_cooldown(deployments), error.get("error_message") + assert (row["model_id"], row["api_base"]) == ("", ""), row + return error + + +def provider_row(call_id: str) -> tuple[str, str, str, JsonValue]: + row: Final = failure_row(call_id) + error: Final = object_value(row["error"]) + return str(row["model_id"]), str(row["api_base"]), str(error["error_code"]), error.get("error_cooldown_deployments") + + +def assert_provider_rows(history: tuple[Outcome, ...], deployments: Mapping[str, str]) -> None: + provider: Final = history[:-1] + assert all(outcome.status == 429 and NO_DEPLOYMENTS not in outcome.detail for outcome in provider), [ + (o.status, o.detail[:200]) for o in provider + ] + rows: Final = tuple(provider_row(outcome.call_id) for outcome in provider) + assert {identity for identity, _, _, _ in rows} == set(deployments), rows + assert all(base.startswith(deployments[identity]) for identity, base, _, _ in rows), rows + assert all((code, cooldown) == ("429", None) for _, _, code, cooldown in rows), rows + + +@pytest.mark.parametrize(("path", "stream"), ENDPOINT_CELLS, ids=ENDPOINT_GROUPS) +def test_all_cooled_refusal_row_records_429_and_every_cooled_deployment(rig: Rig, path: str, stream: bool) -> None: + name: Final = ENDPOINT_GROUPS[ENDPOINT_CELLS.index((path, stream))] + history: Final = until_router_refusal(lambda: send(rig.gateway, path, rig.group(name), stream)) + refusal: Final = history[-1] + assert refusal.status == 429, (refusal.status, refusal.detail[:300]) + assert_refusal_row(refusal.call_id, rig.deployments(name)) + assert_provider_rows(history, rig.deployments(name)) + + +def openai_chat(rig: Rig, group: str) -> Outcome: + client: Final = openai.OpenAI(base_url=f"{rig.gateway.client.base_url}/v1", api_key=rig.gateway.key, max_retries=0) + with pytest.raises(openai.RateLimitError) as raised: + client.chat.completions.create(model=group, messages=[{"role": "user", "content": uuid.uuid4().hex}]) + return Outcome(429, raised.value.message, raised.value.response.headers["x-litellm-call-id"]) + + +def openai_async_stream(rig: Rig, group: str) -> Outcome: + async def call() -> Outcome: + client: Final = openai.AsyncOpenAI( + base_url=f"{rig.gateway.client.base_url}/v1", api_key=rig.gateway.key, max_retries=0 + ) + try: + with pytest.raises(openai.RateLimitError) as raised: + await client.chat.completions.create( + model=group, messages=[{"role": "user", "content": uuid.uuid4().hex}], stream=True + ) + return Outcome(429, raised.value.message, raised.value.response.headers["x-litellm-call-id"]) + finally: + await client.close() + + return asyncio.run(call()) + + +def anthropic_message(rig: Rig, group: str) -> Outcome: + client: Final = anthropic.Anthropic( + base_url=str(rig.gateway.client.base_url), api_key=rig.gateway.key, max_retries=0 + ) + with pytest.raises(anthropic.RateLimitError) as raised: + client.messages.create(model=group, max_tokens=8, messages=[{"role": "user", "content": uuid.uuid4().hex}]) + return Outcome(429, raised.value.message, raised.value.response.headers["x-litellm-call-id"]) + + +SDK_CLIENTS: Final = { + "openai-sync": openai_chat, + "openai-async-stream": openai_async_stream, + "anthropic-sync": anthropic_message, +} + + +@pytest.mark.parametrize("client", SDK_CELLS) +def test_sdk_clients_get_429_and_the_row_names_the_cooled_deployments(rig: Rig, client: str) -> None: + history: Final = until_router_refusal(lambda: SDK_CLIENTS[client](rig, rig.group(client))) + assert_refusal_row(history[-1].call_id, rig.deployments(client)) + assert_provider_rows(history, rig.deployments(client)) + + +def test_the_row_lists_only_the_requested_groups_deployments_while_another_group_cools(rig: Rig) -> None: + cool_group(rig, "neighbour") + cool_group(rig, "scoped") + refusal: Final = eventually( + lambda: send(rig.gateway, CHAT, rig.group("scoped")), + lambda outcome: ( + NO_DEPLOYMENTS in outcome.detail + and all(identity in outcome.detail for identity in rig.deployments("neighbour")) + ), + ) + assert_refusal_row(refusal.call_id, rig.deployments("scoped")) + + +def test_a_concurrent_burst_on_a_cooled_group_writes_one_complete_row_per_refusal(rig: Rig) -> None: + cool_group(rig, "burst") + rig.wire.drain() + calls: Final = tuple(ENDPOINT_CELLS[index % len(ENDPOINT_CELLS)] for index in range(30)) + with ( + httpx.Client( + base_url=rig.gateway.client.base_url, timeout=60, trust_env=False, limits=GATEWAY_LIMITS + ) as client, + ThreadPoolExecutor(max_workers=30) as pool, + ): + patient: Final = Gateway(client, rig.gateway.key, rig.gateway.upstream_url) + outcomes: Final = tuple(pool.map(lambda call: send(patient, call[0], rig.group("burst"), call[1]), calls)) + assert all(outcome.status == 429 for outcome in outcomes), [(o.status, o.detail[:200]) for o in outcomes] + assert len({outcome.call_id for outcome in outcomes}) == len(outcomes), outcomes + refusals: Final = tuple(outcome for outcome in outcomes if NO_DEPLOYMENTS in outcome.detail) + reached: Final = tuple(outcome for outcome in outcomes if NO_DEPLOYMENTS not in outcome.detail) + assert refusals, [(o.status, o.detail[:200]) for o in outcomes] + assert len(tuple(request for request in rig.wire.drain() if not is_model_probe(request))) == len(reached) + for outcome in refusals: + assert_refusal_row(outcome.call_id, rig.deployments("burst")) + for outcome in reached: + identity, api_base, code, cooldown = provider_row(outcome.call_id) + assert api_base.startswith(rig.deployments("burst")[identity]), (identity, api_base) + assert (code, cooldown) == ("429", None), outcome + + +def cool_lane(rig: Rig, name: str, lane: str, extra: Mapping[str, JsonValue]) -> None: + identity: Final = f"{rig.group(name)}-{lane}" + outcomes: Final = eventually(lambda: send_chat(rig, name, extra), lambda _: cooled(rig.cache, identity)) + assert NO_DEPLOYMENTS not in outcomes.detail, outcomes.detail[:300] + + +def router_refusal(rig: Rig, name: str, extra: Mapping[str, JsonValue]) -> Outcome: + refusal: Final = eventually(lambda: send_chat(rig, name, extra), lambda outcome: NO_DEPLOYMENTS in outcome.detail) + assert refusal.status == 429, (refusal.status, refusal.detail[:300]) + return refusal + + +def test_a_tag_routed_refusal_row_names_only_the_cooled_deployments_its_tags_reach(rig: Rig) -> None: + deployments: Final = rig.deployments(TAGGED) + tenant: Final = f"{rig.group(TAGGED)}-a" + cool_lane(rig, TAGGED, "a", TENANT_A) + cool_lane(rig, TAGGED, "b", UNTAGGED) + assert_refusal_row(router_refusal(rig, TAGGED, TENANT_A).call_id, {tenant: deployments[tenant]}) + assert_refusal_row(router_refusal(rig, TAGGED, UNTAGGED).call_id, deployments) + + +def test_a_region_restricted_refusal_row_names_only_the_cooled_deployments_in_its_region(rig: Rig) -> None: + deployments: Final = rig.deployments(REGIONAL) + european: Final = f"{rig.group(REGIONAL)}-a" + customer: Final = f"{rig.prefix}-eu-customer" + with ExitStack() as cleanups: + rig.gateway.post("/customer/new", {"user_id": customer, "allowed_model_region": "eu"}) + cleanups.callback(rig.gateway.post, "/customer/delete", {"user_ids": [customer]}) + cool_group(rig, REGIONAL) + assert_refusal_row(router_refusal(rig, REGIONAL, {"user": customer}).call_id, {european: deployments[european]}) diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index d1258e43154..37a985264b7 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -4569,6 +4569,50 @@ def test_get_error_information_falls_back_to_str_when_no_message_attr(): assert result["error_class"] == "ValueError" +def test_get_error_information_records_the_cooled_deployments_of_a_router_refusal(): + from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.types.router import RouterRateLimitError + from litellm.types.utils import CooldownDeploymentInfo + + cooled: Final = ( + CooldownDeploymentInfo(model_id="refused-1", api_base="https://first.example/v1"), + CooldownDeploymentInfo(model_id="refused-2", api_base=None), + ) + refusal: Final = RouterRateLimitError( + model="refused", + cooldown_time=60, + enable_pre_call_checks=False, + cooldown_list=["refused-1", "refused-2", "neighbour-1"], + model_ids=("refused-1", "refused-2"), + cooldown_deployments=cooled, + ) + + result: Final = StandardLoggingPayloadSetup.get_error_information(refusal) + assert (result["error_class"], result["error_code"], result["error_cooldown_deployments"]) == ( + "RouterRateLimitError", + "429", + cooled, + ), result + provider_429: Final = litellm.RateLimitError("scripted 429", "openai", "gpt-5.4") + assert StandardLoggingPayloadSetup.get_error_information(provider_429)["error_cooldown_deployments"] is None + + +def test_get_error_information_handles_a_router_refusal_subclass_that_skips_its_init(): + from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup + from litellm.types.router import RouterRateLimitError + + class PluginRefusal(RouterRateLimitError): + def __init__(self, message: str) -> None: + ValueError.__init__(self, message) + + result: Final = StandardLoggingPayloadSetup.get_error_information(PluginRefusal("plugin refused")) + assert (result["error_class"], result["error_code"], result["error_cooldown_deployments"]) == ( + "PluginRefusal", + "429", + None, + ), result + + # ────────────────────────────────────────────────────────────────────── # Tests for _get_assembled_streaming_response non-streaming early return # ────────────────────────────────────────────────────────────────────── diff --git a/tests/unit/router_utils/test_cooldown_handlers.py b/tests/unit/router_utils/test_cooldown_handlers.py index 2376e0f4d0c..0c9d7ab9f85 100644 --- a/tests/unit/router_utils/test_cooldown_handlers.py +++ b/tests/unit/router_utils/test_cooldown_handlers.py @@ -1,5 +1,7 @@ import asyncio +import copy import importlib +import itertools import random import time from collections.abc import Mapping @@ -15,6 +17,7 @@ from litellm import Router from litellm._internal_context import current_service_target from litellm.caching.dual_cache import DualCache from litellm.caching.in_memory_cache import InMemoryCache +from litellm.litellm_core_utils.litellm_logging import mask_api_base_credentials from litellm.router_utils.cooldown_cache import CooldownCache, CooldownCacheValue from litellm.router_utils.cooldown_callbacks import router_cooldown_event_callback from litellm.router_utils.cooldown_handlers import ( @@ -44,8 +47,9 @@ from litellm.types.router import ( AllowedFailsPolicy, DeploymentTypedDict, LiteLLMParamsTypedDict, + RouterRateLimitError, ) -from litellm.types.utils import CredentialItem +from litellm.types.utils import CooldownDeploymentInfo, CredentialItem from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome @@ -3004,3 +3008,358 @@ def test_shared_mode_upstream_429_still_cools_down_the_deployment(single_deploym is True ) assert _should_run_cooldown_logic(single_deployment_router, "dep-1", 429, exc) is True + + +REFUSAL_MESSAGES: Final = [{"role": "user", "content": "cooldown"}] +SECRET_API_BASE: Final = "https://first.example/v1?key=sk-scripted-secret" +FIRST_REFUSED: Final = CooldownDeploymentInfo(model_id="refused-1", api_base=mask_api_base_credentials(SECRET_API_BASE)) +SECOND_REFUSED: Final = CooldownDeploymentInfo(model_id="refused-2", api_base=None) +THIRD_REFUSED: Final = CooldownDeploymentInfo(model_id="refused-3", api_base=None) +OTHER_TEAMS_REFUSED: Final = CooldownDeploymentInfo(model_id="refused-team-b", api_base=None) +FIRST_WILDCARD: Final = CooldownDeploymentInfo(model_id="wildcard-1", api_base="https://wildcard.example/v1") +SECOND_WILDCARD: Final = CooldownDeploymentInfo(model_id="wildcard-2", api_base=None) +ALIAS_TARGET: Final = CooldownDeploymentInfo(model_id="tier-1", api_base="https://tier.example/v1") +PROVIDER_MODEL: Final = CooldownDeploymentInfo(model_id="friendly-1", api_base=None) +EVERY_DEPLOYMENT: Final = ( + "refused-1", + "refused-2", + "refused-3", + "refused-team-b", + "neighbour-1", + "wildcard-1", + "wildcard-2", + "tier-1", + "friendly-1", +) + + +def _cooled_router(routing_strategy: str, cooled: tuple[str, ...]) -> Router: + router: Final = Router( + routing_strategy=routing_strategy, + model_group_alias={"entry": "tier", "tier": "refused"}, + model_list=[ + { + "model_name": "refused", + "litellm_params": {"model": "openai/gpt-5.4", "api_base": SECRET_API_BASE, "use_in_pass_through": True}, + "model_info": {"id": "refused-1"}, + }, + { + "model_name": "refused", + "litellm_params": {"model": "openai/gpt-5.4", "use_in_pass_through": True, "tpm": 1}, + "model_info": {"id": "refused-2"}, + }, + { + "model_name": "refused", + "litellm_params": {"model": "openai/gpt-5.4", "tpm": 1}, + "model_info": {"id": "refused-3"}, + }, + { + "model_name": "refused", + "litellm_params": {"model": "openai/gpt-5.4", "use_in_pass_through": True}, + "model_info": {"id": "refused-team-b", "team_id": "team-b"}, + }, + { + "model_name": "neighbour", + "litellm_params": {"model": "openai/gpt-5.4", "api_base": "https://neighbour.example/v1"}, + "model_info": {"id": "neighbour-1"}, + }, + { + "model_name": "openai/*", + "litellm_params": {"model": "openai/*", "api_base": "https://wildcard.example/v1"}, + "model_info": {"id": "wildcard-1"}, + }, + { + "model_name": "openai/*", + "litellm_params": {"model": "openai/*"}, + "model_info": {"id": "wildcard-2"}, + }, + { + "model_name": "tier", + "litellm_params": {"model": "openai/gpt-5.4", "api_base": "https://tier.example/v1"}, + "model_info": {"id": "tier-1"}, + }, + { + "model_name": "friendly", + "litellm_params": {"model": "anthropic/claude-friendly"}, + "model_info": {"id": "friendly-1"}, + }, + ], + ) + for model_id in cooled: + router.cooldown_cache.add_deployment_to_cooldown( + model_id=model_id, + original_exception=litellm.RateLimitError("scripted 429", "openai", "gpt-5.4"), + exception_status=429, + cooldown_time=60, + ) + return router + + +def _team_request(team_id: str | None) -> dict[str, object]: + return {"metadata": {"user_api_key_team_id": team_id} if team_id else {}} + + +def _select(router: Router, entrypoint: str, model: str, request_kwargs: dict[str, object]) -> object: + match entrypoint: + case "async": + return asyncio.run( + router.async_get_available_deployment( + model=model, messages=REFUSAL_MESSAGES, request_kwargs=request_kwargs + ) + ) + case "async-pass-through": + return asyncio.run( + router.async_get_available_deployment_for_pass_through(model=model, request_kwargs=request_kwargs) + ) + case "pass-through": + return router.get_available_deployment_for_pass_through(model=model, request_kwargs=request_kwargs) + case "pass-through-chat": + return router.get_available_deployment_for_pass_through( + model=model, messages=REFUSAL_MESSAGES, request_kwargs=request_kwargs + ) + case _: + return router.get_available_deployment( + model=model, messages=REFUSAL_MESSAGES, request_kwargs=request_kwargs + ) + + +ALL_COOLED: Final = (FIRST_REFUSED, SECOND_REFUSED, THIRD_REFUSED) +STRATEGY_COOLED: Final = ("refused-1", "refused-team-b", "neighbour-1") +WILDCARD_COOLED: Final = (FIRST_WILDCARD, SECOND_WILDCARD) + + +@pytest.mark.parametrize( + ("routing_strategy", "cooled", "entrypoint", "model", "team_id", "expected"), + ( + ("simple-shuffle", EVERY_DEPLOYMENT, "sync", "refused", "team-a", ALL_COOLED), + ("simple-shuffle", EVERY_DEPLOYMENT, "async", "refused", "team-a", ALL_COOLED), + ("simple-shuffle", EVERY_DEPLOYMENT, "pass-through", "refused", "team-a", (FIRST_REFUSED, SECOND_REFUSED)), + ("simple-shuffle", EVERY_DEPLOYMENT, "async-pass-through", "refused", "team-a", ALL_COOLED), + ("simple-shuffle", EVERY_DEPLOYMENT, "sync", "refused", "team-b", (*ALL_COOLED, OTHER_TEAMS_REFUSED)), + ("simple-shuffle", EVERY_DEPLOYMENT, "async", "refused", None, ALL_COOLED), + ("cost-based-routing", STRATEGY_COOLED, "sync", "refused", "team-a", None), + ("cost-based-routing", STRATEGY_COOLED, "async", "refused", "team-a", None), + ("cost-based-routing", STRATEGY_COOLED, "pass-through", "refused", "team-a", None), + ("simple-shuffle", EVERY_DEPLOYMENT, "sync", "openai/gpt-5.4-mini", "team-a", WILDCARD_COOLED), + ("simple-shuffle", EVERY_DEPLOYMENT, "async", "openai/gpt-5.4-mini", "team-a", WILDCARD_COOLED), + ("simple-shuffle", EVERY_DEPLOYMENT, "sync", "entry", "team-a", (ALIAS_TARGET,)), + ("simple-shuffle", EVERY_DEPLOYMENT, "async", "entry", "team-a", (ALIAS_TARGET,)), + ("simple-shuffle", EVERY_DEPLOYMENT, "sync", "anthropic/claude-friendly", "team-a", (PROVIDER_MODEL,)), + ("simple-shuffle", EVERY_DEPLOYMENT, "async", "anthropic/claude-friendly", "team-a", (PROVIDER_MODEL,)), + ), + ids=( + "all-cooled-sync", + "all-cooled-async", + "all-cooled-pass-through", + "all-cooled-async-pass-through", + "all-cooled-owning-team", + "all-cooled-async-no-team", + "strategy-refused-sync", + "strategy-refused-async", + "strategy-refused-pass-through", + "wildcard-sync", + "wildcard-async", + "alias-chain-sync", + "alias-chain-async", + "provider-model-sync", + "provider-model-async", + ), +) +def test_no_deployment_error_names_the_cooled_deployments_the_request_could_route_to( + routing_strategy: str, + cooled: tuple[str, ...], + entrypoint: str, + model: str, + team_id: str | None, + expected: tuple[CooldownDeploymentInfo, ...] | None, +) -> None: + router: Final = _cooled_router(routing_strategy, cooled) + with pytest.raises(RouterRateLimitError) as refusal: + _select(router, entrypoint, model, _team_request(team_id)) + assert refusal.value.cooldown_deployments == expected, refusal.value.cooldown_list + + +@pytest.mark.parametrize("entrypoint", ("sync", "async")) +def test_no_deployment_error_names_only_the_cooled_part_of_a_route_emptied_by_other_filters(entrypoint: str) -> None: + router: Final = _cooled_router("simple-shuffle", STRATEGY_COOLED) + request_kwargs: Final = {**_team_request("team-a"), "_excluded_deployment_ids": ["refused-2", "refused-3"]} + with pytest.raises(RouterRateLimitError) as refusal: + _select(router, entrypoint, "refused", request_kwargs) + assert refusal.value.cooldown_deployments == (FIRST_REFUSED,), refusal.value.cooldown_list + assert "sk-scripted-secret" not in repr(refusal.value.cooldown_deployments) + + +TENANT_A: Final = CooldownDeploymentInfo(model_id="tenant-a-1", api_base="https://tenant-a.example/v1") +TENANT_B: Final = CooldownDeploymentInfo(model_id="tenant-b-1", api_base="https://tenant-b.example/v1") +SHARED: Final = CooldownDeploymentInfo(model_id="shared-1", api_base="https://shared.example/v1") +EVERY_TENANT: Final = (TENANT_A, TENANT_B, SHARED) +TENANT_ROUTES: Final = ((TENANT_A, "tenant-a", "eu"), (TENANT_B, "tenant-b", "us"), (SHARED, "default", "eu")) + + +def _tenant_router(cooled: tuple[CooldownDeploymentInfo, ...], **router_settings: bool) -> Router: + router: Final = Router( + model_list=[ + { + "model_name": "tenants", + "litellm_params": { + "model": "openai/gpt-5.4", + "api_base": deployment["api_base"], + "tags": [tag], + "region_name": region, + "use_in_pass_through": True, + }, + "model_info": {"id": deployment["model_id"]}, + } + for deployment, tag, region in TENANT_ROUTES + ], + **router_settings, + ) + for deployment in cooled: + router.cooldown_cache.add_deployment_to_cooldown( + model_id=deployment["model_id"], + original_exception=litellm.RateLimitError("scripted 429", "openai", "gpt-5.4"), + exception_status=429, + cooldown_time=60, + ) + return router + + +TAGS_ON: Final = {"enable_tag_filtering": True} +REGION_CHECKS_ON: Final = {"enable_pre_call_checks": True} +BOTH_ON: Final = {**TAGS_ON, **REGION_CHECKS_ON} + + +@pytest.mark.parametrize( + ("router_settings", "entrypoint", "request_kwargs", "expected"), + ( + (TAGS_ON, "async", {"metadata": {"tags": ["tenant-a"]}}, (TENANT_A,)), + (TAGS_ON, "async", {"metadata": {"tags": ["tenant-b"]}}, (TENANT_B,)), + (TAGS_ON, "async", {"metadata": {}}, EVERY_TENANT), + (TAGS_ON, "async", {}, EVERY_TENANT), + (TAGS_ON, "async", {"metadata": "unparsed"}, ()), + (TAGS_ON, "async", {"litellm_metadata": {"tags": ["tenant-a"]}}, (TENANT_A,)), + (TAGS_ON, "async", {"metadata": {"tags": ["!tenant-a", "&tenant-b"]}}, (TENANT_B,)), + (TAGS_ON, "async", {"metadata": {"tags": ["!tenant-a"]}}, (TENANT_B, SHARED)), + (TAGS_ON, "async", {"metadata": {"tags": ["tenant-z"]}}, (SHARED,)), + (TAGS_ON, "async-pass-through", {"metadata": {"tags": ["tenant-a"]}}, (TENANT_A,)), + ({}, "async", {"metadata": {"tags": ["tenant-a"]}, "enable_tag_filtering": True}, (TENANT_A,)), + ({}, "async", {"metadata": {"tags": ["tenant-a"]}}, EVERY_TENANT), + (REGION_CHECKS_ON, "async", {"metadata": {}, "allowed_model_region": "eu"}, (TENANT_A, SHARED)), + (REGION_CHECKS_ON, "sync", {"metadata": {}, "allowed_model_region": "eu"}, (TENANT_A, SHARED)), + (REGION_CHECKS_ON, "pass-through-chat", {"metadata": {}, "allowed_model_region": "eu"}, (TENANT_A, SHARED)), + (REGION_CHECKS_ON, "pass-through", {"metadata": {}, "allowed_model_region": "eu"}, EVERY_TENANT), + ({}, "sync", {"metadata": {}, "allowed_model_region": "eu"}, EVERY_TENANT), + (REGION_CHECKS_ON, "sync", {"metadata": {}, "allowed_model_region": 1}, ()), + ( + {**TAGS_ON, **REGION_CHECKS_ON}, + "async", + {"metadata": {"tags": ["tenant-a"]}, "allowed_model_region": "us"}, + (), + ), + ), + ids=( + "tag-a", + "tag-b", + "untagged-falls-back-past-the-default", + "no-metadata", + "metadata-not-a-mapping", + "tag-litellm-metadata", + "tag-constraints", + "tag-exclusion-falls-back-in-tiers", + "unknown-tag-gets-default", + "tag-async-pass-through", + "tag-request-level-switch", + "tags-off", + "region-async", + "region-sync", + "region-pass-through", + "region-pass-through-without-input", + "region-without-pre-call-checks", + "region-not-a-string", + "region-and-tag-allow-none", + ), +) +def test_no_deployment_error_names_only_the_cooled_deployments_tag_and_region_routing_allow( + router_settings: dict[str, bool], + entrypoint: str, + request_kwargs: dict[str, object], + expected: tuple[CooldownDeploymentInfo, ...], +) -> None: + router: Final = _tenant_router(EVERY_TENANT, **router_settings) + with pytest.raises(RouterRateLimitError) as refusal: + _select(router, entrypoint, "tenants", request_kwargs) + assert refusal.value.cooldown_deployments == expected, refusal.value.cooldown_list + + +def test_naming_tag_routed_deployments_leaves_the_request_metadata_untouched() -> None: + metadata: Final[dict[str, object]] = {"tags": ["tenant-a"]} + with pytest.raises(RouterRateLimitError) as refusal: + _select(_tenant_router(EVERY_TENANT, **TAGS_ON), "async", "tenants", {"metadata": metadata}) + assert refusal.value.cooldown_deployments == (TENANT_A,) + assert metadata == {"tags": ["tenant-a"]} + + +@pytest.mark.parametrize( + ("has_request_input", "expected"), ((True, (TENANT_A, SHARED)), (False, EVERY_TENANT)), ids=("chat", "no-input") +) +def test_region_routable_deployments_follow_the_pre_call_region_check( + has_request_input: bool, expected: tuple[CooldownDeploymentInfo, ...] +) -> None: + router: Final = _tenant_router((), **REGION_CHECKS_ON) + kept: Final = router._region_routable_deployments( + router.get_model_list(model_name="tenants") or [], {"allowed_model_region": "eu"}, has_request_input + ) + assert router._deployment_ids(kept) == frozenset(deployment["model_id"] for deployment in expected) + + +def test_routable_cooled_deployment_ids_skip_healthy_deployments_and_peel_tag_fallbacks() -> None: + router: Final = _tenant_router((TENANT_A, SHARED), **TAGS_ON) + routable: Final = asyncio.run( + router._async_routable_cooled_deployment_ids( + model="tenants", + request_deployments=router.get_model_list(model_name="tenants") or [], + cooldown_deployments=(TENANT_A["model_id"], SHARED["model_id"]), + request_kwargs={"metadata": {}}, + has_request_input=True, + ) + ) + assert routable == frozenset((TENANT_A["model_id"], SHARED["model_id"])) + + +REACHABILITY_REQUESTS: Final = ( + {"metadata": {"tags": ["tenant-a"]}}, + {"metadata": {}}, + {"metadata": {"tags": ["!tenant-a"]}}, + {"metadata": {"tags": ["tenant-z"]}}, + {"metadata": {"tags": ["!tenant-a", "&tenant-b"]}}, + {"metadata": {}, "allowed_model_region": "eu"}, + {"metadata": {"tags": ["tenant-a"]}, "allowed_model_region": "us"}, +) +HEALTH_STATES: Final = tuple( + itertools.chain.from_iterable( + itertools.combinations(EVERY_TENANT, size) for size in range(1, len(EVERY_TENANT) + 1) + ) +) + + +def _routable_while_up(up: tuple[CooldownDeploymentInfo, ...], request_kwargs: dict[str, object]) -> set[str]: + router: Final = _tenant_router(tuple(deployment for deployment in EVERY_TENANT if deployment not in up), **BOTH_ON) + try: + pool: Final = asyncio.run( + router.async_get_healthy_deployments( + model="tenants", request_kwargs=copy.deepcopy(request_kwargs), messages=REFUSAL_MESSAGES + ) + ) + except ValueError: + return set() + return {str(deployment["model_info"]["id"]) for deployment in pool} + + +@pytest.mark.parametrize("request_kwargs", REACHABILITY_REQUESTS) +def test_the_refusal_names_each_cooled_deployment_some_cooldown_state_routes_the_request_to( + request_kwargs: dict[str, object], +) -> None: + reachable: Final = set().union(*(_routable_while_up(up, request_kwargs) for up in HEALTH_STATES)) + with pytest.raises(RouterRateLimitError) as refusal: + _select(_tenant_router(EVERY_TENANT, **BOTH_ON), "async", "tenants", copy.deepcopy(request_kwargs)) + assert {deployment["model_id"] for deployment in refusal.value.cooldown_deployments or ()} == reachable From 89a194bbd0fd25c57df5456ccdc223978e4ac084 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Mon, 19 Oct 2026 13:20:50 -0700 Subject: [PATCH 17/20] test(integration): hold the upstream until the disconnect is settled before reading its spend row (#47303) test_client_disconnect_before_any_content_sends_no_response_scan released the gated upstream right after the client closed, so the upstream's next chunk raced the disconnect into the proxy. When the chunk won, the proxy had " tail" on record for the client and the cut-short scan recorded a post_call success on the spend row, which the test asserts is absent (CircleCI integration-extensions job 2263560 on 68db022d5704: `AssertionError: (('success',),)`). The test now reads the spend row first, which is written once the proxy has settled the disconnect, and releases the upstream after it. The proxy's behavior is unchanged; only the test's ordering is. Co-authored-by: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> --- tests/integration/observability/test_guardrail_effects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/observability/test_guardrail_effects.py b/tests/integration/observability/test_guardrail_effects.py index b3481057b3b..086baac6ee5 100644 --- a/tests/integration/observability/test_guardrail_effects.py +++ b/tests/integration/observability/test_guardrail_effects.py @@ -2836,9 +2836,9 @@ def test_client_disconnect_before_any_content_sends_no_response_scan(gateway: Ga with _disconnect_rig(gateway, tmp_path, shape="empty") as rig: try: _stream_and_close(rig, "/v1/chat/completions", _chat_body(rig, 0), "data: ") + rows: Final = _post_call_statuses(rig, rig.model) finally: rig.gate.set() - rows: Final = _post_call_statuses(rig, rig.model) assert rig.scans.response_scans(rig.secret()) == (), rig.scans.seen assert len(rows) == 1 and "success" not in rows[0], rows From 6e148255d921de26381aed7fa199af0e68bd5bb6 Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 13:30:25 -0700 Subject: [PATCH 18/20] chore(pricing): add meta sam-3.1 image segmentation price (#47395) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- .../model_prices_and_context_window_backup.json | 17 +++++++++++++++++ model_prices_and_context_window.json | 17 +++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 6e573402427..9d5e0aa8021 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -76135,5 +76135,22 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true + }, + "meta/sam-3.1": { + "input_cost_per_image": 0.0025, + "litellm_provider": "meta", + "mode": "responses", + "source": "https://dev.meta.ai/docs/pricing-rate-limits.md", + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "text" + ] } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 6e573402427..9d5e0aa8021 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -76135,5 +76135,22 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true + }, + "meta/sam-3.1": { + "input_cost_per_image": 0.0025, + "litellm_provider": "meta", + "mode": "responses", + "source": "https://dev.meta.ai/docs/pricing-rate-limits.md", + "supported_endpoints": [ + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image", + "video" + ], + "supported_output_modalities": [ + "text" + ] } } From 9bafafc2df3c8871bb30277f47d9fb6600e707e4 Mon Sep 17 00:00:00 2001 From: litellm-stampbot Date: Mon, 19 Oct 2026 13:35:04 -0700 Subject: [PATCH 19/20] feat(search): add DeepSeek as a /v1/search provider (#47401) * feat(search): add DeepSeek as a /v1/search provider Adds DeepSeekSearchConfig for POST https://api.deepseek.com/v1/tools/search, registers it under search_provider=deepseek, prices deepseek/search at 0.00075 per query and marks search support in provider_endpoints_support.json * fix(search): reject DeepSeek queries over the 128 character limit before the call * test(search): escape regex metacharacters in DeepSeek query limit match --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/deepseek/search/__init__.py | 7 + .../llms/deepseek/search/transformation.py | 172 ++++++++++++++++++ ...odel_prices_and_context_window_backup.json | 5 + litellm/types/utils.py | 1 + litellm/utils.py | 2 + model_prices_and_context_window.json | 5 + provider_endpoints_support.json | 3 +- tests/unit/llms/deepseek/search/__init__.py | 0 .../test_deepseek_search_transformation.py | 143 +++++++++++++++ 9 files changed, 337 insertions(+), 1 deletion(-) create mode 100644 litellm/llms/deepseek/search/__init__.py create mode 100644 litellm/llms/deepseek/search/transformation.py create mode 100644 tests/unit/llms/deepseek/search/__init__.py create mode 100644 tests/unit/llms/deepseek/search/test_deepseek_search_transformation.py diff --git a/litellm/llms/deepseek/search/__init__.py b/litellm/llms/deepseek/search/__init__.py new file mode 100644 index 00000000000..528dceb223f --- /dev/null +++ b/litellm/llms/deepseek/search/__init__.py @@ -0,0 +1,7 @@ +""" +DeepSeek Search API module. +""" + +from litellm.llms.deepseek.search.transformation import DeepSeekSearchConfig + +__all__ = ["DeepSeekSearchConfig"] diff --git a/litellm/llms/deepseek/search/transformation.py b/litellm/llms/deepseek/search/transformation.py new file mode 100644 index 00000000000..b6bb1c343d8 --- /dev/null +++ b/litellm/llms/deepseek/search/transformation.py @@ -0,0 +1,172 @@ +""" +DeepSeek Search API. +Endpoint: POST https://api.deepseek.com/v1/tools/search +Docs: https://api-docs.deepseek.com/guides/search/ +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from typing import Final + +import httpx +from pydantic import ConfigDict, TypeAdapter, ValidationError + +from litellm.exceptions import BadRequestError +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.search.transformation import ( + BaseSearchConfig, + SearchResponse, + SearchResult, +) +from litellm.secret_managers.main import get_secret_str + +_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) +_JSON_OBJECTS: Final = TypeAdapter(Sequence[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True)) +_STR_LIST: Final = TypeAdapter(list[str]) + +DEEPSEEK_SEARCH_MAX_COUNT: Final = 50 +DEEPSEEK_SEARCH_MAX_QUERY_CHARS: Final = 128 +_DEEPSEEK_SEARCH_PATH: Final = "/v1/tools/search" +# DeepSeek chat defaults DEEPSEEK_API_BASE to the /beta prefix; the search route lives at the root. +_CHAT_ONLY_BASE_SUFFIXES: Final = ("/beta", "/v1") + + +class DeepSeekSearchConfig(BaseSearchConfig): + DEEPSEEK_API_BASE = "https://api.deepseek.com" + + @staticmethod + def ui_friendly_name() -> str: + return "DeepSeek" + + def validate_environment( + self, + headers: dict[str, str], # mutable-ok: BaseSearchConfig.validate_environment signature + api_key: str | None = None, + api_base: str | None = None, + **kwargs: object, # kwargs-ok: BaseSearchConfig.validate_environment signature + ) -> dict[str, str]: # mutable-ok: the http handler passes this straight to httpx as headers + resolved_key: Final = self.resolve_server_api_key( + caller_api_key=api_key, + caller_api_base=api_base, + key_env_vars=("DEEPSEEK_API_KEY",), + base_env_var="DEEPSEEK_API_BASE", + default_api_base=self.DEEPSEEK_API_BASE, + ) + if not resolved_key: + raise ValueError("DEEPSEEK_API_KEY is not set. Set `DEEPSEEK_API_KEY` environment variable.") + return {**headers, "Authorization": f"Bearer {resolved_key}", "Content-Type": "application/json"} + + def get_complete_url( + self, + api_base: str | None, + optional_params: dict[str, object], # mutable-ok: BaseSearchConfig.get_complete_url signature + data: dict[str, object] | list[dict[str, object]] | None = None, # mutable-ok: base signature + **kwargs: object, # kwargs-ok: BaseSearchConfig.get_complete_url signature + ) -> str: + resolved_base: Final = api_base or get_secret_str("DEEPSEEK_API_BASE") or self.DEEPSEEK_API_BASE + clean_base: Final = resolved_base.rstrip("/") + if clean_base.endswith(_DEEPSEEK_SEARCH_PATH): + return clean_base + return f"{_strip_chat_suffix(clean_base)}{_DEEPSEEK_SEARCH_PATH}" + + def transform_search_request( + self, + query: str | list[str], # mutable-ok: BaseSearchConfig.transform_search_request signature + optional_params: dict[str, object], # mutable-ok: base signature + **kwargs: object, # kwargs-ok: BaseSearchConfig.transform_search_request signature + ) -> dict[str, object]: # mutable-ok: the http handler sends this as the JSON body + """ + Unified params -> DeepSeek body: ``query`` (list joined with spaces, ``search_domain_filter`` + folded in as ``site:`` operators, the whole string capped at 128 characters by DeepSeek), + ``max_results`` -> ``count`` clamped to [1, 50]. + ``country`` and ``max_tokens_per_page`` have no DeepSeek equivalent and are dropped. + Everything outside the unified spec (``need_content``, ``ori_question``, ``localization``) + is forwarded as-is. + """ + joined_query: Final = " ".join(query) if isinstance(query, list) else query + domains: Final = _domain_filter(optional_params.get("search_domain_filter")) + count: Final = _count(optional_params.get("max_results")) + unified: Final = self.get_supported_perplexity_optional_params() + passthrough: Final = {k: v for k, v in optional_params.items() if k not in unified} + return { + **passthrough, + "query": _checked_query(_with_site_filters(joined_query, domains), domains), + **({"count": count} if count is not None else {}), + } + + def transform_search_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + **kwargs: object, # kwargs-ok: BaseSearchConfig.transform_search_response signature + ) -> SearchResponse: + response_json: Final = _JSON_OBJECT.validate_python(raw_response.json()) + raw_results: Final = _JSON_OBJECTS.validate_python(response_json.get("results", ())) + return SearchResponse( + results=[_to_search_result(item) for item in raw_results], + object="search", + ) + + +def _strip_chat_suffix(base: str) -> str: + return next( + (base.removesuffix(suffix) for suffix in _CHAT_ONLY_BASE_SUFFIXES if base.endswith(suffix)), + base, + ) + + +def _domain_filter(raw: object) -> tuple[str, ...]: + try: + return tuple(_STR_LIST.validate_python(raw)) + except ValidationError: + return () + + +def _with_site_filters(query: str, domains: tuple[str, ...]) -> str: + if not domains: + return query + site_clause: Final = " OR ".join(f"site:{domain}" for domain in domains) + return f"{query} {site_clause}" + + +def _checked_query(query: str, domains: tuple[str, ...]) -> str: + if len(query) <= DEEPSEEK_SEARCH_MAX_QUERY_CHARS: + return query + filter_note: Final = ( + f" ({len(domains)} search_domain_filter entries are sent as site: operators inside the query)" + if domains + else "" + ) + raise BadRequestError( + message=( + f"DeepSeek Search accepts a query of at most {DEEPSEEK_SEARCH_MAX_QUERY_CHARS} characters, " + f"got {len(query)}{filter_note}. Shorten the query or the domain filter." + ), + model="deepseek/search", + llm_provider="deepseek", + ) + + +def _count(raw: object) -> int | None: + if isinstance(raw, bool) or not isinstance(raw, int): + return None + return max(1, min(raw, DEEPSEEK_SEARCH_MAX_COUNT)) + + +def _to_search_result(item: Mapping[str, object]) -> SearchResult: + date: Final = item.get("date") + return SearchResult.model_validate( + { + **item, + "title": _as_str(item.get("title")), + "url": _as_str(item.get("url")), + "snippet": _as_str(item.get("snippet")), + "date": date if isinstance(date, str) else None, + "last_updated": None, + } + ) + + +def _as_str(value: object) -> str: + return value if isinstance(value, str) else "" diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 9d5e0aa8021..f5eec206e04 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -24366,6 +24366,11 @@ "notes": "Grounding with Bing Search (G1 SKU): $14 per 1,000 transactions. Tokens for the Foundry model deployment that runs the grounded search are billed separately on that deployment." } }, + "deepseek/search": { + "input_cost_per_query": 0.00075, + "litellm_provider": "deepseek", + "mode": "search" + }, "tinyfish/search": { "input_cost_per_query": 0.0, "litellm_provider": "tinyfish", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 2e05107f82d..41581d702f3 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4469,6 +4469,7 @@ class SearchProviders(str, Enum): AGENTCORE = "agentcore" NIMBLE = "nimble" BING_GROUNDING = "bing_grounding" + DEEPSEEK = "deepseek" # Create a set of all search provider values for quick lookup diff --git a/litellm/utils.py b/litellm/utils.py index ff2c9012ec8..cf3b8756e2f 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -10297,6 +10297,7 @@ class ProviderConfigManager: from litellm.llms.bedrock.search.transformation import AgentCoreSearchConfig from litellm.llms.brave.search.transformation import BraveSearchConfig from litellm.llms.dataforseo.search.transformation import DataForSEOSearchConfig + from litellm.llms.deepseek.search.transformation import DeepSeekSearchConfig from litellm.llms.duckduckgo.search.transformation import DuckDuckGoSearchConfig from litellm.llms.exa_ai.search.transformation import ExaAISearchConfig from litellm.llms.fastcrw.search.transformation import FastCRWSearchConfig @@ -10336,6 +10337,7 @@ class ProviderConfigManager: SearchProviders.AGENTCORE: AgentCoreSearchConfig, SearchProviders.NIMBLE: NimbleSearchConfig, SearchProviders.BING_GROUNDING: BingGroundingSearchConfig, + SearchProviders.DEEPSEEK: DeepSeekSearchConfig, } config_class: Final = PROVIDER_TO_CONFIG_MAP.get(provider, None) if config_class is None: diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 9d5e0aa8021..f5eec206e04 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -24366,6 +24366,11 @@ "notes": "Grounding with Bing Search (G1 SKU): $14 per 1,000 transactions. Tokens for the Foundry model deployment that runs the grounded search are billed separately on that deployment." } }, + "deepseek/search": { + "input_cost_per_query": 0.00075, + "litellm_provider": "deepseek", + "mode": "search" + }, "tinyfish/search": { "input_cost_per_query": 0.0, "litellm_provider": "tinyfish", diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index c2cec2c5490..d65da3f41fd 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -923,7 +923,8 @@ "batches": false, "rerank": false, "a2a": true, - "interactions": true + "interactions": true, + "search": true } }, "edenai": { diff --git a/tests/unit/llms/deepseek/search/__init__.py b/tests/unit/llms/deepseek/search/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/deepseek/search/test_deepseek_search_transformation.py b/tests/unit/llms/deepseek/search/test_deepseek_search_transformation.py new file mode 100644 index 00000000000..7539876bd37 --- /dev/null +++ b/tests/unit/llms/deepseek/search/test_deepseek_search_transformation.py @@ -0,0 +1,143 @@ +""" +Tests for the DeepSeek Search provider (POST https://api.deepseek.com/v1/tools/search). +""" + +from typing import Final +import re +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +import litellm +from litellm.llms.deepseek.search.transformation import DeepSeekSearchConfig + + +@pytest.mark.asyncio +async def test_asearch_sends_deepseek_body_and_maps_results(monkeypatch): + monkeypatch.setenv("DEEPSEEK_API_KEY", "test-deepseek-key") + monkeypatch.delenv("DEEPSEEK_API_BASE", raising=False) + mock_response: Final = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "results": [ + { + "rerank_score": 2.4, + "title": "DeepSeek V4 model released", + "url": "https://example.com/deepseek-v4", + "snippet": "short snippet", + "summary": "long summary for the model", + "content": "full page content", + "site_name": None, + "date": "2026-06-01T00:00:00+00:00", + "authority": 3, + }, + {"title": None, "url": "https://example.com/degraded", "snippet": None, "date": None}, + ] + } + + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_post: + response: Final = await litellm.asearch( + query=["DeepSeek V4", "release"], + search_provider="deepseek", + max_results=5, + search_domain_filter=["github.com", "huggingface.co"], + need_content=True, + ) + + assert mock_post.call_count == 1 + call: Final = mock_post.call_args + assert call.kwargs["url"] == "https://api.deepseek.com/v1/tools/search" + assert call.kwargs["headers"]["Authorization"] == "Bearer test-deepseek-key" + assert call.kwargs["headers"]["Content-Type"] == "application/json" + assert call.kwargs["json"] == { + "query": "DeepSeek V4 release site:github.com OR site:huggingface.co", + "count": 5, + "need_content": True, + } + + assert response.object == "search" + assert len(response.results) == 2 + first: Final = response.results[0] + assert (first.title, first.url, first.snippet) == ( + "DeepSeek V4 model released", + "https://example.com/deepseek-v4", + "short snippet", + ) + assert first.date == "2026-06-01T00:00:00+00:00" + assert first.last_updated is None + assert first.model_extra["summary"] == "long summary for the model" + assert first.model_extra["content"] == "full page content" + assert first.model_extra["rerank_score"] == 2.4 + degraded: Final = response.results[1] + assert (degraded.title, degraded.snippet, degraded.date) == ("", "", None) + + +@pytest.mark.parametrize( + ("max_results", "expected_count"), + [(0, 1), (3, 3), (50, 50), (500, 50)], +) +def test_max_results_clamps_to_deepseek_count_range(max_results, expected_count): + body: Final = DeepSeekSearchConfig().transform_search_request( + query="litellm", optional_params={"max_results": max_results} + ) + assert body == {"query": "litellm", "count": expected_count} + + +def test_query_over_deepseek_limit_fails_before_the_call(): + with pytest.raises( + litellm.BadRequestError, match=re.escape("at most 128 characters, got 144 (3 search_domain_filter") + ): + DeepSeekSearchConfig().transform_search_request( + query="litellm proxy search endpoint configuration guide for enterprise deployments with", + optional_params={"search_domain_filter": ["docs.litellm.ai", "github.com", "huggingface.co"]}, + ) + + +def test_unified_params_without_deepseek_equivalent_are_not_sent(): + body: Final = DeepSeekSearchConfig().transform_search_request( + query="litellm", + optional_params={"country": "US", "max_tokens_per_page": 1024, "ori_question": "what is litellm"}, + ) + assert body == {"query": "litellm", "ori_question": "what is litellm"} + + +@pytest.mark.parametrize( + "api_base", + [ + None, + "https://api.deepseek.com", + "https://api.deepseek.com/", + "https://api.deepseek.com/beta", + "https://api.deepseek.com/v1", + "https://api.deepseek.com/v1/tools/search", + ], +) +def test_complete_url_resolves_search_route_from_chat_style_api_base(api_base, monkeypatch): + monkeypatch.delenv("DEEPSEEK_API_BASE", raising=False) + url: Final = DeepSeekSearchConfig().get_complete_url(api_base=api_base, optional_params={}) + assert url == "https://api.deepseek.com/v1/tools/search" + + +def test_missing_api_key_raises(monkeypatch): + monkeypatch.delenv("DEEPSEEK_API_KEY", raising=False) + with pytest.raises(ValueError, match="DEEPSEEK_API_KEY"): + DeepSeekSearchConfig().validate_environment(headers={}) + + +def test_search_cost_comes_from_cost_map(monkeypatch): + monkeypatch.setenv("DEEPSEEK_API_KEY", "test-deepseek-key") + priced: Final[float] = litellm.model_cost["deepseek/search"]["input_cost_per_query"] + assert priced > 0 + mock_response: Final = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = {"results": [{"title": "t", "url": "https://e.com", "snippet": "s"}]} + with patch( + "litellm.llms.custom_httpx.http_handler.HTTPHandler.post", + return_value=mock_response, + ): + response: Final = litellm.search(query="litellm", search_provider="deepseek") + assert response._hidden_params["response_cost"] == pytest.approx(priced) From c0d88b46c9b5a462405e557782341d3d07663aa9 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:35:27 +0000 Subject: [PATCH 20/20] fix(azure): cache the remaining per-request Entra credentials and keep token refresh off the event loop (#47348) * fix(azure): cache the remaining per-request Entra credentials and keep token refresh off the event loop Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(realtime): await the now async azure health-check auth header helper Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(azure): await async azure_ad_token_provider results in the threaded wrapper * fix(azure): keep transport params when validating async Azure client params on pydantic 2.11 * test(azure): assert the token and validation thread hops by thread identity instead of sleeps * fix(azure): give the OIDC token exchange its own short-lived HTTP client The exchange now runs from worker threads, so it no longer shares litellm.module_level_client's connection pool with other threads * test(azure): check the OIDC exchange closes its client through an in-process transport * refactor(responses): merge caller extra_headers into a new dict instead of updating validated headers in place --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/azure/common_utils.py | 121 +++++++++++----- litellm/llms/azure/search/transformation.py | 5 +- litellm/llms/custom_httpx/llm_http_handler.py | 71 ++++++---- litellm/realtime_api/main.py | 23 ++- litellm/secret_managers/main.py | 9 +- .../test_health_check_helpers.py | 4 +- ...st_bing_grounding_search_transformation.py | 22 +++ .../llms/azure/test_azure_common_utils.py | 133 +++++++++++++++++- .../custom_httpx/test_llm_http_handler.py | 36 +++++ 9 files changed, 348 insertions(+), 76 deletions(-) diff --git a/litellm/llms/azure/common_utils.py b/litellm/llms/azure/common_utils.py index f9f636244f4..52e3cdf380e 100644 --- a/litellm/llms/azure/common_utils.py +++ b/litellm/llms/azure/common_utils.py @@ -3,13 +3,15 @@ import hashlib import json import os import re -from collections.abc import Callable, Mapping +from collections.abc import Awaitable, Callable, Mapping +from contextlib import closing from functools import lru_cache from types import MappingProxyType from typing import Final, Literal, NamedTuple, cast import httpx from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI +from pydantic import ConfigDict, TypeAdapter, with_config from typing_extensions import ReadOnly, TypedDict import litellm @@ -17,6 +19,7 @@ from litellm._logging import verbose_logger from litellm.caching.caching import DualCache from litellm.constants import DEFAULT_MAX_RETRIES from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.llms.openai.common_utils import BaseOpenAILLM, OpenAIAsyncHTTPClient, OpenAIHTTPClient from litellm.secret_managers.get_azure_ad_token_provider import ( get_azure_ad_token_provider, @@ -38,6 +41,16 @@ class _AzureV1ClientParams(TypedDict, total=False, extra_items=object): base_url: ReadOnly[str] +@with_config(ConfigDict(extra="allow")) +class _AsyncAzureClientParams(TypedDict, total=False, extra_items=object): + api_key: ReadOnly[str | None] + azure_ad_token: ReadOnly[str | None] + azure_ad_token_provider: ReadOnly[Callable[[], str | Awaitable[str]] | None] + + +_ASYNC_AZURE_CLIENT_PARAMS_ADAPTER: Final = TypeAdapter(_AsyncAzureClientParams) + + class _AzureGatewayClientParams(TypedDict, total=False, extra_items=object): api_version: ReadOnly[str] base_url: ReadOnly[str] @@ -98,11 +111,42 @@ def _cached_entra_id_token_provider( return get_bearer_token_provider(ClientSecretCredential(tenant_id, client_id, client_secret), scope) +@lru_cache(maxsize=128) +def _cached_username_password_token_provider( + client_id: str, + azure_username: str, + azure_password: str, + scope: str, +) -> Callable[[], str]: + from azure.identity import UsernamePasswordCredential, get_bearer_token_provider + + credential: Final = UsernamePasswordCredential( + client_id=client_id, + username=azure_username, + password=azure_password, + ) + return get_bearer_token_provider(credential, scope) + + @lru_cache(maxsize=128) def _cached_azure_ad_token_refresh_provider(scope: str) -> Callable[[], str]: return get_azure_ad_token_provider(azure_scope=scope) +def cached_azure_ad_token_refresh_provider(scope: str) -> Callable[[], str]: + return _cached_azure_ad_token_refresh_provider(scope) + + +def _async_azure_ad_token_provider( + provider: Callable[[], str | Awaitable[str]], +) -> Callable[[], Awaitable[str]]: + async def async_provider() -> str: + token: Final = await asyncio.to_thread(provider) + return token if isinstance(token, str) else await token + + return async_provider + + def get_azure_ad_token_from_entra_id( tenant_id: str, client_id: str, @@ -177,22 +221,19 @@ def get_azure_ad_token_from_username_password( Returns: callable that returns a bearer token. """ - from azure.identity import UsernamePasswordCredential, get_bearer_token_provider - verbose_logger.debug( "client_id=%s, azure_username=[set=%s], azure_password=[set=%s]", client_id, azure_username is not None, azure_password is not None, ) - credential: Final = UsernamePasswordCredential( + token_provider: Final = _cached_username_password_token_provider( client_id=client_id, - username=azure_username, - password=azure_password, + azure_username=azure_username, + azure_password=azure_password, + scope=scope, ) - token_provider: Final = get_bearer_token_provider(credential, scope) - verbose_logger.debug("token_provider %s", token_provider) return token_provider @@ -248,18 +289,17 @@ def get_azure_ad_token_from_oidc( if azure_ad_token_access_token is not None: return azure_ad_token_access_token - client: Final = litellm.module_level_client - - req_token: Final = client.post( - f"{azure_authority_host}/{azure_tenant_id}/oauth2/v2.0/token", - data={ - "client_id": azure_client_id, - "grant_type": "client_credentials", - "scope": scope, - "client_assertion_type": "urn:ietf:params:oauth:client-assertion-type:jwt-bearer", - "client_assertion": oidc_token, - }, - ) + with closing(HTTPHandler(timeout=litellm.request_timeout)) as client: + req_token: Final = client.post( + f"{azure_authority_host}/{azure_tenant_id}/oauth2/v2.0/token", + data={ + "client_id": azure_client_id, + "grant_type": "client_credentials", + "scope": scope, + "client_assertion_type": "urn:ietf:params:oauth:client-assertion-type:jwt-bearer", + "client_assertion": oidc_token, + }, + ) if req_token.status_code != 200: raise AzureOpenAIError( @@ -560,22 +600,19 @@ class BaseAzureLLM(BaseOpenAILLM): # on every request (via `_refresh_api_key`), so passing # `azure_ad_token_provider` directly preserves Azure AD token refresh # behavior that the regular AzureOpenAI client provides. - v1_api_key: str | Callable[[], object] | None = ( - azure_client_params.get("api_key") - or azure_client_params.get("azure_ad_token_provider") - or azure_client_params.get("azure_ad_token") + v1_client_params: Final[_AsyncAzureClientParams] = _ASYNC_AZURE_CLIENT_PARAMS_ADAPTER.validate_python( + azure_client_params + ) + v1_token_provider: Final = ( + v1_client_params.get("api_key") + or v1_client_params.get("azure_ad_token_provider") + or v1_client_params.get("azure_ad_token") + ) + v1_api_key: Final[str | Callable[[], str | Awaitable[str]] | None] = ( + _async_azure_ad_token_provider(v1_token_provider) + if _is_async is True and callable(v1_token_provider) + else v1_token_provider ) - if _is_async is True and callable(v1_api_key): - # AsyncOpenAI expects an async provider; wrap the sync provider - # returned by azure-identity. Offload to a thread so a token - # refresh (blocking HTTP call to AAD on cache miss) does not - # stall the event loop. - _sync_provider: Final = v1_api_key - - async def _async_v1_api_key() -> str: - return await asyncio.to_thread(_sync_provider) - - v1_api_key = _async_v1_api_key v1_params: Final[_AzureV1ClientParams] = { "api_key": v1_api_key, @@ -597,7 +634,19 @@ class BaseAzureLLM(BaseOpenAILLM): else: # Traditional Azure API uses AzureOpenAI client if _is_async is True: - openai_client = AsyncAzureOpenAI(**azure_client_params) + async_azure_client_params: Final[_AsyncAzureClientParams] = ( + _ASYNC_AZURE_CLIENT_PARAMS_ADAPTER.validate_python(azure_client_params) + ) + async_token_provider_candidate: Final = async_azure_client_params.get("azure_ad_token_provider") + async_azure_client_params_with_threaded_provider: Final[_AsyncAzureClientParams] = ( + { + **async_azure_client_params, + "azure_ad_token_provider": _async_azure_ad_token_provider(async_token_provider_candidate), + } + if async_token_provider_candidate is not None + else async_azure_client_params + ) + openai_client = AsyncAzureOpenAI(**async_azure_client_params_with_threaded_provider) else: openai_client = AzureOpenAI(**azure_client_params) diff --git a/litellm/llms/azure/search/transformation.py b/litellm/llms/azure/search/transformation.py index 4264dae018a..f2bb2ecc68c 100644 --- a/litellm/llms/azure/search/transformation.py +++ b/litellm/llms/azure/search/transformation.py @@ -35,6 +35,7 @@ from typing import TYPE_CHECKING, Final, Literal import httpx from pydantic import ConfigDict, ValidationError +from litellm.llms.azure.common_utils import cached_azure_ad_token_refresh_provider from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.search.transformation import ( BaseSearchConfig, @@ -263,9 +264,7 @@ def _search_tool(optional_params: Mapping[str, object]) -> _BingGroundingTool | def _default_entra_token_minter() -> str: - from litellm.secret_managers.get_azure_ad_token_provider import get_azure_ad_token_provider - - return get_azure_ad_token_provider(azure_scope=ENTRA_SCOPE)() + return cached_azure_ad_token_refresh_provider(ENTRA_SCOPE)() class BingGroundingSearchConfig(BaseSearchConfig): diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index b03b8050299..48858dfa441 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -2953,14 +2953,15 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client - headers = responses_api_provider_config.validate_environment( - headers=response_api_optional_request_params.get("extra_headers", {}) or {}, - model=model, - litellm_params=litellm_params, - ) - - if extra_headers: - headers.update(extra_headers) + validated_headers: Final = { + **await asyncio.to_thread( + responses_api_provider_config.validate_environment, + headers=response_api_optional_request_params.get("extra_headers", {}) or {}, + model=model, + litellm_params=litellm_params, + ), + **(extra_headers or {}), + } # Check if streaming is requested stream = response_api_optional_request_params.get("stream", False) @@ -2976,7 +2977,7 @@ class BaseLLMHTTPHandler: input=input, response_api_optional_request_params=response_api_optional_request_params, litellm_params=litellm_params, - headers=headers, + headers=validated_headers, ) data = BaseResponsesAPIConfig.normalize_responses_api_request_dict(data) @@ -3011,7 +3012,7 @@ class BaseLLMHTTPHandler: headers, signed_body = await sign_request_off_loop_if_aws( responses_api_provider_config, responses_api_provider_config.sign_request, - headers=headers, + headers=validated_headers, optional_params=dict(litellm_params), request_data=data, api_base=api_base, @@ -3145,8 +3146,11 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client - headers: Final = responses_api_provider_config.validate_environment( - headers=extra_headers or {}, model="None", litellm_params=litellm_params + headers: Final = await asyncio.to_thread( + responses_api_provider_config.validate_environment, + headers=extra_headers or {}, + model="None", + litellm_params=litellm_params, ) if extra_headers: @@ -3399,8 +3403,11 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client - headers: Final = responses_api_provider_config.validate_environment( - headers=extra_headers or {}, model="None", litellm_params=litellm_params + headers: Final = await asyncio.to_thread( + responses_api_provider_config.validate_environment, + headers=extra_headers or {}, + model="None", + litellm_params=litellm_params, ) if extra_headers: @@ -3563,8 +3570,11 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client - headers: Final = responses_api_provider_config.validate_environment( - headers=extra_headers or {}, model="None", litellm_params=litellm_params + headers: Final = await asyncio.to_thread( + responses_api_provider_config.validate_environment, + headers=extra_headers or {}, + model="None", + litellm_params=litellm_params, ) if extra_headers: @@ -4620,8 +4630,11 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client - headers: Final = responses_api_provider_config.validate_environment( - headers=extra_headers or {}, model="None", litellm_params=litellm_params + headers: Final = await asyncio.to_thread( + responses_api_provider_config.validate_environment, + headers=extra_headers or {}, + model="None", + litellm_params=litellm_params, ) if extra_headers: @@ -4795,12 +4808,15 @@ class BaseLLMHTTPHandler: else: async_httpx_client = client - headers = responses_api_provider_config.validate_environment( - headers=extra_headers or {}, model=model, litellm_params=litellm_params - ) - - if extra_headers: - headers.update(extra_headers) + validated_headers: Final = { + **await asyncio.to_thread( + responses_api_provider_config.validate_environment, + headers=extra_headers or {}, + model=model, + litellm_params=litellm_params, + ), + **(extra_headers or {}), + } api_base: Final = responses_api_provider_config.get_complete_url( api_base=litellm_params.api_base, @@ -4816,14 +4832,14 @@ class BaseLLMHTTPHandler: response_api_optional_request_params=response_api_optional_request_params, api_base=api_base, litellm_params=litellm_params, - headers=headers, + headers=validated_headers, ) data = BaseResponsesAPIConfig.normalize_responses_api_request_dict(data) headers, signed_body = await sign_request_off_loop_if_aws( responses_api_provider_config, responses_api_provider_config.sign_request, - headers=headers, + headers=validated_headers, optional_params=dict(litellm_params), request_data=data, api_base=url, @@ -6991,7 +7007,8 @@ class BaseLLMHTTPHandler: **kwargs, } ) - headers: Final = responses_api_provider_config.validate_environment( + headers: Final = await asyncio.to_thread( + responses_api_provider_config.validate_environment, headers={}, model=model, litellm_params=litellm_params, diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index 96b4b081c05..ea47d744cae 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -562,7 +562,12 @@ async def _arealtime( configured_realtime_protocol, query_params=query_params, websocket=websocket ) resolved_azure_ad_token: Final = ( - None if api_key else get_azure_ad_token(GenericLiteLLMParams(**kwargs, azure_ad_token=azure_ad_token)) + None + if api_key + else await asyncio.to_thread( + get_azure_ad_token, + GenericLiteLLMParams(**kwargs, azure_ad_token=azure_ad_token), + ) ) await azure_realtime.async_realtime( model=model, @@ -743,15 +748,21 @@ def _azure_realtime_health_protocol( return "GA", query_params -def _realtime_health_check_auth_headers( +async def _realtime_health_check_auth_headers( custom_llm_provider: str, api_key: str | None, model_params: Mapping[str, object] ) -> Mapping[str, str]: if custom_llm_provider == "azure": + azure_ad_token: Final = ( + None + if api_key + else await asyncio.to_thread( + get_azure_ad_token, + GenericLiteLLMParams.model_validate(dict(model_params)), + ) + ) return azure_realtime.get_auth_headers( api_key=api_key, - azure_ad_token=( - None if api_key else get_azure_ad_token(GenericLiteLLMParams.model_validate(dict(model_params))) - ), + azure_ad_token=azure_ad_token, ) if api_key is None: return _EMPTY_AUTH_HEADERS @@ -807,7 +818,7 @@ async def realtime_health_check( if custom_llm_provider == LlmProviders.OPENAI.value else resolved_api_key ) - auth_headers: Final = _realtime_health_check_auth_headers( + auth_headers: Final = await _realtime_health_check_auth_headers( custom_llm_provider=custom_llm_provider, api_key=health_check_api_key, model_params=resolved_params, diff --git a/litellm/secret_managers/main.py b/litellm/secret_managers/main.py index 716407d6def..d12e200772f 100644 --- a/litellm/secret_managers/main.py +++ b/litellm/secret_managers/main.py @@ -3,6 +3,8 @@ import base64 import os import time import traceback +from collections.abc import Callable +from functools import lru_cache from typing import Final import httpx @@ -23,6 +25,11 @@ oidc_cache: Final = DualCache() _PARSED_LITERAL: Final = TypeAdapter(object) +@lru_cache(maxsize=128) +def _cached_azure_oidc_token_provider(audience: str) -> Callable[[], str]: + return get_azure_ad_token_provider(azure_scope=audience) + + _OIDC_TOKEN_EXPIRY_MARGIN_SECONDS: Final = 60 @@ -283,7 +290,7 @@ def get_secret( verbose_logger.warning( "AZURE_FEDERATED_TOKEN_FILE not found in environment will use Azure AD token provider" ) - azure_token_provider: Final = get_azure_ad_token_provider(azure_scope=oidc_aud) + azure_token_provider: Final = _cached_azure_oidc_token_provider(oidc_aud) try: oidc_token = azure_token_provider() if oidc_token is None: diff --git a/tests/unit/litellm_core_utils/test_health_check_helpers.py b/tests/unit/litellm_core_utils/test_health_check_helpers.py index 4f62c6fb096..c23749f7a20 100644 --- a/tests/unit/litellm_core_utils/test_health_check_helpers.py +++ b/tests/unit/litellm_core_utils/test_health_check_helpers.py @@ -534,12 +534,12 @@ async def test_ocr_health_check_sends_the_document_kind_the_provider_config_acce assert document[expected_document_type].startswith(expected_uri_prefix) -def test_realtime_health_check_azure_ad_params_drop_reserved_keys(): +async def test_realtime_health_check_azure_ad_params_drop_reserved_keys(): from litellm.realtime_api import main as realtime_main seen = [] with patch.object(realtime_main, "get_azure_ad_token", lambda params: seen.append(params) or "ad-token"): - headers = realtime_main._realtime_health_check_auth_headers( + headers = await realtime_main._realtime_health_check_auth_headers( "azure", None, MappingProxyType({"api_base": "https://x.openai.azure.com", "self": 1, "params": 2, "__class__": 3}), diff --git a/tests/unit/llms/azure/search/test_bing_grounding_search_transformation.py b/tests/unit/llms/azure/search/test_bing_grounding_search_transformation.py index f8d042c14fd..dfd1474211f 100644 --- a/tests/unit/llms/azure/search/test_bing_grounding_search_transformation.py +++ b/tests/unit/llms/azure/search/test_bing_grounding_search_transformation.py @@ -1,5 +1,6 @@ import json, litellm from pathlib import Path +from typing import Final from unittest.mock import AsyncMock, Mock, patch import pytest @@ -110,6 +111,27 @@ def test_validate_environment_entra_minter_failure_names_the_options(): assert "no az login" in message +def test_default_entra_minter_reuses_provider(monkeypatch: pytest.MonkeyPatch) -> None: + import importlib + + from litellm.llms.azure import common_utils + from litellm.llms.azure.search.transformation import _default_entra_token_minter + + provider_module: Final = importlib.import_module("litellm.secret_managers.get_azure_ad_token_provider") + provider_factory: Final = Mock(side_effect=lambda **_: lambda: "entra-token") + common_utils._cached_azure_ad_token_refresh_provider.cache_clear() + monkeypatch.setattr(common_utils, "get_azure_ad_token_provider", provider_factory) + monkeypatch.setattr(provider_module, "get_azure_ad_token_provider", provider_factory) + + try: + first: Final = _default_entra_token_minter() + second: Final = _default_entra_token_minter() + assert first == second == "entra-token" + assert provider_factory.call_count == 1 + finally: + common_utils._cached_azure_ad_token_refresh_provider.cache_clear() + + def test_validate_environment_does_not_mutate_and_is_idempotent(): config = _config() caller_headers = {"X-Custom": "keep-me"} diff --git a/tests/unit/llms/azure/test_azure_common_utils.py b/tests/unit/llms/azure/test_azure_common_utils.py index 80449909b3f..41ad27d6f40 100644 --- a/tests/unit/llms/azure/test_azure_common_utils.py +++ b/tests/unit/llms/azure/test_azure_common_utils.py @@ -1,9 +1,12 @@ import json import os +import threading import traceback -from typing import Callable, Optional +from typing import Callable, Final, Optional +from urllib.parse import parse_qs from unittest.mock import MagicMock, patch +import httpx import pytest import litellm @@ -13,6 +16,8 @@ from litellm.llms.azure.common_utils import ( _cached_entra_id_token_provider, get_azure_ad_token, get_azure_ad_token_from_entra_id, + get_azure_ad_token_from_oidc, + get_azure_ad_token_from_username_password, ) from litellm.secret_managers.get_azure_ad_token_provider import ( get_azure_ad_token_provider, @@ -2141,6 +2146,132 @@ class TestEntraIdTokenProviderCache: assert mock_credential.call_count == 2 +def test_username_password_token_provider_reuses_same_credentials_and_separates_passwords() -> None: + with ( + patch("azure.identity.UsernamePasswordCredential") as mock_credential, + patch( + "azure.identity.get_bearer_token_provider", + side_effect=lambda credential, scope: lambda: "token", + ), + ): + first: Final = get_azure_ad_token_from_username_password( + client_id="client", + azure_username="user", + azure_password="password", + scope="https://cognitiveservices.azure.com/.default", + ) + second: Final = get_azure_ad_token_from_username_password( + client_id="client", + azure_username="user", + azure_password="password", + scope="https://cognitiveservices.azure.com/.default", + ) + changed_password: Final = get_azure_ad_token_from_username_password( + client_id="client", + azure_username="user", + azure_password="different-password", + scope="https://cognitiveservices.azure.com/.default", + ) + + assert first() == second() == changed_password() == "token" + assert mock_credential.call_count == 2 + + +@pytest.mark.asyncio +async def test_non_v1_azure_token_refresh_runs_off_the_event_loop_thread() -> None: + from openai import AsyncAzureOpenAI + + def token_provider_naming_its_thread() -> str: + return str(threading.get_ident()) + + client: Final = BaseAzureLLM().get_azure_openai_client( + api_key=None, + api_base="https://test.openai.azure.com", + api_version="2024-02-01", + litellm_params={"azure_ad_token_provider": token_provider_naming_its_thread}, + _is_async=True, + ) + + assert isinstance(client, AsyncAzureOpenAI) + try: + token_thread: Final = await client._get_azure_ad_token() + finally: + await client.close() + + assert token_thread is not None + assert token_thread != str(threading.get_ident()) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("api_version", ["2024-02-01", "v1"]) +async def test_async_azure_ad_token_provider_reaches_azure_as_bearer_token( + monkeypatch: pytest.MonkeyPatch, api_version: str +) -> None: + def echo_authorization(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={ + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 0, + "model": "gpt", + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": request.headers["authorization"]}, + } + ], + }, + ) + + async def async_token_provider() -> str: + return "async-token" + + monkeypatch.setattr( + litellm, "aclient_session", httpx.AsyncClient(transport=httpx.MockTransport(echo_authorization)) + ) + client: Final = BaseAzureLLM().get_azure_openai_client( + api_key=None, + api_base=f"https://async-token-provider-{api_version}.openai.azure.com", + api_version=api_version, + litellm_params={"azure_ad_token_provider": async_token_provider}, + _is_async=True, + ) + + response: Final = await client.chat.completions.create(model="gpt", messages=[{"role": "user", "content": "hi"}]) + + assert response.choices[0].message.content == "Bearer async-token" + + +def test_oidc_token_exchange_closes_its_own_http_client(monkeypatch: pytest.MonkeyPatch) -> None: + class TokenEndpoint(httpx.BaseTransport): + def __init__(self) -> None: + self.closed = False + + def handle_request(self, request: httpx.Request) -> httpx.Response: + form: Final = parse_qs(request.read().decode()) + return httpx.Response( + 200, json={"access_token": f"minted-for-{form['client_assertion'][0]}", "expires_in": 3600} + ) + + def close(self) -> None: + self.closed = True + + token_endpoint: Final = TokenEndpoint() + monkeypatch.setattr(litellm, "sync_transport", token_endpoint, raising=False) + monkeypatch.setenv("AZURE_TEST_OIDC_ASSERTION", "workload-jwt") + + token: Final = get_azure_ad_token_from_oidc( + azure_ad_token="os.environ/AZURE_TEST_OIDC_ASSERTION", + azure_client_id="client", + azure_tenant_id="tenant-closes-its-client", + ) + + assert token == "minted-for-workload-jwt" + assert token_endpoint.closed + + def test_evicting_an_azure_client_built_on_the_callers_session_leaves_it_open(monkeypatch): """`initialize_azure_sdk_client` puts `litellm.aclient_session` on the SDK client. diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index 23c142b61ab..4e529764229 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -476,6 +476,42 @@ async def test_async_response_api_handler_streams_when_provider_transform_adds_s assert client.post.call_args.kwargs["json"]["stream"] is True +@pytest.mark.asyncio +async def test_async_response_api_handler_validates_environment_off_the_event_loop() -> None: + def validate_environment(**_kwargs: object) -> dict[str, str]: + return {"validated-on-thread": str(threading.get_ident()), "x-override": "provider"} + + config: Final = Mock() + config.validate_environment.side_effect = validate_environment + config.get_complete_url.return_value = "https://responses.example.com/v1/responses" + config.async_transform_responses_api_request = AsyncMock(return_value={"model": "model", "input": "hi"}) + config.sign_request.return_value = ({}, None) + client: Final = AsyncMock(spec=AsyncHTTPHandler) + client.post.return_value = httpx.Response( + 200, + request=httpx.Request("POST", "https://responses.example.com/v1/responses"), + ) + logging_obj: Final = Mock() + logging_obj.dynamic_success_callbacks = None + + await BaseLLMHTTPHandler().async_response_api_handler( + model="model", + input="hi", + responses_api_provider_config=config, + response_api_optional_request_params={}, + custom_llm_provider="test", + litellm_params=GenericLiteLLMParams(), + logging_obj=logging_obj, + client=client, + extra_headers={"x-override": "caller"}, + ) + + validated_headers: Final = config.async_transform_responses_api_request.await_args.kwargs["headers"] + assert validated_headers["validated-on-thread"] != str(threading.get_ident()) + assert validated_headers["x-override"] == "caller" + client.post.assert_awaited_once() + + _CHATGPT_SSE_BODY = ( "event: response.output_item.done\n" 'data: {"type": "response.output_item.done", "output_index": 0, "item": {"type": "message", '