From 4b0aa0f44679e3aa1dd58f8208ff5ccfcbdc1c27 Mon Sep 17 00:00:00 2001 From: tessellark <349031000+tessellark@users.noreply.github.com> Date: Wed, 28 Oct 2026 03:01:40 -0700 Subject: [PATCH 01/12] fix(cost-map): drop the gpt-5-nano priority input price OpenAI does not sell (#48763) gpt-5-nano and gpt-5-nano-2025-08-07 carried gpt-5's priority input price, 50x the nano standard rate. OpenAI sells no priority tier for nano and answers a nano priority request with service_tier default, so proxy spend on real calls was already billed at the standard rate. The wrong price showed in /model/info, in the tier pricing a custom-priced nano deployment inherited, and in SDK cost estimates like litellm.cost_per_token with service_tier priority. The key is removed, and a new test keeps every priority rate between 1x and 5x of its standard rate in both cost map files --- ...odel_prices_and_context_window_backup.json | 2 - model_prices_and_context_window.json | 2 - .../test_model_prices_and_context_window.py | 46 ++++++++++++++++++- 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 5774e699ad7..d053300e299 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -37504,7 +37504,6 @@ "input_cost_per_token": 5e-08, "input_cost_per_token_batches": 2.5e-08, "input_cost_per_token_flex": 2.5e-08, - "input_cost_per_token_priority": 2.5e-06, "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, @@ -37552,7 +37551,6 @@ "deprecation_date": "2026-12-11", "input_cost_per_token": 5e-08, "input_cost_per_token_batches": 2.5e-08, - "input_cost_per_token_priority": 2.5e-06, "input_cost_per_token_flex": 2.5e-08, "litellm_provider": "openai", "max_input_tokens": 272000, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 5774e699ad7..d053300e299 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -37504,7 +37504,6 @@ "input_cost_per_token": 5e-08, "input_cost_per_token_batches": 2.5e-08, "input_cost_per_token_flex": 2.5e-08, - "input_cost_per_token_priority": 2.5e-06, "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, @@ -37552,7 +37551,6 @@ "deprecation_date": "2026-12-11", "input_cost_per_token": 5e-08, "input_cost_per_token_batches": 2.5e-08, - "input_cost_per_token_priority": 2.5e-06, "input_cost_per_token_flex": 2.5e-08, "litellm_provider": "openai", "max_input_tokens": 272000, diff --git a/tests/unit/test_model_prices_and_context_window.py b/tests/unit/test_model_prices_and_context_window.py index 1f84be7710f..08a39b94c12 100644 --- a/tests/unit/test_model_prices_and_context_window.py +++ b/tests/unit/test_model_prices_and_context_window.py @@ -2,7 +2,7 @@ from __future__ import annotations import json import re -from collections.abc import Mapping +from collections.abc import Iterator, Mapping from pathlib import Path from types import MappingProxyType from typing import Final @@ -224,6 +224,50 @@ def test_dated_variants_carry_base_alias_service_tier_pricing(prices: dict): ) +PRIORITY_SUFFIX: Final = "_priority" +MIN_PRIORITY_MARKUP: Final = 1.0 +MAX_PRIORITY_MARKUP: Final = 5.0 + + +def _rate(value: JsonValue | None) -> float | None: + return float(value) if isinstance(value, (int, float)) and not isinstance(value, bool) else None + + +def _priority_markup(entry: Mapping[str, JsonValue], key: str) -> float | None: + if not key.endswith(PRIORITY_SUFFIX): + return None + priority: Final = _rate(entry[key]) + standard: Final = _rate(entry.get(key.removesuffix(PRIORITY_SUFFIX))) + if priority is None or standard is None or standard <= 0: + return None + return priority / standard + + +def priority_markups(prices: Mapping[str, JsonValue]) -> Iterator[tuple[str, str, float]]: + for name, entry in prices.items(): + if not isinstance(entry, dict): + continue + for key in entry: + if (markup := _priority_markup(entry, key)) is not None: + yield name, key, markup + + +@pytest.mark.parametrize("path", (PRICES_PATH, BACKUP_PRICES_PATH), ids=("main", "backup")) +def test_priority_rates_stay_within_a_premium_of_their_standard_rate(path: Path) -> None: + prices: Final = TypeAdapter(dict[str, JsonValue]).validate_json(path.read_bytes()) + markups: Final = tuple(priority_markups(prices)) + assert markups + overpriced: Final = tuple( + f"{name}.{key} is {markup:.1f}x its standard rate" + for name, key, markup in markups + if not MIN_PRIORITY_MARKUP <= markup <= MAX_PRIORITY_MARKUP + ) + assert overpriced == (), ( + "a priority rate is a premium on the standard rate, so one below it or far above it was likely copied from " + "another model; drop the key when the provider sells no priority tier for the model:\n" + "\n".join(overpriced) + ) + + # Azure Foundry GPT-6.1 Sol announcement, 2026-09-29: https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-6-1-sol-in-microsoft-foundry-advanced-intelligence-optimized-for/4560811 AZURE_GPT_6_1_SOL_REGIONAL_PREMIUM: Final = MappingProxyType({"us": 1.1, "eu": 1.2, "apac": 1.2}) From d84a2d4452d37420876573234e810bf4588882b0 Mon Sep 17 00:00:00 2001 From: norkelbyte <348963847+norkelbyte@users.noreply.github.com> Date: Wed, 28 Oct 2026 03:04:11 -0700 Subject: [PATCH 02/12] test(spend): audit #47815 (#48774) Adds an integration cell proving a failed /v1/responses request counts its input from the newest compaction item on its failure spend row Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../spend/test_failed_dispatch_tokens.py | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tests/integration/spend/test_failed_dispatch_tokens.py b/tests/integration/spend/test_failed_dispatch_tokens.py index b92606ebc88..c9875be562c 100644 --- a/tests/integration/spend/test_failed_dispatch_tokens.py +++ b/tests/integration/spend/test_failed_dispatch_tokens.py @@ -167,3 +167,51 @@ def test_provider_500_failure_rows_count_deepseek_v4_prompt_tokens_with_deepseek short: Final = _failure_row_prompt_tokens(gateway, model, key, SHORT_TEXT) assert wire.drain() assert_counted_with_deepseek(chinese - short, chinese_minus_short) + + +COMPACTION_INSTRUCTIONS: Final[JsonValue] = {"role": "system", "content": "answer briefly"} +COMPACTED_HISTORY: Final[tuple[JsonValue, ...]] = ( + {"role": "user", "content": "a long turn " * 200}, + {"role": "assistant", "content": "a long reply " * 200}, +) +COMPACTION_ITEM: Final[JsonValue] = { + "type": "compaction", + "encrypted_content": json.dumps({"type": "compaction", "content": "the cat is named Biscuit"}), +} +TURN_AFTER_COMPACTION: Final[JsonValue] = {"role": "user", "content": "what is the cat's name?"} + + +def _failed_responses_prompt_tokens(gateway: Gateway, model: str, key: str, items: list[JsonValue]) -> int: + failed: Final = gateway.request("POST", "/v1/responses", {"model": model, "input": items}, key=key) + assert failed.status_code == 500 and "synthetic provider outage" in failed.text, failed.text + rows: Final = eventually( + lambda: read_rows( + 'SELECT status, prompt_tokens FROM "LiteLLM_SpendLogs" WHERE request_id=%s', + (failed.headers["x-litellm-call-id"],), + ), + lambda values: len(values) == 1, + seconds=70, + ) + assert rows[0]["status"] == "failure", rows + prompt_tokens: Final = rows[0]["prompt_tokens"] + assert isinstance(prompt_tokens, int) and prompt_tokens > 0, rows + return prompt_tokens + + +def test_a_failed_responses_request_counts_its_input_from_the_newest_compaction_item(gateway: Gateway) -> None: + with wire_server(_provider_500) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(api_base=wire.url + "/v1", num_retries=0) + key: Final = scenario.key(models=[model]) + resent: Final = _failed_responses_prompt_tokens( + gateway, model, key, [COMPACTION_INSTRUCTIONS, *COMPACTED_HISTORY, COMPACTION_ITEM, TURN_AFTER_COMPACTION] + ) + compacted: Final = _failed_responses_prompt_tokens( + gateway, model, key, [COMPACTION_INSTRUCTIONS, COMPACTION_ITEM, TURN_AFTER_COMPACTION] + ) + uncompacted: Final = _failed_responses_prompt_tokens( + gateway, model, key, [COMPACTION_INSTRUCTIONS, *COMPACTED_HISTORY, TURN_AFTER_COMPACTION] + ) + assert len(wire.drain()) == 3 + + assert resent == compacted + assert resent < uncompacted From f6f49d16cc2422d2a83c93c074e0ac4d6a889cc5 Mon Sep 17 00:00:00 2001 From: islomjon-aiand Date: Wed, 28 Oct 2026 19:15:56 +0900 Subject: [PATCH 03/12] feat: add ai& (aiand) provider (#38958) * feat: add aiand provider * feat: expose aiand provider in admin UI * feat(aiand): add ai& model pricing to the cost map * feat(aiand): list ai& models for aiand/* wildcards and mark /v1/messages supported * feat(aiand): send Anthropic /v1/messages to ai& natively * fix(aiand): drop the retired max_tokens field and regenerate endpoint support --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/__init__.py | 5 + litellm/constants.py | 2 + litellm/llms/openai_like/providers.json | 6 + ...odel_prices_and_context_window_backup.json | 195 ++++++++++++++++++ .../provider_endpoints_support_backup.json | 26 +++ .../provider_create_fields.json | 28 +++ litellm/types/utils.py | 1 + model_prices_and_context_window.json | 195 ++++++++++++++++++ provider_endpoints_support.json | 26 +++ .../llms/openai_like/test_aiand_provider.py | 145 +++++++++++++ 10 files changed, 629 insertions(+) create mode 100644 tests/unit/llms/openai_like/test_aiand_provider.py diff --git a/litellm/__init__.py b/litellm/__init__.py index 41e3fa34323..2cff21bf29a 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -710,6 +710,7 @@ transcribe_models: Set = set() gigachat_models: Set = set() llamagate_models: Set = set() coralbricks_models: Set[str] = set() +aiand_models: Set[str] = set() reducto_models: Set = set() bedrock_mantle_models: Set = set() @@ -1022,6 +1023,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje llamagate_models.add(key) elif value.get("litellm_provider") == "coralbricks": coralbricks_models.add(key) + elif value.get("litellm_provider") == "aiand": + aiand_models.add(key) elif value.get("litellm_provider") == "reducto": reducto_models.add(key) elif value.get("litellm_provider") == "bedrock_mantle": @@ -1159,6 +1162,7 @@ model_list = list( | reducto_models | bedrock_mantle_models | coralbricks_models + | aiand_models | set(clarifai_models) ) @@ -1283,6 +1287,7 @@ def _build_models_by_provider() -> dict: "gigachat": gigachat_models, "llamagate": llamagate_models, "coralbricks": coralbricks_models, + "aiand": aiand_models, "reducto": reducto_models, "bedrock_mantle": bedrock_mantle_models, } diff --git a/litellm/constants.py b/litellm/constants.py index 8cc51d0ab5e..0d6dfb5c0d4 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1035,6 +1035,7 @@ openai_compatible_endpoints: Final[list[str]] = [ "https://api-inference.modelscope.cn/v1", "https://api.moonshot.ai/v1", "https://api.publicai.co/v1", + "https://api.aiand.com/v1", "https://api.synthetic.new/openai/v1", "https://serverless.tensormesh.ai/v1", "https://api.stima.tech/v1", @@ -1100,6 +1101,7 @@ openai_compatible_providers: Final[list[str]] = [ "novita", "meta_llama", "publicai", # PublicAI - JSON-configured provider + "aiand", # ai& - JSON-configured provider "synthetic", # Synthetic - JSON-configured provider "tensormesh", # Tensormesh - JSON-configured provider "apertis", # Apertis - JSON-configured provider diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index 115d914d3ac..0034ed2b9d4 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -107,6 +107,12 @@ "api_key_env": "AIHUBMIX_API_KEY", "api_base_env": "AIHUBMIX_API_BASE" }, + "aiand": { + "base_url": "https://api.aiand.com/v1", + "api_key_env": "AIAND_API_KEY", + "api_base_env": "AIAND_API_BASE", + "supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/messages"] + }, "crusoe": { "base_url": "https://api.inference.crusoecloud.com/v1", "api_key_env": "CRUSOE_API_KEY", diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index d053300e299..8207c3122f9 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -69825,6 +69825,201 @@ "supports_reasoning": true, "supports_vision": true }, + "aiand/deepseek-ai/deepseek-v4-flash": { + "cache_read_input_token_cost": 8e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "aiand", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "aiand/deepseek-ai/deepseek-v4-pro": { + "cache_read_input_token_cost": 2.5e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "aiand", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "aiand/deepseek-ai/deepseek-v4.1-flash": { + "cache_read_input_token_cost": 2e-08, + "input_cost_per_token": 3e-07, + "litellm_provider": "aiand", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 6e-07, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "aiand/google/gemma-4-31b-it": { + "cache_read_input_token_cost": 5e-08, + "input_cost_per_token": 2e-07, + "litellm_provider": "aiand", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "aiand/moonshotai/kimi-k2.7-code": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 7.5e-07, + "litellm_provider": "aiand", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.5e-06, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "aiand/moonshotai/kimi-k3": { + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 3e-06, + "litellm_provider": "aiand", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.25e-05, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "aiand/motif-technologies/motif-3": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 5e-07, + "litellm_provider": "aiand", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 2e-06, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "aiand/openai/gpt-oss-120b": { + "cache_read_input_token_cost": 8e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "aiand", + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 6e-07, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "aiand/qwen/qwen3.6-27b": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 3.2e-07, + "litellm_provider": "aiand", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 3.2e-06, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "aiand/qwen/qwen3.8-27b": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 4e-07, + "litellm_provider": "aiand", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 3e-06, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "aiand/zai-org/glm-5.2": { + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "aiand", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "aiand/zai-org/glm-5.3": { + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "aiand", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "aiand/zai-org/glm-5.3-flash": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "aiand", + "max_input_tokens": 1048550, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, "bespoke/nimble-latest": { "input_cost_per_token": 0.0, "litellm_provider": "bespoke", diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index 9ba6ed342ea..1ab03fb3fdc 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -189,6 +189,32 @@ "generateContent": false } }, + "aiand": { + "display_name": "ai& (`aiand`)", + "url": "https://docs.aiand.com/api/chat-completions/", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } + }, "amazon_nova": { "display_name": "Amazon Nova (`amazon_nova`)", "url": "https://docs.litellm.ai/docs/providers/amazon_nova", diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index d96f2976508..0dd96c856f8 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -1,4 +1,32 @@ [ + { + "provider": "AIAND", + "provider_display_name": "ai&", + "litellm_provider": "aiand", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://api.aiand.com/v1", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "aiand/openai/gpt-oss-120b" + }, { "provider": "SCALEDOWN", "provider_display_name": "ScaleDown", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 2869f9b2577..055c18f2e36 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4573,6 +4573,7 @@ class LlmProviders(str, Enum): POE = "poe" CHUTES = "chutes" NEOSANTARA = "neosantara" + AIAND = "aiand" PARASAIL = "parasail" XIAOMI_MIMO = "xiaomi_mimo" TENSORMESH = "tensormesh" diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index d053300e299..8207c3122f9 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -69825,6 +69825,201 @@ "supports_reasoning": true, "supports_vision": true }, + "aiand/deepseek-ai/deepseek-v4-flash": { + "cache_read_input_token_cost": 8e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "aiand", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "aiand/deepseek-ai/deepseek-v4-pro": { + "cache_read_input_token_cost": 2.5e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "aiand", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "aiand/deepseek-ai/deepseek-v4.1-flash": { + "cache_read_input_token_cost": 2e-08, + "input_cost_per_token": 3e-07, + "litellm_provider": "aiand", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 6e-07, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "aiand/google/gemma-4-31b-it": { + "cache_read_input_token_cost": 5e-08, + "input_cost_per_token": 2e-07, + "litellm_provider": "aiand", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "aiand/moonshotai/kimi-k2.7-code": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 7.5e-07, + "litellm_provider": "aiand", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.5e-06, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "aiand/moonshotai/kimi-k3": { + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 3e-06, + "litellm_provider": "aiand", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.25e-05, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "aiand/motif-technologies/motif-3": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 5e-07, + "litellm_provider": "aiand", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 2e-06, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "aiand/openai/gpt-oss-120b": { + "cache_read_input_token_cost": 8e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "aiand", + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 6e-07, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "aiand/qwen/qwen3.6-27b": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 3.2e-07, + "litellm_provider": "aiand", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 3.2e-06, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "aiand/qwen/qwen3.8-27b": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 4e-07, + "litellm_provider": "aiand", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 3e-06, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "aiand/zai-org/glm-5.2": { + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "aiand", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "aiand/zai-org/glm-5.3": { + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "aiand", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "aiand/zai-org/glm-5.3-flash": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "aiand", + "max_input_tokens": 1048550, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, "bespoke/nimble-latest": { "input_cost_per_token": 0.0, "litellm_provider": "bespoke", diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 9ba6ed342ea..1ab03fb3fdc 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -189,6 +189,32 @@ "generateContent": false } }, + "aiand": { + "display_name": "ai& (`aiand`)", + "url": "https://docs.aiand.com/api/chat-completions/", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } + }, "amazon_nova": { "display_name": "Amazon Nova (`amazon_nova`)", "url": "https://docs.litellm.ai/docs/providers/amazon_nova", diff --git a/tests/unit/llms/openai_like/test_aiand_provider.py b/tests/unit/llms/openai_like/test_aiand_provider.py new file mode 100644 index 00000000000..804f3a3b6eb --- /dev/null +++ b/tests/unit/llms/openai_like/test_aiand_provider.py @@ -0,0 +1,145 @@ +import json +from typing import Final + +import httpx +import pytest +import respx + +import litellm +from litellm.caching.llm_caching_handler import LLMClientCache +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.llms.openai_like.json_loader import JSONProviderRegistry +from litellm.proxy.auth.model_checks import get_known_models_from_wildcard +from litellm.utils import ProviderConfigManager + +AIAND_API_BASE = "https://api.aiand.com/v1" + + +def test_aiand_chat_provider(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("AIAND_API_KEY", "test-key") + + provider_config = JSONProviderRegistry.get("aiand") + assert provider_config is not None + assert litellm.LlmProviders.AIAND.value == "aiand" + assert "aiand" in litellm.openai_compatible_providers + assert provider_config.base_url == AIAND_API_BASE + assert provider_config.api_key_env == "AIAND_API_KEY" + assert provider_config.api_base_env == "AIAND_API_BASE" + + model, provider, api_key, api_base = get_llm_provider("aiand/openai/gpt-oss-120b") + assert model == "openai/gpt-oss-120b" + assert provider == "aiand" + assert api_key == "test-key" + assert api_base == AIAND_API_BASE + + +def test_aiand_completion_request(respx_mock: respx.MockRouter): + messages = [{"role": "user", "content": "Hello"}] + route = respx_mock.post(f"{AIAND_API_BASE}/chat/completions").mock( + return_value=httpx.Response( + 200, + json={ + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1, + "model": "openai/gpt-oss-120b", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Hello"}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 1, + "completion_tokens": 1, + "total_tokens": 2, + }, + }, + ) + ) + + litellm.completion( + model="aiand/openai/gpt-oss-120b", + api_key="test-key", + messages=messages, + ) + + assert route.called + request = route.calls.last.request + assert str(request.url) == f"{AIAND_API_BASE}/chat/completions" + assert request.headers["Authorization"] == "Bearer test-key" + request_body = json.loads(request.read()) + assert request_body["model"] == "openai/gpt-oss-120b" + assert request_body["messages"] == messages + + +def test_aiand_responses_provider(): + config = ProviderConfigManager.get_provider_responses_api_config( + provider="aiand", + model="aiand/openai/gpt-oss-120b", + ) + + assert config is not None + assert config.custom_llm_provider == "aiand" + assert config.get_complete_url(api_base=None, litellm_params={}) == f"{AIAND_API_BASE}/responses" + + +def _aiand_catalog() -> frozenset[str]: + return frozenset(name for name, info in litellm.model_cost.items() if info.get("litellm_provider") == "aiand") + + +def test_aiand_wildcard_lists_its_priced_models(): + catalog: Final = _aiand_catalog() + + assert catalog + assert frozenset(get_known_models_from_wildcard("aiand/*")) == catalog + + +def test_aiand_calls_are_billed_from_their_cost_rows(): + catalog: Final = _aiand_catalog() + assert catalog + + for model in sorted(catalog): + row: Final = litellm.model_cost[model] + prompt_cost, completion_cost = litellm.cost_per_token(model=model, prompt_tokens=1000, completion_tokens=500) + + assert prompt_cost == pytest.approx(1000 * row["input_cost_per_token"]) + assert completion_cost == pytest.approx(500 * row["output_cost_per_token"]) + assert prompt_cost > 0 and completion_cost > 0 + + +@pytest.mark.asyncio +async def test_aiand_anthropic_messages_reach_ai_and_with_the_thinking_budget(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + thinking: Final = {"type": "enabled", "budget_tokens": 1024} + with respx.mock() as upstream: + route: Final = upstream.post(f"{AIAND_API_BASE}/messages").respond( + 200, + json={ + "id": "msg_aiand", + "type": "message", + "role": "assistant", + "model": "zai-org/glm-5.3", + "content": [{"type": "thinking", "thinking": "A greeting", "signature": ""}, {"type": "text", "text": "Hello"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 4, "output_tokens": 3}, + }, + ) + response: Final = await litellm.anthropic.messages.acreate( + model="aiand/zai-org/glm-5.3", + messages=[{"role": "user", "content": "Say hello"}], + max_tokens=2048, + thinking=thinking, + api_key="aiand-test-key", + ) + + request: Final = route.calls.last.request + body: Final = json.loads(request.content) + assert route.call_count == 1 + assert body["model"] == "zai-org/glm-5.3" + assert body["thinking"] == thinking + assert "reasoning_effort" not in body + assert [block["type"] for block in response["content"]] == ["thinking", "text"] From ff65785717e7b682d5b174bfca3448c8511c5057 Mon Sep 17 00:00:00 2001 From: Max Azatian <66359507+HardMax71@users.noreply.github.com> Date: Wed, 28 Oct 2026 11:19:26 +0100 Subject: [PATCH 04/12] fix(router)!: count an explicit allowed_fails of 3 and keep fallback deployments out of cooldown (#46183) * fix(router): count an explicit allowed_fails equal to the default as set Router(allowed_fails=3) stored the same value as an unset Router, so _is_allowed_fails_set_on_router treated it as "not set" and kept the rate-based cooldown logic. Keep the passed value in explicit_allowed_fails, like _explicit_timeout, and count it as set. * chore(router): drop the comment on explicit_allowed_fails * fix(router): keep the fallback deployment out of cooldown when no deployment can be picked When every deployment of the requested model is in cooldown, the router raises "No deployments available" and schedules the failure callbacks on the request's logging object. By the time they run, the fallback has already written its own deployment into that object, so the 429 counted as a failure of the healthy fallback deployment and cooled it down too, leaving the model group with nothing to serve. Tag the selection error and skip cooldown accounting for it, the same way advisor orchestration failures are skipped. * test(router): annotate the new cooldown test variables with Final * test(router): pin that a saved allowed_fails of 3 stays unset and type the fallback helper * test(router): use a fixed start time in the fallback helper * test(router): cover a fallback group staying up after the primary's selection fails --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/router.py | 10 +- litellm/router_utils/cooldown_handlers.py | 20 ++- .../router_utils/test_cooldown_handlers.py | 141 +++++++++++++++++- 3 files changed, 164 insertions(+), 7 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index c8153b596d6..18c5c67abce 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -238,11 +238,12 @@ from litellm.router_utils.cooldown_handlers import ( async_get_cooldown_deployments_with_debug_info, get_cooldown_deployment_info, get_cooldown_deployments, - is_advisor_orchestration_failure, is_background_response_poll_not_found, is_caller_scoped_auth_failure, is_caller_timeout_408, + is_failure_outside_selected_deployment, is_video_lookup_not_found, + mark_deployment_selection_failure, set_cooldown_deployments, ) from litellm.router_utils.deployment_pricing import deployment_pricing_models @@ -1519,6 +1520,7 @@ class Router: self.allowed_fails = allowed_fails else: self.allowed_fails = litellm.allowed_fails + self.explicit_allowed_fails = allowed_fails self.cooldown_time = cooldown_time or DEFAULT_COOLDOWN_TIME_SECONDS self.cooldown_cache = CooldownCache(cache=self.cache, default_cooldown_time=self.cooldown_time) self.disable_cooldowns = disable_cooldowns @@ -9579,10 +9581,10 @@ class Router: try: exception: Final = kwargs.get("exception", None) - if is_advisor_orchestration_failure(exception): + if is_failure_outside_selected_deployment(exception): verbose_router_logger.debug( "Router: Exiting 'deployment_callback_on_failure' without cooldown. " - "Failure originated from advisor orchestration, not the selected deployment." + "Failure came from advisor orchestration or deployment selection, not the selected deployment." ) return False @@ -15473,6 +15475,7 @@ class Router: phase_event(_DEPLOYMENT_SELECTED_EVENT, pick_attributes) return deployment except Exception as e: + mark_deployment_selection_failure(e) traceback_exception: Final = traceback.format_exc() # if router rejects call -> log to langfuse/otel/etc. if request_kwargs is not None: @@ -15641,6 +15644,7 @@ class Router: return deployment except Exception as e: + mark_deployment_selection_failure(e) traceback_exception: Final = traceback.format_exc() if request_kwargs is not None: logging_obj: Final = request_kwargs.get("litellm_logging_obj", None) diff --git a/litellm/router_utils/cooldown_handlers.py b/litellm/router_utils/cooldown_handlers.py index 330500789a4..e1baf66b201 100644 --- a/litellm/router_utils/cooldown_handlers.py +++ b/litellm/router_utils/cooldown_handlers.py @@ -57,6 +57,7 @@ else: Span = Any _ADVISOR_ORCHESTRATION_FAILURE_ATTR: Final = "_litellm_advisor_orchestration_failure" +_DEPLOYMENT_SELECTION_FAILURE_ATTR: Final = "_litellm_deployment_selection_failure" _CREDENTIAL_VALUES_ADAPTER: Final = TypeAdapter(Mapping[str, object]) @@ -79,6 +80,16 @@ def is_advisor_orchestration_failure(exception: BaseException | None) -> bool: return bool(getattr(exception, _ADVISOR_ORCHESTRATION_FAILURE_ATTR, False)) +def mark_deployment_selection_failure(exception: BaseException) -> None: + setattr(exception, _DEPLOYMENT_SELECTION_FAILURE_ATTR, True) + + +def is_failure_outside_selected_deployment(exception: BaseException | None) -> bool: + return is_advisor_orchestration_failure(exception) or bool( + getattr(exception, _DEPLOYMENT_SELECTION_FAILURE_ATTR, False) + ) + + _BACKGROUND_RESPONSE_POLL_CALL_ORIGINS: Final[frozenset[InternalCallOrigin]] = frozenset( {BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, BACKGROUND_RESPONSE_STATUS_POLL_CALL_ORIGIN} ) @@ -781,14 +792,17 @@ def _is_allowed_fails_set_on_router( litellm_router_instance: LitellmRouter, ) -> bool: """ - Check if Router.allowed_fails is set or is Non-default Value + Check if Router.allowed_fails was passed to the Router or is a Non-default Value Returns: - - True if Router.allowed_fails is set or is Non-default Value - - False if Router.allowed_fails is None or is Default Value + - True if allowed_fails was passed to the Router, even when it equals the default + - True if Router.allowed_fails is a Non-default Value + - False if Router.allowed_fails is None, or is the Default Value and was not passed """ if litellm_router_instance.allowed_fails is None: return False + if litellm_router_instance.explicit_allowed_fails is not None: + return True if litellm_router_instance.allowed_fails != litellm.allowed_fails: return True return False diff --git a/tests/unit/router_utils/test_cooldown_handlers.py b/tests/unit/router_utils/test_cooldown_handlers.py index bab6d257579..4299b61c703 100644 --- a/tests/unit/router_utils/test_cooldown_handlers.py +++ b/tests/unit/router_utils/test_cooldown_handlers.py @@ -1,10 +1,12 @@ import asyncio import copy +import datetime import importlib import itertools import random import time -from collections.abc import Mapping +import uuid +from collections.abc import Mapping, Sequence from typing import Final from unittest.mock import MagicMock, patch @@ -682,6 +684,45 @@ class TestDeploymentLevelAllowedFails: ) +def _gpt_4_group_with_a_sibling() -> Sequence[Mapping[str, object]]: + return ( + {"model_name": "gpt-4", "litellm_params": {"model": "openai/gpt-4"}, "model_info": {"id": "primary"}}, + {"model_name": "gpt-4", "litellm_params": {"model": "openai/gpt-4"}, "model_info": {"id": "secondary"}}, + ) + + +def _primary_cooldown_decisions_on_repeated_503s(router: Router) -> tuple[bool, ...]: + exception: Final = litellm.ServiceUnavailableError("down", "openai", "gpt-4") + return tuple( + _should_cooldown_deployment( + litellm_router_instance=router, + deployment="primary", + exception_status=503, + original_exception=exception, + ) + for _ in range(litellm.allowed_fails + 1) + ) + + +@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") +class TestRouterLevelAllowedFailsEqualToDefault: + def test_allowed_fails_equal_to_the_default_cools_down_after_that_many_fails(self): + router: Final = _make_router(model_list=list(_gpt_4_group_with_a_sibling()), allowed_fails=litellm.allowed_fails) + + assert _primary_cooldown_decisions_on_repeated_503s(router) == (False,) * litellm.allowed_fails + (True,) + + def test_unset_allowed_fails_keeps_the_rate_based_default(self): + router: Final = _make_router(model_list=list(_gpt_4_group_with_a_sibling())) + + assert _primary_cooldown_decisions_on_repeated_503s(router) == (False,) * (litellm.allowed_fails + 1) + + def test_default_allowed_fails_saved_through_update_settings_keeps_the_rate_based_default(self): + router: Final = _make_router(model_list=list(_gpt_4_group_with_a_sibling())) + router.update_settings(allowed_fails=litellm.allowed_fails) + + assert _primary_cooldown_decisions_on_repeated_503s(router) == (False,) * (litellm.allowed_fails + 1) + + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestDeploymentLevelAllowedFailsPolicyByExceptionType: def test_rate_limit_error_triggers_cooldown_with_zero_threshold(self): @@ -2931,6 +2972,104 @@ async def test_bad_request_error_does_not_cool_down_a_deployment( assert tuple(cooldown_deployments) == () +async def _complete_like_the_proxy(router: Router, model: str) -> str: + logging_obj, request = litellm.utils.function_setup( + original_function="acompletion", + rules_obj=litellm.utils.Rules(), + start_time=datetime.datetime(2026, 1, 1), + model=model, + messages=[{"role": "user", "content": "hi"}], + litellm_call_id=str(uuid.uuid4()), + ) + response: Final = await router.acompletion(**request, litellm_logging_obj=logging_obj) + model_id: Final = response._hidden_params["model_id"] + assert isinstance(model_id, str) + return model_id + + +_CHAT_COMPLETION_OK: Final = { + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, +} + + +def _openai_deployment(model_name: str, host: str, deployment_id: str) -> Mapping[str, object]: + return { + "model_name": model_name, + "litellm_params": {"model": "openai/gpt-4o", "api_key": "k", "api_base": f"https://{host}/v1"}, + "model_info": {"id": deployment_id}, + } + + +@pytest.mark.asyncio +async def test_fallback_deployment_stays_up_while_the_primary_is_in_cooldown( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://primary.test/v1/chat/completions").mock( + return_value=httpx.Response(503, json={"error": {"message": "down", "type": "server_error"}}) + ) + backup_route: Final = respx_mock.post("https://backup.test/v1/chat/completions").mock( + return_value=httpx.Response(200, json=_CHAT_COMPLETION_OK) + ) + router: Final = litellm.Router( + model_list=[ + _openai_deployment("primary", "primary.test", "primary-dep"), + _openai_deployment("backup", "backup.test", "backup-dep"), + ], + fallbacks=[{"primary": ["backup"]}], + allowed_fails=litellm.allowed_fails, + num_retries=0, + ) + + served_by: Final = [await _complete_like_the_proxy(router, "primary") for _ in range(20)] + cooldown_deployments: Final = await async_get_cooldown_deployments( + litellm_router_instance=router, + parent_otel_span=None, + ) + + assert served_by == ["backup-dep"] * 20 + assert len(backup_route.calls) == 20 + assert tuple(cooldown_deployments) == ("primary-dep",) + + +@pytest.mark.asyncio +async def test_fallback_group_stays_up_after_a_bad_key_cools_the_primary_under_the_rate_based_default( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://primary.test/v1/chat/completions").mock( + return_value=httpx.Response(401, json={"error": {"message": "bad key", "type": "invalid_request_error"}}) + ) + respx_mock.post("https://backup.test/v1/chat/completions").mock( + return_value=httpx.Response(200, json=_CHAT_COMPLETION_OK) + ) + router: Final = litellm.Router( + model_list=[ + _openai_deployment("primary", "primary.test", "primary-dep"), + _openai_deployment("backup", "backup.test", "backup-1"), + _openai_deployment("backup", "backup.test", "backup-2"), + ], + fallbacks=[{"primary": ["backup"]}], + num_retries=0, + ) + + served_by: Final = [await _complete_like_the_proxy(router, "primary") for _ in range(12)] + cooldown_deployments: Final = await async_get_cooldown_deployments( + litellm_router_instance=router, + parent_otel_span=None, + ) + + assert set(served_by) <= {"backup-1", "backup-2"} + assert tuple(cooldown_deployments) == ("primary-dep",) + + TIMED_OUT_DEPLOYMENT: Final = { "model_name": "test-model", "litellm_params": {"model": "openai/test-model", "api_key": "test-key"}, From 5dc18fe909a840c527b80592ce1b93a7354de929 Mon Sep 17 00:00:00 2001 From: tavren27 <348969968+tavren27@users.noreply.github.com> Date: Wed, 28 Oct 2026 03:23:34 -0700 Subject: [PATCH 05/12] test(proxy): audit #47418 (#48766) * test(proxy): audit #47418 * test(proxy): hold the stream open past its first chunk and require cost headers in the audit cells --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../test_compressed_request_bodies.py | 392 ++++++++++++++++++ 1 file changed, 392 insertions(+) create mode 100644 tests/integration/compatibility/test_compressed_request_bodies.py diff --git a/tests/integration/compatibility/test_compressed_request_bodies.py b/tests/integration/compatibility/test_compressed_request_bodies.py new file mode 100644 index 00000000000..ef2490add40 --- /dev/null +++ b/tests/integration/compatibility/test_compressed_request_bodies.py @@ -0,0 +1,392 @@ +from __future__ import annotations + +import gzip +import json +import os +import secrets +import threading +import uuid +from collections.abc import Callable, Iterator, Mapping +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from functools import partial +from pathlib import Path +from types import MappingProxyType +from typing import Final + +import httpx +import pytest +import yaml +from integration._support.budget import case_timeout_seconds +from integration._support.client import ( + JSON_OBJECT, + Gateway, + eventually, + gateway_from_environment, + list_value, + object_value, + string_value, +) +from integration._support.process import graceful_stop_seconds, owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue + +MIB: Final = 1024 * 1024 +CHAT: Final = "/v1/chat/completions" +CHAT_MODEL: Final = "compressed-chat" +OTHER_MODEL: Final = "compressed-chat-other" +ANTHROPIC_MODEL: Final = "claude-haiku-5-5" +GATED_PREFIX: Final = "gated-" +ORIGIN: Final = "http://browser.example" +MIDDLEWARE_HEADERS: Final = ( + "access-control-allow-origin", + "x-frame-options", + "content-security-policy", + "x-content-type-options", +) +ZSTD_MAGIC: Final = b"\x28\xb5\x2f\xfd" +ZSTD_SINGLE_SEGMENT_FOUR_BYTE_SIZE: Final = b"\xa0" +ZSTD_MAX_BLOCK: Final = 128 * 1024 +NO_HEADERS: Final[Mapping[str, str]] = MappingProxyType({}) + +pytestmark: Final = [ + pytest.mark.timeout(2 * graceful_stop_seconds() + 120), + pytest.mark.xdist_group("compressed-request-bodies"), +] + + +def zstd_frame(data: bytes) -> bytes: + blocks: Final = tuple(data[start : start + ZSTD_MAX_BLOCK] for start in range(0, len(data), ZSTD_MAX_BLOCK)) + framed: Final = blocks or (b"",) + return ( + ZSTD_MAGIC + + ZSTD_SINGLE_SEGMENT_FOUR_BYTE_SIZE + + len(data).to_bytes(4, "little") + + b"".join( + (len(block) << 3 | (index == len(framed) - 1)).to_bytes(3, "little") + block + for index, block in enumerate(framed) + ) + ) + + +def gzip_body(data: bytes) -> bytes: + return gzip.compress(data, mtime=0) + + +def as_sent(data: bytes) -> bytes: + return data + + +@dataclass(frozen=True, slots=True) +class Rig: + gateway: Gateway + upstream: Wire + gate: threading.Event + limited_key: str + + def calls(self) -> tuple[Request, ...]: + return tuple(request for request in self.upstream.drain() if request.method == "POST") + + +def _chat_completion() -> bytes: + return json.dumps( + { + "id": f"chatcmpl-{uuid.uuid4().hex}", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "scripted"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}, + } + ).encode() + + +def _stream_frame(identity: str, delta: Mapping[str, JsonValue], finish_reason: str | None) -> bytes: + chunk: Final = { + "id": identity, + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "delta": dict(delta), "finish_reason": finish_reason}], + } + return b"data: " + json.dumps(chunk).encode() + b"\n\n" + + +def _chat_stream(gate: threading.Event | None) -> Reply: + identity: Final = f"chatcmpl-{uuid.uuid4().hex}" + return Reply( + content_type="text/event-stream", + chunks=( + _stream_frame(identity, {"role": "assistant", "content": "streamed "}, None), + _stream_frame(identity, {"content": "answer"}, "stop"), + b"data: [DONE]\n\n", + ), + gate_after_first=gate, + ) + + +def _anthropic_message() -> bytes: + return json.dumps( + { + "id": f"msg_{uuid.uuid4().hex}", + "type": "message", + "role": "assistant", + "model": ANTHROPIC_MODEL, + "content": [{"type": "text", "text": "scripted"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 3, "output_tokens": 2}, + } + ).encode() + + +def _respond(gate: threading.Event, request: Request) -> Reply: + if request.method == "GET": + return Reply(body=b'{"object":"list","data":[]}') + if request.target == "/v1/messages": + return Reply(body=_anthropic_message()) + body: Final = JSON_OBJECT.validate_json(request.body) + if body.get("stream") is not True: + return Reply(body=_chat_completion()) + text: Final = string_value(object_value(list_value(body["messages"])[0])["content"]) + return _chat_stream(gate if text.startswith(GATED_PREFIX) else None) + + +def _deployment(name: str, upstream: Wire) -> Mapping[str, JsonValue]: + return { + "model_name": name, + "litellm_params": {"model": "openai/gpt-4o-mini", "api_base": f"{upstream.url}/v1", "api_key": "sk-fixture"}, + } + + +def _config(directory: Path, upstream: Wire) -> Path: + base: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + configuration: Final = { + **base, + "model_list": [_deployment(CHAT_MODEL, upstream), _deployment(OTHER_MODEL, upstream)], + "general_settings": { + **object_value(base.get("general_settings", {})), + "max_request_size_mb": 1, + "enable_claude_code_gateway": True, + }, + "router_settings": {**object_value(base.get("router_settings", {})), "num_retries": 0}, + } + path: Final = directory / "compressed-request-bodies.yaml" + path.write_text(yaml.safe_dump(configuration)) + return path + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + assert os.environ.get("LITELLM_LICENSE"), ( + "LITELLM_LICENSE must be set: max_request_size_mb is an enterprise feature" + ) + gate: Final = threading.Event() + directory: Final = tmp_path_factory.mktemp("compressed-request-bodies") + with gateway_from_environment() as root, wire_server(partial(_respond, gate)) as upstream: + overrides: Final = { + "ANTHROPIC_API_BASE": upstream.url, + "ANTHROPIC_API_KEY": "sk-ant-synthetic", + "MAX_DECOMPRESSED_REQUEST_BODIES_IN_FLIGHT_MB": "1", + } + with ( + owned_proxy_process(root, directory, overrides, config=_config(directory, upstream), workers=1) as owned, + owned.gateway.scenario() as scenario, + ): + yield Rig(owned.gateway, upstream, gate, scenario.key(models=[CHAT_MODEL])) + + +@pytest.fixture(autouse=True) +def empty_upstream_log(rig: Rig) -> None: + rig.upstream.drain() + + +def _headers(key: str, coding: str | None, extra: Mapping[str, str] = NO_HEADERS) -> Mapping[str, str]: + return { + "Authorization": f"Bearer {key}", + "content-type": "application/json", + **({"content-encoding": coding} if coding is not None else {}), + **extra, + } + + +def _send( + rig: Rig, + data: bytes, + coding: str | None, + *, + key: str | None = None, + extra: Mapping[str, str] = NO_HEADERS, +) -> httpx.Response: + return rig.gateway.client.post( + CHAT, + content=data, + headers=_headers(rig.gateway.key if key is None else key, coding, extra), + timeout=case_timeout_seconds(), + ) + + +def _chat_body(model: str, text: str, *, stream: bool = False) -> bytes: + return json.dumps( + { + "model": model, + "messages": [{"role": "user", "content": text}], + "stream": stream, + "cache": {"no-cache": True}, + } + ).encode() + + +def _error_type(response: httpx.Response) -> JsonValue: + return object_value(JSON_OBJECT.validate_json(response.content)["error"])["type"] + + +def _answer(response: httpx.Response) -> JsonValue: + choice: Final = object_value(list_value(JSON_OBJECT.validate_json(response.content)["choices"])[0]) + return object_value(choice["message"])["content"] + + +def _stream(gateway: Gateway, data: bytes, first_text: threading.Event) -> tuple[int, str]: + with ( + httpx.Client(base_url=gateway.client.base_url, timeout=case_timeout_seconds(), trust_env=False) as client, + client.stream("POST", CHAT, content=data, headers=_headers(gateway.key, "gzip")) as response, + ): + pieces: Final = response.iter_text() + first: Final = next(pieces, "") + first_text.set() + return response.status_code, first + "".join(pieces) + + +CODINGS: Final = ( + pytest.param("zstd", zstd_frame, id="zstd"), + pytest.param("gzip", gzip_body, id="gzip"), + pytest.param("gzip", as_sent, id="plain-labeled-gzip"), +) + + +@pytest.mark.parametrize(("coding", "encode"), CODINGS) +def test_a_compressed_chat_body_reaches_the_provider_as_its_plain_twin( + rig: Rig, coding: str, encode: Callable[[bytes], bytes] +) -> None: + body: Final = _chat_body(CHAT_MODEL, f"twin {uuid.uuid4().hex}") + plain: Final = _send(rig, body, None) + compressed: Final = _send(rig, encode(body), coding) + assert plain.status_code == 200, plain.text + assert compressed.status_code == 200, compressed.text + assert _answer(compressed) == _answer(plain) + assert compressed.headers["x-litellm-response-cost"] == plain.headers["x-litellm-response-cost"] + calls: Final = rig.calls() + assert len(calls) == 2, [call.target for call in calls] + assert JSON_OBJECT.validate_json(calls[1].body) == JSON_OBJECT.validate_json(calls[0].body) + assert "content-encoding" not in calls[1].headers, calls[1].headers + + +def test_auth_reads_the_model_from_a_compressed_body(rig: Rig) -> None: + refused: Final = _send(rig, zstd_frame(_chat_body(OTHER_MODEL, "outside the key")), "zstd", key=rig.limited_key) + assert refused.status_code == 403, refused.text + assert _error_type(refused) == "key_model_access_denied", refused.text + assert rig.calls() == () + served: Final = _send(rig, zstd_frame(_chat_body(CHAT_MODEL, "inside the key")), "zstd", key=rig.limited_key) + assert served.status_code == 200, served.text + assert len(rig.calls()) == 1 + + +@pytest.mark.parametrize( + ("coding", "encode"), + (pytest.param("zstd", zstd_frame, id="zstd"), pytest.param("gzip", as_sent, id="plain-labeled-gzip")), +) +def test_a_compressed_pass_through_body_reaches_the_vendor_decoded( + rig: Rig, coding: str, encode: Callable[[bytes], bytes] +) -> None: + body: Final = json.dumps( + { + "model": ANTHROPIC_MODEL, + "max_tokens": 8, + "messages": [{"role": "user", "content": f"pass through {uuid.uuid4().hex}"}], + } + ).encode() + response: Final = rig.gateway.client.post( + "/anthropic/v1/messages", + content=encode(body), + headers={ + "x-litellm-api-key": rig.gateway.key, + "anthropic-version": "2023-06-01", + "content-type": "application/json", + "content-encoding": coding, + }, + timeout=case_timeout_seconds(), + ) + assert response.status_code == 200, response.text + calls: Final = rig.calls() + assert [call.target for call in calls] == ["/v1/messages"] + assert JSON_OBJECT.validate_json(calls[0].body) == JSON_OBJECT.validate_json(body) + assert "content-encoding" not in calls[0].headers, calls[0].headers + + +def test_a_gzip_bomb_gets_a_413_with_the_headers_other_errors_carry(rig: Rig) -> None: + origin: Final = {"origin": ORIGIN} + rejected: Final = _send(rig, gzip_body(b"\0" * 800_000), "gzip", extra=origin) + unauthorized: Final = _send( + rig, _chat_body(CHAT_MODEL, "unknown key"), None, key=f"sk-unknown-{uuid.uuid4().hex}", extra=origin + ) + assert rejected.status_code == 413, rejected.text + assert unauthorized.status_code == 401, unauthorized.text + assert unauthorized.headers.get("access-control-allow-origin") is not None, dict(unauthorized.headers) + assert {name: rejected.headers.get(name) for name in MIDDLEWARE_HEADERS} == { + name: unauthorized.headers.get(name) for name in MIDDLEWARE_HEADERS + } + assert rig.calls() == () + + +def test_the_operator_size_limit_holds_a_compressed_body_to_its_plain_twins_limit(rig: Rig) -> None: + over: Final = _chat_body(CHAT_MODEL, secrets.token_hex(600_000)) + compressed_over: Final = gzip_body(over) + assert len(compressed_over) < MIB < len(over) + plain: Final = _send(rig, over, None) + rejected: Final = _send(rig, compressed_over, "gzip") + served: Final = _send(rig, gzip_body(_chat_body(CHAT_MODEL, secrets.token_hex(300_000))), "gzip") + assert plain.status_code == 413, plain.text + assert rejected.status_code == plain.status_code, rejected.text + assert served.status_code == 200, served.text + assert len(rig.calls()) == 1 + + +def test_the_decoded_body_budget_turns_away_a_second_body_until_the_stream_ends(rig: Rig) -> None: + held: Final = gzip_body(_chat_body(CHAT_MODEL, GATED_PREFIX + secrets.token_hex(300_000), stream=True)) + second: Final = gzip_body(_chat_body(CHAT_MODEL, secrets.token_hex(300_000))) + keyless: Final = _send(rig, second, "gzip", key=f"sk-unknown-{uuid.uuid4().hex}") + assert keyless.status_code == 401, keyless.text + first_text: Final = threading.Event() + with ThreadPoolExecutor(1) as pool: + streaming: Final = pool.submit(_stream, rig.gateway, held, first_text) + try: + eventually(lambda: first_text.is_set() or streaming.done(), bool, seconds=case_timeout_seconds()) + reached: Final = rig.calls() + turned_away: Final = _send(rig, second, "gzip") + finally: + rig.gate.set() + status, streamed = streaming.result(timeout=case_timeout_seconds()) + assert len(reached) == 1, streamed + assert turned_away.status_code == 503, turned_away.text + assert _error_type(turned_away) == "overloaded_error", turned_away.text + assert turned_away.headers.get("retry-after") == "1", dict(turned_away.headers) + assert status == 200, streamed + assert "streamed " in streamed and "answer" in streamed and "data: [DONE]" in streamed, streamed + assert rig.calls() == () + after_the_stream: Final = _send(rig, second, "gzip") + assert after_the_stream.status_code == 200, after_the_stream.text + assert len(rig.calls()) == 1 + + +def test_a_claude_code_otlp_export_keeps_a_two_member_gzip_body(rig: Rig) -> None: + response: Final = rig.gateway.client.post( + "/claude_code_gateway/v1/metrics", + content=gzip_body(b"first member") + gzip_body(b"second member"), + headers={ + "Authorization": f"Bearer {rig.gateway.key}", + "content-type": "application/x-protobuf", + "content-encoding": "gzip", + }, + timeout=case_timeout_seconds(), + ) + assert response.status_code == 200, response.text From b97fcf1af0c68767e6c42990f2a6815df972cfb1 Mon Sep 17 00:00:00 2001 From: Aidan Singh <95709291+aidanasingh@users.noreply.github.com> Date: Wed, 28 Oct 2026 06:24:26 -0400 Subject: [PATCH 06/12] fix(router): let affinity pins yield to deployments that already failed this request (#32309) * fix(router): apply weighted-failover exclusion before deployment-affinity callback * fix(router): let affinity pins yield to deployments that already failed this request Moving the weighted failover exclusion ahead of the affinity check also changed what tag, order, plugin and managed-file filters saw. Restore the original filter order and have the affinity check ignore session and API key pins that point at a deployment already excluded or retry-skipped in this request. previous_response_id pins still win. * fix(router): release affinity pins on deployments that failed this request Delete a session or API key pin whose deployment already failed the request, so the deployment that serves it claims the pin. Prompt caching pins skip failed deployments too, and the proxy strips the internal failed-id fields from client bodies. * fix(router): keep affinity pins in place and skip failed origins in encrypted content affinity A pin now only yields for the request whose pinned deployment failed. Deleting it moved pins after one 429 or a request-caused 400 and added a Redis call on the routing path. Encrypted content affinity also ignores deployments that failed this request, so a failed origin takes the strip-and-dispatch path. * fix(router): keep encrypted reasoning when a retry can only go back to its origin The encrypted content check now drops failed deployments the way the router does: a weighted failover exclusion always holds, and a retry skip that would leave nothing falls back to the whole group. The proxy also drops client-sent metadata._failover_excluded_ids, and the encrypted content tests use a real Router * fix(router): keep encrypted reasoning when previous_response_id binds the failed origin --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/proxy/litellm_pre_call_utils.py | 3 + litellm/router.py | 11 +- litellm/router_utils/common_utils.py | 28 ++++- .../deployment_affinity_check.py | 8 +- .../encrypted_content_affinity_check.py | 39 ++++++- .../prompt_caching_deployment_check.py | 5 +- .../unit/proxy/test_litellm_pre_call_utils.py | 4 + .../test_deployment_affinity_check.py | 90 +++++++++++++++ .../test_encrypted_content_affinity_check.py | 109 ++++++++++++++++++ .../test_prompt_caching_deployment_check.py | 21 ++++ tests/unit/test_router/test_router.py | 4 +- tests/unit/test_router_weighted_failover.py | 77 ++++++++++++- 12 files changed, 382 insertions(+), 17 deletions(-) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 1e7cdc44c70..0f8ac58861b 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -363,6 +363,8 @@ _UNTRUSTED_ROOT_CONTROL_FIELDS: Final = ( "fallback_depth", "_target_order", "attempted_targets", + "_excluded_deployment_ids", + "_retry_skipped_deployment_ids", "proxy_server_request", "standard_logging_object", "secret_fields", @@ -444,6 +446,7 @@ _UNTRUSTED_METADATA_CONTROL_FIELDS: Final = ( "_pipeline_managed_guardrails", "client_disconnected", "error_information", + "_failover_excluded_ids", PRE_CALL_EXECUTED_GUARDRAILS_KEY, ) diff --git a/litellm/router.py b/litellm/router.py index 18c5c67abce..e53bab4659e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -212,6 +212,7 @@ from litellm.router_utils.clientside_credential_handler import ( ) from litellm.router_utils.common_utils import ( MID_STREAM_CONTINUATION_KWARG, + as_retry_skipped_deployment_ids, deployments_the_caller_may_use, filter_team_based_models, filter_web_search_deployments, @@ -685,10 +686,6 @@ def _configured_model_info(deployment: Mapping[str, object]) -> Mapping[str, obj return _MODEL_INFO_ADAPTER.validate_python(deployment.get("model_info") or {}) -def _as_retry_skipped_deployment_ids(value: object) -> tuple[str, ...]: - return tuple(item for item in value if isinstance(item, str)) if isinstance(value, tuple) else () - - def _silent_experiment_targets(silent_model: object) -> tuple[str, ...]: if silent_model is None: return () @@ -8942,7 +8939,7 @@ class Router: return () if litellm.should_retry(status_code): return () - already_skipped_ids: Final = _as_retry_skipped_deployment_ids(already_skipped) + already_skipped_ids: Final = as_retry_skipped_deployment_ids(already_skipped) skipped: Final = tuple(sorted(frozenset((*already_skipped_ids, failed_deployment_id)))) verbose_router_logger.debug( "Retry skips deployments that already answered %s to this request: %s", status_code, skipped @@ -15186,7 +15183,7 @@ class Router: ## RETRY SKIP ## -> drop deployments that already refused this request with a ## non-retryable status, unless that leaves nothing, so the caller still gets ## the provider's own error instead of a no-deployments error. - _retry_skipped_deployment_ids: Final = _as_retry_skipped_deployment_ids( + _retry_skipped_deployment_ids: Final = as_retry_skipped_deployment_ids( request_kwargs.pop("_retry_skipped_deployment_ids", None) if request_kwargs else None ) healthy_deployments = ( @@ -16418,7 +16415,7 @@ class Router: ) ## RETRY SKIP ## -> see async counterpart in async_get_healthy_deployments. - _retry_skipped_deployment_ids: Final = _as_retry_skipped_deployment_ids( + _retry_skipped_deployment_ids: Final = as_retry_skipped_deployment_ids( request_kwargs.pop("_retry_skipped_deployment_ids", None) if request_kwargs else None ) healthy_deployments = ( diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index a00d84babf3..91b9e5cb114 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -4,7 +4,7 @@ from collections.abc import Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Final, TypeVar -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError if TYPE_CHECKING: from litellm.types.llms.openai import OpenAIFileObject @@ -38,6 +38,7 @@ _OBJECT_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[object, object]]] = TypeAdapt _DEPLOYMENT_ROWS_ADAPTER: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter( tuple[Mapping[str, object], ...] ) +_DEPLOYMENT_IDS_ADAPTER: Final = TypeAdapter(tuple[str, ...]) def with_admin_api_key_secret_reference(deployment: Deployment) -> Deployment: @@ -57,6 +58,31 @@ def without_router_only_kwargs(kwargs: Mapping[str, _V]) -> dict[str, _V]: return {key: value for key, value in kwargs.items() if key not in ROUTER_ONLY_CALL_KWARGS} +def as_retry_skipped_deployment_ids(value: object) -> tuple[str, ...]: + if not isinstance(value, tuple): + return () + items: Final[tuple[object, ...]] = value + return tuple(item for item in items if isinstance(item, str)) + + +def as_excluded_deployment_ids(value: object) -> tuple[str, ...]: + if value is None: + return () + try: + return _DEPLOYMENT_IDS_ADAPTER.validate_python(value) + except ValidationError: + return () + + +def deployment_ids_failed_in_request(request_kwargs: Mapping[str, object]) -> frozenset[str]: + return frozenset( + ( + *as_excluded_deployment_ids(request_kwargs.get("_excluded_deployment_ids")), + *as_retry_skipped_deployment_ids(request_kwargs.get("_retry_skipped_deployment_ids")), + ) + ) + + def request_user_api_key_auth(request_kwargs: Mapping[str, object]) -> object: """The caller's auth object, from whichever metadata bucket this surface writes to.""" metadata_value: Final = request_kwargs.get("metadata") diff --git a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py index 8693eaa5151..daf7d6bb345 100644 --- a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py @@ -31,6 +31,7 @@ from litellm.caching.dual_cache import DualCache from litellm.constants import SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger, Span from litellm.responses.utils import ResponsesAPIRequestUtils +from litellm.router_utils.common_utils import deployment_ids_failed_in_request from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import CallTypes @@ -429,6 +430,9 @@ class DeploymentAffinityCheck(CustomLogger): session_affinity_active: Final = ( enable_session_id or self._get_marker_session_affinity_ttl(request_kwargs=request_kwargs) is not None ) + failed_deployment_ids: Final = deployment_ids_failed_in_request( + cast(Mapping[str, object], request_kwargs) # cast-ok: request kwargs keys are param names + ) user_key: Final = ( self.get_user_key_from_request_kwargs(request_kwargs=request_kwargs) if (session_affinity_active or enable_user_key) @@ -458,7 +462,7 @@ class DeploymentAffinityCheck(CustomLogger): elif isinstance(session_cache_result, str): session_model_id = session_cache_result - if session_model_id: + if session_model_id and session_model_id not in failed_deployment_ids: session_deployment: Final = self._find_deployment_by_model_id( healthy_deployments=typed_healthy_deployments, model_id=session_model_id, @@ -493,7 +497,7 @@ class DeploymentAffinityCheck(CustomLogger): # Backwards / safety: allow raw string values. model_id = cache_result - if not model_id: + if not model_id or model_id in failed_deployment_ids: return typed_healthy_deployments deployment = self._find_deployment_by_model_id( diff --git a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py index 5d47bf63a38..806c3db7601 100644 --- a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py @@ -51,6 +51,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( strip_encrypted_reasoning_from_messages, ) from litellm.responses.utils import ResponsesAPIRequestUtils +from litellm.router_utils.common_utils import as_excluded_deployment_ids, as_retry_skipped_deployment_ids from litellm.types.llms.openai import AllMessageValues from litellm.types.router import Deployment from litellm.utils import get_order_filtered_deployments @@ -182,6 +183,14 @@ class EncryptedContentAffinityCheck(CustomLogger): None, ) + @staticmethod + def _deployment_model_id(deployment: Mapping[str, object]) -> str | None: + model_info: Final = deployment.get("model_info") + if not isinstance(model_info, Mapping): + return None + deployment_id: Final = cast(Mapping[str, object], model_info).get("id") # cast-ok: narrowed by isinstance + return None if deployment_id is None else str(deployment_id) + @staticmethod def _find_deployment_by_model_id(healthy_deployments: list[dict], model_id: str) -> dict | None: for deployment in healthy_deployments: @@ -345,7 +354,8 @@ class EncryptedContentAffinityCheck(CustomLogger): typed_healthy_deployments: Final = cast(list[dict[str, object]], healthy_deployments) if not self._is_enabled_for_model_group(model): return typed_healthy_deployments - target_order: Final = _REQUEST_KWARGS_ADAPTER.validate_python(request_kwargs).get("_target_order") + typed_request_kwargs: Final = _REQUEST_KWARGS_ADAPTER.validate_python(request_kwargs) + target_order: Final = typed_request_kwargs.get("_target_order") candidates: Final = ( get_order_filtered_deployments(typed_healthy_deployments, target_order=target_order) if isinstance(target_order, int) @@ -375,8 +385,31 @@ class EncryptedContentAffinityCheck(CustomLogger): model_id, ) + previous_response_id: Final = typed_request_kwargs.get("previous_response_id") + previous_response_origin: Final = ResponsesAPIRequestUtils.get_model_id_from_response_id( + None if previous_response_id is None else str(previous_response_id) + ) + previous_response_binds_origin: Final = previous_response_origin == model_id + excluded_ids: Final[frozenset[str]] = ( + frozenset() + if previous_response_binds_origin + else frozenset(as_excluded_deployment_ids(typed_request_kwargs.get("_excluded_deployment_ids"))) + ) + retry_skipped_ids: Final[frozenset[str]] = ( + frozenset() + if previous_response_binds_origin + else frozenset(as_retry_skipped_deployment_ids(typed_request_kwargs.get("_retry_skipped_deployment_ids"))) + ) + not_excluded_candidates: Final = [ + candidate for candidate in candidates if self._deployment_model_id(candidate) not in excluded_ids + ] + untried_candidates: Final = [ + candidate + for candidate in not_excluded_candidates + if self._deployment_model_id(candidate) not in retry_skipped_ids + ] or not_excluded_candidates deployment: Final = self._find_deployment_by_model_id( - healthy_deployments=candidates, + healthy_deployments=untried_candidates, model_id=model_id, ) if deployment is not None: @@ -392,7 +425,7 @@ class EncryptedContentAffinityCheck(CustomLogger): # Follow-up switched model_name (LIT-2531): pin by Azure resource instead. boundary_matches, _originating = self._find_deployments_on_same_encryption_boundary( - healthy_deployments=candidates, + healthy_deployments=untried_candidates, model_id=model_id, ) if boundary_matches: diff --git a/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py b/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py index bb3475cdcbe..c04bab4d360 100644 --- a/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py +++ b/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py @@ -19,6 +19,7 @@ from litellm.integrations.anthropic_cache_control_hook import ( from litellm.integrations.custom_logger import CustomLogger, Span from litellm.litellm_core_utils.token_counter import offload_token_count from litellm.llms.bedrock.common_utils import bedrock_cache_point_ttl_supported, bedrock_capability_model +from litellm.router_utils.common_utils import deployment_ids_failed_in_request from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam from litellm.types.utils import CallTypes, StandardLoggingPayload from litellm.utils import ( @@ -152,7 +153,9 @@ class PromptCachingDeploymentCheck(CustomLogger): request_tools, ), ) - if model_id_dict is not None: + if model_id_dict is not None and model_id_dict["model_id"] not in deployment_ids_failed_in_request( + request_data or {} + ): model_id: Final = model_id_dict["model_id"] for deployment in healthy_deployments: if deployment["model_info"]["id"] == model_id: diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index a44373b87c8..f743a14c7a2 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -1393,6 +1393,8 @@ async def test_add_litellm_data_to_request_strips_string_encoded_admin_injection ("fallback_depth", True), ("_target_order", 2), ("attempted_targets", ["forged"]), + ("_excluded_deployment_ids", ["pinned-deployment"]), + ("_retry_skipped_deployment_ids", ["pinned-deployment"]), ], ) async def test_add_litellm_data_to_request_strips_forged_fallback_hop_state( @@ -1477,6 +1479,7 @@ async def test_add_litellm_data_to_request_strips_user_control_fields(): "_managed_background_response": True, "_guardrail_pipelines": [{"name": "spoofed"}], "_pipeline_managed_guardrails": ["evaded"], + "_failover_excluded_ids": ["pinned-deployment"], "safe_user_metadata": "kept", } data = { @@ -1537,6 +1540,7 @@ async def test_add_litellm_data_to_request_strips_user_control_fields(): "_managed_background_response", "_guardrail_pipelines", "_pipeline_managed_guardrails", + "_failover_excluded_ids", } assert "litellm_metadata" not in updated for stripped_key in stripped_keys: diff --git a/tests/unit/router_utils/pre_call_checks/test_deployment_affinity_check.py b/tests/unit/router_utils/pre_call_checks/test_deployment_affinity_check.py index d8bc4c45ab8..fa75f980cc0 100644 --- a/tests/unit/router_utils/pre_call_checks/test_deployment_affinity_check.py +++ b/tests/unit/router_utils/pre_call_checks/test_deployment_affinity_check.py @@ -9,6 +9,7 @@ import pytest import litellm from litellm.caching.dual_cache import DualCache +from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.router_utils.pre_call_checks.deployment_affinity_check import ( DeploymentAffinityCheck, ) @@ -1209,3 +1210,92 @@ async def test_virtual_key_hash_wins_over_user_id_for_affinity(): parent_otel_span=None, ) assert other_key_same_user == healthy_deployments + + +class _ForbiddenError(Exception): + status_code: Final = 403 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("pre_call_check", "metadata"), + [ + ("session_affinity", {"session_id": "sess-retry"}), + ("deployment_affinity", {"user_api_key_hash": "hashed-key-retry"}), + ], +) +async def test_retry_reaches_healthy_sibling_when_affinity_pins_deployment_that_refused( + pre_call_check: str, metadata: dict[str, str] +): + router: Final = litellm.Router( + model_list=[ + { + "model_name": "test-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": "bad", + "mock_response": _ForbiddenError("A refuses"), + "weight": 1, + }, + "model_info": {"id": "A"}, + }, + { + "model_name": "test-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": "good", + "mock_response": "ok from B", + "weight": 0, + }, + "model_info": {"id": "B"}, + }, + ], + routing_strategy="simple-shuffle", + num_retries=1, + retry_after=0, + disable_cooldowns=True, + optional_pre_call_checks=[pre_call_check], + ) + + responses: Final = [ + await router.acompletion( + model="test-model", + messages=[{"role": "user", "content": "hi"}], + metadata=dict(metadata), + ) + for _ in range(2) + ] + + assert [response._hidden_params["model_id"] for response in responses] == ["B", "B"] + assert [response._hidden_params["additional_headers"]["x-litellm-attempted-retries"] for response in responses] == [ + 1, + 1, + ] + + +@pytest.mark.asyncio +async def test_previous_response_id_pin_holds_even_after_its_deployment_failed(): + callback: Final = DeploymentAffinityCheck( + cache=DualCache(), + ttl_seconds=60, + enable_user_key_affinity=True, + enable_responses_api_affinity=True, + ) + healthy_deployments: Final = [ + {"model_name": "gpt-4o", "litellm_params": {"model": "gpt-4o"}, "model_info": {"id": "A"}}, + {"model_name": "gpt-4o", "litellm_params": {"model": "gpt-4o"}, "model_info": {"id": "B"}}, + ] + + assert await callback.async_filter_deployments( + model="gpt-4o", + healthy_deployments=healthy_deployments, + messages=None, + request_kwargs={ + "previous_response_id": ResponsesAPIRequestUtils._build_responses_api_response_id( + custom_llm_provider="openai", model_id="A", response_id="resp_upstream" + ), + "_excluded_deployment_ids": ["A"], + "metadata": {"user_api_key_hash": "hashed-key"}, + }, + parent_otel_span=None, + ) == [healthy_deployments[0]] diff --git a/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py b/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py index 87fbf4433a1..a3745c7b23f 100644 --- a/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py +++ b/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py @@ -16,6 +16,7 @@ The mechanism works without any cache and supports two encoding strategies: """ import time +from typing import Final from unittest.mock import AsyncMock, patch import pytest @@ -2606,3 +2607,111 @@ async def test_affinity_strips_for_team_and_pattern_routes(): assert result is sibling_pool assert not request_kwargs["input"][1].get("encrypted_content") + + +def _region_deployment(region: str) -> dict[str, object]: + return { + "model_name": "gpt-5.4", + "litellm_params": { + "model": "openai/gpt-5.4", + "api_base": f"https://{region}.example.com/v1", + "api_key": f"key-{region}", + }, + "model_info": {"id": region}, + } + + +def _reasoning_input_minted_by(origin: str) -> list[dict[str, object]]: + return [ + { + "type": "reasoning", + "id": ResponsesAPIRequestUtils._build_encrypted_item_id(origin, "rs_test"), + "encrypted_content": ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id("gAAAAA-blob", origin), + "summary": [{"type": "summary_text", "text": "scattering"}], + }, + {"role": "user", "content": "and sunsets?"}, + ] + + +def _previous_response_id_from(origin: str) -> str: + return ResponsesAPIRequestUtils._build_responses_api_response_id("openai", origin, "resp_test") + + +_ORIGIN_FAILED_THIS_REQUEST: Final = pytest.mark.parametrize( + "failed_in_request", + [{"_excluded_deployment_ids": ["region-a"]}, {"_retry_skipped_deployment_ids": ("region-a",)}], +) + + +@pytest.mark.asyncio +@_ORIGIN_FAILED_THIS_REQUEST +@pytest.mark.parametrize("continuing", [{}, {"previous_response_id": _previous_response_id_from("region-b")}]) +async def test_affinity_strips_and_dispatches_when_origin_already_failed_this_request( + failed_in_request: dict[str, object], continuing: dict[str, object] +): + from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( + EncryptedContentAffinityCheck, + ) + + router: Final = litellm.Router(model_list=[_region_deployment("region-a"), _region_deployment("region-b")]) + group: Final = router.model_list + reasoning_input: Final = _reasoning_input_minted_by("region-a") + + result: Final = await EncryptedContentAffinityCheck(router=router).async_filter_deployments( + model="gpt-5.4", + healthy_deployments=group, + messages=None, + request_kwargs={"litellm_metadata": {}, "input": reasoning_input, **failed_in_request, **continuing}, + ) + + assert result is group + assert reasoning_input[0] == {"type": "reasoning", "summary": [{"type": "summary_text", "text": "scattering"}]} + + +@pytest.mark.asyncio +@_ORIGIN_FAILED_THIS_REQUEST +async def test_affinity_keeps_origin_and_reasoning_when_previous_response_id_binds_the_failed_origin( + failed_in_request: dict[str, object], +): + from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( + EncryptedContentAffinityCheck, + ) + + pool: Final = (_region_deployment("region-a"), _region_deployment("region-b")) + reasoning_input: Final = _reasoning_input_minted_by("region-a") + original_reasoning: Final = dict(reasoning_input[0]) + + assert await EncryptedContentAffinityCheck(router=litellm.Router(model_list=list(pool))).async_filter_deployments( + model="gpt-5.4", + healthy_deployments=list(pool), + messages=None, + request_kwargs={ + "litellm_metadata": {}, + "input": reasoning_input, + "previous_response_id": _previous_response_id_from("region-a"), + **failed_in_request, + }, + ) == [pool[0]] + assert reasoning_input[0] == original_reasoning + + +@pytest.mark.asyncio +@pytest.mark.parametrize("regions", [("region-a",), ("region-a", "region-b")]) +async def test_affinity_keeps_reasoning_when_every_deployment_already_refused_the_request(regions: tuple[str, ...]): + from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( + EncryptedContentAffinityCheck, + ) + + router: Final = litellm.Router(model_list=[_region_deployment(region) for region in regions]) + reasoning_input: Final = _reasoning_input_minted_by("region-a") + original_reasoning: Final = dict(reasoning_input[0]) + + result: Final = await EncryptedContentAffinityCheck(router=router).async_filter_deployments( + model="gpt-5.4", + healthy_deployments=router.model_list, + messages=None, + request_kwargs={"litellm_metadata": {}, "input": reasoning_input, "_retry_skipped_deployment_ids": regions}, + ) + + assert [deployment["model_info"]["id"] for deployment in result] == ["region-a"] + assert reasoning_input[0] == original_reasoning diff --git a/tests/unit/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py b/tests/unit/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py index 8d893160780..9487e4d6321 100644 --- a/tests/unit/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py +++ b/tests/unit/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py @@ -313,6 +313,27 @@ async def test_async_filter_deployments_does_not_pin_when_target_order_is_set(): assert filtered == deployments +@pytest.mark.asyncio +@pytest.mark.parametrize( + "failed_in_request", + [{"_excluded_deployment_ids": ["dep-2"]}, {"_retry_skipped_deployment_ids": ("dep-2",)}], +) +async def test_async_filter_deployments_does_not_pin_to_a_deployment_that_failed_this_request( + failed_in_request: dict[str, object], +): + cache: Final = DualCache() + check: Final = PromptCachingDeploymentCheck(cache=cache) + messages: Final = _messages(word_count=5000) + await PromptCachingCache(cache=cache).async_add_model_id(model_id="dep-2", messages=messages, tools=None) + + assert await check.async_filter_deployments( + model=MODEL_GROUP_ALIAS, + healthy_deployments=_deployments("anthropic/claude-opus-4-6", "anthropic/claude-opus-4-6"), + messages=messages, + request_kwargs=failed_in_request, + ) == _deployments("anthropic/claude-opus-4-6", "anthropic/claude-opus-4-6") + + @pytest.mark.asyncio async def test_async_filter_deployments_narrows_for_group_whose_model_minimum_is_lower(): """ diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index d9b655a0aff..30d675e130d 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -21239,9 +21239,9 @@ def test_router_retry_skip_stamp_feeds_deployment_ids_to_skip_on_retry( ], ) def test_router_as_retry_skipped_deployment_ids_keeps_only_a_tuple_of_strings(value, expected): - from litellm.router import _as_retry_skipped_deployment_ids + from litellm.router_utils.common_utils import as_retry_skipped_deployment_ids - assert _as_retry_skipped_deployment_ids(value) == expected + assert as_retry_skipped_deployment_ids(value) == expected @pytest.mark.parametrize( diff --git a/tests/unit/test_router_weighted_failover.py b/tests/unit/test_router_weighted_failover.py index 9cb38b00e1e..8b7fc81dbde 100644 --- a/tests/unit/test_router_weighted_failover.py +++ b/tests/unit/test_router_weighted_failover.py @@ -8,13 +8,14 @@ cross-group fallback runs. """ from collections import Counter -from typing import Optional +from typing import Final, Optional from unittest.mock import AsyncMock, patch import pytest import litellm from litellm import Router +from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo from litellm.utils import get_excluded_filtered_deployments @@ -944,3 +945,77 @@ async def test_failover_falls_through_to_external_fallback_when_remaining_in_coo ) assert response._hidden_params["model_id"] == "fallback" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("pre_call_check", "metadata"), + [ + ("session_affinity", {"session_id": "sess-1"}), + ("deployment_affinity", {"user_api_key_hash": "hashed-key-1"}), + ], +) +async def test_failover_reaches_healthy_sibling_when_affinity_pins_failed_deployment( + pre_call_check: str, metadata: dict[str, str] +): + router: Final = Router( + model_list=[ + { + "model_name": "test-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": "bad", + "mock_response": Exception("A down"), + "weight": 1, + }, + "model_info": {"id": "A"}, + }, + { + "model_name": "test-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": "good", + "mock_response": "ok from B", + "weight": 0, + }, + "model_info": {"id": "B"}, + }, + { + "model_name": "fallback-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": "good", + "mock_response": "ok from fallback", + }, + "model_info": {"id": "fallback"}, + }, + ], + routing_strategy="simple-shuffle", + num_retries=0, + disable_cooldowns=True, + enable_weighted_failover=True, + optional_pre_call_checks=[pre_call_check], + fallbacks=[{"test-model": ["fallback-model"]}], + ) + + while_a_is_down: Final = await router.acompletion( + model="test-model", + messages=[{"role": "user", "content": "hi"}], + metadata=dict(metadata), + ) + router.upsert_deployment( + Deployment( + model_name="test-model", + litellm_params=LiteLLM_Params(model="gpt-4o", api_key="bad", mock_response="ok from A", weight=1), + model_info=ModelInfo(id="A"), + ) + ) + after_a_recovers: Final = await router.acompletion( + model="test-model", + messages=[{"role": "user", "content": "hi"}], + metadata=dict(metadata), + ) + + assert while_a_is_down._hidden_params["model_id"] == "B" + assert while_a_is_down._hidden_params["additional_headers"]["x-litellm-attempted-fallbacks"] == 1 + assert after_a_recovers._hidden_params["model_id"] == "A" From de40c0aa39e26d318eaa3fc0337a788165862972 Mon Sep 17 00:00:00 2001 From: tessellark <349031000+tessellark@users.noreply.github.com> Date: Wed, 28 Oct 2026 03:38:21 -0700 Subject: [PATCH 07/12] test(llms): audit #47535 (#48786) Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../providers/_api_base_url_join_support.py | 9 ++ ...test_api_base_url_join_host_owned_proxy.py | 37 ++++++- .../providers/test_api_base_url_join_wire.py | 99 +++++++++++++++++-- 3 files changed, 134 insertions(+), 11 deletions(-) diff --git a/tests/integration/providers/_api_base_url_join_support.py b/tests/integration/providers/_api_base_url_join_support.py index 44bd6ec4dca..5711cc77309 100644 --- a/tests/integration/providers/_api_base_url_join_support.py +++ b/tests/integration/providers/_api_base_url_join_support.py @@ -13,6 +13,7 @@ from pydantic import JsonValue COUNTED_TOKENS: Final = 7 RELEVANCE: Final = 0.75 +AZURE_DEPLOYMENT: Final = "gpt-url-join" _USAGE: Final = {"input_tokens": 3, "output_tokens": 2} @@ -146,12 +147,20 @@ def _cohere_rerank(request: Request) -> Reply: ) +def _titan_embedding(_: Request) -> Reply: + return _json({"embedding": list(VECTOR), "inputTextTokenCount": 2}) + + _BY_PREFIX: Final[Mapping[str, Callable[[Request], Reply]]] = MappingProxyType( { "an": _anthropic, "bi": lambda request: _json(anthropic_message(marker_of(request))), "bc": _bedrock_converse, + "be": _titan_embedding, "gm": _gemini, + "vx": _gemini, + "fd": _anthropic, + "az": _openai_like(AZURE_DEPLOYMENT), "mi": _openai_like("mistral-small-latest"), "db": _openai_like("databricks-meta-llama-3-3-70b-instruct"), "co": _cohere_rerank, diff --git a/tests/integration/providers/test_api_base_url_join_host_owned_proxy.py b/tests/integration/providers/test_api_base_url_join_host_owned_proxy.py index 4a075138305..687c15283f4 100644 --- a/tests/integration/providers/test_api_base_url_join_host_owned_proxy.py +++ b/tests/integration/providers/test_api_base_url_join_host_owned_proxy.py @@ -1,6 +1,7 @@ import os from collections.abc import Callable, Iterator from pathlib import Path +from types import MappingProxyType from typing import Final import pytest @@ -8,6 +9,7 @@ import yaml from integration._support.client import JSON_OBJECT, Gateway, gateway_from_environment from integration._support.forward_proxy import ForwardProxy, refusing_forward_proxy from integration._support.process import graceful_stop_seconds, owned_proxy +from integration._support.vertex import service_account_json from integration.providers._api_base_trailing_slash_support import KEY, new_marker from pydantic import JsonValue @@ -17,6 +19,12 @@ _RERANK: Final = "host-rerank" _AZURE_CLAUDE: Final = "host-azure-claude" _AZURE_COUNT: Final = "host-azure-count" _EMPTY_BASE: Final = "host-empty-base" +_VERTEX_HOST: Final = "vertex-gateway" +_VERTEX_PROJECT: Final = "url-join-project" +_VERTEX_LOCATION: Final = "us-east5" +_VERTEX_CLAUDE_BASES: Final = MappingProxyType( + {"host-vertex-claude": f"http://{_VERTEX_HOST}", "host-vertex-claude-slash": f"http://{_VERTEX_HOST}/"} +) _SCRUBBED_PREFIXES: Final = ("ANTHROPIC_", "AZURE_AI_", "INFINITY_") _PROXY_VARIABLES: Final = frozenset({"HTTPS_PROXY", "HTTP_PROXY", "ALL_PROXY", "NO_PROXY"}) _load_yaml: Final[Callable[[str], object]] = yaml.safe_load @@ -28,7 +36,20 @@ def _deployment(name: str, model: str, api_base: str) -> dict[str, JsonValue]: return {"model_name": name, "litellm_params": {"model": model, "api_base": api_base, "api_key": KEY}} -def _config(directory: Path) -> Path: +def _vertex_claude_deployment(name: str, api_base: str, token_url: str) -> dict[str, JsonValue]: + return { + "model_name": name, + "litellm_params": { + "model": "vertex_ai/claude-haiku-5-5", + "api_base": api_base, + "vertex_project": _VERTEX_PROJECT, + "vertex_location": _VERTEX_LOCATION, + "vertex_credentials": service_account_json(_VERTEX_PROJECT, token_url), + }, + } + + +def _config(directory: Path, token_url: str) -> Path: base: Final = JSON_OBJECT.validate_python(_load_yaml(Path("tests/integration/proxy_config.yaml").read_text())) configuration: Final = { **base, @@ -38,6 +59,7 @@ def _config(directory: Path) -> Path: _deployment(_AZURE_CLAUDE, "azure_ai/claude-haiku-5-5", f"http://{_AZURE_HOST}/anthropic"), _deployment(_AZURE_COUNT, "azure_ai/claude-haiku-5-5", f"http://{_AZURE_HOST}"), _deployment(_EMPTY_BASE, "anthropic/claude-haiku-5-5", ""), + *(_vertex_claude_deployment(name, base, token_url) for name, base in _VERTEX_CLAUDE_BASES.items()), ], "router_settings": {"disable_cooldowns": True, "num_retries": 0}, } @@ -72,7 +94,7 @@ def isolated(forward: ForwardProxy, tmp_path_factory: pytest.TempPathFactory) -> "NO_PROXY": "127.0.0.1,localhost", "PROXY_BATCH_POLLING_ENABLED": "false", }, - config=_config(directory), + config=_config(directory, shared.upstream_url.rstrip("/")), remove_environment=_scrubbed(), ) as candidate: yield candidate @@ -138,3 +160,14 @@ def test_azure_claude_count_tokens_on_a_bare_host_starting_with_anthropic_adds_t def test_empty_api_base_falls_back_to_the_provider_default_host(isolated: Gateway, forward: ForwardProxy) -> None: targets: Final = _targets_after(forward, lambda: _chat(isolated, _EMPTY_BASE)) assert targets == {"api.anthropic.com:443"} + + +@pytest.mark.parametrize("model", tuple(_VERTEX_CLAUDE_BASES), ids=("no-slash", "slash")) +def test_vertex_partner_claude_on_a_bare_host_reaches_one_single_slash_url( + isolated: Gateway, forward: ForwardProxy, model: str +) -> None: + targets: Final = _targets_after(forward, lambda: _chat(isolated, model)) + assert targets == { + f"http://{_VERTEX_HOST}/v1/projects/{_VERTEX_PROJECT}/locations/{_VERTEX_LOCATION}" + "/publishers/anthropic/models/claude-haiku-5-5:rawPredict" + } diff --git a/tests/integration/providers/test_api_base_url_join_wire.py b/tests/integration/providers/test_api_base_url_join_wire.py index 42e53ff9f95..74f8dde3cf7 100644 --- a/tests/integration/providers/test_api_base_url_join_wire.py +++ b/tests/integration/providers/test_api_base_url_join_wire.py @@ -7,6 +7,7 @@ import openai import pytest from integration._support.client import Gateway, Scenario, eventually, list_value, object_value, string_value from integration._support.database import read_rows +from integration._support.vertex import service_account_json from integration._support.wire import Reply, Request, wire_server from integration.providers._api_base_trailing_slash_support import ( KEY, @@ -22,7 +23,7 @@ from integration.providers._api_base_trailing_slash_support import ( reply_text, request_json, ) -from integration.providers._api_base_url_join_support import COUNTED_TOKENS, RELEVANCE, join_peer +from integration.providers._api_base_url_join_support import AZURE_DEPLOYMENT, COUNTED_TOKENS, RELEVANCE, join_peer from pydantic import JsonValue _HAIKU: Final = "anthropic.claude-haiku-5-5" @@ -49,10 +50,20 @@ _RESPONSES_ROWS: Final = (_ANTHROPIC, _GEMINI) _EMBEDDING_ROWS: Final = ( Row("mistral/mistral-embed", "/mi/v1", "/embeddings"), Row("gemini/gemini-embedding-2", "/gm/v1beta", "/models/gemini-embedding-2:batchEmbedContents"), + Row("bedrock/amazon.titan-embed-text-v2:0", "/be", "/model/amazon.titan-embed-text-v2%3A0/invoke"), ) _TRANSCRIPTION: Final = Row("mistral/voxtral-mini-latest", "/mi/v1", "/audio/transcriptions") -_FULL_ROUTE_SUFFIXES: Final = ("", "/", "?beta=true") -_FULL_ROUTE_IDS: Final = ("exact", "slash", "query") +_FULL_ROUTE_SUFFIXES: Final = (("", ""), ("/", ""), ("?beta=true", "?beta=true"), ("/?beta=true", "?beta=true")) +_FULL_ROUTE_IDS: Final = ("exact", "slash", "query", "slash-query") +_VERTEX_PROJECT: Final = "url-join-project" +_VERTEX_LOCATION: Final = "us-east5" +_VERTEX_GEMINI_PATH: Final = ( + f"/vx/v1/projects/{_VERTEX_PROJECT}/locations/{_VERTEX_LOCATION}/publishers/google/models/gemini-3.8-flash" +) +_AZURE_V1_BASES: Final = ("/az", "/az/", "/az/openai/v1", "/az/openai/v1/") +_AZURE_V1_IDS: Final = ("host", "host-slash", "route", "route-slash") +_AZURE_ANTHROPIC_BASES: Final = ("/fd/anthropic", "/fd/anthropic/v1/messages") +_AZURE_ANTHROPIC_IDS: Final = ("anthropic-root", "messages-route") def _ids(rows: tuple[Row, ...]) -> tuple[str, ...]: @@ -65,6 +76,17 @@ def _deploy(scenario: Scenario, row: Row, api_base: str) -> str: return scenario.model(model=row.model, api_base=api_base, api_key=KEY) +def _deploy_vertex(gateway: Gateway, scenario: Scenario, model: str, api_base: str) -> str: + return scenario.model( + model=model, + api_base=api_base, + api_key=None, + vertex_project=_VERTEX_PROJECT, + vertex_location=_VERTEX_LOCATION, + vertex_credentials=service_account_json(_VERTEX_PROJECT, gateway.upstream_url.rstrip("/")), + ) + + def _openai(gateway: Gateway) -> openai.OpenAI: return openai.OpenAI(base_url=str(gateway.client.base_url) + "/v1", api_key=gateway.key, max_retries=0) @@ -80,6 +102,13 @@ def _chat(gateway: Gateway, model: str, marker: str) -> str | None: return completion.choices[0].message.content +def _streamed_chat(gateway: Gateway, model: str, marker: str) -> str: + stream: Final = _openai(gateway).chat.completions.create( + model=model, messages=[{"role": "user", "content": marker}], stream=True + ) + return "".join(chunk.choices[0].delta.content or "" for chunk in stream if chunk.choices) + + def _message_text(gateway: Gateway, model: str, marker: str) -> list[str]: message: Final = _anthropic(gateway).messages.create( model=model, max_tokens=16, messages=[{"role": "user", "content": marker}] @@ -270,22 +299,74 @@ def test_cache_hit_on_a_slash_base_skips_the_upstream_and_logs_its_own_row(gatew assert "//" not in urlsplit(string_value(hit["api_base"])).path, hit -@pytest.mark.parametrize("suffix", _FULL_ROUTE_SUFFIXES, ids=_FULL_ROUTE_IDS) -def test_chat_base_that_already_names_the_route_keeps_its_path_and_query(gateway: Gateway, suffix: str) -> None: +@pytest.mark.parametrize(("suffix", "sent"), _FULL_ROUTE_SUFFIXES, ids=_FULL_ROUTE_IDS) +def test_chat_base_that_already_names_the_route_keeps_its_path_and_query( + gateway: Gateway, suffix: str, sent: str +) -> None: marker: Final = new_marker() with wire_server(join_peer) as wire, gateway.scenario() as scenario: model: Final = _deploy(scenario, _ANTHROPIC, f"{wire.url}/an/v1/messages{suffix}") assert _chat(gateway, model, marker) == reply_text(marker) - assert_one_request(wire.drain(), f"/an/v1/messages{suffix.rstrip('/')}", marker) + assert_one_request(wire.drain(), f"/an/v1/messages{sent}", marker) -@pytest.mark.parametrize("suffix", _FULL_ROUTE_SUFFIXES, ids=_FULL_ROUTE_IDS) -def test_messages_base_that_already_names_the_route_keeps_its_path_and_query(gateway: Gateway, suffix: str) -> None: +@pytest.mark.parametrize(("suffix", "sent"), _FULL_ROUTE_SUFFIXES, ids=_FULL_ROUTE_IDS) +def test_messages_base_that_already_names_the_route_keeps_its_path_and_query( + gateway: Gateway, suffix: str, sent: str +) -> None: marker: Final = new_marker() with wire_server(join_peer) as wire, gateway.scenario() as scenario: model: Final = _deploy(scenario, _ANTHROPIC, f"{wire.url}/an/v1/messages{suffix}") assert _message_text(gateway, model, marker) == [reply_text(marker)] - assert_one_request(wire.drain(), f"/an/v1/messages{suffix.rstrip('/')}", marker) + assert_one_request(wire.drain(), f"/an/v1/messages{sent}", marker) + + +@pytest.mark.parametrize("trailing", TRAILING, ids=TRAILING_IDS) +def test_vertex_model_url_base_calls_its_method_with_no_slash_before_the_colon(gateway: Gateway, trailing: str) -> None: + marker: Final = new_marker() + with wire_server(join_peer) as wire, gateway.scenario() as scenario: + model: Final = _deploy_vertex( + gateway, scenario, "vertex_ai/gemini-3.8-flash", f"{wire.url}{_VERTEX_GEMINI_PATH}{trailing}" + ) + assert _chat(gateway, model, marker) == reply_text(marker) + assert_one_request(wire.drain(), f"{_VERTEX_GEMINI_PATH}:generateContent", marker, keyed=False) + + +@pytest.mark.parametrize("trailing", TRAILING, ids=TRAILING_IDS) +def test_vertex_model_url_base_streams_with_no_slash_before_the_colon(gateway: Gateway, trailing: str) -> None: + marker: Final = new_marker() + with wire_server(join_peer) as wire, gateway.scenario() as scenario: + model: Final = _deploy_vertex( + gateway, scenario, "vertex_ai/gemini-3.8-flash", f"{wire.url}{_VERTEX_GEMINI_PATH}{trailing}" + ) + assert _streamed_chat(gateway, model, marker) == reply_text(marker) + assert_one_request(wire.drain(), f"{_VERTEX_GEMINI_PATH}:streamGenerateContent?alt=sse", marker, keyed=False) + + +@pytest.mark.parametrize("base", _AZURE_V1_BASES, ids=_AZURE_V1_IDS) +def test_azure_v1_chat_reaches_one_openai_v1_route(gateway: Gateway, base: str) -> None: + marker: Final = new_marker() + with wire_server(join_peer) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"azure/{AZURE_DEPLOYMENT}", api_base=f"{wire.url}{base}", api_key=KEY, api_version="v1" + ) + assert _chat(gateway, model, marker) == reply_text(marker) + assert_one_request(wire.drain(), "/az/openai/v1/chat/completions", marker) + + +@pytest.mark.parametrize("trailing", TRAILING, ids=TRAILING_IDS) +@pytest.mark.parametrize("base", _AZURE_ANTHROPIC_BASES, ids=_AZURE_ANTHROPIC_IDS) +def test_azure_ai_count_tokens_reaches_one_count_tokens_route(gateway: Gateway, base: str, trailing: str) -> None: + marker: Final = new_marker() + with wire_server(join_peer) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model="azure_ai/claude-haiku-5-5", api_base=f"{wire.url}{base}{trailing}", api_key=KEY + ) + response: Final = gateway.request( + "POST", "/v1/messages/count_tokens", {"model": model, "messages": [{"role": "user", "content": marker}]} + ) + assert body_of(response)["input_tokens"] == COUNTED_TOKENS + assert_one_request(wire.drain(), "/fd/anthropic/v1/messages/count_tokens", marker) @pytest.mark.parametrize("suffix", ("", "/", "?tier=a"), ids=("exact", "slash", "query")) From 91db681532d35f318726df20c605e14c82dd2219 Mon Sep 17 00:00:00 2001 From: norkelbyte <348963847+norkelbyte@users.noreply.github.com> Date: Wed, 28 Oct 2026 03:50:24 -0700 Subject: [PATCH 08/12] test(responses): audit #47504 (#48787) Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../test_background_response_billed_once.py | 31 ++++++++++++++++--- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/tests/integration/spend/test_background_response_billed_once.py b/tests/integration/spend/test_background_response_billed_once.py index 1b7fc975217..b207e46204a 100644 --- a/tests/integration/spend/test_background_response_billed_once.py +++ b/tests/integration/spend/test_background_response_billed_once.py @@ -642,6 +642,15 @@ def _key_budget(scenario: Scenario, budget_owner: str, model_budget: JsonValue) return {} +def _assert_the_next_create_exceeds_the_model_budget(scenario: Scenario, key: str, model: str) -> None: + refused: Final = eventually( + lambda: _next_create(scenario, key, model), + lambda response: response.status_code != 200, + seconds=SPEND_LANDING_SECONDS, + ) + assert f"exceeded budget for model={model}" in refused.text, refused.text + + @pytest.mark.timeout(240) @pytest.mark.parametrize("budget_owner", ["key", "team", "user", "budget_tier"]) def test_a_billed_background_response_counts_toward_its_creators_project_and_model_budget( @@ -663,12 +672,24 @@ def test_a_billed_background_response_counts_toward_its_creators_project_and_mod lambda spends: spends[0] > 0 and costs(spends[0])(spends[1]), seconds=SPEND_LANDING_SECONDS, ) - refused: Final = eventually( - lambda: _next_create(scenario, key, upstream.model), - lambda response: response.status_code != 200, - seconds=SPEND_LANDING_SECONDS, + _assert_the_next_create_exceeds_the_model_budget(scenario, key, upstream.model) + + +@pytest.mark.timeout(240) +def test_a_billed_background_response_on_a_team_model_counts_toward_the_budget_of_the_model_name_it_was_created_with( + gateway: Gateway, +) -> None: + with gateway.scenario() as scenario: + team_id: Final = scenario.team() + upstream: Final = scripted_upstream(scenario, background_routes("completed"), model_info={"team_id": team_id}) + key: Final = scenario.key( + team_id=team_id, model_max_budget={upstream.model: {"max_budget": PRICE / 2, "budget_duration": "1d"}} ) - assert f"exceeded budget for model={upstream.model}" in refused.text, refused.text + response_id: Final = create_background_response(scenario, key, upstream.model) + read: Final = read_response(gateway, key, response_id) + assert (read,) in (settling_reads("completed", 1), free_reads("completed", 1)) + eventually(lambda: key_spend(key), lambda spend: spend > 0, seconds=SPEND_LANDING_SECONDS) + _assert_the_next_create_exceeds_the_model_budget(scenario, key, upstream.model) @pytest.mark.timeout(240) From 6ba77b7013666c5b16ddc369c88f926b6318f486 Mon Sep 17 00:00:00 2001 From: rj-cairn <349020452+rj-cairn@users.noreply.github.com> Date: Wed, 28 Oct 2026 03:50:50 -0700 Subject: [PATCH 09/12] test(spend): audit #47353 (#48785) A DeepSeek V4 stream that ends without a usage block now has a spend-row cell proving its prompt_tokens are counted with DeepSeek's tokenizer, the one path #47353's own cells did not reach Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../spend/test_streamed_usage_estimate.py | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 tests/integration/spend/test_streamed_usage_estimate.py diff --git a/tests/integration/spend/test_streamed_usage_estimate.py b/tests/integration/spend/test_streamed_usage_estimate.py new file mode 100644 index 00000000000..c7bcdf4401a --- /dev/null +++ b/tests/integration/spend/test_streamed_usage_estimate.py @@ -0,0 +1,73 @@ +import json +import uuid +from hashlib import sha256 +from typing import Final + +from integration._support.client import Gateway, eventually +from integration._support.database import read_rows +from integration._support.deepseek_tokens import ( + CHINESE_TEXT, + SHORT_TEXT, + assert_counted_with_deepseek, + chinese_minus_short, +) +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue + +_DEEPSEEK_V4: Final = "DeepSeek-V4-Flash-0731" + + +def _data_frame(frame: dict[str, JsonValue]) -> bytes: + return b"data: " + json.dumps(frame).encode() + b"\n\n" + + +def _stream_without_usage(request: Request) -> Reply: + assert request.method == "POST" + chunk: Final[dict[str, JsonValue]] = { + "id": f"chatcmpl-{uuid.uuid4().hex}", + "object": "chat.completion.chunk", + "created": 1, + "model": _DEEPSEEK_V4, + } + return Reply( + content_type="text/event-stream", + chunks=( + _data_frame({**chunk, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "OK"}}]}), + _data_frame({**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}), + b"data: [DONE]\n\n", + ), + ) + + +def _streamed_prompt_tokens(gateway: Gateway, key: str, model: str, text: str) -> int: + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": text}], "stream": True}, + key=key, + ) + assert response.status_code == 200, response.text + assert response.text.rstrip().endswith("data: [DONE]"), response.text + rows: Final = eventually( + lambda: read_rows( + 'SELECT status, prompt_tokens FROM "LiteLLM_SpendLogs" WHERE api_key=%s', + (sha256(key.encode()).hexdigest(),), + ), + lambda values: len(values) == 1, + seconds=70, + ) + assert rows[0]["status"] == "success", rows + prompt_tokens: Final = rows[0]["prompt_tokens"] + assert isinstance(prompt_tokens, int), rows + return prompt_tokens + + +def test_a_deepseek_v4_stream_without_usage_bills_prompt_tokens_counted_with_deepseeks_tokenizer( + gateway: Gateway, +) -> None: + with wire_server(_stream_without_usage) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"azure_ai/{_DEEPSEEK_V4}", api_base=wire.url, num_retries=0) + chinese: Final = _streamed_prompt_tokens(gateway, scenario.key(models=[model]), model, CHINESE_TEXT) + short: Final = _streamed_prompt_tokens(gateway, scenario.key(models=[model]), model, SHORT_TEXT) + assert len(wire.drain()) == 2 + assert_counted_with_deepseek(chinese - short, chinese_minus_short) From e2d78b69cd8b8e51569c9334eda6dd2aa491b002 Mon Sep 17 00:00:00 2001 From: m6atlas <349022726+m6atlas@users.noreply.github.com> Date: Wed, 28 Oct 2026 03:55:25 -0700 Subject: [PATCH 10/12] test(proxy): audit #46815 (#48768) * test(proxy): audit #46815 Add integration tests for every count route #46815 changed, and count a Claude subscription caller's tokens with their forwarded token when the deployment has no configured key, which #46815 turned into a 401 * test(proxy): accept the forwarded subscription token in the count endpoint mocks --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../anthropic/count_tokens/token_counter.py | 10 +- .../proxy/anthropic_endpoints/endpoints.py | 6 +- litellm/proxy/litellm_pre_call_utils.py | 39 +++-- litellm/proxy/proxy_server.py | 13 +- .../test_count_tokens_error_shapes_wire.py | 143 +++++++++++++++++- .../llms/anthropic/test_count_tokens_oauth.py | 39 +++++ .../unit/proxy/test_litellm_pre_call_utils.py | 20 +++ tests/unit/proxy/test_proxy_token_counter.py | 5 + 8 files changed, 250 insertions(+), 25 deletions(-) diff --git a/litellm/llms/anthropic/count_tokens/token_counter.py b/litellm/llms/anthropic/count_tokens/token_counter.py index 0764fb41941..368bb929eeb 100644 --- a/litellm/llms/anthropic/count_tokens/token_counter.py +++ b/litellm/llms/anthropic/count_tokens/token_counter.py @@ -27,6 +27,9 @@ anthropic_count_tokens_handler: Final = AnthropicCountTokensHandler() class AnthropicTokenCounter(BaseTokenCounter): """Token counter implementation for Anthropic provider using the CountTokens API.""" + def __init__(self, client_oauth_token: str | None = None) -> None: + self.client_oauth_token: Final = client_oauth_token + def should_use_token_counting_api( self, custom_llm_provider: str | None = None, @@ -95,12 +98,17 @@ class AnthropicTokenCounter(BaseTokenCounter): api_base: Final = litellm_params.get("api_base") try: - auth_header: Final = await AnthropicModelInfo.aget_auth_header( + configured_auth_header: Final = await AnthropicModelInfo.aget_auth_header( api_key=litellm_params.get("api_key") or litellm.anthropic_key or litellm.api_key, api_base=api_base, litellm_params=litellm_params, allow_workload_identity=True, ) + auth_header: Final = ( + configured_auth_header + if configured_auth_header is not None or self.client_oauth_token is None + else AnthropicModelInfo.get_auth_header(api_key=self.client_oauth_token, api_base=api_base) + ) if auth_header is None: return TokenCountResponse( total_tokens=0, diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 13a1844a6de..1d2045eb743 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -50,7 +50,10 @@ from litellm.proxy.common_utils.openai_error_payload import ( with_litellm_call_id, ) from litellm.proxy.common_utils.token_count_settings import count_source_fields -from litellm.proxy.litellm_pre_call_utils import should_auto_drop_params_for_agentic_cli +from litellm.proxy.litellm_pre_call_utils import ( + forwarded_anthropic_oauth_token, + should_auto_drop_params_for_agentic_cli, +) from litellm.router import Router router: Final = APIRouter() @@ -412,6 +415,7 @@ async def count_tokens( request=token_request, call_endpoint=True, messages_format="anthropic", + client_oauth_token=forwarded_anthropic_oauth_token(request.headers), **_count_tokens_body_options(data, auto_drop_params), ) except UnsupportedContentTypeError as e: diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 0f8ac58861b..314f14b6279 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1415,12 +1415,34 @@ def _tenant_service_name(user_api_key_dict: UserAPIKeyAuth) -> str | None: ) +def litellm_auth_header_name(headers: Headers) -> str: + if "x-litellm-api-key" in headers: + return "x-litellm-api-key" + if "authorization" in headers: + return "authorization" + return "x-api-key" + + +def forwarded_anthropic_oauth_token(headers: Headers) -> str | None: + from litellm.llms.anthropic.common_utils import is_anthropic_oauth_key + + forwarded: Final = clean_headers(headers, authenticated_with_header=litellm_auth_header_name(headers)) + return next( + ( + value.removeprefix("Bearer ") + for header, value in forwarded.items() + if header.lower() == "authorization" and is_anthropic_oauth_key(value) + ), + None, + ) + + def clean_headers( headers: Headers, litellm_key_header_name: str | None = None, forward_llm_provider_auth_headers: bool = False, authenticated_with_header: str | None = None, -) -> dict: +) -> dict[str, str]: """ Removes litellm api key from headers @@ -2513,26 +2535,13 @@ async def add_litellm_data_to_request( forward_llm_auth = general_settings.get("forward_llm_provider_auth_headers", False) if not forward_llm_auth: forward_llm_auth = getattr(litellm, "forward_llm_provider_auth_headers", False) - # Determine which header was used for authentication - # This enables forwarding provider keys (e.g., x-api-key) when they weren't used for LiteLLM auth - authenticated_with_header = None - if "x-litellm-api-key" in request.headers: - # If x-litellm-api-key is present, it was used for auth - authenticated_with_header = "x-litellm-api-key" - elif "authorization" in request.headers: - # Authorization header was used for auth - authenticated_with_header = "authorization" - else: - # x-api-key or another header was used for auth - authenticated_with_header = "x-api-key" - _headers: Final[dict[str, str]] = clean_headers( request.headers, litellm_key_header_name=( general_settings.get("litellm_key_header_name") if general_settings is not None else None ), forward_llm_provider_auth_headers=forward_llm_auth, - authenticated_with_header=authenticated_with_header, + authenticated_with_header=litellm_auth_header_name(request.headers), ) _strip_untrusted_request_header_controls( _headers, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 3b36ea5afce..04f84f1f7b5 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -14628,6 +14628,7 @@ async def run_thread( # ) # async def get_available_routes(user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth)): from litellm.litellm_core_utils.tool_fields import DropPolicy, drop_policy +from litellm.llms.anthropic.count_tokens.token_counter import AnthropicTokenCounter from litellm.llms.base_llm.base_utils import BaseTokenCounter from litellm.llms.openai.responses.count_tokens.token_counter import OpenAITokenCounter from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient @@ -14649,6 +14650,14 @@ from litellm.responses.litellm_completion_transformation.count_request import ( ) +def _with_client_oauth_token( + counter: BaseTokenCounter | None, client_oauth_token: str | None +) -> BaseTokenCounter | None: + if client_oauth_token is None or not isinstance(counter, AnthropicTokenCounter): + return counter + return AnthropicTokenCounter(client_oauth_token=client_oauth_token) + + def _get_provider_token_counter( deployment: dict, model_to_use: str ) -> tuple[BaseTokenCounter | None, str | None, str | None]: @@ -15264,6 +15273,7 @@ async def count_request_tokens( context_management: JsonValue = None, request_litellm_params: Mapping[str, JsonValue] = MappingProxyType({}), responses_request: Mapping[str, object] | None = None, + client_oauth_token: str | None = None, ) -> TokenCountResponse: prompt: Final = request.prompt messages: Final = request.messages @@ -15321,7 +15331,8 @@ async def count_request_tokens( custom_llm_provider: str | None = None if call_endpoint is True and deployment is not None: # Auto-route to the correct provider based on model - provider_counter, _model, custom_llm_provider = _get_provider_token_counter(deployment, model_to_use) + routed_counter, _model, custom_llm_provider = _get_provider_token_counter(deployment, model_to_use) + provider_counter = _with_client_oauth_token(routed_counter, client_oauth_token) if _model is not None: model_to_use = _model diff --git a/tests/integration/providers/test_count_tokens_error_shapes_wire.py b/tests/integration/providers/test_count_tokens_error_shapes_wire.py index a6c3a6dc62f..59114e4aced 100644 --- a/tests/integration/providers/test_count_tokens_error_shapes_wire.py +++ b/tests/integration/providers/test_count_tokens_error_shapes_wire.py @@ -1,5 +1,6 @@ import json -from collections.abc import Iterator +from collections.abc import Callable, Iterator +from types import MappingProxyType from typing import Final import anthropic @@ -8,6 +9,7 @@ import pytest from integration._support.client import Gateway, gateway_from_environment from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process from integration._support.wire import Reply, Request, wire_server +from litellm.types.llms.anthropic import ANTHROPIC_OAUTH_BETA_HEADER from pydantic import JsonValue, TypeAdapter pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120) @@ -26,6 +28,14 @@ _PROVIDER_KEY_VARIABLES: Final = ( "AZURE_TENANT_ID", "OPENAI_API_KEY", ) +_ENVIRONMENT_KEYS: Final = MappingProxyType( + { + "ANTHROPIC_API_KEY": "synthetic-anthropic-env-key", + "AZURE_API_KEY": "synthetic-azure-env-key", + "OPENAI_API_KEY": "synthetic-openai-env-key", + } +) +_SUBSCRIPTION_TOKEN: Final = "sk-ant-oat01-synthetic-subscription-token" @pytest.fixture(scope="module") @@ -44,14 +54,14 @@ def keyless_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[OwnedPro @pytest.fixture(scope="module") -def azure_environment_key_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[OwnedProxy]: +def environment_key_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[OwnedProxy]: with ( gateway_from_environment() as gateway, owned_proxy_process( gateway, - tmp_path_factory.mktemp("count-tokens-azure-environment-key"), - {"AZURE_API_KEY": "synthetic-azure-env-key"}, - remove_environment=tuple(name for name in _PROVIDER_KEY_VARIABLES if name != "AZURE_API_KEY"), + tmp_path_factory.mktemp("count-tokens-environment-keys"), + _ENVIRONMENT_KEYS, + remove_environment=tuple(name for name in _PROVIDER_KEY_VARIABLES if name not in _ENVIRONMENT_KEYS), workers=1, ) as owned, ): @@ -147,9 +157,9 @@ def test_anthropic_count_without_a_provider_credential_is_an_anthropic_401( def test_azure_ai_anthropic_count_sends_the_azure_environment_key_as_x_api_key( - azure_environment_key_proxy: OwnedProxy, + environment_key_proxy: OwnedProxy, ) -> None: - gateway: Final = azure_environment_key_proxy.gateway + gateway: Final = environment_key_proxy.gateway def count(request: Request) -> Reply: assert request.headers.get("x-api-key") == "synthetic-azure-env-key", request.headers @@ -257,3 +267,122 @@ def test_google_count_tokens_errors_use_the_google_envelope(keyless_proxy: Owned assert _JSON_OBJECT.validate_json(malformed.content) == { "error": {"code": 400, "message": "Invalid Google countTokens request body", "status": "INVALID_ARGUMENT"} } + + +def _counted(tokens: int) -> Reply: + return Reply(body=json.dumps({"object": "response.input_tokens", "input_tokens": tokens}).encode()) + + +def _anthropic_count(model: str) -> tuple[str, dict[str, JsonValue]]: + return "/v1/messages/count_tokens", {"model": model, "messages": _MESSAGES} + + +def _responses_count(model: str) -> tuple[str, dict[str, JsonValue]]: + return "/v1/responses/input_tokens", {"model": model, "input": "Count this synthetic request."} + + +@pytest.mark.parametrize( + ("provider_model", "base_path", "count_request", "header", "credential"), + ( + ("anthropic/claude-sonnet-5-5", "", _anthropic_count, "x-api-key", "synthetic-anthropic-env-key"), + ("openai/gpt-4o-mini", "/v1", _responses_count, "authorization", "Bearer synthetic-openai-env-key"), + ), + ids=("anthropic", "openai"), +) +def test_count_uses_a_provider_key_set_only_in_the_proxy_environment( + environment_key_proxy: OwnedProxy, + provider_model: str, + base_path: str, + count_request: Callable[[str], tuple[str, dict[str, JsonValue]]], + header: str, + credential: str, +) -> None: + gateway: Final = environment_key_proxy.gateway + with wire_server(lambda _request: _counted(47)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=provider_model, api_base=f"{wire.url}{base_path}", api_key=None, num_retries=0 + ) + path, body = count_request(model) + counted: Final = gateway.request("POST", path, body) + assert counted.status_code == 200, counted.text + assert _JSON_OBJECT.validate_json(counted.content)["input_tokens"] == 47, counted.text + received: Final = wire.drain() + assert [request.headers.get(header) for request in received] == [credential], received + + +def _google_count(model: str) -> tuple[str, dict[str, JsonValue]]: + return f"/v1beta/models/{model}:countTokens", { + "contents": [{"role": "user", "parts": [{"text": "Count this synthetic request."}]}] + } + + +def _utils_count(model: str) -> tuple[str, dict[str, JsonValue]]: + return "/utils/token_counter?call_endpoint=true", {"model": model, "messages": _MESSAGES} + + +@pytest.mark.parametrize( + ("count_request", "google_status"), + ((_google_count, "UNAUTHENTICATED"), (_utils_count, None)), + ids=("google", "utils"), +) +def test_count_entrypoints_report_a_missing_provider_key_in_their_own_shape( + keyless_proxy: OwnedProxy, + count_request: Callable[[str], tuple[str, dict[str, JsonValue]]], + google_status: str | None, +) -> None: + gateway: Final = keyless_proxy.gateway + with wire_server(_never_called) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model="anthropic/claude-sonnet-5-5", api_base=wire.url, api_key=None, num_retries=0 + ) + path, body = count_request(model) + rejected: Final = gateway.request("POST", path, body) + assert rejected.status_code == 401, rejected.text + error: Final = _JSON_OBJECT.validate_json(rejected.content)["error"] + assert isinstance(error, dict), rejected.text + assert f"No Anthropic API key is set for model {model}" in str(error["message"]), rejected.text + if google_status is not None: + assert (error["code"], error["status"]) == (401, google_status), rejected.text + assert wire.drain() == () + + +def test_anthropic_count_uses_the_callers_subscription_token_when_the_deployment_has_no_key( + keyless_proxy: OwnedProxy, +) -> None: + gateway: Final = keyless_proxy.gateway + with wire_server(lambda _request: _counted(41)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model="anthropic/claude-sonnet-5-5", api_base=wire.url, api_key=None, num_retries=0 + ) + counted: Final = gateway.request( + "POST", + "/v1/messages/count_tokens", + {"model": model, "messages": _MESSAGES}, + headers={"Authorization": f"Bearer {_SUBSCRIPTION_TOKEN}", "x-litellm-api-key": f"Bearer {gateway.key}"}, + ) + assert counted.status_code == 200, counted.text + assert _JSON_OBJECT.validate_json(counted.content)["input_tokens"] == 41, counted.text + (received,) = wire.drain() + assert received.headers.get("authorization") == f"Bearer {_SUBSCRIPTION_TOKEN}", received.headers + assert "x-api-key" not in received.headers, received.headers + assert ANTHROPIC_OAUTH_BETA_HEADER in received.headers.get("anthropic-beta", ""), received.headers + + +def test_anthropic_count_keeps_the_deployments_key_over_the_callers_subscription_token( + keyless_proxy: OwnedProxy, +) -> None: + gateway: Final = keyless_proxy.gateway + with wire_server(lambda _request: _counted(43)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model="anthropic/claude-sonnet-5-5", api_base=wire.url, api_key="synthetic-deployment-key", num_retries=0 + ) + counted: Final = gateway.request( + "POST", + "/v1/messages/count_tokens", + {"model": model, "messages": _MESSAGES}, + headers={"Authorization": f"Bearer {_SUBSCRIPTION_TOKEN}", "x-litellm-api-key": f"Bearer {gateway.key}"}, + ) + assert counted.status_code == 200, counted.text + (received,) = wire.drain() + assert received.headers.get("x-api-key") == "synthetic-deployment-key", received.headers + assert _SUBSCRIPTION_TOKEN not in json.dumps(dict(received.headers)), received.headers diff --git a/tests/unit/llms/anthropic/test_count_tokens_oauth.py b/tests/unit/llms/anthropic/test_count_tokens_oauth.py index 16ec513916c..b8a7c001b37 100644 --- a/tests/unit/llms/anthropic/test_count_tokens_oauth.py +++ b/tests/unit/llms/anthropic/test_count_tokens_oauth.py @@ -221,6 +221,45 @@ class TestCountTokensUsesWorkloadIdentity: assert result.total_tokens == 13 assert route.calls.last.request.headers["x-api-key"] == FAKE_REGULAR_KEY + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("deployment_key", "expected_credentials"), + [ + (None, {"authorization": f"Bearer {FAKE_OAUTH_TOKEN}", "x-api-key": None}), + (FAKE_REGULAR_KEY, {"authorization": None, "x-api-key": FAKE_REGULAR_KEY}), + ], + ids=["keyless-deployment-uses-the-callers-token", "deployment-key-wins-over-the-callers-token"], + ) + async def test_a_callers_subscription_token_counts_only_when_no_key_is_configured( + self, + monkeypatch: pytest.MonkeyPatch, + httpx_transport_clients: None, + deployment_key: str | None, + expected_credentials: dict[str, str | None], + ) -> None: + for name in ("ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_API_BASE", "ANTHROPIC_BASE_URL"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setattr(litellm, "anthropic_key", None) + monkeypatch.setattr(litellm, "api_key", None) + + with respx.mock(assert_all_called=True) as router: + route = router.post("https://api.anthropic.com/v1/messages/count_tokens").mock( + return_value=httpx.Response(200, json={"input_tokens": 17}) + ) + result = await AnthropicTokenCounter(client_oauth_token=FAKE_OAUTH_TOKEN).count_tokens( + model_to_use="claude-sonnet-4-5", + messages=[{"role": "user", "content": "hi"}], + contents=None, + deployment={"litellm_params": {"model": "anthropic/claude-sonnet-4-5", "api_key": deployment_key}}, + request_model="claude-sonnet-4-5", + ) + + assert result is not None + assert result.total_tokens == 17 + sent = route.calls.last.request.headers + assert {name: sent.get(name) for name in expected_credentials} == expected_credentials + assert (ANTHROPIC_OAUTH_BETA_HEADER in sent["anthropic-beta"].split(",")) is (deployment_key is None) + @pytest.mark.asyncio async def test_a_failed_mint_degrades_like_an_anthropic_error(self, monkeypatch): from litellm.llms.anthropic.count_tokens import token_counter as token_counter_module diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index f743a14c7a2..a32c40e5a43 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -53,6 +53,7 @@ from litellm.proxy.litellm_pre_call_utils import ( add_provider_specific_headers_to_request, check_if_token_is_service_account, clean_headers, + forwarded_anthropic_oauth_token, get_dynamic_logging_metadata, get_metadata_variable_name, move_guardrails_to_metadata, @@ -5966,6 +5967,25 @@ def test_clean_headers_strips_x_api_key_when_byok_disabled(): assert "x-api-key" not in result +@pytest.mark.parametrize( + ("headers", "expected_token"), + [ + ( + {"authorization": "Bearer sk-ant-oat01-caller", "x-litellm-api-key": "Bearer sk-proxy"}, + "sk-ant-oat01-caller", + ), + ({"authorization": "Bearer sk-ant-oat01-caller"}, None), + ({"authorization": "Bearer sk-ant-api03-caller", "x-litellm-api-key": "Bearer sk-proxy"}, None), + ({"x-litellm-api-key": "Bearer sk-proxy"}, None), + ], + ids=["oauth-beside-proxy-key", "oauth-used-as-proxy-key", "non-oauth-authorization", "no-authorization"], +) +def test_forwarded_anthropic_oauth_token_follows_chats_forwarding_rule( + headers: dict[str, str], expected_token: str | None +) -> None: + assert forwarded_anthropic_oauth_token(Headers(headers)) == expected_token + + def test_clean_headers_strips_x_api_key_when_byok_enabled_but_x_api_key_was_auth_header(): """ Anti-replay regression: even when forward_llm_provider_auth_headers=True, diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index fa0c9eda02d..ebe3c7407d8 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -364,6 +364,7 @@ async def test_anthropic_messages_count_tokens_endpoint(): output_format=None, context_management=None, request_litellm_params=None, + client_oauth_token=None, ): assert call_endpoint == True, "Should be called with call_endpoint=True for Anthropic endpoint" assert request.model == "claude-3-sonnet-20240229" @@ -444,6 +445,7 @@ async def test_anthropic_messages_count_tokens_with_non_anthropic_model(): output_format=None, context_management=None, request_litellm_params=None, + client_oauth_token=None, ): assert call_endpoint == True, "Should be called with call_endpoint=True for Anthropic endpoint" assert request.model == "gpt-4" @@ -1262,6 +1264,7 @@ async def test_anthropic_endpoint_returns_anthropic_error_format(): output_format=None, context_management=None, request_litellm_params=None, + client_oauth_token=None, ): raise ProxyException( message='{"detail":{"message":"Input is too long for requested model."}}', @@ -1319,6 +1322,7 @@ async def test_anthropic_endpoint_403_permission_error_format(): output_format=None, context_management=None, request_litellm_params=None, + client_oauth_token=None, ): raise ProxyException( message='{"Message":"Bearer Token has expired"}', @@ -1376,6 +1380,7 @@ async def test_anthropic_endpoint_429_rate_limit_error_format(): output_format=None, context_management=None, request_litellm_params=None, + client_oauth_token=None, ): raise ProxyException( message="Rate limit exceeded", From a0c490ea5471c8ef2c09b26e0ea744fe1de9961a Mon Sep 17 00:00:00 2001 From: m6atlas <349022726+m6atlas@users.noreply.github.com> Date: Wed, 28 Oct 2026 04:04:39 -0700 Subject: [PATCH 11/12] fix(bedrock): retry without strict when Bedrock rejects strict tool schemas under drop_params (#48733) * fix(bedrock): retry without strict when Bedrock rejects strict tool schemas under drop_params * test(bedrock): cover the strict tools 400 retry under drop_params * fix(bedrock): anchor the strict tools rejection match and strip toolSpec strict * test(bedrock): return captured requests from the strict tools test helper * fix(bedrock): retry without strict only when a tool sent strict to Bedrock A strict-limit 400 on a request whose tools carry strict: false, or on a model litellm never sends strict to, was sent again with an identical body. The fallback now takes the model and returns nothing unless a tool puts strict on the wire. Also types the fallback test parameter and records the mock transport's requests on the recorder instead of a mutated list * fix(bedrock): decide the strict retry on the profile's base model and retry grammar timeouts An application inference profile with base_model sends strict on the wire, so the retry check now resolves the capability model the way the request transformation does. Bedrock's "Grammar compilation timed out" 400 is a strict-grammar rejection too, so it takes the same single retry under drop_params. * test(bedrock): annotate the strict-tools retry test helper's values as Final --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../bedrock/chat/native_output_fallback.py | 48 +++++-- litellm/llms/bedrock/common_utils.py | 54 ++++++++ litellm/main.py | 13 +- .../chat/test_native_output_fallback.py | 102 ++++++++++++++ .../llms/bedrock/test_bedrock_common_utils.py | 79 +++++++++++ tests/unit/test_main.py | 128 +++++++++++++++++- 6 files changed, 412 insertions(+), 12 deletions(-) diff --git a/litellm/llms/bedrock/chat/native_output_fallback.py b/litellm/llms/bedrock/chat/native_output_fallback.py index 1b8ce5fcda4..6892cd37beb 100644 --- a/litellm/llms/bedrock/chat/native_output_fallback.py +++ b/litellm/llms/bedrock/chat/native_output_fallback.py @@ -1,11 +1,14 @@ from collections.abc import Callable, Coroutine +from functools import partial from typing import Final, TypeAlias from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.bedrock.common_utils import ( NativeOutputFallback, is_bedrock_native_output_rejection, + is_bedrock_strict_tools_rejection, warn_bedrock_native_output_rejected, + warn_bedrock_strict_tools_dropped, ) from litellm.types.utils import ModelResponse from litellm.utils import CustomStreamWrapper @@ -14,12 +17,13 @@ NativeOutputResponse: TypeAlias = ModelResponse | CustomStreamWrapper NativeOutputAttemptResult: TypeAlias = NativeOutputResponse | Coroutine[object, object, NativeOutputResponse] NativeOutputAttempt: TypeAlias = Callable[[NativeOutputFallback | None], NativeOutputAttemptResult] NativeOutputFallbackFactory: TypeAlias = Callable[[], NativeOutputFallback | None] +_RetryFor: TypeAlias = Callable[[BaseLLMException], NativeOutputFallback | None] -def _fallback_for( - error: BaseLLMException, fallback: NativeOutputFallbackFactory, model: str +def _native_output_retry( + error: BaseLLMException, fallback: NativeOutputFallbackFactory | None, model: str ) -> NativeOutputFallback | None: - if not is_bedrock_native_output_rejection(error.status_code, error.message): + if fallback is None or not is_bedrock_native_output_rejection(error.status_code, error.message): return None retry: Final = fallback() if retry is not None: @@ -27,16 +31,38 @@ def _fallback_for( return retry +def _strict_tools_retry( + error: BaseLLMException, fallback: NativeOutputFallbackFactory | None, model: str +) -> NativeOutputFallback | None: + if fallback is None or not is_bedrock_strict_tools_rejection(error.status_code, error.message): + return None + retry: Final = fallback() + if retry is not None: + warn_bedrock_strict_tools_dropped(model) + return retry + + +def _fallback_for( + error: BaseLLMException, + fallback: NativeOutputFallbackFactory | None, + strict_tools_fallback: NativeOutputFallbackFactory | None, + model: str, +) -> NativeOutputFallback | None: + native_output_retry: Final = _native_output_retry(error, fallback, model) + if native_output_retry is not None: + return native_output_retry + return _strict_tools_retry(error, strict_tools_fallback, model) + + async def _await_with_native_output_fallback( first_attempt: Coroutine[object, object, NativeOutputResponse], attempt: NativeOutputAttempt, - fallback: NativeOutputFallbackFactory, - model: str, + retry_for: _RetryFor, ) -> NativeOutputResponse: try: return await first_attempt except BaseLLMException as error: - retry: Final = _fallback_for(error, fallback, model) + retry: Final = retry_for(error) if retry is None: raise second_attempt: Final = attempt(retry) @@ -47,16 +73,20 @@ def call_with_native_output_fallback( attempt: NativeOutputAttempt, fallback: NativeOutputFallbackFactory | None, model: str, + strict_tools_fallback: NativeOutputFallbackFactory | None = None, ) -> NativeOutputAttemptResult: - if fallback is None: + if fallback is None and strict_tools_fallback is None: return attempt(None) + retry_for: Final = partial( + _fallback_for, fallback=fallback, strict_tools_fallback=strict_tools_fallback, model=model + ) try: first_attempt: Final = attempt(None) except BaseLLMException as error: - retry: Final = _fallback_for(error, fallback, model) + retry: Final = retry_for(error) if retry is None: raise return attempt(retry) if isinstance(first_attempt, Coroutine): - return _await_with_native_output_fallback(first_attempt, attempt, fallback, model) + return _await_with_native_output_fallback(first_attempt, attempt, retry_for) return first_attempt diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 28888f1a23a..fde995c7907 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -362,6 +362,60 @@ class NativeOutputFallback: litellm_params: Mapping[str, object] +_BEDROCK_STRICT_TOOLS_REJECTION: Final = re.compile( + r"Too many strict tools \(\d+\)" + r"|Schemas contains too many (?:optional parameters|parameters with union types)" + r"|[Cc]ompiled grammar (?:is too large|size)" + r"|tools\.\d+\.custom(?:\.strict)?: " + r"|Grammar compilation timed out" +) +_TOOLS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...]) +_TOOL_SPEC_ADAPTER: Final = TypeAdapter(Mapping[str, object]) + + +def is_bedrock_strict_tools_rejection(status_code: int, error_text: str) -> bool: + return status_code == 400 and _BEDROCK_STRICT_TOOLS_REJECTION.search(error_text) is not None + + +def warn_bedrock_strict_tools_dropped(model: str) -> None: + verbose_logger.warning( + "Bedrock rejected the strict tool schemas for model=%s, so litellm sent the request again without `strict` " + "because drop_params is set.", + model, + ) + + +def _without_strict(tool: Mapping[str, object]) -> Mapping[str, object]: + spec_key: Final = next((key for key in ("function", "toolSpec") if key in tool), None) + if spec_key is None: + return tool + spec: Final = _TOOL_SPEC_ADAPTER.validate_python(tool[spec_key]) + return {**tool, spec_key: {key: value for key, value in spec.items() if key != "strict"}} + + +def _sends_strict(tool: Mapping[str, object], supports_strict_tools: bool) -> bool: + if "toolSpec" in tool: + return "strict" in _TOOL_SPEC_ADAPTER.validate_python(tool["toolSpec"]) + if "function" not in tool: + return False + return supports_strict_tools and bool(_TOOL_SPEC_ADAPTER.validate_python(tool["function"]).get("strict")) + + +def bedrock_strict_tools_fallback( + model: str, optional_params: Mapping[str, object], litellm_params: Mapping[str, object] +) -> NativeOutputFallback | None: + tools: Final = _TOOLS_ADAPTER.validate_python(optional_params.get("tools", ())) + supports_strict_tools: Final = bedrock_converse_supports_strict_tools( + bedrock_capability_model(model, litellm_params.get("base_model")) + ) + if not any(_sends_strict(tool, supports_strict_tools) for tool in tools): + return None + return NativeOutputFallback( + optional_params={**optional_params, "tools": [_without_strict(tool) for tool in tools]}, + litellm_params=litellm_params, + ) + + def requests_output_config_format(optional_params: Mapping[str, object]) -> bool: match optional_params: case {"output_config": {"format": object()}}: diff --git a/litellm/main.py b/litellm/main.py index 9f412e1f24b..39a0a0cd3c1 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -123,6 +123,7 @@ from litellm.llms.bedrock.common_utils import ( BedrockModelInfo, NativeOutputFallback, bedrock_route_for_request, + bedrock_strict_tools_fallback, deployment_model_info, without_bedrock_native_output_opt_in, without_bedrock_route_prefix, @@ -206,7 +207,12 @@ from litellm.utils import ( from ._logging import verbose_logger from .caching.caching import disable_cache, enable_cache, update_cache -from .litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params, safe_deep_copy +from .litellm_core_utils.core_helpers import ( + as_str_mapping, + drops_unsupported_params, + normalize_drop_params, + safe_deep_copy, +) from .litellm_core_utils.fallback_utils import ( async_completion_with_fallbacks, completion_with_fallbacks, @@ -4378,6 +4384,11 @@ def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResu else None ), converse_model, + strict_tools_fallback=( + partial(bedrock_strict_tools_fallback, converse_model, converse_params, dict(litellm_params)) + if drops_unsupported_params(litellm_params.get("drop_params")) + else None + ), ) elif bedrock_route == "converse_like": model = model.replace("converse_like/", "") diff --git a/tests/unit/llms/bedrock/chat/test_native_output_fallback.py b/tests/unit/llms/bedrock/chat/test_native_output_fallback.py index 301fdaeb245..9ee8c14f5d6 100644 --- a/tests/unit/llms/bedrock/chat/test_native_output_fallback.py +++ b/tests/unit/llms/bedrock/chat/test_native_output_fallback.py @@ -121,3 +121,105 @@ def test_a_rejection_logs_the_retry_and_advises_opting_out_only_when_native_outp assert _MODEL in caplog.text assert ("supports_bedrock_native_structured_output: false" in caplog.text) is advises_opting_out + + +_STRICT_FALLBACK: Final = NativeOutputFallback(optional_params={"tools": "without_strict"}, litellm_params={}) +# Bedrock's 400s for strict tools, seen live on us.anthropic.claude-sonnet-4-6 from 2026-10-20 to 2026-10-28 and quoted +# in issue #34388, each answered 200 once the request dropped strict +_STRICT_TOOLS_REJECTIONS: Final = tuple( + BedrockError(status_code=400, message=f'{{"message":"The model returned the following errors: {text}"}}') + for text in ( + "Too many strict tools (21). The maximum number of strict tools supported is 20.", + "Schemas contains too many optional parameters (27), which would make grammar compilation inefficient.", + "Schemas contains too many parameters with union types (18 parameters with type arrays or anyOf). " + "This causes exponential compilation cost.", + "The compiled grammar is too large, which would cause performance issues.", + "tools.0.custom: Invalid schema: Schema type is missing for schema: {'description': 'tag'}", + "tools.3.custom: For 'integer' type, property 'minimum' is not supported", + "Compiled grammar size (410.4MB) exceeds maximum allowed size (300MB)", + "tools.1.custom.strict: Extra inputs are not permitted", + "Grammar compilation timed out.", + ) +) +_TOO_MANY_STRICT_TOOLS: Final = _STRICT_TOOLS_REJECTIONS[0] +# Bedrock's 400s for requests with a strict tool that fail for other reasons, seen live on +# us.anthropic.claude-sonnet-4-6 on 2026-10-21, which echo tool names and tool call ids +_OTHER_400S_WITH_A_STRICT_TOOL: Final = tuple( + BedrockError(status_code=400, message=f'{{"message":"{text}"}}') + for text in ( + "The tool check_grammar is already defined at toolConfig.tools.1.", + "Expected toolResult blocks at messages.0.content for the following Ids: call_grammar_9", + "tool_use ids were found without tool_result blocks immediately after: call_grammar_1", + ) +) + + +def _call_with_strict_tools_fallback( + bedrock: ScriptedBedrock, fallback: NativeOutputFallback | None, strict_tools_fallback: NativeOutputFallback | None +) -> object: + return _resolve( + call_with_native_output_fallback( + bedrock, + None if fallback is None else lambda: fallback, + _MODEL, + strict_tools_fallback=None if strict_tools_fallback is None else lambda: strict_tools_fallback, + ) + ) + + +@_MODES +@pytest.mark.parametrize("rejection", _STRICT_TOOLS_REJECTIONS, ids=lambda error: error.message[60:100]) +def test_a_strict_tools_rejection_is_sent_again_once_with_the_strict_tools_fallback( + asynchronous: bool, rejection: BedrockError +) -> None: + bedrock: Final = ScriptedBedrock(errors=(rejection, None), asynchronous=asynchronous) + + response: Final = _call_with_strict_tools_fallback(bedrock, None, _STRICT_FALLBACK) + + assert isinstance(response, ModelResponse) and response.id == "answer-2" + assert bedrock.sent == (None, _STRICT_FALLBACK) + + +@_MODES +@pytest.mark.parametrize( + ("error", "strict_tools_fallback"), + [ + (_OTHER_400, _STRICT_FALLBACK), + *((error, _STRICT_FALLBACK) for error in _OTHER_400S_WITH_A_STRICT_TOOL), + (BedrockError(status_code=500, message=_TOO_MANY_STRICT_TOOLS.message), _STRICT_FALLBACK), + (_TOO_MANY_STRICT_TOOLS, None), + ], + ids=[ + "other_400", + "duplicate_tool_name", + "missing_tool_result", + "orphan_tool_use", + "server_error", + "no_strict_tools_fallback", + ], +) +def test_no_second_request_without_both_a_strict_tools_rejection_and_a_strict_tools_fallback( + asynchronous: bool, error: BedrockError, strict_tools_fallback: NativeOutputFallback | None +) -> None: + bedrock: Final = ScriptedBedrock(errors=(error,), asynchronous=asynchronous) + + with pytest.raises(BedrockError) as raised: + _call_with_strict_tools_fallback(bedrock, None, strict_tools_fallback) + + assert raised.value is error + assert bedrock.sent == (None,) + + +@pytest.mark.parametrize( + ("error", "retried_with"), + [(_REJECTION, _FALLBACK), (_TOO_MANY_STRICT_TOOLS, _STRICT_FALLBACK)], + ids=["native_output_rejection", "strict_tools_rejection"], +) +def test_each_rejection_is_sent_again_with_its_own_fallback_when_both_are_set( + error: BedrockError, retried_with: NativeOutputFallback +) -> None: + bedrock: Final = ScriptedBedrock(errors=(error, None), asynchronous=False) + + _call_with_strict_tools_fallback(bedrock, _FALLBACK, _STRICT_FALLBACK) + + assert bedrock.sent == (None, retried_with) diff --git a/tests/unit/llms/bedrock/test_bedrock_common_utils.py b/tests/unit/llms/bedrock/test_bedrock_common_utils.py index 7bde1805e18..dd89d9c46b6 100644 --- a/tests/unit/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/unit/llms/bedrock/test_bedrock_common_utils.py @@ -6,7 +6,9 @@ from typing import Final from litellm.llms.bedrock.common_utils import( AmazonBedrockGlobalConfig, BedrockModelInfo, + NativeOutputFallback, bedrock_native_output_is_opt_in, + bedrock_strict_tools_fallback, deployment_model_info, extract_model_name_from_bedrock_arn, get_bedrock_base_model, @@ -2150,3 +2152,80 @@ def test_bedrock_response_stream_shape_is_unavailable_when_botocore_describes_th assert shape is None (logged,) = warning.call_args_list assert "only-in-the-description" not in logged.args[0] % logged.args[1:] + + +def test_bedrock_strict_tools_fallback_drops_only_strict_and_keeps_everything_else(): + strict_tool: Final = { + "type": "function", + "cache_control": {"type": "ephemeral"}, + "function": {"name": "lookup", "description": "Look up", "strict": True, "parameters": {"type": "object"}}, + } + plain_tool: Final = {"type": "function", "function": {"name": "plain", "parameters": {"type": "object"}}} + anthropic_tool: Final = {"name": "native", "input_schema": {"type": "object"}} + bedrock_tool: Final = {"toolSpec": {"name": "spec", "strict": True, "inputSchema": {"json": {"type": "object"}}}} + litellm_params: Final = {"drop_params": True} + + fallback: Final = bedrock_strict_tools_fallback( + "us.anthropic.claude-sonnet-4-6", + {"tools": [strict_tool, plain_tool, anthropic_tool, bedrock_tool], "maxTokens": 16, "stream": True}, + litellm_params, + ) + + assert fallback == NativeOutputFallback( + optional_params={ + "tools": [ + {**strict_tool, "function": {"name": "lookup", "description": "Look up", "parameters": {"type": "object"}}}, + plain_tool, + anthropic_tool, + {"toolSpec": {"name": "spec", "inputSchema": {"json": {"type": "object"}}}}, + ], + "maxTokens": 16, + "stream": True, + }, + litellm_params=litellm_params, + ) + assert strict_tool["function"]["strict"] is True + assert bedrock_tool["toolSpec"]["strict"] is True + + +@pytest.mark.parametrize( + "optional_params", + [ + {"maxTokens": 16}, + {"tools": [{"type": "function", "function": {"name": "plain", "parameters": {}}}]}, + {"tools": [{"toolSpec": {"name": "spec", "inputSchema": {"json": {}}}}]}, + ], + ids=["no_tools", "no_strict_tool", "no_strict_tool_spec"], +) +def test_bedrock_strict_tools_fallback_is_none_without_a_strict_tool(optional_params: dict[str, object]) -> None: + assert bedrock_strict_tools_fallback("us.anthropic.claude-sonnet-4-6", optional_params, {}) is None + + +_CLAUDE_PROFILE_ARN: Final = "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/claude" + + +@pytest.mark.parametrize( + ("model", "litellm_params"), + [ + ("us.anthropic.claude-sonnet-4-6", {}), + ("us.amazon.nova-pro-v1:0", {}), + (_CLAUDE_PROFILE_ARN, {"base_model": "bedrock/us.anthropic.claude-sonnet-4-6"}), + (_CLAUDE_PROFILE_ARN, {}), + ], + ids=["claude", "nova", "claude_profile_with_base_model", "profile_without_base_model"], +) +@pytest.mark.parametrize("strict", [True, False]) +def test_bedrock_strict_tools_fallback_retries_only_when_the_request_sent_strict( + model: str, litellm_params: dict[str, object], strict: bool +) -> None: + from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig + + tools: Final = [{"type": "function", "function": {"name": "lookup", "strict": strict, "parameters": {}}}] + request: Final = AmazonConverseConfig().transform_request( + model, [{"role": "user", "content": "hi"}], {"tools": tools}, {**litellm_params}, {} + ) + sent_strict: Final = any("strict" in tool["toolSpec"] for tool in request["toolConfig"]["tools"]) + + fallback: Final = bedrock_strict_tools_fallback(model, {"tools": tools}, litellm_params) + + assert (fallback is not None) == sent_strict diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 70b268bf2a2..f12129af1fc 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -14,7 +14,7 @@ from dataclasses import dataclass from datetime import datetime from importlib import import_module from pathlib import Path -from typing import Final +from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -44,12 +44,13 @@ from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kw from litellm.litellm_core_utils.get_litellm_params import stored_control_options from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt -from litellm.llms.custom_httpx.http_handler import HTTPHandler +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.types.litellm_params import ControlOptions from litellm.types.llms.openai import AllMessageValues, HttpxBinaryResponseContent from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.utils import Delta, ModelResponseStream, StandardCallbackDynamicParams, StreamingChoices, Usage from tests.unit.llms.bedrock.slow_upstream import ( + CONVERSE_ANSWER, STREAM_TIMEOUT_SECONDS, UPSTREAM_FIRST_BYTE_SECONDS, slow_upstream_sync_client, @@ -9950,3 +9951,126 @@ async def test_converse_fallback_prices_the_current_attempt( else: client.close() configuration.reset_rust_configuration() + + +_LOOKUP_FUNCTION: Final = { + "name": "lookup_record", + "description": "Look up a record", + "parameters": { + "type": "object", + "properties": {"query": {"type": "string"}}, + "required": ["query"], + "additionalProperties": False, + }, +} +_STRICT_LOOKUP_TOOL: Final = {"type": "function", "function": {**_LOOKUP_FUNCTION, "strict": True}} +# Bedrock's 400 for a request past Claude's strict tool limits, seen live on us.anthropic.claude-sonnet-4-6 on 2026-10-20 +_TOO_MANY_STRICT_TOOLS: Final = ( + "The model returned the following errors: Too many strict tools (21). The maximum number of strict tools " + "supported is 20. Try reducing the number of tools marked as strict." +) + + +@dataclass(slots=True) +class _StrictToolsLimitRejectionFirst: + stream: Literal["off", "fake", "real"] + sent: tuple[tuple[str, dict[str, object]], ...] = () + + def __call__(self, request: httpx.Request) -> httpx.Response: + self.sent = (*self.sent, (request.url.path.rsplit("/", 1)[-1], json.loads(request.content))) + if len(self.sent) == 1: + return httpx.Response(400, json={"message": _TOO_MANY_STRICT_TOOLS}) + return httpx.Response(200, content=b"" if self.stream == "real" else CONVERSE_ANSWER) + + +async def _converse_with_a_strict_tools_limit_rejection_first( + asynchronous: bool, + stream: Literal["off", "fake", "real"], + drop_params: bool | None, + tool: Mapping[str, object] = _STRICT_LOOKUP_TOOL, +) -> tuple[tuple[tuple[str, dict[str, object]], ...], litellm.BadRequestError | None]: + respond: Final = _StrictToolsLimitRejectionFirst(stream) + kwargs: Final = { + "model": "bedrock/us.anthropic.claude-sonnet-4-6", + "messages": [{"role": "user", "content": "Find invoice 42"}], + "tools": [tool], + "aws_access_key_id": "AKIASTRICTTOOLS", + "aws_secret_access_key": "strict-tools-secret", + "aws_region_name": "us-east-1", + "num_retries": 0, + "drop_params": drop_params, + "stream": stream != "off", + "fake_stream": stream == "fake", + } + try: + chunks: Final = ( + await _acompletion_chunks(AsyncHTTPHandler(transport=httpx.MockTransport(respond)), kwargs, stream) + if asynchronous + else _completion_chunks( + HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))), kwargs, stream + ) + ) + except litellm.BadRequestError as error: + return respond.sent, error + assert chunks + return respond.sent, None + + +async def _acompletion_chunks( + client: AsyncHTTPHandler, kwargs: Mapping[str, object], stream: Literal["off", "fake", "real"] +) -> tuple[object, ...]: + response: Final = await litellm.acompletion(client=client, **kwargs) + return tuple([chunk async for chunk in response]) if stream == "fake" else (response,) + + +def _completion_chunks( + client: HTTPHandler, kwargs: Mapping[str, object], stream: Literal["off", "fake", "real"] +) -> tuple[object, ...]: + response: Final = litellm.completion(client=client, **kwargs) + return tuple(response) if stream == "fake" else (response,) + + +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("stream", ["off", "fake", "real"], ids=["completion", "fake_stream", "stream"]) +async def test_bedrock_strict_tools_limit_rejection_is_sent_again_without_strict_under_drop_params( + asynchronous: bool, stream: Literal["off", "fake", "real"], local_model_cost_map: None +) -> None: + sent, error = await _converse_with_a_strict_tools_limit_rejection_first(asynchronous, stream, drop_params=True) + + assert error is None + (first_path, first_body), (second_path, second_body) = sent + first_tool_spec: Final = first_body["toolConfig"]["tools"][0]["toolSpec"] + assert first_tool_spec["strict"] is True + assert first_path == second_path == ("converse-stream" if stream == "real" else "converse") + assert second_body == { + **first_body, + "toolConfig": { + **first_body["toolConfig"], + "tools": [{"toolSpec": {key: value for key, value in first_tool_spec.items() if key != "strict"}}], + }, + } + + +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +async def test_bedrock_strict_tools_limit_rejection_reaches_the_caller_without_drop_params( + asynchronous: bool, local_model_cost_map: None +) -> None: + sent, error = await _converse_with_a_strict_tools_limit_rejection_first(asynchronous, stream="off", drop_params=None) + + assert error is not None and "Too many strict tools" in str(error) + assert len(sent) == 1 + + +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +async def test_bedrock_strict_tools_limit_rejection_is_not_sent_again_when_no_tool_sent_strict( + asynchronous: bool, local_model_cost_map: None +) -> None: + sent, error = await _converse_with_a_strict_tools_limit_rejection_first( + asynchronous, + stream="off", + drop_params=True, + tool={"type": "function", "function": {**_LOOKUP_FUNCTION, "strict": False}}, + ) + + assert error is not None and "Too many strict tools" in str(error) + assert len(sent) == 1 From 5a18e6b6da67f273c14fcbd47000145294ef6a40 Mon Sep 17 00:00:00 2001 From: zoveki <349028329+zoveki@users.noreply.github.com> Date: Wed, 28 Oct 2026 04:13:03 -0700 Subject: [PATCH 12/12] test(files): audit #40621 (#48788) * test(files): audit #40621 Pin the raw cloud storage file id gate on /v1/files content and delete against a running proxy: twice-encoded gs:// ids stay blocked for non-admin keys, proxy admin keys read s3:// objects, and ordinary provider ids keep working * test(files): check the full SigV4 signature on the admin raw s3 read --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../security/test_files_batches_access.py | 126 +++++++++++++++--- 1 file changed, 104 insertions(+), 22 deletions(-) diff --git a/tests/integration/security/test_files_batches_access.py b/tests/integration/security/test_files_batches_access.py index 7b6fd4652cf..119055c64c7 100644 --- a/tests/integration/security/test_files_batches_access.py +++ b/tests/integration/security/test_files_batches_access.py @@ -48,6 +48,7 @@ VERTEX_ACCESS_TOKEN: Final = "ya29.scripted-delete-token" AWS_ACCESS_KEY_ID: Final = "AKIAIOSFODNN7EXAMPLE" AWS_SECRET_ACCESS_KEY: Final = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" AWS_REGION: Final = "us-east-1" +S3_OBJECT_BODY: Final = b'{"recordId":"r1","modelOutput":{"stop_reason":"end_turn"}}\n' _ForbiddenSurface = Literal[ "upload_sdk_extra_body", "upload_form", @@ -708,6 +709,86 @@ def test_non_admin_cannot_delete_a_raw_gcs_uri_before_vertex_auth_or_storage(gat assert token_requests == (), "non-admin raw GCS delete reached the Vertex token endpoint" +def test_non_admin_still_reads_an_ordinary_provider_file_id(gateway: Gateway) -> None: + with wire_server(_openai_backend()) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model="openai/gpt-4o-mini", api_base=wire.url + "/v1", api_key="provider-key-" + uuid.uuid4().hex[:8] + ) + _wait_until_every_worker_serves(gateway, model) + member: Final = _member(scenario, model) + file_id: Final = "file-" + uuid.uuid4().hex + + read: Final = gateway.request("GET", f"/v1/files/{file_id}/content", key=member.key, params={"model": model}) + assert read.status_code == 200, read.text + assert read.content == b"file-content\n", read.content + assert [ + (request.method, urlsplit(request.target).path) for request in _drained_other_than_model_list_probes(wire) + ] == [("GET", f"/v1/files/{file_id}/content")] + + +@pytest.mark.parametrize( + ("method", "route_suffix", "verb"), + [("GET", "/content", "retrieved"), ("DELETE", "", "deleted")], + ids=["content", "delete"], +) +def test_non_admin_cannot_reach_vertex_with_a_twice_encoded_raw_gcs_uri( + gateway: Gateway, method: Literal["GET", "DELETE"], route_suffix: str, verb: str +) -> None: + with ( + wire_server(_vertex_token_backend) as token_wire, + gateway.scenario() as scenario, + ): + model: Final = scenario.model( + model=VERTEX_MODEL, + api_key=None, + vertex_project=VERTEX_PROJECT, + vertex_location=VERTEX_LOCATION, + vertex_credentials=_vertex_credentials(token_wire.url), + gcs_bucket_name=BUCKET, + ) + _wait_until_every_worker_serves(gateway, model) + caller_user: Final = scenario.user(user_role="internal_user") + caller_key: Final = scenario.key(user_id=caller_user, models=[model]) + object_uri: Final = f"gs://{BUCKET}/litellm-vertex-files/litellm-vertex-files-obj.jsonl" + twice_encoded: Final = quote(quote(object_uri, safe=""), safe="") + token_wire.drain() + + denied: Final = gateway.request( + method, f"/v1/files/{twice_encoded}{route_suffix}", key=caller_key, params={"model": model} + ) + token_requests: Final = token_wire.drain() + assert denied.status_code == 403, f"{denied.text}; token requests={token_requests}" + assert _error_message(denied, 403) == ( + f"Raw cloud storage file ids can only be {verb} by a proxy admin key. " + "Use the LiteLLM managed file id returned when the file was created." + ), denied.text + assert token_requests == (), f"non-admin twice-encoded raw GCS {method} reached the Vertex token endpoint" + + +def _assert_signed_empty_body_s3_request(request: Request, method: Literal["GET", "DELETE"], wire_url: str) -> None: + assert request.method == method, request.method + request_date: Final = request.headers["x-amz-date"] + content_hash: Final = hashlib.sha256(b"").hexdigest() + assert request.headers["x-amz-content-sha256"] == content_hash, request.headers + aws_request: Final = AWSRequest( + method=method, + url=wire_url + request.target, + headers={ + "host": request.headers["host"], + "x-amz-date": request_date, + "x-amz-content-sha256": content_hash, + }, + ) + aws_request.context["timestamp"] = request_date + auth: Final = S3SigV4Auth(Credentials(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY), "s3", AWS_REGION) + signature: Final = auth.signature(auth.string_to_sign(aws_request, auth.canonical_request(aws_request)), aws_request) + expected_authorization: Final = ( + f"AWS4-HMAC-SHA256 Credential={AWS_ACCESS_KEY_ID}/{request_date[:8]}/{AWS_REGION}/s3/aws4_request, " + f"SignedHeaders={auth.signed_headers(auth.headers_to_sign(aws_request))}, Signature={signature}" + ) + assert request.headers["authorization"] == expected_authorization, request.headers["authorization"] + + def test_proxy_admin_deletes_a_raw_s3_uri_with_one_signed_delete(gateway: Gateway) -> None: with wire_server(_s3_backend()) as wire, gateway.scenario() as scenario: model: Final = _bedrock_model(scenario, wire) @@ -721,29 +802,30 @@ def test_proxy_admin_deletes_a_raw_s3_uri_with_one_signed_delete(gateway: Gatewa assert parsed.deleted is True and parsed.id == object_uri, deleted.text (request,) = _drained_other_than_model_list_probes(wire) assert request.target == f"/{BUCKET}/litellm-bedrock-files-obj.jsonl", request.target - request_date: Final = request.headers["x-amz-date"] - content_hash: Final = hashlib.sha256(b"").hexdigest() - assert request.headers["x-amz-content-sha256"] == content_hash, request.headers - aws_request: Final = AWSRequest( - method="DELETE", - url=wire.url + request.target, - headers={ - "host": request.headers["host"], - "x-amz-date": request_date, - "x-amz-content-sha256": content_hash, - }, + _assert_signed_empty_body_s3_request(request, "DELETE", wire.url) + + +def _s3_object_backend(request: Request) -> Reply: + if request.method == "GET": + return Reply(body=S3_OBJECT_BODY, content_type="application/octet-stream") + return _json_reply({"error": {"message": f"unscripted {request.method} {request.target}"}}, 404) + + +def test_proxy_admin_reads_a_raw_s3_uri_with_one_signed_get(gateway: Gateway) -> None: + with wire_server(_s3_object_backend) as wire, gateway.scenario() as scenario: + model: Final = _bedrock_model(scenario, wire) + _wait_until_every_worker_serves(gateway, model) + object_key: Final = f"litellm-batch-outputs/{uuid.uuid4().hex}.jsonl.out" + object_uri: Final = f"s3://{BUCKET}/{object_key}" + + read: Final = gateway.request( + "GET", "/v1/files/" + quote(object_uri, safe="") + "/content", params={"model": model} ) - aws_request.context["timestamp"] = request_date - auth: Final = S3SigV4Auth(Credentials(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY), "s3", AWS_REGION) - signature: Final = auth.signature( - auth.string_to_sign(aws_request, auth.canonical_request(aws_request)), aws_request - ) - expected_authorization: Final = ( - f"AWS4-HMAC-SHA256 Credential={AWS_ACCESS_KEY_ID}/{request_date[:8]}/{AWS_REGION}/s3/aws4_request, " - f"SignedHeaders={auth.signed_headers(auth.headers_to_sign(aws_request))}, Signature={signature}" - ) - assert request.headers["authorization"] == expected_authorization, request.headers["authorization"] - assert request.method == "DELETE", request.method + storage_requests: Final = _drained_other_than_model_list_probes(wire) + assert read.status_code == 200, f"{read.text}; storage requests={storage_requests}" + assert read.content == S3_OBJECT_BODY, read.content + assert [request.target for request in storage_requests] == [f"/{BUCKET}/{object_key}"], storage_requests + _assert_signed_empty_body_s3_request(storage_requests[0], "GET", wire.url) def _granting_vertex_token_backend(request: Request) -> Reply: