From 9ec3b5fd8bc07706e674e5f936212e6ddd723a9f Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Sat, 7 Nov 2026 11:56:47 -0800 Subject: [PATCH 01/39] feat(cost-map): move -latest model aliases into the cost map (#50491) * feat(cost-map): move -latest model aliases into the cost map Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test: load the finalized local cost map in the moonshot alias test and assert alias pricing by behavior Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test: compare alias cost map rows to their targets Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(cost-map): give each -latest alias its own alias_of row Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(cost-map): keep alias_of rows out of alias lists and add them to the registry schema Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> (cherry picked from commit 91e6cf2c27011d45b9f96e941c3189098df9c4f1) * fix(cost-map): let alias lists win over alias_of rows in both loaders and never serve a malformed pointer Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> (cherry picked from commit a54c42218a0e306b2ad4cd1aea9ad70f54392e8f) * feat(cost-map): add -latest alias rows for the tier 1 and 2 hosting providers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(cost-map): derive hosted alias targets from the cost map Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(databricks): skip alias rows when checking cache-write rates Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../get_llm_provider_logic.py | 6 +- .../model_latest_aliases.py | 72 ++++- litellm/model_latest_aliases.json | 21 -- ...odel_prices_and_context_window_backup.json | 294 ++++++++++++++++++ model_prices_and_context_window.json | 294 ++++++++++++++++++ .../test_get_llm_provider_logic.py | 59 +++- .../test_model_latest_aliases.py | 69 +++- .../test_databricks_cost_calculator.py | 2 + .../test_moonshot_chat_transformation.py | 5 +- tests/unit/rust_bridge/test_detectors.py | 4 +- 10 files changed, 760 insertions(+), 66 deletions(-) delete mode 100644 litellm/model_latest_aliases.json diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index b2851345302..bbb1b19819f 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -194,11 +194,7 @@ def get_llm_provider( resolved_model, provider, dynamic_api_key, resolved_api_base = _get_llm_provider( model, custom_llm_provider, api_base, api_key, litellm_params ) - explicit_provider: Final = custom_llm_provider or (litellm_params.custom_llm_provider if litellm_params else None) - target: Final = ( - resolve_model_latest_alias(resolved_model, provider) if explicit_provider or "/" in model else resolved_model - ) - return target, provider, dynamic_api_key, resolved_api_base + return resolve_model_latest_alias(resolved_model, provider), provider, dynamic_api_key, resolved_api_base def _get_llm_provider( diff --git a/litellm/litellm_core_utils/model_latest_aliases.py b/litellm/litellm_core_utils/model_latest_aliases.py index e3b1a21dd5a..ee45ccc9b3c 100644 --- a/litellm/litellm_core_utils/model_latest_aliases.py +++ b/litellm/litellm_core_utils/model_latest_aliases.py @@ -1,33 +1,75 @@ from collections.abc import Mapping -from functools import cache -from importlib.resources import files from types import MappingProxyType -from typing import Final +from typing import Final, cast # noqa: TID251 # litellm.model_cost is an untyped module global -from pydantic import TypeAdapter - -_ALIASES_ADAPTER: Final = TypeAdapter(dict[str, dict[str, str]]) +import litellm +from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry + + +def _cost_map() -> Mapping[str, object]: + return cast("Mapping[str, object]", litellm.model_cost) # cast-ok: per-lookup catalog validation is too slow + + +def _alias_entry(key: str) -> AliasedCostEntry | None: + entry: Final = _cost_map().get(key) + return entry if isinstance(entry, AliasedCostEntry) and entry.names[:1] not in ((), (key,)) else None + + +def _alias_provider(key: str, entry: AliasedCostEntry) -> str | None: + """The provider an alias is requested through: its key prefix for a hosted alias such as + ``bedrock/claude-haiku-latest`` (whose target row belongs to ``bedrock_converse``), else the target's provider""" + prefix, separator, _ = key.partition("/") + if separator and prefix in litellm.provider_list: + return prefix + provider: Final = entry.get("litellm_provider") + return provider if isinstance(provider, str) else None + + +def _alias_target(key: str, provider: str) -> str | None: + entry: Final = _alias_entry(key) + if entry is None or _alias_provider(key, entry) != provider: + return None + return entry.names[0].removeprefix(f"{provider}/") -@cache def get_model_latest_aliases() -> Mapping[str, Mapping[str, str]]: - aliases: Final = _ALIASES_ADAPTER.validate_json(files("litellm").joinpath("model_latest_aliases.json").read_bytes()) - return MappingProxyType({provider: MappingProxyType(models) for provider, models in aliases.items()}) + """``{provider: {alias: target}}`` for every alias row in the cost map, with provider prefixes stripped""" + rows: Final = tuple( + (provider, key, entry.names[0]) + for key in _cost_map() + if (entry := _alias_entry(key)) is not None and (provider := _alias_provider(key, entry)) is not None + ) + return MappingProxyType( + { + provider: MappingProxyType( + { + alias.removeprefix(f"{provider}/"): target.removeprefix(f"{provider}/") + for row_provider, alias, target in rows + if row_provider == provider + } + ) + for provider in dict.fromkeys(provider for provider, _, _ in rows) + } + ) def resolve_model_latest_alias(model: str, provider: str) -> str: - aliases: Final = get_model_latest_aliases().get(provider) - return aliases.get(model, model) if aliases is not None else model + return _alias_target(f"{provider}/{model}", provider) or _alias_target(model, provider) or model def expand_model_latest_alias(model: str) -> tuple[str, ...]: provider, separator, name = model.partition("/") - target: Final = resolve_model_latest_alias(name, provider) if separator else name - return (model, f"{provider}/{target}") if separator and target != name else (model,) + if not separator: + entry: Final = _alias_entry(model) + return (model,) if entry is None else (model, entry.names[0]) + target: Final = resolve_model_latest_alias(name, provider) + return (model, f"{provider}/{target}") if target != name else (model,) def is_model_latest_alias(model: str, custom_llm_provider: str | None = None) -> bool: prefix, separator, name = model.partition("/") - provider: Final = custom_llm_provider or prefix + provider: Final = custom_llm_provider or (prefix if separator else None) + if provider is None: + return _alias_entry(model) is not None candidate: Final = name if separator and prefix == provider else model - return bool(custom_llm_provider or separator) and resolve_model_latest_alias(candidate, provider) != candidate + return resolve_model_latest_alias(candidate, provider) != candidate diff --git a/litellm/model_latest_aliases.json b/litellm/model_latest_aliases.json deleted file mode 100644 index 7b0fa5d3d39..00000000000 --- a/litellm/model_latest_aliases.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "openai": { - "gpt-luna-latest": "gpt-6-luna", - "gpt-sol-latest": "gpt-6.1-sol", - "gpt-terra-latest": "gpt-6.1-terra", - "gpt-astra-latest": "gpt-6-astra", - "gpt-mini-latest": "gpt-5.4-mini" - }, - "anthropic": { - "claude-opus-latest": "claude-opus-5-5", - "claude-sonnet-latest": "claude-sonnet-5-5", - "claude-haiku-latest": "claude-haiku-5-5", - "claude-fable-latest": "claude-fable-5-1" - }, - "xai": {"grok-latest": "grok-4.7"}, - "moonshot": {"kimi-latest": "kimi-k3"}, - "zai": {"glm-latest": "glm-5.3", "glm-flash-latest": "glm-5.3-flash"}, - "deepseek": {"deepseek-pro-latest": "deepseek-v4-pro", "deepseek-flash-latest": "deepseek-v4-flash"}, - "dashscope": {"qwen-max-latest": "qwen3.8-max", "qwen-flash-latest": "qwen3.8-flash"}, - "minimax": {"minimax-latest": "MiniMax-M3"} -} diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index ac5e99c9313..b3412518cd7 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1969,6 +1969,9 @@ "prompt_cache_min_tokens": 512, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/claude-fable-latest": { + "alias_of": "global.anthropic.claude-fable-5-1" + }, "us.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.375e-05, "cache_creation_input_token_cost_above_1hr": 2.2e-05, @@ -2305,6 +2308,9 @@ "region": "us-east-1" } }, + "bedrock/claude-opus-latest": { + "alias_of": "global.anthropic.claude-opus-5-5" + }, "us.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, @@ -4063,6 +4069,9 @@ "deprecation_date": "2027-12-05", "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement" }, + "azure_ai/claude-fable-latest": { + "alias_of": "azure_ai/claude-fable-5-1" + }, "azure_ai/claude-mythos-5-1": { "input_cost_per_token": 1e-05, "output_cost_per_token": 5e-05, @@ -4165,6 +4174,9 @@ "prompt_cache_min_tokens": 512, "source": "https://management.azure.com/subscriptions/c873328e-b572-4770-8dff-aaeb6f1f0e79/providers/Microsoft.CognitiveServices/locations/eastus2/models?api-version=2024-10-01" }, + "azure_ai/claude-opus-latest": { + "alias_of": "azure_ai/claude-opus-5-5" + }, "azure_ai/claude-opus-4-8": { "deprecation_date": "2027-09-01", "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", @@ -4382,6 +4394,9 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "azure_ai/gpt-astra-latest": { + "alias_of": "azure_ai/gpt-6-astra" + }, "azure_ai/gpt-6-luna": { "cache_creation_input_token_cost": 1.25e-07, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, @@ -4430,6 +4445,9 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "azure_ai/gpt-luna-latest": { + "alias_of": "azure_ai/gpt-6-luna" + }, "azure_ai/gpt-6-sol": { "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, @@ -4526,6 +4544,9 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "azure_ai/gpt-sol-latest": { + "alias_of": "azure_ai/gpt-6.1-sol" + }, "azure_ai/gpt-6.1-terra": { "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-06, @@ -4574,6 +4595,9 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "azure_ai/gpt-terra-latest": { + "alias_of": "azure_ai/gpt-6.1-terra" + }, "azure_ai/gpt-5.5": { "deprecation_date": "2027-10-26", "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", @@ -5036,6 +5060,9 @@ "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false }, + "azure_ai/gpt-mini-latest": { + "alias_of": "azure_ai/gpt-5.4-mini" + }, "azure_ai/gpt-5.4-mini-2026-03-17": { "deprecation_date": "2027-09-21", "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", @@ -9080,6 +9107,9 @@ "cache_read_input_token_cost_flex": 5e-07, "cache_creation_input_token_cost_flex": 6.25e-06 }, + "azure/gpt-astra-latest": { + "alias_of": "azure/gpt-6-astra" + }, "azure/gpt-6-astra-2026-09-03": { "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-05, @@ -9184,6 +9214,9 @@ "output_cost_per_token_above_272k_tokens_flex": 3.75e-07, "output_cost_per_token_flex": 2.5e-07 }, + "azure/gpt-luna-latest": { + "alias_of": "azure/gpt-6-luna" + }, "azure/gpt-6-luna-2026-09-22": { "cache_creation_input_token_cost": 1.25e-07, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, @@ -9403,6 +9436,9 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "azure/gpt-terra-latest": { + "alias_of": "azure/gpt-6.1-terra" + }, "azure/gpt-6.1-sol": { "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, @@ -9455,6 +9491,9 @@ "cache_read_input_token_cost_above_272k_tokens_flex": 1e-07, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06 }, + "azure/gpt-sol-latest": { + "alias_of": "azure/gpt-6.1-sol" + }, "azure/gpt-6.1-sol-2026-09-29": { "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, @@ -11146,6 +11185,9 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_xhigh_reasoning_effort": true }, + "azure/gpt-mini-latest": { + "alias_of": "azure/gpt-5.4-mini" + }, "azure/gpt-5.4-mini-2026-03-17": { "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_flex": 3.75e-08, @@ -13212,6 +13254,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "azure_ai/kimi-latest": { + "alias_of": "azure_ai/FW-Kimi-K3" + }, "azure_ai/FW-MiniMax-M2.5": { "deprecation_date": "2027-07-01", "cache_read_input_token_cost": 3.3e-08, @@ -13251,6 +13296,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "azure_ai/minimax-latest": { + "alias_of": "azure_ai/FW-MiniMax-M3" + }, "azure_ai/FW-Nemotron-Lightning-3.5-30B-A3B": { "cache_read_input_token_cost": 1e-08, "input_cost_per_token": 6e-08, @@ -13656,6 +13704,9 @@ "cache_read_input_token_cost": 1.45e-07, "supports_prompt_caching": true }, + "azure_ai/deepseek-pro-latest": { + "alias_of": "azure_ai/deepseek-v4-pro" + }, "azure_ai/deepseek-v4-flash": { "deprecation_date": "2028-02-20", "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", @@ -13674,6 +13725,9 @@ "cache_read_input_token_cost": 2.8e-08, "supports_prompt_caching": true }, + "azure_ai/deepseek-flash-latest": { + "alias_of": "azure_ai/deepseek-v4-flash" + }, "azure_ai/DeepSeek-V4-Flash-0731": { "cache_read_input_token_cost": 1.4e-08, "deprecation_date": "2026-12-03", @@ -16623,6 +16677,9 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing", "supports_web_search": true }, + "claude-sonnet-latest": { + "alias_of": "claude-sonnet-5-5" + }, "claude-sonnet-4-6": { "supports_web_search": true, "supports_anthropic_compaction": true, @@ -17024,6 +17081,9 @@ "supports_native_structured_output": true, "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, + "claude-fable-latest": { + "alias_of": "claude-fable-5-1" + }, "claude-opus-5-5": { "supports_web_search": true, "supports_anthropic_compaction": true, @@ -17069,6 +17129,9 @@ "prompt_cache_min_tokens": 512, "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, + "claude-opus-latest": { + "alias_of": "claude-opus-5-5" + }, "claude-opus-5": { "supports_web_search": true, "supports_anthropic_compaction": true, @@ -18868,6 +18931,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "dashscope/qwen-max-latest": { + "alias_of": "dashscope/qwen3.8-max" + }, "dashscope/qwen3.8-flash": { "cache_creation_input_token_cost": 2e-07, "cache_read_input_token_cost": 1.6e-08, @@ -18887,6 +18953,9 @@ "supports_vision": true, "supports_web_search": true }, + "dashscope/qwen-flash-latest": { + "alias_of": "dashscope/qwen3.8-flash" + }, "dashscope/qwen3.8-omni-flash": { "cache_read_input_token_cost": 1.6e-08, "input_cost_per_token": 1.5e-07, @@ -21767,6 +21836,9 @@ "supports_vision": true, "thinking_always_on": true }, + "databricks/claude-fable-latest": { + "alias_of": "databricks/databricks-claude-fable-5-1" + }, "databricks/databricks-claude-haiku-4-5": { "cache_creation_input_token_cost": 1.24999e-06, "cache_creation_input_token_cost_above_1hr": 1.99997e-06, @@ -22007,6 +22079,9 @@ "supports_xhigh_reasoning_effort": true, "thinking_always_on": true }, + "databricks/claude-opus-latest": { + "alias_of": "databricks/databricks-claude-opus-5-5" + }, "databricks/databricks-claude-sonnet-4": { "cache_creation_input_token_cost": 3.74997e-06, "cache_read_input_token_cost": 3.0002e-07, @@ -22724,6 +22799,9 @@ "cache_read_input_token_cost_priority": 4.55e-07, "cache_creation_input_token_cost_priority": 2.45e-06 }, + "databricks/glm-latest": { + "alias_of": "databricks/databricks-glm-5-3" + }, "databricks/databricks-glm-5-3-flash": { "cache_creation_input_token_cost": 1.5001e-07, "cache_read_input_token_cost": 3.003e-08, @@ -22757,6 +22835,9 @@ "cache_read_input_token_cost_priority": 5.25e-08, "cache_creation_input_token_cost_priority": 2.625e-07 }, + "databricks/glm-flash-latest": { + "alias_of": "databricks/databricks-glm-5-3-flash" + }, "databricks/databricks-gpt-5": { "cache_creation_input_token_cost": 1.24999e-06, "cache_read_input_token_cost": 1.2502e-07, @@ -22919,6 +23000,9 @@ "cache_read_input_token_cost_priority": 1.5001e-07, "cache_creation_input_token_cost_priority": 1.50003e-06 }, + "databricks/gpt-mini-latest": { + "alias_of": "databricks/databricks-gpt-5-4-mini" + }, "databricks/databricks-gpt-5-4-nano": { "cache_creation_input_token_cost": 1.9999e-07, "cache_read_input_token_cost": 2.002e-08, @@ -23315,6 +23399,9 @@ "cache_read_input_token_cost_priority": 5.25e-07, "cache_creation_input_token_cost_priority": 5.25e-06 }, + "databricks/kimi-latest": { + "alias_of": "databricks/databricks-kimi-k3" + }, "databricks/system.ai.kimi-k3": { "litellm_provider": "databricks", "mode": "chat", @@ -28843,6 +28930,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "fireworks_ai/minimax-latest": { + "alias_of": "fireworks_ai/accounts/fireworks/models/minimax-m3" + }, "fireworks_ai/accounts/fireworks/models/mixtral-8x22b-instruct-hf": { "input_cost_per_token": 1.2e-06, "litellm_provider": "fireworks_ai", @@ -33789,6 +33879,9 @@ "supports_vision": true, "thinking_always_on": true }, + "github_copilot/claude-fable-latest": { + "alias_of": "github_copilot/claude-fable-5.1" + }, "github_copilot/claude-haiku-4.5": { "cache_creation_input_token_cost": 1.25e-06, "cache_read_input_token_cost": 1e-07, @@ -33832,6 +33925,9 @@ "supports_vision": true, "thinking_always_on": false }, + "github_copilot/claude-haiku-latest": { + "alias_of": "github_copilot/claude-haiku-5.5" + }, "github_copilot/claude-opus-4.8": { "cache_creation_input_token_cost": 6.25e-06, "cache_read_input_token_cost": 5e-07, @@ -33920,6 +34016,9 @@ "supports_vision": true, "thinking_always_on": true }, + "github_copilot/claude-opus-latest": { + "alias_of": "github_copilot/claude-opus-5.5" + }, "github_copilot/claude-sonnet-5": { "cache_creation_input_token_cost": 2.5e-06, "cache_read_input_token_cost": 2e-07, @@ -33963,6 +34062,9 @@ "supports_vision": true, "thinking_always_on": true }, + "github_copilot/claude-sonnet-latest": { + "alias_of": "github_copilot/claude-sonnet-5.5" + }, "github_copilot/gemini-3.7-flash": { "cache_read_input_token_cost": 7.5e-08, "deprecation_date": "2026-10-19", @@ -34308,6 +34410,9 @@ "supports_response_schema": true, "supports_vision": true }, + "github_copilot/gpt-astra-latest": { + "alias_of": "github_copilot/gpt-6-astra" + }, "github_copilot/gpt-6-luna": { "cache_creation_input_token_cost": 1.25e-07, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, @@ -34331,6 +34436,9 @@ "supports_response_schema": true, "supports_vision": true }, + "github_copilot/gpt-luna-latest": { + "alias_of": "github_copilot/gpt-6-luna" + }, "github_copilot/gpt-6-sol": { "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, @@ -34377,6 +34485,9 @@ "supports_response_schema": true, "supports_vision": true }, + "github_copilot/gpt-sol-latest": { + "alias_of": "github_copilot/gpt-6.1-sol" + }, "github_copilot/kimi-k3": { "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, @@ -34394,6 +34505,9 @@ "supports_response_schema": true, "supports_vision": true }, + "github_copilot/kimi-latest": { + "alias_of": "github_copilot/kimi-k3" + }, "github_copilot/mai-code-1.1-flash": { "cache_read_input_token_cost": 2e-08, "input_cost_per_token": 2e-07, @@ -37687,6 +37801,9 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "gpt-astra-latest": { + "alias_of": "gpt-6-astra" + }, "gpt-6-sol": { "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_12hr": 5e-06, @@ -37884,6 +38001,9 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "gpt-luna-latest": { + "alias_of": "gpt-6-luna" + }, "gpt-5.6": { "cache_creation_input_token_cost": 5e-06, "cache_creation_input_token_cost_above_12hr": 1e-05, @@ -39067,6 +39187,9 @@ "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false }, + "gpt-mini-latest": { + "alias_of": "gpt-5.4-mini" + }, "gpt-5.4-mini-2026-03-17": { "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_batches": 3.75e-08, @@ -43025,6 +43148,9 @@ "max_input_tokens": 1000000, "max_output_tokens": 128000 }, + "minimax/minimax-latest": { + "alias_of": "minimax/MiniMax-M3" + }, "minimax/MiniMax-H3": { "litellm_provider": "minimax", "mode": "video_generation", @@ -44535,6 +44661,9 @@ "supports_video_input": true, "supports_vision": true }, + "moonshot/kimi-latest": { + "alias_of": "moonshot/kimi-k3" + }, "moonshot/moonshot-v1-128k": { "deprecation_date": "2026-08-31", "input_cost_per_token": 2e-06, @@ -50136,6 +50265,9 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "together_ai/deepseek-pro-latest": { + "alias_of": "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813" + }, "together_ai/meta-models/Muse-Glimmer-30B": { "cache_read_input_token_cost": 4e-08, "input_cost_per_token": 3.5e-07, @@ -50182,6 +50314,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "together_ai/kimi-latest": { + "alias_of": "together_ai/moonshotai/Kimi-K3" + }, "together_ai/thinkingmachines/Inkling": { "cache_read_input_token_cost": 1.7e-07, "deprecation_date": "2026-11-06", @@ -50232,6 +50367,9 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "together_ai/glm-latest": { + "alias_of": "together_ai/zai-org/GLM-5.3" + }, "together_ai/zai-org/GLM-5.3-Flash": { "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 1.5e-07, @@ -50249,6 +50387,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "together_ai/glm-flash-latest": { + "alias_of": "together_ai/zai-org/GLM-5.3-Flash" + }, "tts-1": { "deprecation_date": "2027-01-06", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", @@ -53019,6 +53160,9 @@ "output_cost_per_token_batches": 2.5e-05, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, + "vertex_ai/claude-fable-latest": { + "alias_of": "vertex_ai/claude-fable-5-1" + }, "vertex_ai/claude-fable-5@default": { "deprecation_date": "2027-06-08", "regional_endpoint_uplift_multiplier": 1.1, @@ -53217,6 +53361,9 @@ "prompt_cache_min_tokens": 512, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, + "vertex_ai/claude-opus-latest": { + "alias_of": "vertex_ai/claude-opus-5-5" + }, "vertex_ai/claude-opus-5-5@default": { "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, @@ -55765,6 +55912,9 @@ "supports_vision": true, "supports_web_search": true }, + "xai/grok-latest": { + "alias_of": "xai/grok-4.7" + }, "zai.glm-4.7": { "deprecation_date": "2027-03-30", "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", @@ -55866,6 +56016,9 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "zai/glm-latest": { + "alias_of": "zai/glm-5.3" + }, "zai/glm-5.3-flash": { "cache_creation_input_token_cost": 0, "cache_read_input_token_cost": 3e-08, @@ -55883,6 +56036,9 @@ "source": "https://docs.z.ai/guides/overview/pricing", "supports_vision": true }, + "zai/glm-flash-latest": { + "alias_of": "zai/glm-5.3-flash" + }, "zai/glm-5v-turbo": { "cache_read_input_token_cost": 2.4e-07, "input_cost_per_token": 1.2e-06, @@ -61519,6 +61675,9 @@ "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html", "supports_web_search": true }, + "bedrock_mantle/gpt-astra-latest": { + "alias_of": "bedrock_mantle/openai.gpt-6-astra" + }, "bedrock_mantle/openai.gpt-6-sol": { "input_cost_per_token": 2.2e-06, "input_cost_per_token_above_272k_tokens": 4.4e-06, @@ -61599,6 +61758,9 @@ "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-cards-openai.html", "supports_web_search": true }, + "bedrock_mantle/gpt-luna-latest": { + "alias_of": "bedrock_mantle/openai.gpt-6-luna" + }, "us.openai.gpt-6-astra": { "supports_bedrock_runtime_chat_completions_response_format": true, "input_cost_per_token": 1.1e-05, @@ -61767,6 +61929,9 @@ "/v1/responses" ] }, + "bedrock/gpt-astra-latest": { + "alias_of": "global.openai.gpt-6-astra" + }, "openai.gpt-6-sol": { "input_cost_per_token": 2e-06, "input_cost_per_token_above_272k_tokens": 4e-06, @@ -61909,6 +62074,9 @@ "/v1/responses" ] }, + "bedrock/gpt-luna-latest": { + "alias_of": "global.openai.gpt-6-luna" + }, "bedrock_mantle/openai.gpt-5.5": { "input_cost_per_token": 5.5e-06, "input_cost_per_token_above_272k_tokens": 1.1e-05, @@ -62186,6 +62354,9 @@ "supports_forced_tool_use": false, "token_count_model": {"model":"anthropic.claude-opus-5-5","litellm_provider":"bedrock_mantle","region":"us-east-1"} }, + "bedrock_mantle/claude-opus-latest": { + "alias_of": "bedrock_mantle/anthropic.claude-opus-5-5" + }, "bedrock_mantle/anthropic.claude-sonnet-5-5": { "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 2.75e-06, @@ -62226,6 +62397,9 @@ "prompt_cache_min_tokens": 512, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, + "bedrock_mantle/claude-sonnet-latest": { + "alias_of": "bedrock_mantle/anthropic.claude-sonnet-5-5" + }, "us.xai.grok-4.6": { "supports_regex_lookaround": false, "input_cost_per_token": 2.2e-06, @@ -63796,6 +63970,9 @@ "token_count_framing": 23, "token_count_tool_framing": 12 }, + "deepseek/deepseek-flash-latest": { + "alias_of": "deepseek/deepseek-v4-flash" + }, "deepseek/deepseek-v4-flash-vision-exp": { "cache_creation_input_token_cost": 0.0, "cache_read_input_token_cost": 6e-09, @@ -63904,6 +64081,9 @@ "token_count_framing": 76, "token_count_tool_framing": 75 }, + "deepseek/deepseek-pro-latest": { + "alias_of": "deepseek/deepseek-v4-pro" + }, "tencent/deepseek-v4-pro": { "cache_creation_input_token_cost": 0.0, "cache_read_input_token_cost": 1.45e-07, @@ -65327,6 +65507,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "fireworks_ai/kimi-latest": { + "alias_of": "fireworks_ai/accounts/fireworks/models/kimi-k3" + }, "fireworks_ai/deepseek-v4-flash-0731": { "cache_read_input_token_cost": 7e-09, "cache_read_input_token_cost_priority": 8.75e-09, @@ -69000,6 +69183,9 @@ "output_cost_per_token": 4.7e-07, "source": "https://api.together.ai/v1/models" }, + "together_ai/qwen-flash-latest": { + "alias_of": "together_ai/Qwen/Qwen3.8-Flash" + }, "together_ai/moonshotai/Kimi-K2.5-fp4": { "input_cost_per_token": 5e-07, "output_cost_per_token": 2.8e-06, @@ -69216,6 +69402,9 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "azure_ai/glm-latest": { + "alias_of": "azure_ai/FW-GLM-5.3" + }, "azure_ai/FW-GLM-5.3-Flash": { "cache_read_input_token_cost": 3.8e-08, "input_cost_per_token": 1.88e-07, @@ -69231,6 +69420,9 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "azure_ai/glm-flash-latest": { + "alias_of": "azure_ai/FW-GLM-5.3-Flash" + }, "azure_ai/FW-GPT-OSS-120B": { "deprecation_date": "2027-07-01", "cache_read_input_token_cost": 8.2e-08, @@ -71236,6 +71428,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "baseten/glm-latest": { + "alias_of": "baseten/zai-org/GLM-5.3" + }, "baseten/zai-org/GLM-5.3-Fast": { "cache_read_input_token_cost": 2.1e-07, "input_cost_per_token": 2.1e-06, @@ -74425,6 +74620,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "bedrock/kimi-latest": { + "alias_of": "global.moonshotai.kimi-k3" + }, "us.moonshotai.kimi-k3": { "supports_regex_lookaround": false, "cache_creation_input_token_cost": 4.125e-06, @@ -74975,6 +75173,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "baseten/kimi-latest": { + "alias_of": "baseten/moonshotai/Kimi-K3" + }, "baseten/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B": { "cache_read_input_token_cost": 1.2e-07, "input_cost_per_token": 6e-07, @@ -75057,6 +75258,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "baseten/glm-flash-latest": { + "alias_of": "baseten/zai-org/GLM-5.3-Flash" + }, "baseten/deepseek-ai/DeepSeek-V4-Flash-0731": { "cache_read_input_token_cost": 2.8e-08, "input_cost_per_token": 1.3e-07, @@ -75073,6 +75277,9 @@ "supports_tool_choice": true, "supports_vision": false }, + "baseten/deepseek-flash-latest": { + "alias_of": "baseten/deepseek-ai/DeepSeek-V4-Flash-0731" + }, "baseten/deepseek-ai/DeepSeek-V4-Pro": { "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 1.45e-07, @@ -75106,6 +75313,9 @@ "supports_tool_choice": true, "supports_vision": false }, + "baseten/deepseek-pro-latest": { + "alias_of": "baseten/deepseek-ai/DeepSeek-V4-Pro-0813" + }, "baseten/zai-org/GLM-5.2-Fast": { "cache_read_input_token_cost": 2.1e-07, "input_cost_per_token": 2.1e-06, @@ -76097,6 +76307,9 @@ "prompt_cache_min_tokens": 512, "source": "https://management.azure.com/subscriptions/c873328e-b572-4770-8dff-aaeb6f1f0e79/providers/Microsoft.CognitiveServices/locations/eastus2/models?api-version=2024-10-01" }, + "azure_ai/claude-sonnet-latest": { + "alias_of": "azure_ai/claude-sonnet-5-5" + }, "bedrock/us-gov-east-1/anthropic.claude-sonnet-5-5": { "bedrock_converse_supports_strict_tools": false, "bedrock_output_config_effort_ceiling": "xhigh", @@ -76261,6 +76474,9 @@ "prompt_cache_min_tokens": 512, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, + "bedrock/claude-sonnet-latest": { + "alias_of": "global.anthropic.claude-sonnet-5-5" + }, "jp.anthropic.claude-sonnet-5-5": { "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 2.75e-06, @@ -76520,6 +76736,9 @@ "prompt_cache_min_tokens": 512, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, + "vertex_ai/claude-sonnet-latest": { + "alias_of": "vertex_ai/claude-sonnet-5-5" + }, "vertex_ai/claude-sonnet-5-5@default": { "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, @@ -76632,6 +76851,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "bedrock/grok-latest": { + "alias_of": "global.xai.grok-4.7" + }, "us.xai.grok-4.7": { "supports_regex_lookaround": false, "supported_endpoints": ["/v1/chat/completions"], @@ -76804,6 +77026,9 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "gpt-sol-latest": { + "alias_of": "gpt-6.1-sol" + }, "global.openai.gpt-6.1-sol": { "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, @@ -76850,6 +77075,9 @@ "supports_sampling_params": false, "supports_xhigh_reasoning_effort": true }, + "bedrock/gpt-sol-latest": { + "alias_of": "global.openai.gpt-6.1-sol" + }, "bedrock_mantle/openai.gpt-6.1-sol": { "cache_creation_input_token_cost": 2.75e-06, "cache_creation_input_token_cost_above_272k_tokens": 5.5e-06, @@ -76898,6 +77126,9 @@ "supports_xhigh_reasoning_effort": true, "use_openai_responses_path": true }, + "bedrock_mantle/gpt-sol-latest": { + "alias_of": "bedrock_mantle/openai.gpt-6.1-sol" + }, "us.openai.gpt-6.1-sol": { "cache_creation_input_token_cost": 2.75e-06, "cache_creation_input_token_cost_above_272k_tokens": 5.5e-06, @@ -76989,6 +77220,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "vertex_ai/grok-latest": { + "alias_of": "vertex_ai/xai/grok-4.7" + }, "azure_ai/kimi-k2-thinking": { "deprecation_date": "2026-03-29", "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -77046,6 +77280,9 @@ "supports_tool_choice": true, "supports_vision": false }, + "bedrock/glm-latest": { + "alias_of": "global.zai.glm-5.3" + }, "us.zai.glm-5.3": { "cache_creation_input_token_cost": 2.31e-06, "cache_creation_input_token_cost_batches": 1.155e-06, @@ -77269,6 +77506,9 @@ "supports_vision": true, "supports_web_search": true }, + "azure_ai/grok-latest": { + "alias_of": "azure_ai/grok-4.7" + }, "in.moonshotai.kimi-k3": { "cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost_flex": 2.0625e-06, @@ -77355,6 +77595,9 @@ "cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1e-06, "cache_read_input_token_cost_above_100k_tokens": 5e-08 }, + "claude-haiku-latest": { + "alias_of": "claude-haiku-5-5" + }, "bedrock_mantle/anthropic.claude-haiku-5-5": { "input_cost_per_token_above_100k_tokens": 5.5e-07, "output_cost_per_token_above_100k_tokens": 2.75e-06, @@ -77401,6 +77644,9 @@ "prompt_cache_min_tokens": 512, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-haiku-5-5.html" }, + "bedrock_mantle/claude-haiku-latest": { + "alias_of": "bedrock_mantle/anthropic.claude-haiku-5-5" + }, "bedrock_mantle/us-gov-west-1/anthropic.claude-haiku-5-5": { "input_cost_per_token_above_100k_tokens": 6e-07, "output_cost_per_token_above_100k_tokens": 3e-06, @@ -77632,6 +77878,9 @@ "prompt_cache_min_tokens": 512, "source": "https://platform.claude.com/docs/en/models/haiku-5-5/migration-guide" }, + "azure_ai/claude-haiku-latest": { + "alias_of": "azure_ai/claude-haiku-5-5" + }, "bedrock/us-gov-east-1/anthropic.claude-haiku-5-5": { "input_cost_per_token_above_100k_tokens": 6e-07, "output_cost_per_token_above_100k_tokens": 3e-06, @@ -77822,6 +78071,9 @@ "prompt_cache_min_tokens": 512, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, + "bedrock/claude-haiku-latest": { + "alias_of": "global.anthropic.claude-haiku-5-5" + }, "jp.anthropic.claude-haiku-5-5": { "input_cost_per_token_above_100k_tokens": 5.5e-07, "output_cost_per_token_above_100k_tokens": 2.75e-06, @@ -78029,6 +78281,9 @@ "prompt_cache_min_tokens": 512, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" }, + "vertex_ai/claude-haiku-latest": { + "alias_of": "vertex_ai/claude-haiku-5-5" + }, "vertex_ai/claude-haiku-5-5@default": { "input_cost_per_token_above_100k_tokens": 5e-07, "output_cost_per_token_above_100k_tokens": 2.5e-06, @@ -78933,6 +79188,9 @@ "supports_response_schema": true, "supports_vision": true }, + "github_copilot/grok-latest": { + "alias_of": "github_copilot/grok-4.7" + }, "together_ai/deepseek-ai/DeepSeek-V4.1-Flash-1017": { "cache_read_input_token_cost": 6e-09, "input_cost_per_token": 3e-07, @@ -80356,6 +80614,9 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "gpt-terra-latest": { + "alias_of": "gpt-6.1-terra" + }, "chatgpt/gpt-6.1-terra": { "litellm_provider": "chatgpt", "max_input_tokens": 922000, @@ -80414,6 +80675,9 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true }, + "bedrock/gpt-terra-latest": { + "alias_of": "global.openai.gpt-6.1-terra" + }, "us.openai.gpt-6.1-terra": { "cache_creation_input_token_cost": 1.375e-06, "cache_creation_input_token_cost_above_272k_tokens": 2.75e-06, @@ -80493,6 +80757,9 @@ "supports_response_schema": true, "supports_vision": true }, + "github_copilot/gpt-terra-latest": { + "alias_of": "github_copilot/gpt-6.1-terra" + }, "refuseless/glm-5.3-abliterated": { "cache_read_input_token_cost": 1.5e-07, "input_cost_per_token": 1e-06, @@ -80647,6 +80914,9 @@ "supports_xhigh_reasoning_effort": true, "use_openai_responses_path": true }, + "bedrock_mantle/gpt-terra-latest": { + "alias_of": "bedrock_mantle/openai.gpt-6.1-terra" + }, "clf_ai_gateway/glm-5.3": { "metadata": { "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." @@ -82196,6 +82466,9 @@ "supports_vision": true, "thinking_always_on": false }, + "databricks/claude-haiku-latest": { + "alias_of": "databricks/databricks-claude-haiku-5-5" + }, "databricks/databricks-claude-sonnet-5-5": { "cache_creation_input_token_cost": 2.49998e-06, "cache_creation_input_token_cost_above_1hr": 4.00001e-06, @@ -82225,6 +82498,9 @@ "supports_vision": false, "thinking_always_on": true }, + "databricks/claude-sonnet-latest": { + "alias_of": "databricks/databricks-claude-sonnet-5-5" + }, "databricks/databricks-deepseek-v4-1-flash": { "cache_creation_input_token_cost": 3.0002e-07, "cache_read_input_token_cost": 3.003e-08, @@ -82457,6 +82733,9 @@ "cache_read_input_token_cost_above_272k_tokens_priority": 3.9998e-07, "cache_creation_input_token_cost_above_272k_tokens_priority": 9.99999e-06 }, + "databricks/gpt-sol-latest": { + "alias_of": "databricks/databricks-gpt-6-1-sol" + }, "databricks/databricks-gpt-6-1-terra": { "cache_creation_input_token_cost": 1.24999e-06, "cache_creation_input_token_cost_above_272k_tokens": 2.49998e-06, @@ -82497,6 +82776,9 @@ "cache_read_input_token_cost_above_272k_tokens_priority": 1.9999e-07, "cache_creation_input_token_cost_above_272k_tokens_priority": 5.00003e-06 }, + "databricks/gpt-terra-latest": { + "alias_of": "databricks/databricks-gpt-6-1-terra" + }, "databricks/databricks-gpt-6-astra": { "cache_creation_input_token_cost": 1.249997e-05, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-05, @@ -82537,6 +82819,9 @@ "cache_read_input_token_cost_above_272k_tokens_priority": 4.00001e-06, "cache_creation_input_token_cost_above_272k_tokens_priority": 5.000002e-05 }, + "databricks/gpt-astra-latest": { + "alias_of": "databricks/databricks-gpt-6-astra" + }, "databricks/databricks-gpt-6-luna": { "cache_creation_input_token_cost": 1.2502e-07, "cache_creation_input_token_cost_above_272k_tokens": 2.4997e-07, @@ -82577,6 +82862,9 @@ "cache_read_input_token_cost_above_272k_tokens_priority": 3.997e-08, "cache_creation_input_token_cost_above_272k_tokens_priority": 5.0001e-07 }, + "databricks/gpt-luna-latest": { + "alias_of": "databricks/databricks-gpt-6-luna" + }, "databricks/databricks-gpt-6-sol": { "cache_creation_input_token_cost": 2.49998e-06, "cache_creation_input_token_cost_above_272k_tokens": 5.00003e-06, @@ -82707,6 +82995,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "databricks/grok-latest": { + "alias_of": "databricks/databricks-grok-4-7" + }, "databricks/databricks-kimi-k2-7": { "cache_creation_input_token_cost": 9.4997e-07, "cache_read_input_token_cost": 1.8998e-07, @@ -86639,6 +86930,9 @@ "supports_vision": true, "use_openai_responses_path": true }, + "bedrock_mantle/grok-latest": { + "alias_of": "bedrock_mantle/xai.grok-4.7" + }, "together_ai/ricursive-ai/Ripple-65B-Preview": { "cache_read_input_token_cost": 2e-08, "input_cost_per_token": 2e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index ac5e99c9313..b3412518cd7 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1969,6 +1969,9 @@ "prompt_cache_min_tokens": 512, "source": "https://aws.amazon.com/bedrock/pricing/" }, + "bedrock/claude-fable-latest": { + "alias_of": "global.anthropic.claude-fable-5-1" + }, "us.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.375e-05, "cache_creation_input_token_cost_above_1hr": 2.2e-05, @@ -2305,6 +2308,9 @@ "region": "us-east-1" } }, + "bedrock/claude-opus-latest": { + "alias_of": "global.anthropic.claude-opus-5-5" + }, "us.anthropic.claude-opus-5": { "bedrock_converse_supports_strict_tools": false, "supports_adaptive_thinking": true, @@ -4063,6 +4069,9 @@ "deprecation_date": "2027-12-05", "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement" }, + "azure_ai/claude-fable-latest": { + "alias_of": "azure_ai/claude-fable-5-1" + }, "azure_ai/claude-mythos-5-1": { "input_cost_per_token": 1e-05, "output_cost_per_token": 5e-05, @@ -4165,6 +4174,9 @@ "prompt_cache_min_tokens": 512, "source": "https://management.azure.com/subscriptions/c873328e-b572-4770-8dff-aaeb6f1f0e79/providers/Microsoft.CognitiveServices/locations/eastus2/models?api-version=2024-10-01" }, + "azure_ai/claude-opus-latest": { + "alias_of": "azure_ai/claude-opus-5-5" + }, "azure_ai/claude-opus-4-8": { "deprecation_date": "2027-09-01", "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", @@ -4382,6 +4394,9 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "azure_ai/gpt-astra-latest": { + "alias_of": "azure_ai/gpt-6-astra" + }, "azure_ai/gpt-6-luna": { "cache_creation_input_token_cost": 1.25e-07, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, @@ -4430,6 +4445,9 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "azure_ai/gpt-luna-latest": { + "alias_of": "azure_ai/gpt-6-luna" + }, "azure_ai/gpt-6-sol": { "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, @@ -4526,6 +4544,9 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "azure_ai/gpt-sol-latest": { + "alias_of": "azure_ai/gpt-6.1-sol" + }, "azure_ai/gpt-6.1-terra": { "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-06, @@ -4574,6 +4595,9 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "azure_ai/gpt-terra-latest": { + "alias_of": "azure_ai/gpt-6.1-terra" + }, "azure_ai/gpt-5.5": { "deprecation_date": "2027-10-26", "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", @@ -5036,6 +5060,9 @@ "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false }, + "azure_ai/gpt-mini-latest": { + "alias_of": "azure_ai/gpt-5.4-mini" + }, "azure_ai/gpt-5.4-mini-2026-03-17": { "deprecation_date": "2027-09-21", "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", @@ -9080,6 +9107,9 @@ "cache_read_input_token_cost_flex": 5e-07, "cache_creation_input_token_cost_flex": 6.25e-06 }, + "azure/gpt-astra-latest": { + "alias_of": "azure/gpt-6-astra" + }, "azure/gpt-6-astra-2026-09-03": { "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-05, @@ -9184,6 +9214,9 @@ "output_cost_per_token_above_272k_tokens_flex": 3.75e-07, "output_cost_per_token_flex": 2.5e-07 }, + "azure/gpt-luna-latest": { + "alias_of": "azure/gpt-6-luna" + }, "azure/gpt-6-luna-2026-09-22": { "cache_creation_input_token_cost": 1.25e-07, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, @@ -9403,6 +9436,9 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "azure/gpt-terra-latest": { + "alias_of": "azure/gpt-6.1-terra" + }, "azure/gpt-6.1-sol": { "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, @@ -9455,6 +9491,9 @@ "cache_read_input_token_cost_above_272k_tokens_flex": 1e-07, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06 }, + "azure/gpt-sol-latest": { + "alias_of": "azure/gpt-6.1-sol" + }, "azure/gpt-6.1-sol-2026-09-29": { "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, @@ -11146,6 +11185,9 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_xhigh_reasoning_effort": true }, + "azure/gpt-mini-latest": { + "alias_of": "azure/gpt-5.4-mini" + }, "azure/gpt-5.4-mini-2026-03-17": { "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_flex": 3.75e-08, @@ -13212,6 +13254,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "azure_ai/kimi-latest": { + "alias_of": "azure_ai/FW-Kimi-K3" + }, "azure_ai/FW-MiniMax-M2.5": { "deprecation_date": "2027-07-01", "cache_read_input_token_cost": 3.3e-08, @@ -13251,6 +13296,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "azure_ai/minimax-latest": { + "alias_of": "azure_ai/FW-MiniMax-M3" + }, "azure_ai/FW-Nemotron-Lightning-3.5-30B-A3B": { "cache_read_input_token_cost": 1e-08, "input_cost_per_token": 6e-08, @@ -13656,6 +13704,9 @@ "cache_read_input_token_cost": 1.45e-07, "supports_prompt_caching": true }, + "azure_ai/deepseek-pro-latest": { + "alias_of": "azure_ai/deepseek-v4-pro" + }, "azure_ai/deepseek-v4-flash": { "deprecation_date": "2028-02-20", "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement", @@ -13674,6 +13725,9 @@ "cache_read_input_token_cost": 2.8e-08, "supports_prompt_caching": true }, + "azure_ai/deepseek-flash-latest": { + "alias_of": "azure_ai/deepseek-v4-flash" + }, "azure_ai/DeepSeek-V4-Flash-0731": { "cache_read_input_token_cost": 1.4e-08, "deprecation_date": "2026-12-03", @@ -16623,6 +16677,9 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing", "supports_web_search": true }, + "claude-sonnet-latest": { + "alias_of": "claude-sonnet-5-5" + }, "claude-sonnet-4-6": { "supports_web_search": true, "supports_anthropic_compaction": true, @@ -17024,6 +17081,9 @@ "supports_native_structured_output": true, "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, + "claude-fable-latest": { + "alias_of": "claude-fable-5-1" + }, "claude-opus-5-5": { "supports_web_search": true, "supports_anthropic_compaction": true, @@ -17069,6 +17129,9 @@ "prompt_cache_min_tokens": 512, "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, + "claude-opus-latest": { + "alias_of": "claude-opus-5-5" + }, "claude-opus-5": { "supports_web_search": true, "supports_anthropic_compaction": true, @@ -18868,6 +18931,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "dashscope/qwen-max-latest": { + "alias_of": "dashscope/qwen3.8-max" + }, "dashscope/qwen3.8-flash": { "cache_creation_input_token_cost": 2e-07, "cache_read_input_token_cost": 1.6e-08, @@ -18887,6 +18953,9 @@ "supports_vision": true, "supports_web_search": true }, + "dashscope/qwen-flash-latest": { + "alias_of": "dashscope/qwen3.8-flash" + }, "dashscope/qwen3.8-omni-flash": { "cache_read_input_token_cost": 1.6e-08, "input_cost_per_token": 1.5e-07, @@ -21767,6 +21836,9 @@ "supports_vision": true, "thinking_always_on": true }, + "databricks/claude-fable-latest": { + "alias_of": "databricks/databricks-claude-fable-5-1" + }, "databricks/databricks-claude-haiku-4-5": { "cache_creation_input_token_cost": 1.24999e-06, "cache_creation_input_token_cost_above_1hr": 1.99997e-06, @@ -22007,6 +22079,9 @@ "supports_xhigh_reasoning_effort": true, "thinking_always_on": true }, + "databricks/claude-opus-latest": { + "alias_of": "databricks/databricks-claude-opus-5-5" + }, "databricks/databricks-claude-sonnet-4": { "cache_creation_input_token_cost": 3.74997e-06, "cache_read_input_token_cost": 3.0002e-07, @@ -22724,6 +22799,9 @@ "cache_read_input_token_cost_priority": 4.55e-07, "cache_creation_input_token_cost_priority": 2.45e-06 }, + "databricks/glm-latest": { + "alias_of": "databricks/databricks-glm-5-3" + }, "databricks/databricks-glm-5-3-flash": { "cache_creation_input_token_cost": 1.5001e-07, "cache_read_input_token_cost": 3.003e-08, @@ -22757,6 +22835,9 @@ "cache_read_input_token_cost_priority": 5.25e-08, "cache_creation_input_token_cost_priority": 2.625e-07 }, + "databricks/glm-flash-latest": { + "alias_of": "databricks/databricks-glm-5-3-flash" + }, "databricks/databricks-gpt-5": { "cache_creation_input_token_cost": 1.24999e-06, "cache_read_input_token_cost": 1.2502e-07, @@ -22919,6 +23000,9 @@ "cache_read_input_token_cost_priority": 1.5001e-07, "cache_creation_input_token_cost_priority": 1.50003e-06 }, + "databricks/gpt-mini-latest": { + "alias_of": "databricks/databricks-gpt-5-4-mini" + }, "databricks/databricks-gpt-5-4-nano": { "cache_creation_input_token_cost": 1.9999e-07, "cache_read_input_token_cost": 2.002e-08, @@ -23315,6 +23399,9 @@ "cache_read_input_token_cost_priority": 5.25e-07, "cache_creation_input_token_cost_priority": 5.25e-06 }, + "databricks/kimi-latest": { + "alias_of": "databricks/databricks-kimi-k3" + }, "databricks/system.ai.kimi-k3": { "litellm_provider": "databricks", "mode": "chat", @@ -28843,6 +28930,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "fireworks_ai/minimax-latest": { + "alias_of": "fireworks_ai/accounts/fireworks/models/minimax-m3" + }, "fireworks_ai/accounts/fireworks/models/mixtral-8x22b-instruct-hf": { "input_cost_per_token": 1.2e-06, "litellm_provider": "fireworks_ai", @@ -33789,6 +33879,9 @@ "supports_vision": true, "thinking_always_on": true }, + "github_copilot/claude-fable-latest": { + "alias_of": "github_copilot/claude-fable-5.1" + }, "github_copilot/claude-haiku-4.5": { "cache_creation_input_token_cost": 1.25e-06, "cache_read_input_token_cost": 1e-07, @@ -33832,6 +33925,9 @@ "supports_vision": true, "thinking_always_on": false }, + "github_copilot/claude-haiku-latest": { + "alias_of": "github_copilot/claude-haiku-5.5" + }, "github_copilot/claude-opus-4.8": { "cache_creation_input_token_cost": 6.25e-06, "cache_read_input_token_cost": 5e-07, @@ -33920,6 +34016,9 @@ "supports_vision": true, "thinking_always_on": true }, + "github_copilot/claude-opus-latest": { + "alias_of": "github_copilot/claude-opus-5.5" + }, "github_copilot/claude-sonnet-5": { "cache_creation_input_token_cost": 2.5e-06, "cache_read_input_token_cost": 2e-07, @@ -33963,6 +34062,9 @@ "supports_vision": true, "thinking_always_on": true }, + "github_copilot/claude-sonnet-latest": { + "alias_of": "github_copilot/claude-sonnet-5.5" + }, "github_copilot/gemini-3.7-flash": { "cache_read_input_token_cost": 7.5e-08, "deprecation_date": "2026-10-19", @@ -34308,6 +34410,9 @@ "supports_response_schema": true, "supports_vision": true }, + "github_copilot/gpt-astra-latest": { + "alias_of": "github_copilot/gpt-6-astra" + }, "github_copilot/gpt-6-luna": { "cache_creation_input_token_cost": 1.25e-07, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-07, @@ -34331,6 +34436,9 @@ "supports_response_schema": true, "supports_vision": true }, + "github_copilot/gpt-luna-latest": { + "alias_of": "github_copilot/gpt-6-luna" + }, "github_copilot/gpt-6-sol": { "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, @@ -34377,6 +34485,9 @@ "supports_response_schema": true, "supports_vision": true }, + "github_copilot/gpt-sol-latest": { + "alias_of": "github_copilot/gpt-6.1-sol" + }, "github_copilot/kimi-k3": { "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, @@ -34394,6 +34505,9 @@ "supports_response_schema": true, "supports_vision": true }, + "github_copilot/kimi-latest": { + "alias_of": "github_copilot/kimi-k3" + }, "github_copilot/mai-code-1.1-flash": { "cache_read_input_token_cost": 2e-08, "input_cost_per_token": 2e-07, @@ -37687,6 +37801,9 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "gpt-astra-latest": { + "alias_of": "gpt-6-astra" + }, "gpt-6-sol": { "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_12hr": 5e-06, @@ -37884,6 +38001,9 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "gpt-luna-latest": { + "alias_of": "gpt-6-luna" + }, "gpt-5.6": { "cache_creation_input_token_cost": 5e-06, "cache_creation_input_token_cost_above_12hr": 1e-05, @@ -39067,6 +39187,9 @@ "supports_xhigh_reasoning_effort": true, "supports_minimal_reasoning_effort": false }, + "gpt-mini-latest": { + "alias_of": "gpt-5.4-mini" + }, "gpt-5.4-mini-2026-03-17": { "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_batches": 3.75e-08, @@ -43025,6 +43148,9 @@ "max_input_tokens": 1000000, "max_output_tokens": 128000 }, + "minimax/minimax-latest": { + "alias_of": "minimax/MiniMax-M3" + }, "minimax/MiniMax-H3": { "litellm_provider": "minimax", "mode": "video_generation", @@ -44535,6 +44661,9 @@ "supports_video_input": true, "supports_vision": true }, + "moonshot/kimi-latest": { + "alias_of": "moonshot/kimi-k3" + }, "moonshot/moonshot-v1-128k": { "deprecation_date": "2026-08-31", "input_cost_per_token": 2e-06, @@ -50136,6 +50265,9 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "together_ai/deepseek-pro-latest": { + "alias_of": "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813" + }, "together_ai/meta-models/Muse-Glimmer-30B": { "cache_read_input_token_cost": 4e-08, "input_cost_per_token": 3.5e-07, @@ -50182,6 +50314,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "together_ai/kimi-latest": { + "alias_of": "together_ai/moonshotai/Kimi-K3" + }, "together_ai/thinkingmachines/Inkling": { "cache_read_input_token_cost": 1.7e-07, "deprecation_date": "2026-11-06", @@ -50232,6 +50367,9 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "together_ai/glm-latest": { + "alias_of": "together_ai/zai-org/GLM-5.3" + }, "together_ai/zai-org/GLM-5.3-Flash": { "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 1.5e-07, @@ -50249,6 +50387,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "together_ai/glm-flash-latest": { + "alias_of": "together_ai/zai-org/GLM-5.3-Flash" + }, "tts-1": { "deprecation_date": "2027-01-06", "deprecation_source": "https://developers.openai.com/api/docs/deprecations", @@ -53019,6 +53160,9 @@ "output_cost_per_token_batches": 2.5e-05, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, + "vertex_ai/claude-fable-latest": { + "alias_of": "vertex_ai/claude-fable-5-1" + }, "vertex_ai/claude-fable-5@default": { "deprecation_date": "2027-06-08", "regional_endpoint_uplift_multiplier": 1.1, @@ -53217,6 +53361,9 @@ "prompt_cache_min_tokens": 512, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, + "vertex_ai/claude-opus-latest": { + "alias_of": "vertex_ai/claude-opus-5-5" + }, "vertex_ai/claude-opus-5-5@default": { "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, @@ -55765,6 +55912,9 @@ "supports_vision": true, "supports_web_search": true }, + "xai/grok-latest": { + "alias_of": "xai/grok-4.7" + }, "zai.glm-4.7": { "deprecation_date": "2027-03-30", "deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html", @@ -55866,6 +56016,9 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "zai/glm-latest": { + "alias_of": "zai/glm-5.3" + }, "zai/glm-5.3-flash": { "cache_creation_input_token_cost": 0, "cache_read_input_token_cost": 3e-08, @@ -55883,6 +56036,9 @@ "source": "https://docs.z.ai/guides/overview/pricing", "supports_vision": true }, + "zai/glm-flash-latest": { + "alias_of": "zai/glm-5.3-flash" + }, "zai/glm-5v-turbo": { "cache_read_input_token_cost": 2.4e-07, "input_cost_per_token": 1.2e-06, @@ -61519,6 +61675,9 @@ "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html", "supports_web_search": true }, + "bedrock_mantle/gpt-astra-latest": { + "alias_of": "bedrock_mantle/openai.gpt-6-astra" + }, "bedrock_mantle/openai.gpt-6-sol": { "input_cost_per_token": 2.2e-06, "input_cost_per_token_above_272k_tokens": 4.4e-06, @@ -61599,6 +61758,9 @@ "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-cards-openai.html", "supports_web_search": true }, + "bedrock_mantle/gpt-luna-latest": { + "alias_of": "bedrock_mantle/openai.gpt-6-luna" + }, "us.openai.gpt-6-astra": { "supports_bedrock_runtime_chat_completions_response_format": true, "input_cost_per_token": 1.1e-05, @@ -61767,6 +61929,9 @@ "/v1/responses" ] }, + "bedrock/gpt-astra-latest": { + "alias_of": "global.openai.gpt-6-astra" + }, "openai.gpt-6-sol": { "input_cost_per_token": 2e-06, "input_cost_per_token_above_272k_tokens": 4e-06, @@ -61909,6 +62074,9 @@ "/v1/responses" ] }, + "bedrock/gpt-luna-latest": { + "alias_of": "global.openai.gpt-6-luna" + }, "bedrock_mantle/openai.gpt-5.5": { "input_cost_per_token": 5.5e-06, "input_cost_per_token_above_272k_tokens": 1.1e-05, @@ -62186,6 +62354,9 @@ "supports_forced_tool_use": false, "token_count_model": {"model":"anthropic.claude-opus-5-5","litellm_provider":"bedrock_mantle","region":"us-east-1"} }, + "bedrock_mantle/claude-opus-latest": { + "alias_of": "bedrock_mantle/anthropic.claude-opus-5-5" + }, "bedrock_mantle/anthropic.claude-sonnet-5-5": { "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 2.75e-06, @@ -62226,6 +62397,9 @@ "prompt_cache_min_tokens": 512, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, + "bedrock_mantle/claude-sonnet-latest": { + "alias_of": "bedrock_mantle/anthropic.claude-sonnet-5-5" + }, "us.xai.grok-4.6": { "supports_regex_lookaround": false, "input_cost_per_token": 2.2e-06, @@ -63796,6 +63970,9 @@ "token_count_framing": 23, "token_count_tool_framing": 12 }, + "deepseek/deepseek-flash-latest": { + "alias_of": "deepseek/deepseek-v4-flash" + }, "deepseek/deepseek-v4-flash-vision-exp": { "cache_creation_input_token_cost": 0.0, "cache_read_input_token_cost": 6e-09, @@ -63904,6 +64081,9 @@ "token_count_framing": 76, "token_count_tool_framing": 75 }, + "deepseek/deepseek-pro-latest": { + "alias_of": "deepseek/deepseek-v4-pro" + }, "tencent/deepseek-v4-pro": { "cache_creation_input_token_cost": 0.0, "cache_read_input_token_cost": 1.45e-07, @@ -65327,6 +65507,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "fireworks_ai/kimi-latest": { + "alias_of": "fireworks_ai/accounts/fireworks/models/kimi-k3" + }, "fireworks_ai/deepseek-v4-flash-0731": { "cache_read_input_token_cost": 7e-09, "cache_read_input_token_cost_priority": 8.75e-09, @@ -69000,6 +69183,9 @@ "output_cost_per_token": 4.7e-07, "source": "https://api.together.ai/v1/models" }, + "together_ai/qwen-flash-latest": { + "alias_of": "together_ai/Qwen/Qwen3.8-Flash" + }, "together_ai/moonshotai/Kimi-K2.5-fp4": { "input_cost_per_token": 5e-07, "output_cost_per_token": 2.8e-06, @@ -69216,6 +69402,9 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "azure_ai/glm-latest": { + "alias_of": "azure_ai/FW-GLM-5.3" + }, "azure_ai/FW-GLM-5.3-Flash": { "cache_read_input_token_cost": 3.8e-08, "input_cost_per_token": 1.88e-07, @@ -69231,6 +69420,9 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "azure_ai/glm-flash-latest": { + "alias_of": "azure_ai/FW-GLM-5.3-Flash" + }, "azure_ai/FW-GPT-OSS-120B": { "deprecation_date": "2027-07-01", "cache_read_input_token_cost": 8.2e-08, @@ -71236,6 +71428,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "baseten/glm-latest": { + "alias_of": "baseten/zai-org/GLM-5.3" + }, "baseten/zai-org/GLM-5.3-Fast": { "cache_read_input_token_cost": 2.1e-07, "input_cost_per_token": 2.1e-06, @@ -74425,6 +74620,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "bedrock/kimi-latest": { + "alias_of": "global.moonshotai.kimi-k3" + }, "us.moonshotai.kimi-k3": { "supports_regex_lookaround": false, "cache_creation_input_token_cost": 4.125e-06, @@ -74975,6 +75173,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "baseten/kimi-latest": { + "alias_of": "baseten/moonshotai/Kimi-K3" + }, "baseten/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B": { "cache_read_input_token_cost": 1.2e-07, "input_cost_per_token": 6e-07, @@ -75057,6 +75258,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "baseten/glm-flash-latest": { + "alias_of": "baseten/zai-org/GLM-5.3-Flash" + }, "baseten/deepseek-ai/DeepSeek-V4-Flash-0731": { "cache_read_input_token_cost": 2.8e-08, "input_cost_per_token": 1.3e-07, @@ -75073,6 +75277,9 @@ "supports_tool_choice": true, "supports_vision": false }, + "baseten/deepseek-flash-latest": { + "alias_of": "baseten/deepseek-ai/DeepSeek-V4-Flash-0731" + }, "baseten/deepseek-ai/DeepSeek-V4-Pro": { "deprecation_date": "2026-10-16", "cache_read_input_token_cost": 1.45e-07, @@ -75106,6 +75313,9 @@ "supports_tool_choice": true, "supports_vision": false }, + "baseten/deepseek-pro-latest": { + "alias_of": "baseten/deepseek-ai/DeepSeek-V4-Pro-0813" + }, "baseten/zai-org/GLM-5.2-Fast": { "cache_read_input_token_cost": 2.1e-07, "input_cost_per_token": 2.1e-06, @@ -76097,6 +76307,9 @@ "prompt_cache_min_tokens": 512, "source": "https://management.azure.com/subscriptions/c873328e-b572-4770-8dff-aaeb6f1f0e79/providers/Microsoft.CognitiveServices/locations/eastus2/models?api-version=2024-10-01" }, + "azure_ai/claude-sonnet-latest": { + "alias_of": "azure_ai/claude-sonnet-5-5" + }, "bedrock/us-gov-east-1/anthropic.claude-sonnet-5-5": { "bedrock_converse_supports_strict_tools": false, "bedrock_output_config_effort_ceiling": "xhigh", @@ -76261,6 +76474,9 @@ "prompt_cache_min_tokens": 512, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, + "bedrock/claude-sonnet-latest": { + "alias_of": "global.anthropic.claude-sonnet-5-5" + }, "jp.anthropic.claude-sonnet-5-5": { "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 2.75e-06, @@ -76520,6 +76736,9 @@ "prompt_cache_min_tokens": 512, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, + "vertex_ai/claude-sonnet-latest": { + "alias_of": "vertex_ai/claude-sonnet-5-5" + }, "vertex_ai/claude-sonnet-5-5@default": { "regional_endpoint_uplift_multiplier": 1.1, "supports_mid_conversation_system": true, @@ -76632,6 +76851,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "bedrock/grok-latest": { + "alias_of": "global.xai.grok-4.7" + }, "us.xai.grok-4.7": { "supports_regex_lookaround": false, "supported_endpoints": ["/v1/chat/completions"], @@ -76804,6 +77026,9 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "gpt-sol-latest": { + "alias_of": "gpt-6.1-sol" + }, "global.openai.gpt-6.1-sol": { "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, @@ -76850,6 +77075,9 @@ "supports_sampling_params": false, "supports_xhigh_reasoning_effort": true }, + "bedrock/gpt-sol-latest": { + "alias_of": "global.openai.gpt-6.1-sol" + }, "bedrock_mantle/openai.gpt-6.1-sol": { "cache_creation_input_token_cost": 2.75e-06, "cache_creation_input_token_cost_above_272k_tokens": 5.5e-06, @@ -76898,6 +77126,9 @@ "supports_xhigh_reasoning_effort": true, "use_openai_responses_path": true }, + "bedrock_mantle/gpt-sol-latest": { + "alias_of": "bedrock_mantle/openai.gpt-6.1-sol" + }, "us.openai.gpt-6.1-sol": { "cache_creation_input_token_cost": 2.75e-06, "cache_creation_input_token_cost_above_272k_tokens": 5.5e-06, @@ -76989,6 +77220,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "vertex_ai/grok-latest": { + "alias_of": "vertex_ai/xai/grok-4.7" + }, "azure_ai/kimi-k2-thinking": { "deprecation_date": "2026-03-29", "deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models", @@ -77046,6 +77280,9 @@ "supports_tool_choice": true, "supports_vision": false }, + "bedrock/glm-latest": { + "alias_of": "global.zai.glm-5.3" + }, "us.zai.glm-5.3": { "cache_creation_input_token_cost": 2.31e-06, "cache_creation_input_token_cost_batches": 1.155e-06, @@ -77269,6 +77506,9 @@ "supports_vision": true, "supports_web_search": true }, + "azure_ai/grok-latest": { + "alias_of": "azure_ai/grok-4.7" + }, "in.moonshotai.kimi-k3": { "cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost_flex": 2.0625e-06, @@ -77355,6 +77595,9 @@ "cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1e-06, "cache_read_input_token_cost_above_100k_tokens": 5e-08 }, + "claude-haiku-latest": { + "alias_of": "claude-haiku-5-5" + }, "bedrock_mantle/anthropic.claude-haiku-5-5": { "input_cost_per_token_above_100k_tokens": 5.5e-07, "output_cost_per_token_above_100k_tokens": 2.75e-06, @@ -77401,6 +77644,9 @@ "prompt_cache_min_tokens": 512, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-haiku-5-5.html" }, + "bedrock_mantle/claude-haiku-latest": { + "alias_of": "bedrock_mantle/anthropic.claude-haiku-5-5" + }, "bedrock_mantle/us-gov-west-1/anthropic.claude-haiku-5-5": { "input_cost_per_token_above_100k_tokens": 6e-07, "output_cost_per_token_above_100k_tokens": 3e-06, @@ -77632,6 +77878,9 @@ "prompt_cache_min_tokens": 512, "source": "https://platform.claude.com/docs/en/models/haiku-5-5/migration-guide" }, + "azure_ai/claude-haiku-latest": { + "alias_of": "azure_ai/claude-haiku-5-5" + }, "bedrock/us-gov-east-1/anthropic.claude-haiku-5-5": { "input_cost_per_token_above_100k_tokens": 6e-07, "output_cost_per_token_above_100k_tokens": 3e-06, @@ -77822,6 +78071,9 @@ "prompt_cache_min_tokens": 512, "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json" }, + "bedrock/claude-haiku-latest": { + "alias_of": "global.anthropic.claude-haiku-5-5" + }, "jp.anthropic.claude-haiku-5-5": { "input_cost_per_token_above_100k_tokens": 5.5e-07, "output_cost_per_token_above_100k_tokens": 2.75e-06, @@ -78029,6 +78281,9 @@ "prompt_cache_min_tokens": 512, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing" }, + "vertex_ai/claude-haiku-latest": { + "alias_of": "vertex_ai/claude-haiku-5-5" + }, "vertex_ai/claude-haiku-5-5@default": { "input_cost_per_token_above_100k_tokens": 5e-07, "output_cost_per_token_above_100k_tokens": 2.5e-06, @@ -78933,6 +79188,9 @@ "supports_response_schema": true, "supports_vision": true }, + "github_copilot/grok-latest": { + "alias_of": "github_copilot/grok-4.7" + }, "together_ai/deepseek-ai/DeepSeek-V4.1-Flash-1017": { "cache_read_input_token_cost": 6e-09, "input_cost_per_token": 3e-07, @@ -80356,6 +80614,9 @@ "supports_web_search": true, "supports_xhigh_reasoning_effort": true }, + "gpt-terra-latest": { + "alias_of": "gpt-6.1-terra" + }, "chatgpt/gpt-6.1-terra": { "litellm_provider": "chatgpt", "max_input_tokens": 922000, @@ -80414,6 +80675,9 @@ "supports_vision": true, "supports_xhigh_reasoning_effort": true }, + "bedrock/gpt-terra-latest": { + "alias_of": "global.openai.gpt-6.1-terra" + }, "us.openai.gpt-6.1-terra": { "cache_creation_input_token_cost": 1.375e-06, "cache_creation_input_token_cost_above_272k_tokens": 2.75e-06, @@ -80493,6 +80757,9 @@ "supports_response_schema": true, "supports_vision": true }, + "github_copilot/gpt-terra-latest": { + "alias_of": "github_copilot/gpt-6.1-terra" + }, "refuseless/glm-5.3-abliterated": { "cache_read_input_token_cost": 1.5e-07, "input_cost_per_token": 1e-06, @@ -80647,6 +80914,9 @@ "supports_xhigh_reasoning_effort": true, "use_openai_responses_path": true }, + "bedrock_mantle/gpt-terra-latest": { + "alias_of": "bedrock_mantle/openai.gpt-6.1-terra" + }, "clf_ai_gateway/glm-5.3": { "metadata": { "comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends." @@ -82196,6 +82466,9 @@ "supports_vision": true, "thinking_always_on": false }, + "databricks/claude-haiku-latest": { + "alias_of": "databricks/databricks-claude-haiku-5-5" + }, "databricks/databricks-claude-sonnet-5-5": { "cache_creation_input_token_cost": 2.49998e-06, "cache_creation_input_token_cost_above_1hr": 4.00001e-06, @@ -82225,6 +82498,9 @@ "supports_vision": false, "thinking_always_on": true }, + "databricks/claude-sonnet-latest": { + "alias_of": "databricks/databricks-claude-sonnet-5-5" + }, "databricks/databricks-deepseek-v4-1-flash": { "cache_creation_input_token_cost": 3.0002e-07, "cache_read_input_token_cost": 3.003e-08, @@ -82457,6 +82733,9 @@ "cache_read_input_token_cost_above_272k_tokens_priority": 3.9998e-07, "cache_creation_input_token_cost_above_272k_tokens_priority": 9.99999e-06 }, + "databricks/gpt-sol-latest": { + "alias_of": "databricks/databricks-gpt-6-1-sol" + }, "databricks/databricks-gpt-6-1-terra": { "cache_creation_input_token_cost": 1.24999e-06, "cache_creation_input_token_cost_above_272k_tokens": 2.49998e-06, @@ -82497,6 +82776,9 @@ "cache_read_input_token_cost_above_272k_tokens_priority": 1.9999e-07, "cache_creation_input_token_cost_above_272k_tokens_priority": 5.00003e-06 }, + "databricks/gpt-terra-latest": { + "alias_of": "databricks/databricks-gpt-6-1-terra" + }, "databricks/databricks-gpt-6-astra": { "cache_creation_input_token_cost": 1.249997e-05, "cache_creation_input_token_cost_above_272k_tokens": 2.5e-05, @@ -82537,6 +82819,9 @@ "cache_read_input_token_cost_above_272k_tokens_priority": 4.00001e-06, "cache_creation_input_token_cost_above_272k_tokens_priority": 5.000002e-05 }, + "databricks/gpt-astra-latest": { + "alias_of": "databricks/databricks-gpt-6-astra" + }, "databricks/databricks-gpt-6-luna": { "cache_creation_input_token_cost": 1.2502e-07, "cache_creation_input_token_cost_above_272k_tokens": 2.4997e-07, @@ -82577,6 +82862,9 @@ "cache_read_input_token_cost_above_272k_tokens_priority": 3.997e-08, "cache_creation_input_token_cost_above_272k_tokens_priority": 5.0001e-07 }, + "databricks/gpt-luna-latest": { + "alias_of": "databricks/databricks-gpt-6-luna" + }, "databricks/databricks-gpt-6-sol": { "cache_creation_input_token_cost": 2.49998e-06, "cache_creation_input_token_cost_above_272k_tokens": 5.00003e-06, @@ -82707,6 +82995,9 @@ "supports_tool_choice": true, "supports_vision": true }, + "databricks/grok-latest": { + "alias_of": "databricks/databricks-grok-4-7" + }, "databricks/databricks-kimi-k2-7": { "cache_creation_input_token_cost": 9.4997e-07, "cache_read_input_token_cost": 1.8998e-07, @@ -86639,6 +86930,9 @@ "supports_vision": true, "use_openai_responses_path": true }, + "bedrock_mantle/grok-latest": { + "alias_of": "bedrock_mantle/xai.grok-4.7" + }, "together_ai/ricursive-ai/Ripple-65B-Preview": { "cache_read_input_token_cost": 2e-08, "input_cost_per_token": 2e-07, diff --git a/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py b/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py index de252b17f00..b4e3ef2c566 100644 --- a/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py +++ b/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py @@ -30,9 +30,9 @@ CUSTOM_PROVIDER: Final = "test-onprem-llm" @pytest.mark.parametrize( ("model", "provider"), - (("openai/gpt-luna-latest", None), ("gpt-luna-latest", "openai")), + (("openai/gpt-luna-latest", None), ("gpt-luna-latest", "openai"), ("gpt-luna-latest", None)), ) -def test_latest_alias_resolves_to_the_bundled_target(model: str, provider: str | None) -> None: +def test_latest_alias_resolves_to_the_cost_map_target(model: str, provider: str | None) -> None: target: Final = get_model_latest_aliases()["openai"]["gpt-luna-latest"] resolved, resolved_provider, _, api_base = get_llm_provider( model, custom_llm_provider=provider, api_base="https://api.openai.com/v1" @@ -42,7 +42,12 @@ def test_latest_alias_resolves_to_the_bundled_target(model: str, provider: str | @pytest.mark.parametrize( "model", - ("gemini/gemini-pro-latest", "gemini/gemini-flash-latest", "dashscope/qwen-plus-latest", "mistral/mistral-large-latest"), + ( + "gemini/gemini-pro-latest", + "gemini/gemini-flash-latest", + "dashscope/qwen-plus-latest", + "mistral/mistral-large-latest", + ), ) def test_provider_native_latest_alias_passes_through(model: str) -> None: provider, name = model.split("/", 1) @@ -153,6 +158,7 @@ def _vcr_outcome_gate(request, vcr): yield record_vcr_outcome(request, vcr) + @pytest.fixture(scope="function") def isolate_litellm_state(): """ @@ -205,6 +211,7 @@ def isolate_litellm_state(): setattr(litellm, attr, original_value) _invalidate_model_cost_lowercase_map() + _SCALAR_DEFAULTS = { "num_retries": getattr(litellm, "num_retries", None), "num_retries_per_request": getattr(litellm, "num_retries_per_request", None), @@ -225,6 +232,7 @@ _SCALAR_DEFAULTS = { "api_key": getattr(litellm, "api_key", None), } + @pytest.fixture(scope="module") def setup_and_teardown(): """ @@ -247,12 +255,14 @@ def setup_and_teardown(): litellm.in_memory_llm_clients_cache.flush_cache() yield + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_llm_provider(): _, response, _, _ = litellm.get_llm_provider(model="anthropic.claude-v2:1") assert response == "bedrock" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_llm_provider_fireworks(): # tests finetuned fireworks models - https://github.com/BerriAI/litellm/issues/4923 model, custom_llm_provider, _, _ = litellm.get_llm_provider(model="fireworks_ai/accounts/my-test-1234") @@ -260,17 +270,20 @@ def test_get_llm_provider_fireworks(): # tests finetuned fireworks models - htt assert custom_llm_provider == "fireworks_ai" assert model == "accounts/my-test-1234" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_llm_provider_catch_all(): _, response, _, _ = litellm.get_llm_provider(model="*") assert response == "openai" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_llm_provider_gpt_instruct(): _, response, _, _ = litellm.get_llm_provider(model="gpt-3.5-turbo-instruct-0914") assert response == "text-completion-openai" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_llm_provider_mistral_custom_api_base(): model, custom_llm_provider, dynamic_api_key, api_base = litellm.get_llm_provider( @@ -281,6 +294,7 @@ def test_get_llm_provider_mistral_custom_api_base(): assert model == "mistral-large-fr" assert api_base == "https://mistral-large-fr-ishaan.francecentral.inference.ai.azure.com/v1" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_llm_provider_deepseek_custom_api_base(): os.environ["DEEPSEEK_API_BASE"] = "MY-FAKE-BASE" @@ -293,6 +307,7 @@ def test_get_llm_provider_deepseek_custom_api_base(): os.environ.pop("DEEPSEEK_API_BASE") + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_llm_provider_vertex_ai_image_models(monkeypatch): monkeypatch.setattr(litellm, "vertex_ai_image_models", set()) @@ -310,6 +325,7 @@ def test_get_llm_provider_vertex_ai_image_models(monkeypatch): ) assert custom_llm_provider == "vertex_ai" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_llm_provider_ai21_chat(): model, custom_llm_provider, dynamic_api_key, api_base = litellm.get_llm_provider( @@ -319,6 +335,7 @@ def test_get_llm_provider_ai21_chat(): assert model == "jamba-1.5-large" assert api_base == "https://api.ai21.com/studio/v1" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_llm_provider_ai21_chat_test2(): """ @@ -335,6 +352,7 @@ def test_get_llm_provider_ai21_chat_test2(): assert model == "jamba-1.5-large" assert api_base == "https://api.ai21.com/studio/v1" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_llm_provider_cohere_chat_test2(): """ @@ -350,6 +368,7 @@ def test_get_llm_provider_cohere_chat_test2(): assert custom_llm_provider == "cohere_chat" assert model == "command-r-plus-08-2024" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_llm_provider_azure_o1(): @@ -359,6 +378,7 @@ def test_get_llm_provider_azure_o1(): assert custom_llm_provider == "azure" assert model == "o1-mini" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_hosted_vllm_default_api_key(): from litellm.litellm_core_utils.get_llm_provider_logic import ( @@ -373,6 +393,7 @@ def test_hosted_vllm_default_api_key(): ) assert dynamic_api_key == "fake-api-key" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_llm_provider_jina_ai(): model, custom_llm_provider, dynamic_api_key, api_base = litellm.get_llm_provider( @@ -382,6 +403,7 @@ def test_get_llm_provider_jina_ai(): assert api_base == "https://api.jina.ai/v1" assert model == "jina-embeddings-v3" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_llm_provider_hosted_vllm(): model, custom_llm_provider, dynamic_api_key, api_base = litellm.get_llm_provider( @@ -391,6 +413,7 @@ def test_get_llm_provider_hosted_vllm(): assert model == "llama-3.1-70b-instruct" assert dynamic_api_key == "fake-api-key" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_llm_provider_llamafile(): model, custom_llm_provider, dynamic_api_key, api_base = litellm.get_llm_provider( @@ -401,6 +424,7 @@ def test_get_llm_provider_llamafile(): assert dynamic_api_key == "fake-api-key" assert api_base == "http://127.0.0.1:8080/v1" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_llm_provider_watson_text(): model, custom_llm_provider, dynamic_api_key, api_base = litellm.get_llm_provider( @@ -409,6 +433,7 @@ def test_get_llm_provider_watson_text(): assert custom_llm_provider == "watsonx_text" assert model == "watson-text-to-speech" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_azure_global_standard_get_llm_provider(): model, custom_llm_provider, dynamic_api_key, api_base = litellm.get_llm_provider( @@ -418,6 +443,7 @@ def test_azure_global_standard_get_llm_provider(): ) assert custom_llm_provider == "azure_ai" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_nova_bedrock_converse(): model, custom_llm_provider, dynamic_api_key, api_base = litellm.get_llm_provider( @@ -426,6 +452,7 @@ def test_nova_bedrock_converse(): assert custom_llm_provider == "bedrock" assert model == "amazon.nova-micro-v1:0" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_bedrock_invoke_anthropic(): model, custom_llm_provider, dynamic_api_key, api_base = litellm.get_llm_provider( @@ -434,6 +461,7 @@ def test_bedrock_invoke_anthropic(): assert custom_llm_provider == "bedrock" assert model == "invoke/anthropic.claude-haiku-4-5-20251001-v1:0" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") @pytest.mark.parametrize("model", ["xai/grok-2-vision-latest", "grok-2-vision-latest"]) def test_xai_api_base(model): @@ -450,6 +478,7 @@ def test_xai_api_base(model): assert api_base == "https://api.x.ai/v1" assert dynamic_api_key == "xai-my-specialkey" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_litellm_proxy_custom_llm_provider(): """ @@ -479,6 +508,7 @@ def test_get_litellm_proxy_custom_llm_provider(): assert key == expected_api_key assert base == expected_api_base + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_litellm_proxy_with_args_override_env_vars(): """ @@ -510,6 +540,7 @@ def test_get_litellm_proxy_with_args_override_env_vars(): assert key == arg_api_key assert base == arg_api_base + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_litellm_proxy_model_prefix_stripping(): """ @@ -540,6 +571,7 @@ def test_get_litellm_proxy_model_prefix_stripping(): assert key == expected_api_key assert base == expected_api_base + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_llm_provider_LITELLM_PROXY_ALWAYS_true(): """ @@ -568,6 +600,7 @@ def test_get_llm_provider_LITELLM_PROXY_ALWAYS_true(): assert key == proxy_api_key assert base == proxy_api_base + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_llm_provider_LITELLM_PROXY_ALWAYS_true_model_prefix(): """ @@ -594,6 +627,7 @@ def test_get_llm_provider_LITELLM_PROXY_ALWAYS_true_model_prefix(): assert key == proxy_api_key assert base == proxy_api_base + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_llm_provider_use_proxy_arg_true(): """ @@ -623,6 +657,7 @@ def test_get_llm_provider_use_proxy_arg_true(): assert key == proxy_api_key assert base == proxy_api_base + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_get_llm_provider_use_proxy_arg_true_with_direct_args(): """ @@ -659,6 +694,7 @@ def test_get_llm_provider_use_proxy_arg_true_with_direct_args(): assert key == arg_api_key # Should use the argument key assert base == arg_api_base # Should use the argument base + @pytest.fixture def shipped_generalizations(): """Install the rules shipped in the bundled backup, then restore. @@ -682,6 +718,7 @@ def shipped_generalizations(): finally: set_fallback_generalizations(previous) + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") class TestClaudeModelPatternMatching: """ @@ -749,9 +786,7 @@ class TestClaudeModelPatternMatching: @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") class TestTypesafeJevPatternMatching: - def test_every_priced_jev_id_routes_to_typesafe( - self, shipped_generalizations: None - ) -> None: + def test_every_priced_jev_id_routes_to_typesafe(self, shipped_generalizations: None) -> None: from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap priced_ids: Final = tuple( @@ -760,9 +795,9 @@ class TestTypesafeJevPatternMatching: if key.startswith("typesafe/") ) assert "jev-latest" in priced_ids - assert { - model: litellm.get_llm_provider(model=model)[1] for model in priced_ids - } == dict.fromkeys(priced_ids, "typesafe") + assert {model: litellm.get_llm_provider(model=model)[1] for model in priced_ids} == dict.fromkeys( + priced_ids, "typesafe" + ) @pytest.mark.parametrize( "model", @@ -837,7 +872,9 @@ def test_metadata_callers_never_log_in_for_a_declared_chatgpt_deployment( assert get_api_base("chatgpt/gpt-5.4", {"model": "chatgpt/gpt-5.4"}) == "https://chatgpt.com/backend-api/codex" identity = _resolve_model("chatgpt/gpt-5.4", None) assert identity is not None and identity.provider == "chatgpt" - group = Router(model_list=[{"model_name": "codex", "litellm_params": {"model": "chatgpt/gpt-5.4"}}]).registry().group( - "codex" + group = ( + Router(model_list=[{"model_name": "codex", "litellm_params": {"model": "chatgpt/gpt-5.4"}}]) + .registry() + .group("codex") ) assert group is not None and group.providers == ["chatgpt"] diff --git a/tests/unit/litellm_core_utils/test_model_latest_aliases.py b/tests/unit/litellm_core_utils/test_model_latest_aliases.py index c052a24b4dc..c6def915640 100644 --- a/tests/unit/litellm_core_utils/test_model_latest_aliases.py +++ b/tests/unit/litellm_core_utils/test_model_latest_aliases.py @@ -3,14 +3,17 @@ from typing import Final import pytest import litellm +from litellm.litellm_core_utils.get_llm_provider_logic import PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO +from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry from litellm.litellm_core_utils.model_latest_aliases import ( expand_model_latest_alias, get_model_latest_aliases, is_model_latest_alias, + resolve_model_latest_alias, ) -@pytest.mark.parametrize("model", ("", "gpt-luna-latest", "openai/unknown", "gemini/gemini-pro-latest")) +@pytest.mark.parametrize("model", ("", "kimi-latest", "openai/unknown", "gemini/gemini-pro-latest")) def test_non_alias_expansion_keeps_the_requested_name(model: str) -> None: assert expand_model_latest_alias(model) == (model,) @@ -21,15 +24,56 @@ def test_alias_expansion_keeps_requested_and_target_names() -> None: assert expand_model_latest_alias(f"{provider}/{alias}") == (f"{provider}/{alias}", f"{provider}/{target}") -def test_latest_alias_targets_have_chat_pricing_without_shadowing_native_models() -> None: - for provider, aliases in get_model_latest_aliases().items(): - for alias, target in aliases.items(): +def test_bare_alias_expands_to_its_cost_map_owner() -> None: + alias, target = next(iter(get_model_latest_aliases()["openai"].items())) + assert expand_model_latest_alias(alias) == (alias, target) + + +def test_latest_aliases_price_and_route_like_their_targets() -> None: + aliases: Final = get_model_latest_aliases() + assert aliases + for provider, provider_aliases in aliases.items(): + for alias, target in provider_aliases.items(): + prefixed: Final = f"{provider}/{alias}" + alias_key: Final = prefixed if prefixed in litellm.model_cost else alias target_key: Final = f"{provider}/{target}" if f"{provider}/{target}" in litellm.model_cost else target - entry: Final = litellm.model_cost[target_key] - assert (entry["litellm_provider"], entry["mode"]) == (provider, "chat") - assert f"{provider}/{alias}" not in litellm.model_cost - assert litellm.model_cost.get(alias, {}).get("litellm_provider") != provider - assert litellm.get_llm_provider(f"{provider}/{alias}")[:2] == (target, provider) + entry: Final = litellm.model_cost[alias_key] + assert entry["mode"] in ("chat", "responses") + assert entry == litellm.model_cost[target_key] + assert resolve_model_latest_alias(alias, provider) == target + if provider in PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO: + continue + assert litellm.get_llm_provider(prefixed)[:2] == (target, provider) + assert litellm.get_llm_provider(alias, custom_llm_provider=provider)[:2] == (target, provider) + + +@pytest.mark.parametrize( + ("model", "provider", "target_provider"), + ( + ("bedrock/claude-haiku-latest", "bedrock", "bedrock_converse"), + ("vertex_ai/claude-haiku-latest", "vertex_ai", "vertex_ai-anthropic_models"), + ("azure/gpt-mini-latest", "azure", "azure"), + ("fireworks_ai/kimi-latest", "fireworks_ai", "fireworks_ai"), + ), +) +def test_hosted_alias_is_sent_as_the_hosts_model_id(model: str, provider: str, target_provider: str) -> None: + entry: Final = litellm.model_cost[model] + assert isinstance(entry, AliasedCostEntry) + name: Final = model.partition("/")[2] + target: Final = entry.names[0].removeprefix(f"{provider}/") + assert target != name + assert entry["litellm_provider"] == target_provider + assert litellm.get_llm_provider(model)[:2] == (target, provider) + assert is_model_latest_alias(model) + assert resolve_model_latest_alias(name, provider) == target + assert get_model_latest_aliases()[provider][name] == target + + +def test_alias_resolution_is_scoped_to_the_owning_provider() -> None: + alias, target = next(iter(get_model_latest_aliases()["openai"].items())) + assert resolve_model_latest_alias(alias, "openai") == target + assert resolve_model_latest_alias(alias, "anthropic") == alias + assert resolve_model_latest_alias(target, "openai") == target @pytest.mark.parametrize( @@ -37,11 +81,14 @@ def test_latest_alias_targets_have_chat_pricing_without_shadowing_native_models( ( ("openai/gpt-luna-latest", None, True), ("gpt-luna-latest", "openai", True), - ("gpt-luna-latest", None, False), + ("gpt-luna-latest", None, True), + ("gpt-luna-latest", "anthropic", False), + ("moonshot/kimi-latest", None, True), + ("kimi-latest", None, False), ("gemini/gemini-pro-latest", None, False), ("", None, False), ("openai/unknown-model", None, False), ), ) -def test_latest_alias_detection_requires_an_explicit_provider(model: str, provider: str | None, expected: bool) -> None: +def test_latest_alias_detection_follows_the_cost_map_owner(model: str, provider: str | None, expected: bool) -> None: assert is_model_latest_alias(model, provider) is expected diff --git a/tests/unit/llms/databricks/test_databricks_cost_calculator.py b/tests/unit/llms/databricks/test_databricks_cost_calculator.py index c35aa6e4cf0..e9bc75119aa 100644 --- a/tests/unit/llms/databricks/test_databricks_cost_calculator.py +++ b/tests/unit/llms/databricks/test_databricks_cost_calculator.py @@ -6,6 +6,7 @@ from typing import Final import pytest import litellm +from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry from litellm.llms.databricks.cost_calculator import cost_per_token from litellm.types.utils import ModelInfo, Usage @@ -213,6 +214,7 @@ def test_every_model_without_published_cache_dbu_bills_cache_writes_at_its_own_i if model.startswith("databricks/") and info.get("input_cost_per_token") and model not in PUBLISHED_DBU_PER_MILLION + and not (isinstance(info, AliasedCostEntry) and info.names[0] != model) ] for model in without_published_rates: diff --git a/tests/unit/llms/moonshot/test_moonshot_chat_transformation.py b/tests/unit/llms/moonshot/test_moonshot_chat_transformation.py index b054dd84c22..2f58330e427 100644 --- a/tests/unit/llms/moonshot/test_moonshot_chat_transformation.py +++ b/tests/unit/llms/moonshot/test_moonshot_chat_transformation.py @@ -13,7 +13,7 @@ import pytest import litellm import litellm.utils -from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap +from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap, get_model_cost_map from litellm.litellm_core_utils.model_latest_aliases import get_model_latest_aliases from litellm.llms.moonshot.chat.transformation import MoonshotChatConfig @@ -715,7 +715,8 @@ class TestMoonshotReasoningEffort: @pytest.fixture(autouse=True) def force_local_model_cost(self, monkeypatch): - monkeypatch.setattr(litellm, "model_cost", GetModelCostMap.load_local_model_cost_map()) + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", get_model_cost_map(url=litellm.model_cost_map_url)) @pytest.mark.parametrize("model", ["kimi-k3", "kimi-k2.5", "kimi-k2.6"]) def test_reasoning_model_supports_reasoning_effort(self, model): diff --git a/tests/unit/rust_bridge/test_detectors.py b/tests/unit/rust_bridge/test_detectors.py index 3001d23e57e..dfa29e24b2e 100644 --- a/tests/unit/rust_bridge/test_detectors.py +++ b/tests/unit/rust_bridge/test_detectors.py @@ -51,8 +51,10 @@ def call(**kwargs: object) -> NativeCall: ("openai/gpt-luna-latest", None, True), ("gpt-luna-latest", "openai", True), ("gemini/gemini-pro-latest", None, False), - ("gpt-luna-latest", None, False), + ("gpt-luna-latest", None, True), ("anthropic/claude-opus-latest", None, True), + ("moonshot/kimi-latest", None, True), + ("kimi-latest", None, False), ("claude-opus-latest", "anthropic", True), ("", None, False), ), From 5bc25ff8ea5d169463e83b2b4efd17d44131bf78 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 7 Nov 2026 12:01:36 -0800 Subject: [PATCH 02/39] fix(cost-map): update Together Qwen3.7-Max pricing and add Rnj-1 Instruct (#50548) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 13 ++++++++++--- model_prices_and_context_window.json | 13 ++++++++++--- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b3412518cd7..6b704577303 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -50176,13 +50176,13 @@ "supports_reasoning": true }, "together_ai/Qwen/Qwen3.7-Max": { - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 1.5e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 2.5e-06, "litellm_provider": "together_ai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "mode": "chat", - "output_cost_per_token": 4.5e-06, + "output_cost_per_token": 7.5e-06, "source": "https://www.together.ai/pricing", "supports_prompt_caching": true }, @@ -71774,6 +71774,13 @@ "output_cost_per_token": 1.7e-06, "source": "https://api.together.ai/v1/models" }, + "together_ai/essentialai/rnj-1-instruct": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "together_ai", + "mode": "chat", + "output_cost_per_token": 1.5e-07, + "source": "https://www.together.ai/models/rnj-1-instruct" + }, "together_ai/google/gemma-4-31B-it": { "deprecation_date": "2026-09-15", "deprecation_source": "https://docs.together.ai/docs/deprecations", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b3412518cd7..6b704577303 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -50176,13 +50176,13 @@ "supports_reasoning": true }, "together_ai/Qwen/Qwen3.7-Max": { - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 1.5e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 2.5e-06, "litellm_provider": "together_ai", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "mode": "chat", - "output_cost_per_token": 4.5e-06, + "output_cost_per_token": 7.5e-06, "source": "https://www.together.ai/pricing", "supports_prompt_caching": true }, @@ -71774,6 +71774,13 @@ "output_cost_per_token": 1.7e-06, "source": "https://api.together.ai/v1/models" }, + "together_ai/essentialai/rnj-1-instruct": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "together_ai", + "mode": "chat", + "output_cost_per_token": 1.5e-07, + "source": "https://www.together.ai/models/rnj-1-instruct" + }, "together_ai/google/gemma-4-31B-it": { "deprecation_date": "2026-09-15", "deprecation_source": "https://docs.together.ai/docs/deprecations", From 8d9ddb349e5e72ee12f0ef2ff3daad22e4d6c9ab Mon Sep 17 00:00:00 2001 From: Yogeshwaran Ravichandran <96047771+yogeshwaran10@users.noreply.github.com> Date: Sun, 8 Nov 2026 01:59:12 +0530 Subject: [PATCH 03/39] feat(guardrails): add prebuilt credential patterns for OpenAI, Anthropic, and Gemini (#50268) * feat(guardrails): add prebuilt credential patterns for JWT, and OpenAI, Anthropic, Gemini API keys * fix(guardrails): prevent quadratic backtracking in jwt_token pattern * test(guardrails): move credential pattern tests into test_patterns.py * feat(guardrails): remove jwt_token to keep credential patterns strictly bounded * fix(guardrails): mask whole provider keys and keep openai_api_key to OpenAI's key shapes End each pattern so a key ending in "-" is masked in full, require OpenAI's T3BlbkFJ marker on bare sk- keys so other vendors' sk- keys and LiteLLM virtual keys are left alone, add sk-None- user keys, and replace the registration-only tests with whole-match, length-boundary, and negative cases * fix(guardrails): require the T3BlbkFJ marker on every OpenAI key form and match keys after escaped separators openai_api_key now needs the marker on the sk-proj-, sk-svcacct-, sk-admin- and sk-None- forms too, as gitleaks and TruffleHog do. anthropic_api_key and gemini_api_key also start a match right after a literal \n or \t or a percent-encoded byte, so keys in pasted JSON strings and URL-encoded logs are masked. * fix(guardrails): bound the OpenAI key regex, match Gemini prefixes case-sensitively, and hold back long keys in streamed replies * fix(guardrails): size the streaming holdback per pattern, stop OpenAI key tails before the next key, and allow non-ASCII text after a key * fix(guardrails): mask content filter pattern matches found in the original text so overlapping patterns cannot split a key * fix(guardrails): let a block pattern see the text after earlier masking patterns, as before Masking patterns still match the original text and merge overlapping matches into one tag, but a BLOCK pattern checks the text with the masking patterns listed before it applied, so a request main masked and passed is not blocked now. Detections name only the patterns whose tag was written. * fix(guardrails): keep content filter masks in order and hold back the longest OpenAI key Masking patterns run in configured order on the masked text as on main, so a contextual pattern still matches next to an earlier tag, and each also matches the original text so an earlier tag cannot split a key. Detections follow the configured pattern, not its name, and the OpenAI streaming holdback covers the pattern's 419-character longest match. --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../litellm_content_filter/content_filter.py | 157 ++++++++++++---- .../litellm_content_filter/patterns.json | 24 ++- .../litellm_content_filter/patterns.py | 13 +- .../content_filter/test_content_filter.py | 142 ++++++++++++++- .../content_filter/test_patterns.py | 168 ++++++++++++++++++ 5 files changed, 468 insertions(+), 36 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py index b1e3d51bbc1..6f077650a92 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py @@ -6,15 +6,17 @@ to detect and block/mask sensitive content. """ import asyncio +import bisect import contextvars import itertools import json import os import re import time -from collections.abc import AsyncGenerator, AsyncIterable, Callable, Coroutine, Mapping, Sequence +from collections.abc import AsyncGenerator, AsyncIterable, Callable, Coroutine, Iterable, Mapping, Sequence from dataclasses import dataclass, replace from datetime import datetime +from functools import partial from re import Pattern from typing import TYPE_CHECKING, Any, Final, Literal, Optional, TypedDict, TypeVar, cast @@ -79,7 +81,7 @@ from .competitor_intent import ( AirlineCompetitorIntentChecker, BaseCompetitorIntentChecker, ) -from .patterns import PATTERN_EXTRA_CONFIG, get_compiled_pattern +from .patterns import PATTERN_EXTRA_CONFIG, PATTERN_STREAMING_HOLDBACK_CHARS, get_compiled_pattern MAX_KEYWORD_VALUE_GAP_WORDS: Final = 1 _DECODED_ARGUMENTS: Final = TypeAdapter(object) @@ -223,10 +225,70 @@ async def _stream_then_end_of_stream_chunk( @dataclass(frozen=True, slots=True) class _StreamedScanPlan: context_chars: int + holdback_chars: int exception_phrases: tuple[str, ...] conditional_words: tuple[str, ...] +@dataclass(frozen=True, slots=True) +class _RedactionSpan: + start: int + end: int + pattern_index: int + pattern_name: str + + +def _merge_redaction_spans(spans: Iterable[_RedactionSpan]) -> tuple[_RedactionSpan, ...]: + ordered: Final = sorted(spans, key=lambda span: (span.start, -span.end)) + reach: Final = tuple(itertools.accumulate((span.end for span in ordered), max)) + first_indices: Final = tuple( + index for index, span in enumerate(ordered) if index == 0 or span.start >= reach[index - 1] + ) + last_indices: Final = (*(index - 1 for index in first_indices[1:]), len(ordered) - 1) + return tuple(replace(ordered[first], end=reach[last]) for first, last in zip(first_indices, last_indices)) + + +@dataclass(frozen=True, slots=True) +class _RedactedText: + spans: tuple[_RedactionSpan, ...] + tags: tuple[str, ...] + tag_starts: tuple[int, ...] + + def render(self, text: str) -> str: + previous_ends: Final = (0, *(span.end for span in self.spans)) + kept_parts: Final = ( + text[previous_end : span.start] + tag + for previous_end, span, tag in zip(previous_ends, self.spans, self.tags) + ) + return "".join((*kept_parts, text[previous_ends[-1] :])) + + def original_start(self, position: int) -> int: + index: Final = bisect.bisect_right(self.tag_starts, position) - 1 + if index < 0: + return position + tag_end: Final = self.tag_starts[index] + len(self.tags[index]) + if position < tag_end: + return self.spans[index].start + return position - tag_end + self.spans[index].end + + def original_end(self, position: int) -> int: + index: Final = bisect.bisect_left(self.tag_starts, position) - 1 + if index < 0: + return position + tag_end: Final = self.tag_starts[index] + len(self.tags[index]) + if position <= tag_end: + return self.spans[index].end + return position - tag_end + self.spans[index].end + + +def _redacted_text(spans: tuple[_RedactionSpan, ...], redaction_format: str) -> _RedactedText: + tags: Final = tuple(redaction_format.format(pattern_name=span.pattern_name.upper()) for span in spans) + shifts: Final = itertools.accumulate( + (len(tag) - (span.end - span.start) for span, tag in zip(spans, tags)), initial=0 + ) + return _RedactedText(spans, tags, tuple(span.start + shift for span, shift in zip(spans, shifts))) + + class CategoryFileData(TypedDict, total=False): category_name: str description: str @@ -987,21 +1049,6 @@ class ContentFilterGuardrail(CustomGuardrail): merged.append((start, end)) return merged - def _mask_spans(self, text: str, spans: list[tuple[int, int]], redaction: str) -> str: - """Apply masking for the provided spans using the given redaction tag.""" - - if not spans: - return text - - result_parts: Final[list[str]] = [] - previous_end = 0 - for start, end in spans: - result_parts.append(text[previous_end:start]) - result_parts.append(redaction) - previous_end = end - result_parts.append(text[previous_end:]) - return "".join(result_parts) - def _convert_word_number_sequence(self, sequence: str) -> str | None: """Convert a spelled-out digit sequence (e.g., 'One-Two') into digits.""" @@ -1392,10 +1439,8 @@ class ContentFilterGuardrail(CustomGuardrail): self, pattern_name: str, action: ContentFilterAction, - text: str, - spans: list[tuple[int, int]], detections: list[ContentFilterDetection] | None, - ) -> str: + ) -> None: """Handle regex pattern match detection and action.""" if detections is not None: pattern_detection: Final[PatternDetection] = { @@ -1413,11 +1458,49 @@ class ContentFilterGuardrail(CustomGuardrail): detail={"error": error_msg, "pattern": pattern_name}, ) elif action == ContentFilterAction.MASK: - redaction_tag: Final = self.pattern_redaction_format.format(pattern_name=pattern_name.upper()) - text = self._mask_spans(text, spans, redaction_tag) verbose_proxy_logger.info("Masked all %s matches in content", pattern_name) - return text + def _mask_step( + self, + text: str, + spans: tuple[_RedactionSpan, ...], + indexed_entry: tuple[int, CompiledPatternEntry], + ) -> tuple[_RedactionSpan, ...]: + index, pattern_entry = indexed_entry + if pattern_entry["action"] != ContentFilterAction.MASK: + return spans + original_matches: Final = self._find_pattern_spans(text, pattern_entry) + redacted_matches: Final = self._redacted_text_matches(text, spans, pattern_entry) if spans else () + return _merge_redaction_spans( + ( + *spans, + *( + _RedactionSpan(start, end, index, pattern_entry["pattern_name"]) + for start, end in (*original_matches, *redacted_matches) + ), + ) + ) + + def _redacted_text_matches( + self, text: str, spans: tuple[_RedactionSpan, ...], pattern_entry: CompiledPatternEntry + ) -> tuple[tuple[int, int], ...]: + redacted: Final = _redacted_text(spans, self.pattern_redaction_format) + return tuple( + (redacted.original_start(start), redacted.original_end(end)) + for start, end in self._find_pattern_spans(redacted.render(text), pattern_entry) + ) + + def _pattern_detected( + self, + text: str, + index: int, + pattern_entry: CompiledPatternEntry, + span_states: Sequence[tuple[_RedactionSpan, ...]], + ) -> bool: + if pattern_entry["action"] == ContentFilterAction.MASK: + return any(span.pattern_index == index for span in span_states[-1]) + text_after_earlier_masks: Final = _redacted_text(span_states[index], self.pattern_redaction_format).render(text) + return bool(self._find_pattern_spans(text_after_earlier_masks, pattern_entry)) def _handle_blocked_word_match( self, @@ -1506,13 +1589,14 @@ class ContentFilterGuardrail(CustomGuardrail): keyword, category_name, severity, action = category_keyword_match text = self._handle_category_keyword_match(keyword, category_name, severity, action, text, detections) - # Check regex patterns - process ALL patterns, not just first match - for pattern_entry in self.compiled_patterns: - spans = self._find_pattern_spans(text, pattern_entry) - if spans: - pattern_name = pattern_entry["pattern_name"] - action = pattern_entry["action"] - text = self._handle_pattern_match(pattern_name, action, text, spans, detections) + no_spans: Final[tuple[_RedactionSpan, ...]] = () + span_states: Final = tuple( + itertools.accumulate(enumerate(self.compiled_patterns), partial(self._mask_step, text), initial=no_spans) + ) + for index, pattern_entry in enumerate(self.compiled_patterns): + if self._pattern_detected(text, index, pattern_entry, span_states): + self._handle_pattern_match(pattern_entry["pattern_name"], pattern_entry["action"], detections) + text = _redacted_text(span_states[-1], self.pattern_redaction_format).render(text) # Check blocked words - iterate through ALL blocked words text_lower = text.lower() @@ -2071,7 +2155,9 @@ class ContentFilterGuardrail(CustomGuardrail): def _streamed_scan_plan(self) -> _StreamedScanPlan: """ Per-stream inputs for buffer trimming: the retained tail length (the default - context, widened to the longest configured keyword), the category exception + context, widened to the longest configured keyword), the unreleased tail length + (widened to the longest holdback a configured prebuilt pattern declares, since + provider keys run longer than the default), the category exception phrases, which suppress matches anywhere in the scanned text, and the conditional category words, which only match when paired inside one sentence. """ @@ -2081,6 +2167,13 @@ class ContentFilterGuardrail(CustomGuardrail): ) return _StreamedScanPlan( context_chars=max(CONTENT_FILTER_STREAMING_SCAN_CONTEXT_CHARS, longest_keyword), + holdback_chars=max( + ( + PATTERN_STREAMING_HOLDBACK_CHARS.get(entry["pattern_name"], CONTENT_FILTER_STREAMING_HOLDBACK_CHARS) + for entry in self.compiled_patterns + ), + default=CONTENT_FILTER_STREAMING_HOLDBACK_CHARS, + ), exception_phrases=tuple( phrase for category in self.loaded_categories.values() for phrase in category.exceptions ), @@ -2221,7 +2314,7 @@ class ContentFilterGuardrail(CustomGuardrail): safe_to_yield_len = max( previous_state.yielded_masked_text_len, - len(masked_text) - (0 if is_final else CONTENT_FILTER_STREAMING_HOLDBACK_CHARS), + len(masked_text) - (0 if is_final else plan.holdback_chars), ) released_text = masked_text[previous_state.yielded_masked_text_len : safe_to_yield_len] if isinstance(choice, TextChoices): diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.json b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.json index db6dd36726c..dcb2dc5ce77 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.json +++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.json @@ -98,6 +98,28 @@ "category": "Credential Patterns", "description": "Detects generic API key patterns" }, + { + "name": "openai_api_key", + "display_name": "OpenAI API Key", + "pattern": "sk-(?:(?:proj|svcacct|admin|None)-[A-Za-z0-9_\\-]{20,200}T3BlbkFJ(?:(?!sk-(?:proj|svcacct|admin|None)-)[A-Za-z0-9_\\-]){20,200}|[A-Za-z0-9]{20}T3BlbkFJ[A-Za-z0-9]{20}(?![A-Za-z0-9_]))", + "streaming_holdback_chars": 419, + "category": "Credential Patterns", + "description": "Detects OpenAI API keys (sk-proj-..., sk-svcacct-..., sk-admin-..., sk-None-... and legacy sk-... keys, all carrying the T3BlbkFJ marker)" + }, + { + "name": "anthropic_api_key", + "display_name": "Anthropic API Key", + "pattern": "(?:(? str: + return "".join(await cls._collect_streamed_deltas(guardrail, chunks, metadata)) + + @staticmethod + async def _collect_streamed_deltas( + guardrail: ContentFilterGuardrail, + chunks: list[str], + metadata: dict[str, list[StandardLoggingGuardrailInformation]], + ) -> list[str]: from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices async def mock_stream(): @@ -951,7 +1042,7 @@ class TestContentFilterGuardrail: request_data={"messages": [], "model": "gpt-4o", "metadata": metadata}, ): yielded.append(chunk.choices[0].delta.content or "") - return "".join(yielded) + return yielded @pytest.mark.asyncio async def test_streaming_hook_scans_bounded_window_per_chunk(self): @@ -1068,6 +1159,53 @@ class TestContentFilterGuardrail: assert entry["guardrail_status"] == "guardrail_intervened" assert [d["keyword"] for d in entry["guardrail_response"]] == [phrase] + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("pattern_name", "key"), + [ + ("openai_api_key", "sk-proj-" + "a" * 74 + "T3BlbkFJ" + "b" * 74), + ("openai_api_key", "sk-svcacct-" + "a" * 200 + "T3BlbkFJ" + "b" * 200), + ("anthropic_api_key", "sk-ant-api03-" + "a" * 93 + "AA"), + ("gemini_api_key", "AQ." + "a" * 55), + ], + ) + async def test_streaming_hook_masks_whole_provider_key_split_across_chunks(self, pattern_name: str, key: str): + guardrail: Final = ContentFilterGuardrail( + guardrail_name="test-streaming-provider-key", + patterns=[ + ContentFilterPattern( + pattern_type="prebuilt", + pattern_name=pattern_name, + action=ContentFilterAction.MASK, + ) + ], + event_hook=GuardrailEventHooks.post_call, + ) + text: Final = f"Use {key} to call the API." + chunks: Final = [text[i : i + 4] for i in range(0, len(text), 4)] + + streamed: Final = await self._collect_streamed_text(guardrail, chunks, {}) + + assert streamed == f"Use [{pattern_name.upper()}_REDACTED] to call the API." + + @pytest.mark.asyncio + async def test_streaming_hook_keeps_default_holdback_without_provider_key_patterns(self): + guardrail: Final = ContentFilterGuardrail( + guardrail_name="test-streaming-older-credential-patterns", + patterns=[ + ContentFilterPattern(pattern_type="prebuilt", pattern_name=name, action=ContentFilterAction.MASK) + for name in ("aws_access_key", "aws_secret_key", "github_token", "slack_token", "generic_api_key") + ], + event_hook=GuardrailEventHooks.post_call, + ) + text: Final = "plain words with nothing secret in them. " * 12 + chunks: Final = [text[i : i + 10] for i in range(0, len(text), 10)] + + deltas: Final = await self._collect_streamed_deltas(guardrail, chunks, {}) + + assert "".join(deltas) == text + assert len(next(delta for delta in reversed(deltas) if delta)) <= CONTENT_FILTER_STREAMING_HOLDBACK_CHARS + @pytest.mark.asyncio async def test_streaming_hook_blocks_keyword_longer_than_scan_context(self): """ diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/content_filter/test_patterns.py b/tests/unit/proxy/guardrails/guardrail_hooks/content_filter/test_patterns.py index d702b9e0116..3c71e814fe1 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/content_filter/test_patterns.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/content_filter/test_patterns.py @@ -4,6 +4,8 @@ Tests for content filter pattern loading from JSON import json import os +import time +from typing import Final import pytest @@ -183,3 +185,169 @@ def test_eu_patterns_have_category(): assert ( pattern_name in eu_category_patterns ), f"Pattern {pattern_name} not in EU PII Patterns category" + + +def test_provider_key_patterns_are_offered_in_the_ui_as_credential_patterns() -> None: + ui_categories: Final = {pattern["name"]: pattern["category"] for pattern in get_pattern_metadata()} + assert [ui_categories.get(name) for name in ("openai_api_key", "anthropic_api_key", "gemini_api_key")] == [ + "Credential Patterns" + ] * 3 + + +@pytest.mark.parametrize( + ("pattern_name", "key"), + [ + pytest.param("openai_api_key", "sk-" + "a" * 20 + "T3BlbkFJ" + "b" * 20, id="openai-legacy"), + pytest.param("openai_api_key", "sk-proj-" + "a" * 74 + "T3BlbkFJ" + "b" * 74, id="openai-project"), + pytest.param("openai_api_key", "sk-proj-" + "a" * 20 + "T3BlbkFJ" + "b" * 20, id="openai-project-shortest"), + pytest.param("openai_api_key", "sk-svcacct-" + "a_b-" * 15 + "T3BlbkFJ" + "c-d_" * 15, id="openai-service-account"), + pytest.param("openai_api_key", "sk-admin-" + "a" * 58 + "T3BlbkFJ" + "b" * 58, id="openai-admin"), + pytest.param("openai_api_key", "sk-None-" + "a" * 20 + "T3BlbkFJ" + "b" * 20, id="openai-user-key"), + pytest.param("openai_api_key", "sk-proj-" + "a" * 20 + "T3BlbkFJ" + "b" * 19 + "-", id="openai-trailing-hyphen"), + pytest.param( + "anthropic_api_key", "sk-ant-api03-abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOP", id="anthropic-versioned" + ), + pytest.param( + "anthropic_api_key", "sk-ant-abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTU", id="anthropic-unversioned" + ), + pytest.param("anthropic_api_key", "sk-ant-" + "a" * 20, id="anthropic-shortest"), + pytest.param("anthropic_api_key", "sk-ant-api03-" + "a_b-" * 23 + "AA", id="anthropic-urlsafe"), + pytest.param( + "anthropic_api_key", "sk-ant-api03-abcdefghijklmnopqrstuvwxyz0123456789-", id="anthropic-trailing-hyphen" + ), + pytest.param("gemini_api_key", "AIzaSyD-9tSrke72I6_AQp2h8HI4-uE6QNKQV0", id="gemini-legacy"), + pytest.param("gemini_api_key", "AIza" + "a" * 33, id="gemini-legacy-shortest"), + pytest.param("gemini_api_key", "AIza" + "a" * 36, id="gemini-legacy-longest"), + pytest.param("gemini_api_key", "AIza" + "a" * 34 + "-", id="gemini-legacy-trailing-hyphen"), + pytest.param("gemini_api_key", "AQ.Ab8RN6I-7xK9pL2mQ0vW5yZ1uX3tS4rE8qD7cB6aN5mP4oI3uY2", id="gemini-auth-key"), + pytest.param("gemini_api_key", "AQ." + "a" * 45, id="gemini-auth-key-shortest"), + pytest.param("gemini_api_key", "AQ." + "a" * 55, id="gemini-auth-key-longest"), + pytest.param("gemini_api_key", "AQ." + "a" * 49 + "-", id="gemini-auth-key-trailing-hyphen"), + ], +) +def test_credential_pattern_matches_whole_key(pattern_name: str, key: str) -> None: + match: Final = get_compiled_pattern(pattern_name).search(f"I keep getting 401 with {key}, is it expired?") + assert match is not None + assert match.group(0) == key + + +@pytest.mark.parametrize("separator", ["\\n", "\\t", "\\u003c", "Bearer%20", "?key%3D", "?key%253D", "GOOGLE_KEY_"]) +@pytest.mark.parametrize( + ("pattern_name", "key"), + [ + pytest.param("openai_api_key", "sk-proj-" + "a" * 20 + "T3BlbkFJ" + "b" * 20, id="openai"), + pytest.param("anthropic_api_key", "sk-ant-api03-" + "a" * 20, id="anthropic"), + pytest.param("gemini_api_key", "AIza" + "a" * 35, id="gemini-legacy"), + pytest.param("gemini_api_key", "AQ." + "a" * 50, id="gemini-auth-key"), + ], +) +def test_credential_pattern_matches_key_after_escaped_separator(pattern_name: str, key: str, separator: str) -> None: + match: Final = get_compiled_pattern(pattern_name).search(f'{{"log": "retrying with{separator}{key} failed"}}') + assert match is not None + assert match.group(0) == key + + +@pytest.mark.parametrize( + ("pattern_name", "text"), + [ + pytest.param("openai_api_key", "sk-short", id="openai-too-short"), + pytest.param("openai_api_key", "sk-proj-" + "a" * 19, id="openai-project-too-short"), + pytest.param("openai_api_key", "sk-" + "a" * 20 + "T3BlbkFJ" + "b" * 21, id="openai-legacy-too-long"), + pytest.param("openai_api_key", "sk-proj-" + "a" * 48, id="openai-project-without-marker"), + pytest.param("openai_api_key", "sk-proj-" + "a" * 19 + "T3BlbkFJ" + "b" * 20, id="openai-project-marker-too-early"), + pytest.param("openai_api_key", "sk-proj-" + "a" * 20 + "T3BlbkFJ" + "b" * 19, id="openai-project-marker-too-late"), + pytest.param("openai_api_key", "SK-ADMIN-dashboard-redesign-phase-two", id="openai-prefixed-slug"), + pytest.param("openai_api_key", "desk-proj-management-dashboard-v2", id="openai-inside-longer-word"), + pytest.param( + "openai_api_key", "sk-abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRST", id="openai-legacy-without-marker" + ), + pytest.param("openai_api_key", "sk-0123456789abcdef0123456789abcdef", id="openai-other-vendor-sk-key"), + pytest.param("openai_api_key", "sk-Xq3vR9tLm2Pa8KdW5yZgQw", id="openai-litellm-virtual-key"), + pytest.param( + "openai_api_key", + "sk-AbCdEfGhIjKlMnOpQrStUvWxYz0123456789AbCd-_x", + id="openai-litellm-virtual-key-raised-length", + ), + pytest.param("openai_api_key", "not-a-key", id="openai-no-prefix"), + pytest.param( + "openai_api_key", "sk-ant-api03-thisisanthropicnotopenaiabc123", id="openai-ignores-anthropic-key" + ), + pytest.param("openai_api_key", "ordinary sentence without any api key in it", id="openai-plain-text"), + pytest.param("anthropic_api_key", "sk-ant-" + "a" * 19, id="anthropic-too-short"), + pytest.param("anthropic_api_key", "flask-ant-design-system-components", id="anthropic-inside-longer-word"), + pytest.param( + "anthropic_api_key", + "sk-proj-abcdefghijklmnopqrstuvwxyzABCDEF1234567890", + id="anthropic-ignores-openai-project-key", + ), + pytest.param( + "anthropic_api_key", + "sk-admin-abcdefghijklmnopqrstuvwxyzABCDEF123456", + id="anthropic-ignores-openai-admin-key", + ), + pytest.param("anthropic_api_key", "sk-ant", id="anthropic-prefix-only"), + pytest.param("anthropic_api_key", "plain text with nothing sensitive", id="anthropic-plain-text"), + pytest.param("gemini_api_key", "AIza", id="gemini-prefix-only"), + pytest.param("gemini_api_key", "AIza" + "a" * 32, id="gemini-legacy-too-short"), + pytest.param("gemini_api_key", "AIza" + "a" * 37, id="gemini-legacy-too-long"), + pytest.param("gemini_api_key", "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB", id="gemini-wrong-prefix"), + pytest.param("gemini_api_key", "AQ.", id="gemini-auth-prefix-only"), + pytest.param("gemini_api_key", "AQ." + "a" * 44, id="gemini-auth-key-too-short"), + pytest.param("gemini_api_key", "AQ." + "a" * 56, id="gemini-auth-key-too-long"), + pytest.param("gemini_api_key", "AQ-" + "a" * 50, id="gemini-auth-key-wrong-separator"), + pytest.param("gemini_api_key", "x" + "AIza" + "a" * 35, id="gemini-inside-longer-token"), + pytest.param("gemini_api_key", "The AQ. was reported at 42 ppm today.", id="gemini-aq-acronym"), + pytest.param("gemini_api_key", "Is the question AQ. Maybe it is BQ.", id="gemini-aq-sentence-end"), + pytest.param("gemini_api_key", "plain text with no google key", id="gemini-plain-text"), + pytest.param( + "gemini_api_key", + "https://example.com/blog/aizawa-takeshi-interview-2024-tokyo-01", + id="gemini-lowercase-legacy-prefix", + ), + pytest.param( + "gemini_api_key", + "ssh aq.internal-observability-metrics-collector-eu-west-01", + id="gemini-lowercase-auth-prefix", + ), + ], +) +def test_credential_pattern_ignores_non_keys(pattern_name: str, text: str) -> None: + assert get_compiled_pattern(pattern_name).search(text) is None + + +@pytest.mark.parametrize("suffix", ["请检查", "é", "ü der Schlüssel"]) +@pytest.mark.parametrize( + ("pattern_name", "key"), + [ + pytest.param("openai_api_key", "sk-" + "a" * 20 + "T3BlbkFJ" + "b" * 20, id="openai-legacy"), + pytest.param("gemini_api_key", "AIza" + "a" * 35, id="gemini-legacy"), + pytest.param("gemini_api_key", "AQ." + "a" * 50, id="gemini-auth-key"), + ], +) +def test_credential_pattern_matches_key_followed_by_non_ascii_letters(pattern_name: str, key: str, suffix: str) -> None: + match: Final = get_compiled_pattern(pattern_name).search(f"我的密钥是{key}{suffix}") + assert match is not None + assert match.group(0) == key + + +@pytest.mark.parametrize( + "leading", + [ + pytest.param("sk-proj-" + "a" * 74 + "T3BlbkFJ" + "b" * 74, id="another-key"), + pytest.param("sk-proj-" + "x" * 20 + "T3BlbkFJ" + "y" * 192, id="key-shaped-filler"), + pytest.param("sk-proj-" + "x" * 100 + "T3BlbkFJ" + "y" * 100, id="long-key-shaped-filler"), + ], +) +def test_openai_key_pattern_masks_a_key_glued_to_preceding_key_shaped_text(leading: str) -> None: + key: Final = "sk-proj-" + "c" * 74 + "T3BlbkFJ" + "d" * 74 + assert get_compiled_pattern("openai_api_key").sub("[KEY]", leading + key) == "[KEY][KEY]" + + +@pytest.mark.parametrize("prefix", ["sk-proj-", "sk-svcacct-", "sk-admin-", "sk-None-"]) +def test_openai_key_pattern_scans_repeated_prefixes_without_stalling(prefix: str) -> None: + prompt: Final = prefix * (128_000 // len(prefix)) + started: Final = time.perf_counter() + match: Final = get_compiled_pattern("openai_api_key").search(prompt) + elapsed: Final = time.perf_counter() - started + assert match is None + assert elapsed < 1.0 From 02d610f1a9e410e2f43d24431083f84f7d2158bd Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Sat, 7 Nov 2026 12:51:37 -0800 Subject: [PATCH 04/39] fix(anthropic): disable optional default thinking for reasoning effort none (#50541) --- .../crates/inference-messages/src/types.rs | 1 + .../crates/llms/src/anthropic/common_utils.rs | 1 + .../llms/src/anthropic/messages/thinking.rs | 37 +++++++++++++++++-- .../llms/src/base_llm/messages/context.rs | 3 ++ litellm/llms/anthropic/chat/transformation.py | 16 ++++++-- .../messages/native/transformation.py | 12 ++++-- .../bedrock/chat/converse_transformation.py | 8 ++-- .../llms/databricks/chat/transformation.py | 8 ++-- .../rust_bridge/host/model_capabilities.py | 1 + .../test_anthropic_messages_transformation.py | 14 +++++++ .../test_anthropic_reasoning_effort.py | 31 ++++++++++++++++ .../chat/test_converse_transformation.py | 15 ++++++++ .../test_databricks_chat_transformation.py | 15 ++++++++ .../host/test_model_capabilities.py | 13 +++++++ 14 files changed, 158 insertions(+), 17 deletions(-) diff --git a/litellm-rust/crates/inference-messages/src/types.rs b/litellm-rust/crates/inference-messages/src/types.rs index ae346259559..ece17b3380c 100644 --- a/litellm-rust/crates/inference-messages/src/types.rs +++ b/litellm-rust/crates/inference-messages/src/types.rs @@ -144,6 +144,7 @@ mod tests { supports_reasoning: true, supports_adaptive_thinking: true, thinking_always_on: false, + thinking_on_by_default: false, supports_legacy_thinking: false, supports_output_config: true, supports_sampling_params: false, diff --git a/litellm-rust/crates/llms/src/anthropic/common_utils.rs b/litellm-rust/crates/llms/src/anthropic/common_utils.rs index 13653f438f7..0854820358b 100644 --- a/litellm-rust/crates/llms/src/anthropic/common_utils.rs +++ b/litellm-rust/crates/llms/src/anthropic/common_utils.rs @@ -1913,6 +1913,7 @@ mod tests { supports_reasoning: false, supports_adaptive_thinking: false, thinking_always_on: false, + thinking_on_by_default: false, supports_legacy_thinking: false, supports_output_config: false, supports_sampling_params: true, diff --git a/litellm-rust/crates/llms/src/anthropic/messages/thinking.rs b/litellm-rust/crates/llms/src/anthropic/messages/thinking.rs index ae3488a21f9..0ddbc915154 100644 --- a/litellm-rust/crates/llms/src/anthropic/messages/thinking.rs +++ b/litellm-rust/crates/llms/src/anthropic/messages/thinking.rs @@ -2,8 +2,8 @@ use litellm_llms_types::{ formats::{ chat_completions::ReasoningEffort, messages::{ - AdaptiveThinking, EffortLevel, MessagesOptionalParams, MessagesRequest, OutputConfig, - ThinkingConfig, ThinkingDisplay, + AdaptiveThinking, DisabledThinking, EffortLevel, MessagesOptionalParams, + MessagesRequest, OutputConfig, ThinkingConfig, ThinkingDisplay, }, }, recognized::Recognized, @@ -172,7 +172,11 @@ fn translate_reasoning_effort( ) else { return Ok(MessagesRequest { params: MessagesOptionalParams { - thinking: None, + thinking: (context.capabilities.thinking_on_by_default + && !context.capabilities.thinking_always_on) + .then(|| { + Recognized::Known(ThinkingConfig::Disabled(DisabledThinking::default())) + }), output_config: None, ..request.params }, @@ -748,6 +752,33 @@ mod tests { ); } + #[rstest] + #[case::default_thinking(true, false, Some(serde_json::json!({"type": "disabled"}))) ] + #[case::legacy(false, false, None)] + #[case::always_on(true, true, None)] + fn none_disables_only_optional_default_thinking( + #[case] thinking_on_by_default: bool, + #[case] thinking_always_on: bool, + #[case] thinking: Option, + ) { + let capabilities = MessagesModelCapabilities { + thinking_on_by_default, + thinking_always_on, + ..opus_4_7() + }; + let expected = match thinking { + Some(thinking) => serde_json::json!({"max_tokens": 1024, "thinking": thinking}), + None => serde_json::json!({"max_tokens": 1024}), + }; + assert_eq!( + translate( + capabilities, + serde_json::json!({"max_tokens": 1024, "reasoning_effort": "none"}) + ), + Ok(request(expected)) + ); + } + #[rstest] #[case::adaptive_model(opus_4_7())] #[case::effort_capable_model(opus_4_5())] diff --git a/litellm-rust/crates/llms/src/base_llm/messages/context.rs b/litellm-rust/crates/llms/src/base_llm/messages/context.rs index b514e975aa4..3d8b0f41d8c 100644 --- a/litellm-rust/crates/llms/src/base_llm/messages/context.rs +++ b/litellm-rust/crates/llms/src/base_llm/messages/context.rs @@ -32,6 +32,8 @@ pub struct MessagesModelCapabilities { #[serde(default)] pub thinking_always_on: bool, #[serde(default)] + pub thinking_on_by_default: bool, + #[serde(default)] pub supports_legacy_thinking: bool, #[serde(default)] pub supports_output_config: bool, @@ -55,6 +57,7 @@ impl Default for MessagesModelCapabilities { supports_reasoning: false, supports_adaptive_thinking: false, thinking_always_on: false, + thinking_on_by_default: false, supports_legacy_thinking: false, supports_output_config: false, supports_sampling_params: true, diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index e0d313175b7..48bccc7c001 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -1718,7 +1718,13 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): llm_provider: str = "anthropic", ) -> AnthropicThinkingParam | None: """Capability probes read the cost map under ``custom_llm_provider``; ``llm_provider`` only tags raised exceptions.""" - if reasoning_effort is None or reasoning_effort == "none": + if reasoning_effort is None: + return None + if reasoning_effort == "none": + if AnthropicModelInfo.thinks_by_default( + model, custom_llm_provider + ) and not AnthropicModelInfo._is_always_on_thinking_model(model, custom_llm_provider): + return AnthropicThinkingParam(type="disabled") return None if AnthropicConfig._is_adaptive_thinking_model(model, custom_llm_provider): # without display, Anthropic defaults adaptive thinking to @@ -2126,12 +2132,14 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): custom_llm_provider=self._resolved_provider, llm_provider=self._resolved_provider, ) - if mapped_thinking is None: + if mapped_thinking is None or mapped_thinking.get("type") == "disabled": optional_params.pop("thinking", None) optional_params.pop("output_config", None) - else: + if mapped_thinking is not None: optional_params["thinking"] = mapped_thinking - if AnthropicConfig._is_adaptive_thinking_model(model, self._resolved_provider): + if mapped_thinking.get("type") != "disabled" and AnthropicConfig._is_adaptive_thinking_model( + model, self._resolved_provider + ): mapped_effort = REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT.get(effort_value) if mapped_effort is None: AnthropicConfig._raise_invalid_reasoning_effort( diff --git a/litellm/llms/anthropic/messages/native/transformation.py b/litellm/llms/anthropic/messages/native/transformation.py index 5281ad99620..aa0693d6f03 100644 --- a/litellm/llms/anthropic/messages/native/transformation.py +++ b/litellm/llms/anthropic/messages/native/transformation.py @@ -471,7 +471,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): """Map OpenAI-style ``reasoning_effort`` to native Anthropic params. Caller-supplied ``thinking`` / ``output_config`` win over the alias. - ``effort='none'`` clears both. Invalid efforts raise a 400. A mapped + ``effort='none'`` disables optional default thinking and clears effort. Invalid efforts raise a 400. A mapped thinking budget is capped below ``max_tokens`` and dropped when even the minimum budget cannot fit. """ @@ -485,6 +485,10 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): if not isinstance(reasoning_effort, str): return + if reasoning_effort == "none": + optional_params.pop("thinking", None) + optional_params.pop("output_config", None) + try: mapped_thinking: Final = AnthropicConfig.map_reasoning_effort( reasoning_effort=reasoning_effort, @@ -495,8 +499,6 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): raise AnthropicError(message=str(e.message), status_code=400) if mapped_thinking is None: - optional_params.pop("thinking", None) - optional_params.pop("output_config", None) return fitted_thinking: Final = AnthropicConfig.cap_thinking_budget_to_max_tokens(mapped_thinking, max_tokens) @@ -505,7 +507,9 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): return optional_params.setdefault("thinking", fitted_thinking) - if AnthropicModelInfo.is_adaptive_thinking_model(model, custom_llm_provider): + if fitted_thinking.get("type") != "disabled" and AnthropicModelInfo.is_adaptive_thinking_model( + model, custom_llm_provider + ): mapped_effort: Final = REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT.get(reasoning_effort) if mapped_effort is None: raise AnthropicError( diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 12c5031628b..5c82a8d6bc4 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -583,12 +583,14 @@ class AmazonConverseConfig(BaseConfig): custom_llm_provider="bedrock", llm_provider="bedrock_converse", ) - if mapped_thinking is None: + if mapped_thinking is None or mapped_thinking.get("type") == "disabled": optional_params.pop("thinking", None) optional_params.pop("output_config", None) - else: + if mapped_thinking is not None: optional_params["thinking"] = mapped_thinking - if AnthropicConfig.is_adaptive_thinking_model(model, "bedrock"): + if mapped_thinking.get("type") != "disabled" and AnthropicConfig.is_adaptive_thinking_model( + model, "bedrock" + ): mapped_effort = REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT.get(reasoning_effort) if mapped_effort is None: AnthropicConfig.raise_invalid_reasoning_effort( diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index a173cdb50f2..7917a74247e 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -629,12 +629,14 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): custom_llm_provider="databricks", llm_provider="databricks", ) - if mapped_thinking is None: + if mapped_thinking is None or mapped_thinking.get("type") == "disabled": optional_params.pop("thinking", None) optional_params.pop("output_config", None) - else: + if mapped_thinking is not None: optional_params["thinking"] = mapped_thinking - if AnthropicConfig._is_adaptive_thinking_model(model, "databricks"): + if mapped_thinking.get("type") != "disabled" and AnthropicConfig._is_adaptive_thinking_model( + model, "databricks" + ): mapped_effort: str | None = None if isinstance(reasoning_effort_value, str): mapped_effort = REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT.get(reasoning_effort_value) diff --git a/litellm/rust_bridge/host/model_capabilities.py b/litellm/rust_bridge/host/model_capabilities.py index cbf59dfb97b..39eb0b08aa2 100644 --- a/litellm/rust_bridge/host/model_capabilities.py +++ b/litellm/rust_bridge/host/model_capabilities.py @@ -27,6 +27,7 @@ def anthropic_model_capabilities(model: str, custom_llm_provider: str | None) -> "supports_reasoning": supports("supports_reasoning"), "supports_adaptive_thinking": supports("supports_adaptive_thinking"), "thinking_always_on": supports("thinking_always_on"), + "thinking_on_by_default": AnthropicModelInfo.thinks_by_default(model, provider), "supports_legacy_thinking": supports("supports_legacy_thinking"), "supports_output_config": supports("supports_output_config"), "supports_sampling_params": AnthropicModelInfo._supports_sampling_params(resolved_model), # pyright: ignore[reportPrivateUsage] # same gate the handler applies diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py index 54b29ca6aa7..8f86888728f 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py @@ -721,3 +721,17 @@ def test_a_first_party_messages_request_never_fetches_the_remote_beta_headers_co assert request["messages"][1] == {"role": "system", "content": "be terse"} assert not remote_config.called + + +@pytest.mark.parametrize( + "model, expected", + [ + ("claude-sonnet-5", {"type": "disabled"}), + ("claude-haiku-4-5", None), + ("claude-sonnet-5-5", None), + ], +) +def test_none_disables_only_optional_default_thinking(model: str, expected: dict[str, str] | None) -> None: + params: Final[dict[str, object]] = {"reasoning_effort": "none", "output_config": {"effort": "high"}} + AnthropicMessagesConfig._translate_reasoning_effort_to_anthropic(model, params, 1024, "anthropic") + assert params == ({"thinking": expected} if expected is not None else {}) diff --git a/tests/unit/llms/anthropic/test_anthropic_reasoning_effort.py b/tests/unit/llms/anthropic/test_anthropic_reasoning_effort.py index 573f586de1e..1b066e49e3b 100644 --- a/tests/unit/llms/anthropic/test_anthropic_reasoning_effort.py +++ b/tests/unit/llms/anthropic/test_anthropic_reasoning_effort.py @@ -5,11 +5,42 @@ Verifies that reasoning_effort=None returns None for all models, including Claude Opus 4.6. """ +from typing import Final + import pytest +from litellm.exceptions import BadRequestError from litellm.llms.anthropic.chat.transformation import AnthropicConfig +@pytest.mark.parametrize( + "model, expected", + [ + ("claude-sonnet-5", {"type": "disabled"}), + ("claude-opus-5", {"type": "disabled"}), + ("claude-haiku-5-5", {"type": "disabled"}), + ("claude-haiku-4-5", None), + ("claude-sonnet-5-5", None), + ("claude-opus-5-5", None), + ("claude-fable-5", None), + ("claude-mythos-5", None), + ], +) +def test_none_disables_only_optional_default_thinking(model: str, expected: dict[str, str] | None) -> None: + mapped: Final = AnthropicConfig().map_openai_params({"reasoning_effort": "none"}, {}, model, False) + assert mapped.get("thinking") == expected + assert "output_config" not in mapped + + +@pytest.mark.parametrize("effort", [None, "bogus"]) +def test_default_thinking_preserves_absent_and_invalid_effort(effort: str | None) -> None: + if effort is None: + assert AnthropicConfig.map_reasoning_effort(effort, "claude-sonnet-5", "anthropic") is None + else: + with pytest.raises(BadRequestError, match="Invalid reasoning_effort"): + AnthropicConfig().map_openai_params({"reasoning_effort": effort}, {}, "claude-sonnet-5", False) + + class TestMapReasoningEffort: def test_none_returns_none_for_opus_4_6(self): """reasoning_effort=None should return None for Opus 4.6, not adaptive.""" diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 4dcf312737d..23ac5baf1ba 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -12203,6 +12203,21 @@ def test_converse_claude_request_over_the_cache_control_cap_drops_the_oldest_hin assert messages == messages_before +@pytest.mark.parametrize( + "model, expected", + [ + ("us.anthropic.claude-sonnet-5", {"type": "disabled"}), + ("us.anthropic.claude-haiku-4-5", None), + ("us.anthropic.claude-sonnet-5-5", None), + ], +) +def test_none_disables_only_optional_default_thinking(model: str, expected: dict[str, str] | None) -> None: + params: Final[dict[str, object]] = {} + AmazonConverseConfig()._handle_reasoning_effort_parameter(model, "none", params) + assert params.get("thinking") == expected + assert "output_config" not in params + + def test_converse_claude_tool_message_with_several_marked_blocks_counts_as_one_cache_point_against_the_cap( monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None ) -> None: diff --git a/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py b/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py index e04dd35f851..053d60ef0e1 100644 --- a/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py +++ b/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py @@ -3220,3 +3220,18 @@ def test_databricks_claude_cache_control_cap_keeps_room_for_a_top_level_cache_co assert "cache_control" not in json.dumps(body["messages"][0]) assert json.dumps(body["messages"]).count('"cache_control"') == 3 + + +@pytest.mark.parametrize( + "model, expected", + [ + ("databricks-claude-sonnet-5", {"type": "disabled"}), + ("databricks-claude-haiku-4-5", None), + ("databricks-claude-sonnet-5-5", None), + ], +) +def test_none_disables_only_optional_default_thinking(model: str, expected: dict[str, str] | None) -> None: + params: Final = _map_reasoning_effort(model, "none") + assert params.get("thinking") == expected + assert "output_config" not in params + assert "reasoning_effort" not in params diff --git a/tests/unit/rust_bridge/host/test_model_capabilities.py b/tests/unit/rust_bridge/host/test_model_capabilities.py index 4d73a29d078..dffa1b2cde3 100644 --- a/tests/unit/rust_bridge/host/test_model_capabilities.py +++ b/tests/unit/rust_bridge/host/test_model_capabilities.py @@ -73,3 +73,16 @@ def test_capability_source_observes_runtime_registration_changes(monkeypatch: py assert before["supports_output_config"] is True assert after["supports_reasoning"] is False assert after["supports_output_config"] is False + + +@pytest.mark.parametrize( + "model, expected", + [ + ("claude-sonnet-5", True), + ("claude-haiku-4-5", False), + ("claude-sonnet-5-5", True), + ], +) +def test_default_thinking_capability_is_projected(model: str, expected: bool) -> None: + capabilities: Final = anthropic_model_capabilities(model, "anthropic") + assert capabilities["thinking_on_by_default"] is expected From 093caa52fa063ad7cbb25c827531d02bde485f57 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Sat, 7 Nov 2026 13:16:39 -0800 Subject: [PATCH 05/39] test(e2e): add a Codex row where a sub-agent must read the task its parent hands it (#50494) --- tests/e2e/AGENTS.md | 2 +- tests/e2e/codex/harness.py | 39 ++++++++- tests/e2e/codex/rollout.py | 62 +++++++++++-- tests/e2e/codex/test_subagent_e2e.py | 31 +++++++ tests/e2e_harness/codex/test_codex_rollout.py | 87 +++++++++++++++++-- 5 files changed, 205 insertions(+), 16 deletions(-) create mode 100644 tests/e2e/codex/test_subagent_e2e.py diff --git a/tests/e2e/AGENTS.md b/tests/e2e/AGENTS.md index a4c51a53098..89bf24016da 100644 --- a/tests/e2e/AGENTS.md +++ b/tests/e2e/AGENTS.md @@ -50,7 +50,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `gateway/` - proxy configuration only (`litellm-config.yml`). No tests - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture. Ships its own driver/builder/publisher, covered by `tests/e2e_harness/claude_code/`. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`). The CLI-driving path stays bespoke. `pr_gate_version_resolver.py` only forwards to `cli_harness/npm_version_resolver.py`, because the Buildkite runner images still run that path and install @latest without it - `cli_harness/` - no tests, only the plumbing every suite that drives a real coding-agent CLI shares: `npm_version_resolver.py` picks the CLI release to install (the newest npm release at least three days old, run by CI as `python tests/e2e/cli_harness/npm_version_resolver.py `, stdlib only), `lockdown.py` gives the CLI subprocess an allowlisted env and a fresh empty HOME so it never sees the runner's credentials or dotfiles, and `rate_limiter.py` is the cross-process token bucket per upstream that keeps parallel CLI runs under one request rate. Covered by `tests/e2e_harness/cli_harness/`. Code only one CLI suite needs stays in that suite -- `codex/` - the Codex compatibility suite: drives the real `codex exec --json` CLI through the gateway, one class per row (the command round trip, a file edit with the patch tool, a long session that follows a chain of files one command at a time, auto-compaction, where Codex reads a random value, compacts through the gateway and is asked for the value after the file is deleted, and an MCP tool, where Codex calls the tool of `mcp_server.py`, a stdio MCP server the test hands it with `add_mcp_server`, on a first and a resumed turn), each row parametrized over `SETUPS` in `setups.py`, one `pytest.param` per setup (GPT on Bedrock Mantle, Bedrock Runtime, Azure OpenAI and OpenAI, plus Claude on Bedrock). A row with one setup broken by a known bug parametrizes over `with_expected_failure(, reason=...)`, which is `SETUPS` with a strict xfail on that one setup. The `setup` fixture registers that setup's deployment under a fresh name via `/model/new` and the `codex` fixture makes a key scoped to it, and both are deleted on teardown. `harness.py` gives each test a throwaway HOME and CODEX_HOME whose `config.toml` points a custom `responses` provider at the gateway and has Codex read its model list from the gateway's `/v1/models` (`model_catalog_url` on the provider), with the sandbox on (`workspace-write`) and approvals off, and runs the CLI under `cli_harness`'s env lockdown and rate limiter. `run.py` reads the JSONL events into a `CodexRun`: the commands with their output, the patched files, the MCP tool calls with the text they returned, the last agent message as `answer`, the usage and the errors, where an error's `status_code` is what lets the suite's 5xx rerun rule catch a turn Codex gave up on. `codex exec --json` prints nothing when Codex compacts, so `rollout.py` reads the compactions from the thread's rollout file under CODEX_HOME/sessions, and a compaction is remote when its replacement history kept the provider's `compaction` item. `exec` and `resume` take `compact_after_tokens`, which forces Codex's remote compaction (`compaction_trigger` items over `/v1/responses`, the path Codex otherwise only takes for OpenAI and Azure) on the gateway provider and compacts once the context grows past that many tokens. A test passes on a random value the model can only learn through the gateway, never on the model's wording or the exact command it typed. It needs `codex` on PATH and fails without it. Covered by `tests/e2e_harness/codex/` +- `codex/` - the Codex compatibility suite: drives the real `codex exec --json` CLI through the gateway, one class per row (the command round trip, a file edit with the patch tool, a long session that follows a chain of files one command at a time, auto-compaction, where Codex reads a random value, compacts through the gateway and is asked for the value after the file is deleted, and an MCP tool, where Codex calls the tool of `mcp_server.py`, a stdio MCP server the test hands it with `add_mcp_server`, on a first and a resumed turn, and a sub-agent, where Codex's multi-agent v2 hands a helper a random codeword as its task with shell tools off so the helper can only learn it from the task, and the helper's own final reply is read from its rollout), each row parametrized over `SETUPS` in `setups.py`, one `pytest.param` per setup (GPT on Bedrock Mantle, Bedrock Runtime, Azure OpenAI and OpenAI, plus Claude on Bedrock). A row with one setup broken by a known bug parametrizes over `with_expected_failure(, reason=...)`, which is `SETUPS` with a strict xfail on that one setup. The `setup` fixture registers that setup's deployment under a fresh name via `/model/new` and the `codex` fixture makes a key scoped to it, and both are deleted on teardown. `harness.py` gives each test a throwaway HOME and CODEX_HOME whose `config.toml` points a custom `responses` provider at the gateway and has Codex read its model list from the gateway's `/v1/models` (`model_catalog_url` on the provider), with the sandbox on (`workspace-write`) and approvals off, and runs the CLI under `cli_harness`'s env lockdown and rate limiter. `run.py` reads the JSONL events into a `CodexRun`: the commands with their output, the patched files, the MCP tool calls with the text they returned, the last agent message as `answer`, the usage and the errors, where an error's `status_code` is what lets the suite's 5xx rerun rule catch a turn Codex gave up on. `codex exec --json` prints nothing when Codex compacts, so `rollout.py` reads the compactions from the thread's rollout file under CODEX_HOME/sessions, and a compaction is remote when its replacement history kept the provider's `compaction` item. `subagents` reads the sub-agents a thread spawned from their own rollouts, whose `session_meta` names that thread as their parent, each with its final replies and whether it forked the parent's conversation (`forked_from_id`), so the sub-agent row can reject a helper that saw the codeword in inherited history instead of its task. `exec` takes `enable` and `disable`, Codex feature names it passes as `--enable` and `--disable`. `exec` and `resume` take `compact_after_tokens`, which forces Codex's remote compaction (`compaction_trigger` items over `/v1/responses`, the path Codex otherwise only takes for OpenAI and Azure) on the gateway provider and compacts once the context grows past that many tokens. A test passes on a random value the model can only learn through the gateway, never on the model's wording or the exact command it typed. It needs `codex` on PATH and fails without it. Covered by `tests/e2e_harness/codex/` - `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream, see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport. The Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json`. Its specs follow the dashboard's rule against testing what shadcn and Base UI already guarantee, in `ui/litellm-dashboard/AGENTS.md` ## MCP suite: real Datadog only diff --git a/tests/e2e/codex/harness.py b/tests/e2e/codex/harness.py index df005b5b5e7..cc29581f2ce 100644 --- a/tests/e2e/codex/harness.py +++ b/tests/e2e/codex/harness.py @@ -12,7 +12,7 @@ from typing import Final from cli_harness.lockdown import isolated_home, locked_down_env from cli_harness.rate_limiter import RateBucket, RateLimiter -from codex.rollout import Compaction, rollout_compactions +from codex.rollout import Compaction, Subagent, rollout_compactions, rollout_parent_thread, rollout_subagent from codex.run import CodexRun, codex_run from e2e_metadata import step @@ -71,8 +71,28 @@ class CodexHarness: _ = config.write(_mcp_server_table(name, server)) @step('Run Codex on {model} through the proxy with the prompt "{prompt}"') - def exec(self, prompt: str, *, model: str, cwd: Path, compact_after_tokens: int | None = None) -> CodexRun: - return self._run(("exec",), (*_compaction_flags(compact_after_tokens), "--model", model, "--", prompt), cwd) + def exec( + self, + prompt: str, + *, + model: str, + cwd: Path, + compact_after_tokens: int | None = None, + enable: tuple[str, ...] = (), + disable: tuple[str, ...] = (), + ) -> CodexRun: + return self._run( + ("exec",), + ( + *_compaction_flags(compact_after_tokens), + *_feature_flags(enable=enable, disable=disable), + "--model", + model, + "--", + prompt, + ), + cwd, + ) @step('Resume Codex thread {thread_id} on {model} through the proxy with the prompt "{prompt}"') def resume( @@ -88,6 +108,12 @@ class CodexHarness: rollouts: Final = sorted(self.codex_home.glob(f"sessions/**/rollout-*-{thread_id}.jsonl")) return tuple(chain.from_iterable(rollout_compactions(rollout.read_text()) for rollout in rollouts)) + def subagents(self, thread_id: str) -> tuple[Subagent, ...]: + rollouts: Final = tuple( + path.read_text() for path in sorted(self.codex_home.glob("sessions/**/rollout-*.jsonl")) + ) + return tuple(rollout_subagent(rollout) for rollout in rollouts if rollout_parent_thread(rollout) == thread_id) + def _run(self, subcommand: tuple[str, ...], args: tuple[str, ...], cwd: Path) -> CodexRun: _ = self.limiter.acquire(self.rate_bucket) completed: Final = subprocess.run( @@ -116,6 +142,13 @@ def _mcp_server_table(name: str, server: McpServer) -> str: ) +def _feature_flags(*, enable: tuple[str, ...], disable: tuple[str, ...]) -> tuple[str, ...]: + return ( + *chain.from_iterable(("--enable", feature) for feature in enable), + *chain.from_iterable(("--disable", feature) for feature in disable), + ) + + def _compaction_flags(after_tokens: int | None) -> tuple[str, ...]: match after_tokens: case None: diff --git a/tests/e2e/codex/rollout.py b/tests/e2e/codex/rollout.py index fcaeaa7ca42..5b8253fab10 100644 --- a/tests/e2e/codex/rollout.py +++ b/tests/e2e/codex/rollout.py @@ -1,19 +1,23 @@ -"""The compactions Codex recorded in a thread's rollout file, `CODEX_HOME/sessions/**/rollout-*-.jsonl`. +"""What Codex recorded in a thread's rollout file, `CODEX_HOME/sessions/**/rollout-*-.jsonl`. -`codex exec --json` prints no event for a compaction, so the rollout is the only place it shows. Each line is -`{"timestamp", "type", "payload"}` (`codex-rs/history/src/rollout_payload.rs`) and only `compacted` lines are read. +`codex exec --json` prints no event for a compaction or for a sub-agent's own turn, so the rollout is the only place +they show. Each line is `{"timestamp", "type", "payload"}` (`codex-rs/history/src/rollout_payload.rs`). A remote compaction is one whose replacement history carries the provider's `compaction` item, while a local one -replaces the history with a summary Codex wrote itself. +replaces the history with a summary Codex wrote itself. A sub-agent gets a rollout of its own whose `session_meta` +names the thread that spawned it, and carries `forked_from_id` when it inherited that thread's conversation. Each turn +it finishes ends in a `task_complete` event with its last message. """ from __future__ import annotations from dataclasses import dataclass -from typing import Final +from typing import Final, Literal from pydantic import BaseModel, ConfigDict _COMPACTED: Final = "compacted" +_SESSION_META: Final = "session_meta" +_EVENT: Final = "event_msg" _PROVIDER_COMPACTION_ITEM: Final = "compaction" @@ -34,14 +38,58 @@ class _CompactedPayload(_Frozen): replacement_history: tuple[_HistoryItem, ...] | None = None +class _SessionMeta(_Frozen): + parent_thread_id: str | None = None + forked_from_id: str | None = None + + +class _Event(_Frozen): + type: str + + +class _TaskComplete(_Frozen): + type: Literal["task_complete"] + last_agent_message: str | None = None + + @dataclass(frozen=True, slots=True) class Compaction: remote: bool +@dataclass(frozen=True, slots=True) +class Subagent: + forked: bool + replies: tuple[str, ...] + + def rollout_compactions(rollout: str) -> tuple[Compaction, ...]: - lines: Final = tuple(_Line.model_validate_json(line) for line in rollout.splitlines() if line.strip()) - return tuple(_compaction(line.payload) for line in lines if line.type == _COMPACTED) + return tuple(_compaction(line.payload) for line in _lines(rollout) if line.type == _COMPACTED) + + +def rollout_parent_thread(rollout: str) -> str | None: + return _session_meta(rollout).parent_thread_id + + +def rollout_subagent(rollout: str) -> Subagent: + return Subagent(forked=_session_meta(rollout).forked_from_id is not None, replies=rollout_final_messages(rollout)) + + +def rollout_final_messages(rollout: str) -> tuple[str, ...]: + events: Final = (line.payload for line in _lines(rollout) if line.type == _EVENT) + completions: Final = ( + _TaskComplete.model_validate(event) for event in events if _Event.model_validate(event).type == "task_complete" + ) + return tuple(done.last_agent_message for done in completions if done.last_agent_message is not None) + + +def _session_meta(rollout: str) -> _SessionMeta: + meta: Final = next((line.payload for line in _lines(rollout) if line.type == _SESSION_META), None) + return _SessionMeta.model_validate(meta) if meta is not None else _SessionMeta() + + +def _lines(rollout: str) -> tuple[_Line, ...]: + return tuple(_Line.model_validate_json(line) for line in rollout.splitlines() if line.strip()) def _compaction(payload: object) -> Compaction: diff --git a/tests/e2e/codex/test_subagent_e2e.py b/tests/e2e/codex/test_subagent_e2e.py new file mode 100644 index 00000000000..0ad86c8b49a --- /dev/null +++ b/tests/e2e/codex/test_subagent_e2e.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from codex.harness import CodexHarness +from codex.setups import SETUPS, ModelSetup +from e2e_config import unique_marker + +pytestmark = pytest.mark.e2e + + +@pytest.mark.parametrize("deployment", SETUPS) +class TestCodexSubagent: + def test_a_subagent_reads_the_task_its_parent_hands_it(self, codex: CodexHarness, setup: ModelSetup) -> None: + codeword: Final = unique_marker() + run: Final = codex.exec( + 'Spawn exactly one sub-agent with the spawn_agent tool, with fork_turns set to "none" so it does not' + f" inherit this conversation. Its task: reply with exactly this codeword and nothing else: {codeword}." + " Wait for its reply, then tell me the exact reply it sent you, verbatim.", + model=setup.model, + cwd=codex.new_workspace(files={}), + enable=("multi_agent_v2",), + disable=("shell_tool", "unified_exec"), + ) + assert run.exit_code == 0, run + assert run.thread_id is not None, run + subagents: Final = codex.subagents(run.thread_id) + assert subagents, run + assert not any(subagent.forked for subagent in subagents), (subagents, run) + assert any(any(codeword in reply for reply in subagent.replies) for subagent in subagents), (subagents, run) diff --git a/tests/e2e_harness/codex/test_codex_rollout.py b/tests/e2e_harness/codex/test_codex_rollout.py index 2ef27a8a9c5..3db39066204 100644 --- a/tests/e2e_harness/codex/test_codex_rollout.py +++ b/tests/e2e_harness/codex/test_codex_rollout.py @@ -8,7 +8,14 @@ from typing import Final from cli_harness.rate_limiter import RateBucket, RateLimiter from codex.harness import CodexHarness -from codex.rollout import Compaction, rollout_compactions +from codex.rollout import ( + Compaction, + Subagent, + rollout_compactions, + rollout_final_messages, + rollout_parent_thread, + rollout_subagent, +) THREAD_ID: Final = "01a18fea-b2e8-7ed2-a29b-1ce4d2b138c3" USER_PROMPT: Final = { @@ -60,13 +67,83 @@ class TestRolloutCompactions: assert rollout_compactions(_rollout()) == () -def test_the_harness_reads_only_the_rollout_of_the_thread_it_is_asked_about(tmp_path: Path) -> None: - codex: Final = CodexHarness( - home=tmp_path, +def _harness(home: Path) -> CodexHarness: + return CodexHarness( + home=home, api_key="unused", rate_bucket=RateBucket.OPENAI, - limiter=RateLimiter({}, tmp_path, time.monotonic, time.sleep), + limiter=RateLimiter({}, home, time.monotonic, time.sleep), ) + + +def _subagent_rollout( + *, thread_id: str, parent: str | None, final_messages: tuple[str | None, ...], forked: bool = False +) -> str: + meta: Final = ( + {"id": thread_id, "cli_version": "0.165.0"} + | ({"parent_thread_id": parent} if parent else {}) + | ({"forked_from_id": parent} if forked else {}) + ) + inherited: Final = (_line("session_meta", {"id": parent, "cli_version": "0.165.0"}),) if forked else () + return "\n".join( + ( + _line("session_meta", meta), + *inherited, + _line("event_msg", {"type": "task_started", "turn_id": "t1"}), + _line( + "response_item", + {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "thinking"}]}, + ), + *( + _line("event_msg", {"type": "task_complete", "turn_id": "t1", "last_agent_message": message}) + for message in final_messages + ), + ) + ) + + +class TestRolloutSubagents: + def test_a_subagent_rollout_names_the_thread_that_spawned_it(self) -> None: + rollout: Final = _subagent_rollout(thread_id="child", parent=THREAD_ID, final_messages=()) + assert rollout_parent_thread(rollout) == THREAD_ID + + def test_a_top_level_rollout_has_no_parent(self) -> None: + assert rollout_parent_thread(_rollout()) is None + + def test_a_subagent_that_inherited_its_parents_conversation_is_forked(self) -> None: + rollout: Final = _subagent_rollout(thread_id="child", parent=THREAD_ID, final_messages=("hi",), forked=True) + assert rollout_subagent(rollout) == Subagent(forked=True, replies=("hi",)) + + def test_a_subagent_that_started_fresh_is_not_forked(self) -> None: + rollout: Final = _subagent_rollout(thread_id="child", parent=THREAD_ID, final_messages=("hi",)) + assert rollout_subagent(rollout) == Subagent(forked=False, replies=("hi",)) + + def test_the_final_messages_are_the_last_message_of_each_finished_turn(self) -> None: + rollout: Final = _subagent_rollout(thread_id="child", parent=THREAD_ID, final_messages=("first", None, "last")) + assert rollout_final_messages(rollout) == ("first", "last") + + +def test_the_harness_reads_only_the_subagents_of_the_thread_it_is_asked_about(tmp_path: Path) -> None: + codex: Final = _harness(tmp_path) + sessions: Final = codex.codex_home / "sessions" / "2026" / "11" / "06" + sessions.mkdir(parents=True) + rollouts: Final = { + THREAD_ID: _subagent_rollout(thread_id=THREAD_ID, parent=None, final_messages=("the parent's answer",)), + "child-a": _subagent_rollout(thread_id="child-a", parent=THREAD_ID, final_messages=("reply a",)), + "child-b": _subagent_rollout(thread_id="child-b", parent=THREAD_ID, final_messages=("reply b",), forked=True), + "stranger": _subagent_rollout(thread_id="stranger", parent="other-thread", final_messages=("not ours",)), + } + for thread_id, rollout in rollouts.items(): + _ = (sessions / f"rollout-2026-11-06T17-01-05-{thread_id}.jsonl").write_text(rollout) + + assert codex.subagents(THREAD_ID) == ( + Subagent(forked=False, replies=("reply a",)), + Subagent(forked=True, replies=("reply b",)), + ) + + +def test_the_harness_reads_only_the_rollout_of_the_thread_it_is_asked_about(tmp_path: Path) -> None: + codex: Final = _harness(tmp_path) sessions: Final = codex.codex_home / "sessions" / "2026" / "10" / "30" sessions.mkdir(parents=True) _ = (sessions / f"rollout-2026-10-30T20-03-32-{THREAD_ID}.jsonl").write_text(_rollout(REMOTE)) From 152a5cac7b26414f948ef553283403502e5189fc Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Sat, 7 Nov 2026 13:25:59 -0800 Subject: [PATCH 06/39] feat(ui): set deployment weights in key and team router settings (#50552) * feat(ui): set deployment weights in key and team router settings The proxy already applies router_settings.weights per key and team, but no dashboard screen could set them, and saving a team in the UI dropped weights set through the API Co-Authored-By: Claude Opus 5.5 * fix(ui): only offer deployments the key or team may weight Co-Authored-By: Claude Opus 5.5 * fix(ui): show weight load errors, inherited strategy and team deployments on key create Co-Authored-By: Claude Opus 5.5 * fix(ui): keep reliability fields on create forms and label weight inputs by deployment id Co-Authored-By: Claude Opus 5.5 * style(ui): name the accordion value identity object Co-Authored-By: Claude Opus 5.5 * test(ui): expect weights in the key create router settings payload Co-Authored-By: Claude Opus 5.5 * fix(ui): keep weight drafts, loading state and team switches correct Co-Authored-By: Claude Opus 5.5 * fix(ui): only accept weight keystrokes that save as a number Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- ...uterSettingsAccordion.integration.test.tsx | 44 ++++ .../RouterSettingsAccordion.test.tsx | 189 +++++++++++++++- .../RouterSettingsAccordion.tsx | 58 ++++- .../RouterSettingsSummary.test.tsx | 9 + .../RouterSettingsSummary.tsx | 24 ++ .../RouterWeightsSection.tsx | 206 ++++++++++++++++++ .../routerSettingsPayload.test.ts | 9 + .../routerSettingsPayload.ts | 1 + .../common_components/routerWeights.test.ts | 152 +++++++++++++ .../common_components/routerWeights.ts | 105 +++++++++ .../create_key_button.integration.test.tsx | 1 + .../organisms/create_key_button.tsx | 1 + .../src/components/team/TeamInfo.tsx | 2 + ui/litellm-dashboard/src/i18n/locales/en.json | 16 ++ 14 files changed, 804 insertions(+), 13 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.integration.test.tsx create mode 100644 ui/litellm-dashboard/src/components/common_components/RouterWeightsSection.tsx create mode 100644 ui/litellm-dashboard/src/components/common_components/routerWeights.test.ts create mode 100644 ui/litellm-dashboard/src/components/common_components/routerWeights.ts diff --git a/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.integration.test.tsx b/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.integration.test.tsx new file mode 100644 index 00000000000..0f23e91207a --- /dev/null +++ b/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.integration.test.tsx @@ -0,0 +1,44 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { act, render, screen } from "@testing-library/react"; +import { useState } from "react"; +import { describe, expect, it, vi } from "vitest"; +import RouterSettingsAccordion, { RouterSettingsAccordionValue } from "./RouterSettingsAccordion"; + +vi.mock("../networking", () => ({ + getRouterSettingsCall: vi.fn().mockResolvedValue({}), + modelInfoCall: vi.fn().mockResolvedValue({ data: [] }), +})); + +vi.mock("@/components/llm_calls/fetch_models", () => ({ + fetchAvailableModels: vi.fn().mockResolvedValue([{ model_group: "gpt-5" }]), + fetchAvailableModelsForTeam: vi.fn().mockResolvedValue([{ model_group: "gpt-5" }]), +})); + +const EchoingCreateForm = () => { + const [value, setValue] = useState(undefined); + return ; +}; + +describe("RouterSettingsAccordion on a create form", () => { + it("should render every reliability input after the parent echoes the first value back", async () => { + render( + + + , + ); + await act(async () => { + await new Promise((resolve) => setTimeout(resolve, 250)); + }); + + for (const field of [ + "allowed_fails", + "cooldown_time", + "num_retries", + "timeout", + "retry_after", + "model_group_alias", + ]) { + expect(screen.getByRole("textbox", { name: new RegExp(field) })).toBeEnabled(); + } + }); +}); diff --git a/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.test.tsx b/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.test.tsx index 6f819607e3f..827c1b60636 100644 --- a/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.test.tsx +++ b/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.test.tsx @@ -1,13 +1,18 @@ import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { act, fireEvent, render, screen, waitFor } from "@testing-library/react"; -import type { ReactElement, ReactNode } from "react"; +import { createRef, useState, type ReactElement, type ReactNode } from "react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { RouterSettingsFormValue } from "../router_settings/RouterSettingsForm"; import { fetchAvailableModels, fetchAvailableModelsForTeam } from "@/components/llm_calls/fetch_models"; -import RouterSettingsAccordion, { RouterSettingsAccordionValue } from "./RouterSettingsAccordion"; +import { getRouterSettingsCall, modelInfoCall } from "../networking"; +import RouterSettingsAccordion, { + RouterSettingsAccordionRef, + RouterSettingsAccordionValue, +} from "./RouterSettingsAccordion"; vi.mock("../networking", () => ({ getRouterSettingsCall: vi.fn().mockResolvedValue({}), + modelInfoCall: vi.fn(), })); vi.mock("@/components/llm_calls/fetch_models", () => ({ @@ -130,6 +135,186 @@ describe("RouterSettingsAccordion", () => { expect(screen.getByTestId("available-models")).not.toHaveTextContent("slow-team-model"); }); + describe("deployment weights", () => { + const gpt5Deployments = { + data: [ + { model_name: "gpt-5", litellm_params: { model: "openai/gpt-5" }, model_info: { id: "dep-openai" } }, + { model_name: "gpt-5", litellm_params: { model: "azure/gpt-5" }, model_info: { id: "dep-azure" } }, + ], + }; + + beforeEach(() => { + vi.mocked(modelInfoCall).mockResolvedValue(gpt5Deployments); + }); + + it("should save stored weights together with a newly entered one", async () => { + const ref = createRef(); + renderWithQueryClient( + , + ); + + fireEvent.change(await screen.findByLabelText("Weight for azure/gpt-5 (dep-azure)"), { target: { value: "1" } }); + + expect(await screen.findByText("90% of traffic")).toBeInTheDocument(); + expect(screen.getByText("10% of traffic")).toBeInTheDocument(); + expect(ref.current?.getValue().router_settings.weights).toStrictEqual({ + "gpt-5": { "dep-openai": 9, "dep-azure": 1 }, + }); + expect(vi.mocked(modelInfoCall).mock.calls[0][7]).toBe("team-123"); + expect(vi.mocked(modelInfoCall).mock.calls[0][11]).toBe("gpt-5"); + }); + + it("should save null once every weight is cleared, so the stored weights are removed", async () => { + const ref = createRef(); + renderWithQueryClient( + , + ); + + fireEvent.change(await screen.findByLabelText("Weight for openai/gpt-5 (dep-openai)"), { target: { value: "" } }); + + await waitFor(() => expect(ref.current?.getValue().router_settings.weights).toBeNull()); + }); + + it("should keep a model group on screen when the parent echoes back the value it was sent", async () => { + const EchoingParent = () => { + const [value, setValue] = useState({ + router_settings: { weights: { "gpt-5": { "dep-openai": 9 } } }, + }); + return ; + }; + renderWithQueryClient(); + + fireEvent.change(await screen.findByLabelText("Weight for openai/gpt-5 (dep-openai)"), { target: { value: "" } }); + await act(async () => { + vi.advanceTimersByTime(200); + }); + + expect(screen.getByLabelText("Weight for openai/gpt-5 (dep-openai)")).toHaveValue(""); + expect(screen.getByRole("button", { name: "Remove weights for gpt-5" })).toBeInTheDocument(); + }); + + it("should say the deployments failed to load instead of calling saved weights stale", async () => { + vi.mocked(modelInfoCall).mockRejectedValue(new Error("403")); + const ref = createRef(); + renderWithQueryClient( + , + ); + + expect( + await screen.findByText("Could not load this model group's deployments. Saved weights are kept."), + ).toBeInTheDocument(); + expect(screen.queryByText("Deployment no longer exists")).not.toBeInTheDocument(); + expect(ref.current?.getValue().router_settings.weights).toStrictEqual({ "gpt-5": { "dep-openai": 9 } }); + }); + + it("should warn that weights are ignored when the inherited global strategy is not simple-shuffle", async () => { + vi.mocked(getRouterSettingsCall).mockResolvedValueOnce({ + fields: [{ field_name: "routing_strategy", options: ["simple-shuffle"], field_value: "latency-based-routing" }], + }); + renderWithQueryClient( + , + ); + + expect( + await screen.findByText("Weights are ignored with latency-based-routing. Select simple-shuffle to use them."), + ).toBeInTheDocument(); + }); + + it("should accept a decimal weight typed one character at a time", async () => { + const ref = createRef(); + renderWithQueryClient( + , + ); + const input = await screen.findByLabelText("Weight for azure/gpt-5 (dep-azure)"); + + fireEvent.change(input, { target: { value: "1." } }); + expect(input).toHaveValue("1."); + fireEvent.change(input, { target: { value: "1.5" } }); + + expect(input).toHaveValue("1.5"); + expect(ref.current?.getValue().router_settings.weights).toStrictEqual({ + "gpt-5": { "dep-openai": 9, "dep-azure": 1.5 }, + }); + }); + + it("should ignore keystrokes that would not save as a weight, so the field always shows the saved value", async () => { + const ref = createRef(); + renderWithQueryClient( + , + ); + const input = await screen.findByLabelText("Weight for openai/gpt-5 (dep-openai)"); + + fireEvent.change(input, { target: { value: "9x" } }); + fireEvent.change(input, { target: { value: "-1" } }); + + expect(input).toHaveValue("9"); + expect(ref.current?.getValue().router_settings.weights).toStrictEqual({ "gpt-5": { "dep-openai": 9 } }); + }); + + it("should not call saved weights stale while the deployments are still loading", async () => { + vi.mocked(modelInfoCall).mockReturnValue(new Promise(() => {})); + renderWithQueryClient( + , + ); + + expect(await screen.findByText("dep-openai")).toBeInTheDocument(); + expect(screen.queryByText("Deployment no longer exists")).not.toBeInTheDocument(); + }); + + it("should drop weights picked for one team when the form switches to another team", async () => { + const ref = createRef(); + const value = { router_settings: { weights: { "gpt-5": { "dep-openai": 9 } } } }; + const { rerender } = renderWithQueryClient( + , + ); + await screen.findByLabelText("Weight for openai/gpt-5 (dep-openai)"); + + rerender(); + + await waitFor(() => expect(ref.current?.getValue().router_settings.weights).toBeNull()); + }); + + it("should flag a group whose weights are all zero", async () => { + renderWithQueryClient( + , + ); + + expect( + await screen.findByText("Set at least one weight above 0, or remove this model group."), + ).toBeInTheDocument(); + }); + }); + it("does not call onChange when unmounted mid-wait", async () => { const onChange = vi.fn<(value: RouterSettingsAccordionValue) => void>(); const { unmount } = renderWithQueryClient(); diff --git a/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.tsx b/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.tsx index 68fbd294b7c..a89c4138970 100644 --- a/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.tsx +++ b/ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.tsx @@ -9,6 +9,8 @@ import { Fallbacks } from "../Settings/RouterSettings/Fallbacks/AddFallbacks"; import { FallbackSelectionForm } from "../Settings/RouterSettings/Fallbacks/FallbackSelectionForm"; import { FallbackGroup } from "../Settings/RouterSettings/Fallbacks/FallbackGroupConfig"; import { fetchAvailableModels, fetchAvailableModelsForTeam, ModelGroup } from "@/components/llm_calls/fetch_models"; +import RouterWeightsSection from "./RouterWeightsSection"; +import { RouterWeights, routerWeightsFrom, savedRouterWeights } from "./routerWeights"; export interface RouterSettingsAccordionValue { router_settings: { @@ -24,6 +26,7 @@ export interface RouterSettingsAccordionValue { model_group_alias?: { [key: string]: any } | null; enable_tag_filtering?: boolean; routing_strategy_args?: { [key: string]: any } | null; + weights?: RouterWeights | null; }; } @@ -41,20 +44,38 @@ export interface RouterSettingsAccordionRef { const PROPAGATE_WAIT_MS = 100; +const EMPTY_RELIABILITY_SETTINGS = { + allowed_fails: null, + cooldown_time: null, + num_retries: null, + timeout: null, + retry_after: null, + model_group_alias: null, +}; + +const valueKeyOf = (settings: RouterSettingsAccordionValue["router_settings"]): string => { + const { routing_strategy, fallbacks, enable_tag_filtering, weights } = settings; + const identity = { routing_strategy, fallbacks, enable_tag_filtering, weights }; + return JSON.stringify(identity); +}; + const RouterSettingsAccordion = forwardRef( ({ accessToken, value, onChange, modelData, teamId }, ref) => { const { t } = useTranslation(); const [formValue, setFormValue] = useState({ - routerSettings: {}, + routerSettings: EMPTY_RELIABILITY_SETTINGS, selectedStrategy: null, enableTagFiltering: false, }); const [fallbacks, setFallbacks] = useState([]); const [fallbackGroups, setFallbackGroups] = useState([]); + const [weights, setWeights] = useState({}); const [availableRoutingStrategies, setAvailableRoutingStrategies] = useState([]); + const [globalRoutingStrategy, setGlobalRoutingStrategy] = useState(null); const [routerFieldsMetadata, setRouterFieldsMetadata] = useState<{ [key: string]: any }>({}); const [routingStrategyDescriptions, setRoutingStrategyDescriptions] = useState<{ [key: string]: string }>({}); const isInternalUpdateRef = useRef(false); + const weightsTeamIdRef = useRef(teamId); const lastInitializedValueRef = useRef(null); // Convert fallbacks format to groups format @@ -90,13 +111,7 @@ const RouterSettingsAccordion = forwardRef { // Create a stable key from the value to detect actual external changes - const valueKey = value?.router_settings - ? JSON.stringify({ - routing_strategy: value.router_settings.routing_strategy, - fallbacks: value.router_settings.fallbacks, - enable_tag_filtering: value.router_settings.enable_tag_filtering, - }) - : null; + const valueKey = value?.router_settings ? valueKeyOf(value.router_settings) : null; // Skip if this is an internal update (from our own onChange) and the value hasn't actually changed if (isInternalUpdateRef.current && valueKey === lastInitializedValueRef.current) { @@ -118,7 +133,7 @@ const RouterSettingsAccordion = forwardRef { + if (weightsTeamIdRef.current === teamId) return; + weightsTeamIdRef.current = teamId; + setWeights({}); + }, [teamId]); + // Fetch router settings metadata useEffect(() => { if (!accessToken) { @@ -169,6 +192,7 @@ const RouterSettingsAccordion = forwardRef { setFallbackGroups(newGroups); @@ -360,6 +386,16 @@ const RouterSettingsAccordion = forwardRef +
+ +
{ expect(screen.getByText("Number of Retries: 3")).toBeInTheDocument(); }); + it("should list each weighted model group with its deployment weights", () => { + render(); + + expect(screen.getByText("Deployment Weights:")).toBeInTheDocument(); + expect(screen.getByText("gpt-5")).toBeInTheDocument(); + expect(screen.getByText("dep-a: 9, dep-b: 1")).toBeInTheDocument(); + expect(screen.queryByText("empty")).not.toBeInTheDocument(); + }); + it("should show the empty state when every setting is null", () => { render(); diff --git a/ui/litellm-dashboard/src/components/common_components/RouterSettingsSummary.tsx b/ui/litellm-dashboard/src/components/common_components/RouterSettingsSummary.tsx index f99a154fe3d..89ec0873920 100644 --- a/ui/litellm-dashboard/src/components/common_components/RouterSettingsSummary.tsx +++ b/ui/litellm-dashboard/src/components/common_components/RouterSettingsSummary.tsx @@ -2,6 +2,7 @@ import { useTranslation } from "react-i18next"; import { Badge } from "@/components/ui/badge"; import i18n from "@/i18n/i18n"; import { hasRouterSettings } from "./routerSettingsPayload"; +import { routerWeightsFrom, savedRouterWeights } from "./routerWeights"; interface RouterSettingsSummaryProps { routerSettings: Record | null | undefined; @@ -15,6 +16,28 @@ const fallbackEntries = (fallbacks: unknown): Array<[string, string[]]> => { ); }; +export function RouterWeightsSummary({ weights }: { weights: unknown }) { + const { t } = useTranslation(); + const saved = savedRouterWeights(routerWeightsFrom(weights)); + if (!saved) return null; + return ( +
+
{t("routerWeights.summary")}
+
+ {Object.entries(saved).map(([group, groupWeights]) => ( +
+ {group} + -> + {Object.entries(groupWeights) + .map(([deploymentId, weight]) => `${deploymentId}: ${weight}`) + .join(", ")} +
+ ))} +
+
+ ); +} + export default function RouterSettingsSummary({ routerSettings, emptyText = i18n.t("routerSettingsSummary.empty"), @@ -63,6 +86,7 @@ export default function RouterSettingsSummary({ )} + ); } diff --git a/ui/litellm-dashboard/src/components/common_components/RouterWeightsSection.tsx b/ui/litellm-dashboard/src/components/common_components/RouterWeightsSection.tsx new file mode 100644 index 00000000000..32c5e70a71f --- /dev/null +++ b/ui/litellm-dashboard/src/components/common_components/RouterWeightsSection.tsx @@ -0,0 +1,206 @@ +import { useQuery } from "@tanstack/react-query"; +import { AlertCircle, X } from "lucide-react"; +import { useId, useState } from "react"; +import { useTranslation } from "react-i18next"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { SearchSelect } from "@/components/shared/SearchSelect"; +import { modelInfoCall } from "../networking"; +import { + GroupWeights, + RouterWeights, + WeightableDeployment, + hasNoPositiveWeight, + parseWeightInput, + weightRows, + withDeploymentWeight, + withGroup, + withoutGroup, +} from "./routerWeights"; + +const WEIGHTED_STRATEGY = "simple-shuffle"; +const DEPLOYMENTS_PAGE_SIZE = 1000; + +interface RouterWeightsSectionProps { + accessToken: string; + teamId?: string | null; + availableModels: string[]; + selectedStrategy: string | null; + weights: RouterWeights; + onChange: (weights: RouterWeights) => void; +} + +interface WeightGroupProps { + accessToken: string; + teamId?: string | null; + group: string; + groupWeights: GroupWeights; + onWeightChange: (deploymentId: string, weight: number | null) => void; + onRemove: () => void; +} + +interface WeightInputProps { + label: string; + weight: number | null; + onChange: (weight: number | null) => void; +} + +const formatWeight = (weight: number | null): string => (weight === null ? "" : String(weight)); + +function WeightInput({ label, weight, onChange }: WeightInputProps) { + const [draft, setDraft] = useState(formatWeight(weight)); + const [shownWeight, setShownWeight] = useState(weight); + if (weight !== shownWeight) { + setShownWeight(weight); + if (parseWeightInput(draft) !== weight) setDraft(formatWeight(weight)); + } + + const handleChange = (event: React.ChangeEvent) => { + const parsed = parseWeightInput(event.target.value); + if (parsed === undefined) return; + setDraft(event.target.value); + onChange(parsed); + }; + + return ( + + ); +} + +function WeightGroup({ accessToken, teamId, group, groupWeights, onWeightChange, onRemove }: WeightGroupProps) { + const { t } = useTranslation(); + const { + data: deployments = [], + isLoading, + isError, + } = useQuery({ + queryKey: ["routerWeightDeployments", accessToken, teamId ?? null, group], + queryFn: async () => + ( + await modelInfoCall( + accessToken, + "", + "", + 1, + DEPLOYMENTS_PAGE_SIZE, + undefined, + undefined, + teamId ?? undefined, + undefined, + undefined, + undefined, + group, + ) + ).data ?? [], + enabled: Boolean(accessToken), + }); + const rows = weightRows(group, groupWeights, deployments, teamId ?? null); + const deploymentsKnown = !isLoading && !isError; + const noDeployments = deploymentsKnown && rows.length === 0; + + return ( +
+
+ {group} + +
+ {isError && ( +

+ {t("routerWeights.loadFailed")} +

+ )} + {noDeployments &&

{t("routerWeights.noDeployments")}

} + {rows.map((row) => ( +
+
+
+ {row.label ?? (deploymentsKnown ? t("routerWeights.unknownDeployment") : "")} +
+
{row.deploymentId}
+
+ onWeightChange(row.deploymentId, weight)} + /> + + {row.sharePercent === null ? "" : t("routerWeights.share", { percent: row.sharePercent })} + +
+ ))} + {hasNoPositiveWeight(groupWeights) && ( +

+ + {t("routerWeights.noPositiveWeight")} +

+ )} +
+ ); +} + +export default function RouterWeightsSection({ + accessToken, + teamId, + availableModels, + selectedStrategy, + weights, + onChange, +}: RouterWeightsSectionProps) { + const { t } = useTranslation(); + const addGroupInputId = useId(); + const unweightedModels = availableModels.filter((model) => !(model in weights)); + const ignoredByStrategy = Boolean(selectedStrategy) && selectedStrategy !== WEIGHTED_STRATEGY; + + return ( +
+
+

{t("routerWeights.title")}

+

{t("routerWeights.description")}

+
+ {ignoredByStrategy && Object.keys(weights).length > 0 && ( +
+ + {t("routerWeights.ignoredForStrategy", { strategy: selectedStrategy })} +
+ )} + {Object.entries(weights).map(([group, groupWeights]) => ( + + onChange(withDeploymentWeight(weights, group, deploymentId, weight)) + } + onRemove={() => onChange(withoutGroup(weights, group))} + /> + ))} +
+ + ({ label: model, value: model }))} + value={null} + onValueChange={(group) => { + if (group) onChange(withGroup(weights, group)); + }} + placeholder={t("routerWeights.addGroupPlaceholder")} + emptyText={t("routerWeights.noModels")} + /> +
+
+ ); +} diff --git a/ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.test.ts b/ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.test.ts index 542ed17d7a2..05645a50b41 100644 --- a/ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.test.ts +++ b/ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.test.ts @@ -101,9 +101,18 @@ describe("routerSettingsUpdate", () => { model_group_alias: null, enable_tag_filtering: null, routing_strategy_args: null, + weights: null, }); }); + it("should replace stored weights with the edited ones, including clearing them", () => { + const stored = { weights: { "gpt-4": { "dep-a": 9, "dep-b": 1 } }, num_retries: 2 }; + expect(routerSettingsUpdate({ num_retries: 2, weights: { "gpt-4": { "dep-a": 1 } } }, stored)).toMatchObject({ + weights: { "gpt-4": { "dep-a": 1 } }, + }); + expect(routerSettingsUpdate({ num_retries: 2, weights: null }, stored)).toMatchObject({ weights: null }); + }); + it("should send the edited settings when the user configured something", () => { expect(routerSettingsUpdate({ fallbacks }, null)).toMatchObject({ fallbacks }); }); diff --git a/ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.ts b/ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.ts index 1d048425a6d..1bf6d0ef252 100644 --- a/ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.ts +++ b/ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.ts @@ -15,6 +15,7 @@ const EDITOR_OWNED_FIELDS: Record = { model_group_alias: true, enable_tag_filtering: true, routing_strategy_args: true, + weights: true, }; const EDITOR_OWNED_KEYS = Object.keys(EDITOR_OWNED_FIELDS) as Array; diff --git a/ui/litellm-dashboard/src/components/common_components/routerWeights.test.ts b/ui/litellm-dashboard/src/components/common_components/routerWeights.test.ts new file mode 100644 index 00000000000..3df42281bf5 --- /dev/null +++ b/ui/litellm-dashboard/src/components/common_components/routerWeights.test.ts @@ -0,0 +1,152 @@ +import { describe, expect, it } from "vitest"; +import { + hasNoPositiveWeight, + parseWeightInput, + routerWeightsFrom, + savedRouterWeights, + weightRows, + withDeploymentWeight, + withGroup, + withoutGroup, +} from "./routerWeights"; + +const deployment = (id: string, modelName: string, model: string, team?: { id: string; publicName: string }) => ({ + model_name: modelName, + litellm_params: { model }, + model_info: { id, ...(team ? { team_id: team.id, team_public_model_name: team.publicName } : {}) }, +}); + +describe("routerWeightsFrom", () => { + it("should keep well-formed groups and drop malformed weights from the stored blob", () => { + expect( + routerWeightsFrom({ + "gpt-5": { a: 9, b: 1, bad: "1", negative: -1, inf: Infinity }, + broken: [1, 2], + }), + ).toStrictEqual({ "gpt-5": { a: 9, b: 1 } }); + }); + + it("should read missing or non-object weights as none", () => { + expect(routerWeightsFrom(undefined)).toStrictEqual({}); + expect(routerWeightsFrom(null)).toStrictEqual({}); + expect(routerWeightsFrom("x")).toStrictEqual({}); + }); +}); + +describe("weight edits", () => { + it("should set, replace and clear one deployment without touching others", () => { + const start = { "gpt-5": { a: 9 }, other: { c: 2 } }; + const added = withDeploymentWeight(start, "gpt-5", "b", 1); + expect(added).toStrictEqual({ "gpt-5": { a: 9, b: 1 }, other: { c: 2 } }); + expect(withDeploymentWeight(added, "gpt-5", "a", 3)).toStrictEqual({ "gpt-5": { a: 3, b: 1 }, other: { c: 2 } }); + expect(withDeploymentWeight(added, "gpt-5", "a", null)).toStrictEqual({ "gpt-5": { b: 1 }, other: { c: 2 } }); + expect(start).toStrictEqual({ "gpt-5": { a: 9 }, other: { c: 2 } }); + }); + + it("should add an empty group once and remove a group entirely", () => { + const withEmpty = withGroup({ "gpt-5": { a: 9 } }, "claude"); + expect(withEmpty).toStrictEqual({ "gpt-5": { a: 9 }, claude: {} }); + expect(withGroup(withEmpty, "gpt-5")).toBe(withEmpty); + expect(withoutGroup(withEmpty, "gpt-5")).toStrictEqual({ claude: {} }); + }); +}); + +describe("savedRouterWeights", () => { + it("should drop groups with no weights so they are not saved", () => { + expect(savedRouterWeights({ "gpt-5": { a: 0, b: 2 }, claude: {} })).toStrictEqual({ "gpt-5": { a: 0, b: 2 } }); + }); + + it("should save no weights as null so the field is cleared", () => { + expect(savedRouterWeights({})).toBeNull(); + expect(savedRouterWeights({ claude: {} })).toBeNull(); + }); +}); + +describe("hasNoPositiveWeight", () => { + it("should flag a group whose weights are all zero, as the proxy rejects it", () => { + expect(hasNoPositiveWeight({ a: 0, b: 0 })).toBe(true); + expect(hasNoPositiveWeight({ a: 0, b: 0.5 })).toBe(false); + expect(hasNoPositiveWeight({})).toBe(false); + }); +}); + +describe("weightRows", () => { + it("should list the group's deployments with each one's share of traffic", () => { + const rows = weightRows( + "gpt-5", + { a: 9, b: 1 }, + [ + deployment("a", "gpt-5", "openai/gpt-5"), + deployment("b", "gpt-5", "azure/gpt-5"), + deployment("c", "gpt-5", "bedrock/gpt-5"), + ], + null, + ); + expect(rows).toStrictEqual([ + { deploymentId: "a", label: "openai/gpt-5", weight: 9, sharePercent: 90 }, + { deploymentId: "b", label: "azure/gpt-5", weight: 1, sharePercent: 10 }, + { deploymentId: "c", label: "bedrock/gpt-5", weight: null, sharePercent: 0 }, + ]); + }); + + it("should match team deployments by their public model name, not their internal routing name", () => { + const rows = weightRows( + "gpt-5", + {}, + [ + deployment("team-dep", "model_name_team-1_abc", "openai/gpt-5", { id: "team-1", publicName: "gpt-5" }), + deployment("other-group", "gpt-5-mini", "openai/gpt-5-mini"), + ], + "team-1", + ); + expect(rows.map((row) => row.deploymentId)).toStrictEqual(["team-dep"]); + }); + + it("should group a deployment with no team by its model name even if it carries a public name", () => { + const orphanedPublicName = { + model_name: "gpt-5-internal", + litellm_params: { model: "openai/gpt-5" }, + model_info: { id: "x", team_public_model_name: "gpt-5" }, + }; + expect(weightRows("gpt-5", {}, [orphanedPublicName], null)).toStrictEqual([]); + expect(weightRows("gpt-5-internal", {}, [orphanedPublicName], null)).toHaveLength(1); + }); + + it("should leave out another team's deployments, which the proxy rejects", () => { + const deployments = [ + deployment("global", "gpt-5", "openai/gpt-5"), + deployment("team-1-dep", "model_name_team-1_abc", "openai/gpt-5", { id: "team-1", publicName: "gpt-5" }), + deployment("team-2-dep", "model_name_team-2_def", "openai/gpt-5", { id: "team-2", publicName: "gpt-5" }), + ]; + expect(weightRows("gpt-5", {}, deployments, "team-1").map((row) => row.deploymentId)).toStrictEqual([ + "global", + "team-1-dep", + ]); + expect(weightRows("gpt-5", {}, deployments, null).map((row) => row.deploymentId)).toStrictEqual(["global"]); + }); + + it("should show no share until some weight is set", () => { + const rows = weightRows("gpt-5", {}, [deployment("a", "gpt-5", "openai/gpt-5")], null); + expect(rows[0].sharePercent).toBeNull(); + }); + + it("should keep a weighted id whose deployment is gone so it can still be cleared", () => { + const rows = weightRows("gpt-5", { a: 1, deleted: 1 }, [deployment("a", "gpt-5", "openai/gpt-5")], null); + const staleRow = { deploymentId: "deleted", label: null, weight: 1, sharePercent: 50 }; + expect(rows[1]).toStrictEqual(staleRow); + }); +}); + +describe("parseWeightInput", () => { + it("should accept non-negative numbers, clear on empty and reject the rest", () => { + expect(parseWeightInput("2.5")).toBe(2.5); + expect(parseWeightInput("0")).toBe(0); + expect(parseWeightInput(" ")).toBeNull(); + expect(parseWeightInput("-1")).toBeUndefined(); + expect(parseWeightInput("abc")).toBeUndefined(); + expect(parseWeightInput("1.")).toBe(1); + expect(parseWeightInput(".")).toBeUndefined(); + expect(parseWeightInput("1e3")).toBeUndefined(); + expect(parseWeightInput("0x10")).toBeUndefined(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/common_components/routerWeights.ts b/ui/litellm-dashboard/src/components/common_components/routerWeights.ts new file mode 100644 index 00000000000..16f428cade8 --- /dev/null +++ b/ui/litellm-dashboard/src/components/common_components/routerWeights.ts @@ -0,0 +1,105 @@ +export type GroupWeights = Readonly>; +export type RouterWeights = Readonly>; + +export interface WeightableDeployment { + model_name?: string | null; + litellm_params?: { model?: string | null } | null; + model_info?: { id?: string | null; team_id?: string | null; team_public_model_name?: string | null } | null; +} + +export interface WeightRow { + deploymentId: string; + label: string | null; + weight: number | null; + sharePercent: number | null; +} + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); + +const isWeight = (value: unknown): value is number => typeof value === "number" && Number.isFinite(value) && value >= 0; + +export const routerWeightsFrom = (stored: unknown): RouterWeights => + isRecord(stored) + ? Object.fromEntries( + Object.entries(stored) + .filter((entry): entry is [string, Record] => isRecord(entry[1])) + .map(([group, weights]) => [ + group, + Object.fromEntries( + Object.entries(weights).filter((entry): entry is [string, number] => isWeight(entry[1])), + ), + ]), + ) + : {}; + +const deploymentGroup = (deployment: WeightableDeployment): string | null => + (deployment.model_info?.team_id ? deployment.model_info.team_public_model_name : null) || + deployment.model_name || + null; + +const isWeightable = (deployment: WeightableDeployment, group: string, teamId: string | null): boolean => { + const ownerTeamId = deployment.model_info?.team_id ?? null; + const visibleToCaller = ownerTeamId === null || ownerTeamId === teamId; + return deploymentGroup(deployment) === group && Boolean(deployment.model_info?.id) && visibleToCaller; +}; + +export const withGroup = (weights: RouterWeights, group: string): RouterWeights => + group in weights ? weights : { ...weights, [group]: {} }; + +export const withoutGroup = (weights: RouterWeights, group: string): RouterWeights => + Object.fromEntries(Object.entries(weights).filter(([name]) => name !== group)); + +export const withDeploymentWeight = ( + weights: RouterWeights, + group: string, + deploymentId: string, + weight: number | null, +): RouterWeights => { + const others = Object.fromEntries(Object.entries(weights[group] ?? {}).filter(([id]) => id !== deploymentId)); + return { ...weights, [group]: weight === null ? others : { ...others, [deploymentId]: weight } }; +}; + +export const savedRouterWeights = (weights: RouterWeights): RouterWeights | null => { + const nonEmpty = Object.entries(weights).filter(([, group]) => Object.keys(group).length > 0); + return nonEmpty.length > 0 ? Object.fromEntries(nonEmpty) : null; +}; + +export const hasNoPositiveWeight = (group: GroupWeights): boolean => + Object.keys(group).length > 0 && !Object.values(group).some((weight) => weight > 0); + +export const weightRows = ( + group: string, + groupWeights: GroupWeights, + deployments: readonly WeightableDeployment[], + teamId: string | null, +): WeightRow[] => { + const total = Object.values(groupWeights).reduce((sum, weight) => sum + weight, 0); + const row = (deploymentId: string, label: string | null): WeightRow => { + const weight = groupWeights[deploymentId] ?? null; + return { + deploymentId, + label, + weight, + sharePercent: total > 0 ? Math.round(((weight ?? 0) / total) * 100) : null, + }; + }; + const current = deployments + .filter((deployment) => isWeightable(deployment, group, teamId)) + .map((deployment) => row(deployment.model_info!.id!, deployment.litellm_params?.model ?? null)); + const currentIds = new Set(current.map(({ deploymentId }) => deploymentId)); + const stale = Object.keys(groupWeights) + .filter((deploymentId) => !currentIds.has(deploymentId)) + .map((deploymentId) => row(deploymentId, null)); + return [...current, ...stale]; +}; + +const WEIGHT_DRAFT = /^(\d+\.?\d*|\.\d+)?$/; + +export const parseWeightInput = (raw: string): number | null | undefined => { + const trimmed = raw.trim(); + if (!WEIGHT_DRAFT.test(trimmed)) return undefined; + if (trimmed === "") return null; + const weight = Number(trimmed); + return isWeight(weight) ? weight : undefined; +}; diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx index 0fbe043c0d1..bbc849167cc 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx @@ -173,6 +173,7 @@ const ROUTER_SETTINGS_DEFAULT = { model_group_alias: null, enable_tag_filtering: false, routing_strategy_args: null, + weights: null, }; const SECTION_PAYLOAD_ADDITIONS: Record> = { diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx index ce780a221fd..dee936c4be0 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx @@ -1784,6 +1784,7 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp key={routerSettingsKey} ref={routerSettingsRef} accessToken={accessToken || ""} + teamId={selectedCreateKeyTeam?.team_id} value={routerSettings || undefined} onChange={setRouterSettings} modelData={ diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx index c6b485a57a1..3d67bed5d93 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx @@ -114,6 +114,7 @@ import SearchToolSelector from "../search_tools/SearchToolSelector"; import SkillSelector from "../skills/SkillSelector"; import EditLoggingSettings from "./EditLoggingSettings"; import RouterSettingsAccordion, { RouterSettingsAccordionRef } from "../common_components/RouterSettingsAccordion"; +import { RouterWeightsSummary } from "../common_components/RouterSettingsSummary"; import { StorePromptsToggle } from "../common_components/StorePromptsToggle"; import MemberModal from "./EditMembership"; import MemberPermissions from "./member_permissions"; @@ -2673,6 +2674,7 @@ const TeamInfoView: React.FC = ({ {info.router_settings.enable_tag_filtering && (
{t("teams.teamInfo.settingsView.router.tagFilteringEnabled")}
)} + ) : (
{t("teams.teamInfo.settingsView.router.none")}
diff --git a/ui/litellm-dashboard/src/i18n/locales/en.json b/ui/litellm-dashboard/src/i18n/locales/en.json index d4ed30d51b2..06b3e4d6b19 100644 --- a/ui/litellm-dashboard/src/i18n/locales/en.json +++ b/ui/litellm-dashboard/src/i18n/locales/en.json @@ -20380,5 +20380,21 @@ "query": "query", "request": "request" } + }, + "routerWeights": { + "title": "Deployment Weights", + "description": "Split a model group's traffic across its deployments. A deployment with no weight gets no traffic once any weight in its group is set. Weights only apply with the simple-shuffle routing strategy.", + "ignoredForStrategy": "Weights are ignored with {{strategy}}. Select simple-shuffle to use them.", + "addGroup": "Add a model group", + "addGroupPlaceholder": "Select a model group to weight", + "noModels": "No model groups found", + "noDeployments": "No deployments found for this model group", + "unknownDeployment": "Deployment no longer exists", + "weightFor": "Weight for {{deployment}}", + "share": "{{percent}}% of traffic", + "noPositiveWeight": "Set at least one weight above 0, or remove this model group.", + "removeGroup": "Remove weights for {{group}}", + "summary": "Deployment Weights:", + "loadFailed": "Could not load this model group's deployments. Saved weights are kept." } } From b603e84c020cbfb554d4857dd81bde0a0572e0dc Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Sat, 7 Nov 2026 13:33:27 -0800 Subject: [PATCH 07/39] fix(mcp): derive MCP bridge and session keys with the FIPS provider's KBKDF (#50032) * fix(mcp): derive MCP bridge and session keys with the FIPS provider's KBKDF With OpenSSL enforcing FIPS, hashlib.scrypt raises, so every MCP session token and bridge envelope request failed with a 500. Under FIPS the keys now come from LITELLM_SALT_KEY through SP 800-108r1 KBKDF run by the provider's EVP_KDF. Without FIPS the scrypt keys are unchanged * fix(mcp): keep master key rotation invalidating MCP tokens under FIPS Pass the master key as the KBKDF context, so rotating it changes the three FIPS-mode keys as it changes the scrypt keys * test(mcp): annotate new FIPS key test locals with Final * fix(mcp): derive the FIPS MCP keys only when general_settings.fips_mode is on On a FIPS OpenSSL 1.1.1 host (RHEL 8) scrypt still works, so keying the switch on openssl_enforces_fips() sent those hosts to the OpenSSL 3 only KBKDF and broke MCP sessions and bridge envelopes that worked on main. fips_mode_active() is the same reader B1 adds, set by enforce_fips_mode at startup. * refactor(fips): move the salt key KBKDF into encrypt_decrypt_utils The KBKDF helper derives keys from LITELLM_SALT_KEY, so it now sits next to the other salt key helpers, and fips.py goes back to main's version. * Pin the MCP key KBKDF fetch to the FIPS provider Fetch KBKDF with the property query fips=yes, so a change to the process's default properties after boot can't move the derivation to a non-FIPS provider. The shared fips_openssl test fixture drops the pin, since the test OpenSSL has no FIPS provider. --- .../common_utils/encrypt_decrypt_utils.py | 69 +++++++++++++++++++ .../bridge_credentials.py | 10 +++ .../session_credentials.py | 9 +++ .../test_encrypt_decrypt_utils.py | 30 ++++++++ tests/unit/proxy/conftest.py | 1 + .../outbound_credentials/conftest.py | 20 ++++++ .../test_bridge_credentials.py | 31 +++++++++ .../test_session_credentials.py | 63 +++++++++++++++++ 8 files changed, 233 insertions(+) create mode 100644 tests/unit/proxy/mcp_server/outbound_credentials/conftest.py diff --git a/litellm/proxy/common_utils/encrypt_decrypt_utils.py b/litellm/proxy/common_utils/encrypt_decrypt_utils.py index e5f6bc5dda8..644a35ef4ef 100644 --- a/litellm/proxy/common_utils/encrypt_decrypt_utils.py +++ b/litellm/proxy/common_utils/encrypt_decrypt_utils.py @@ -1,11 +1,14 @@ import base64 +import ctypes import os +import re from collections.abc import Mapping from typing import Final, Literal, cast from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_proxy_logger +from litellm.proxy.auth.master_key_boot_check import SALT_KEY_ENV_VAR # Versioned ciphertext marker for AES-256-GCM values. # Format: "v2:gcm:" + base64url(nonce(12) || ciphertext || tag(16)). @@ -42,6 +45,72 @@ def get_salt_key() -> str | None: _get_salt_key: Final = get_salt_key +_SALT_KEY_HEX: Final = re.compile(r"(?:sk-)?((?:[0-9a-fA-F]{2}){32,})") +_OSSL_PARAM_UTF8_STRING: Final = 4 +_OSSL_PARAM_OCTET_STRING: Final = 5 +_OSSL_PARAM_UNMODIFIED: Final = ctypes.c_size_t(-1).value +_DERIVED_KEY_BYTES: Final = 32 +_KBKDF_PROPERTIES: Final = b"fips=yes" + + +class _OsslParam(ctypes.Structure): + _fields_ = ( + ("key", ctypes.c_char_p), + ("data_type", ctypes.c_uint), + ("data", ctypes.c_char_p), + ("data_size", ctypes.c_size_t), + ("return_size", ctypes.c_size_t), + ) + + +def _param(name: bytes, data_type: int, data: bytes) -> _OsslParam: + return _OsslParam(name, data_type, data, len(data), _OSSL_PARAM_UNMODIFIED) + + +def _kbkdf_hmac_sha256(key: bytes, label: bytes, context: bytes) -> bytes: + import _hashlib + + libcrypto: Final = ctypes.CDLL(_hashlib.__file__) + kdf_fetch: Final = ctypes.CFUNCTYPE(ctypes.c_void_p, ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p)( + ("EVP_KDF_fetch", libcrypto) + ) + ctx_new: Final = ctypes.CFUNCTYPE(ctypes.c_void_p, ctypes.c_void_p)(("EVP_KDF_CTX_new", libcrypto)) + derive: Final = ctypes.CFUNCTYPE( + ctypes.c_int, ctypes.c_void_p, ctypes.c_char_p, ctypes.c_size_t, ctypes.POINTER(_OsslParam) + )(("EVP_KDF_derive", libcrypto)) + ctx_free: Final = ctypes.CFUNCTYPE(None, ctypes.c_void_p)(("EVP_KDF_CTX_free", libcrypto)) + kdf_free: Final = ctypes.CFUNCTYPE(None, ctypes.c_void_p)(("EVP_KDF_free", libcrypto)) + params: Final = (_OsslParam * 6)( + _param(b"mac", _OSSL_PARAM_UTF8_STRING, b"HMAC"), + _param(b"digest", _OSSL_PARAM_UTF8_STRING, b"SHA256"), + _param(b"key", _OSSL_PARAM_OCTET_STRING, key), + _param(b"salt", _OSSL_PARAM_OCTET_STRING, label), + _param(b"info", _OSSL_PARAM_OCTET_STRING, context), + ) + derived: Final = ctypes.create_string_buffer(_DERIVED_KEY_BYTES) + kdf: Final = kdf_fetch(None, b"KBKDF", _KBKDF_PROPERTIES) + ctx: Final = ctx_new(kdf) + try: + if not ctx or derive(ctx, derived, _DERIVED_KEY_BYTES, params) != 1: + raise ValueError("OpenSSL could not run KBKDF") + return derived.raw + finally: + ctx_free(ctx) + kdf_free(kdf) + + +def derive_key_from_salt_key(label: bytes, context: bytes) -> str: + """Hex of 32 bytes derived from LITELLM_SALT_KEY under label and context by SP 800-108r1 KBKDF in counter mode + with HMAC-SHA256, run by EVP_KDF from the FIPS provider of the OpenSSL that hashlib uses. Raises ValueError when + LITELLM_SALT_KEY is not at least 32 bytes of hex, optionally prefixed with sk-, or when no FIPS provider runs it.""" + salt_key: Final = _SALT_KEY_HEX.fullmatch(os.getenv(SALT_KEY_ENV_VAR, "")) + if salt_key is None: + raise ValueError( + f"With general_settings.fips_mode on, {SALT_KEY_ENV_VAR} must be at least 32 bytes of hex, such as " + "sk-$(openssl rand -hex 32)" + ) + return _kbkdf_hmac_sha256(bytes.fromhex(salt_key.group(1)), label, context).hex() + def _get_encryption_algorithm() -> str: """ diff --git a/litellm/proxy/mcp_server/outbound_credentials/bridge_credentials.py b/litellm/proxy/mcp_server/outbound_credentials/bridge_credentials.py index 2326bcebdc3..cc3e99263c9 100644 --- a/litellm/proxy/mcp_server/outbound_credentials/bridge_credentials.py +++ b/litellm/proxy/mcp_server/outbound_credentials/bridge_credentials.py @@ -16,6 +16,8 @@ from typing import Final, Literal, TypeAlias from pydantic import ConfigDict, SecretStr +from litellm.proxy.common_utils.encrypt_decrypt_utils import derive_key_from_salt_key +from litellm.proxy.common_utils.fips import fips_mode_active from litellm.proxy.mcp_server.outbound_credentials.envelope import ( EnvelopeIdentity, EnvelopeKeys, @@ -61,7 +63,15 @@ def envelope_keys_from_master_key(master_key: str) -> EnvelopeKeys: fixed for a process), so the KDF runs once per key and adds nothing to the per-request admission path. The derivation is deterministic; rotating ``master_key`` invalidates every outstanding envelope, which is the intended behavior for a signing-key change. + + When general_settings.fips_mode is on, both keys come from :func:`derive_key_from_salt_key` with the master key as + context instead. """ + if fips_mode_active(): + return EnvelopeKeys( + signing_key=SecretStr(derive_key_from_salt_key(_SIGNING_KEY_DOMAIN, master_key.encode())), + encryption_key=SecretStr(derive_key_from_salt_key(_ENCRYPTION_KEY_DOMAIN, master_key.encode())), + ) signing: Final = hashlib.scrypt( master_key.encode(), salt=_SIGNING_KEY_DOMAIN, diff --git a/litellm/proxy/mcp_server/outbound_credentials/session_credentials.py b/litellm/proxy/mcp_server/outbound_credentials/session_credentials.py index 6e3d09f4dd2..7cb89e53cdb 100644 --- a/litellm/proxy/mcp_server/outbound_credentials/session_credentials.py +++ b/litellm/proxy/mcp_server/outbound_credentials/session_credentials.py @@ -22,6 +22,8 @@ from typing import Final, Literal, TypeAlias from pydantic import ConfigDict, Field, SecretStr, ValidationError +from litellm.proxy.common_utils.encrypt_decrypt_utils import derive_key_from_salt_key +from litellm.proxy.common_utils.fips import fips_mode_active from litellm.proxy.mcp_server.outbound_credentials.session_token import ( AsymmetricSessionKeys, OpenedSessionToken, @@ -59,7 +61,14 @@ def session_keys_from_master_key(master_key: str) -> SessionKeys: into the other. The result is cached (the master key is fixed for a process); rotating ``master_key`` invalidates every outstanding session, which is the intended behavior for a signing-key change. + + When general_settings.fips_mode is on, the key comes from :func:`derive_key_from_salt_key` with the master key as + context instead. """ + if fips_mode_active(): + return SessionKeys( + signing_key=SecretStr(derive_key_from_salt_key(_SESSION_SIGNING_KEY_DOMAIN, master_key.encode())) + ) signing: Final = hashlib.scrypt( master_key.encode(), salt=_SESSION_SIGNING_KEY_DOMAIN, diff --git a/tests/unit/proxy/common_utils/test_encrypt_decrypt_utils.py b/tests/unit/proxy/common_utils/test_encrypt_decrypt_utils.py index 1163090c510..5b0d5f014f8 100644 --- a/tests/unit/proxy/common_utils/test_encrypt_decrypt_utils.py +++ b/tests/unit/proxy/common_utils/test_encrypt_decrypt_utils.py @@ -6,8 +6,11 @@ gate, and the backward-compatibility guarantees that let legacy XSalsa20-Poly130 (nacl) ciphertext and new AES values coexist and decrypt correctly. """ +import _hashlib import base64 +import hmac import re +from typing import Final import pytest @@ -17,6 +20,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_bearer_token, decrypt_if_encrypted_with, decrypt_value_helper, + derive_key_from_salt_key, encrypt_bearer_token, encrypt_value, encrypt_value_helper, @@ -266,3 +270,29 @@ def test_bearer_token_uses_only_header_safe_characters(length: int): assert re.fullmatch(r"kind_a_[A-Za-z0-9_-]+", token), token assert decrypt_bearer_token(token, prefix="kind_a_") == "x" * length + + +_needs_shared_openssl: Final = pytest.mark.skipif( + not hasattr(_hashlib, "__file__"), reason="this Python links OpenSSL statically, out of ctypes reach" +) + + +@_needs_shared_openssl +@pytest.mark.usefixtures("fips_openssl") +@pytest.mark.parametrize("salt_key", ["sk-" + bytes(range(32)).hex(), bytes(range(32)).hex().upper()]) +def test_derive_key_from_salt_key_is_sp800_108_counter_mode_hmac_sha256( + monkeypatch: pytest.MonkeyPatch, salt_key: str +) -> None: + label: Final = b"litellm-test:label:" + context: Final = b"sk-test-master-key" + monkeypatch.setenv("LITELLM_SALT_KEY", salt_key) + fixed_input: Final = (1).to_bytes(4, "big") + label + b"\x00" + context + (256).to_bytes(4, "big") + assert derive_key_from_salt_key(label, context) == hmac.digest(bytes(range(32)), fixed_input, "sha256").hex() + + +@_needs_shared_openssl +@pytest.mark.skipif(_hashlib.get_fips_mode() == 1, reason="this OpenSSL runs a FIPS provider") +def test_derive_key_from_salt_key_refuses_an_openssl_without_a_fips_provider(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_SALT_KEY", "sk-" + bytes(range(32)).hex()) + with pytest.raises(ValueError, match="OpenSSL could not run KBKDF"): + derive_key_from_salt_key(b"litellm-test:label:", b"sk-test-master-key") diff --git a/tests/unit/proxy/conftest.py b/tests/unit/proxy/conftest.py index d08a98242ef..8d8781be0a8 100644 --- a/tests/unit/proxy/conftest.py +++ b/tests/unit/proxy/conftest.py @@ -523,6 +523,7 @@ def postgres_span_names() -> Iterator[Callable[[], Awaitable[tuple[str, ...]]]]: def fips_openssl(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(_hashlib, "get_fips_mode", lambda: 1) monkeypatch.setattr(_hashlib, "scrypt", MagicMock(side_effect=ValueError("[digital envelope routines] unsupported"))) + monkeypatch.setattr("litellm.proxy.common_utils.encrypt_decrypt_utils._KBKDF_PROPERTIES", None) @pytest.fixture diff --git a/tests/unit/proxy/mcp_server/outbound_credentials/conftest.py b/tests/unit/proxy/mcp_server/outbound_credentials/conftest.py new file mode 100644 index 00000000000..bd80cba1c62 --- /dev/null +++ b/tests/unit/proxy/mcp_server/outbound_credentials/conftest.py @@ -0,0 +1,20 @@ +from collections.abc import Iterator + +import pytest + +from litellm.proxy.mcp_server.outbound_credentials.bridge_credentials import envelope_keys_from_master_key +from litellm.proxy.mcp_server.outbound_credentials.session_credentials import session_keys_from_master_key + + +@pytest.fixture +def openssl_enforcing_fips(fips_openssl: None) -> Iterator[None]: + envelope_keys_from_master_key.cache_clear() + session_keys_from_master_key.cache_clear() + yield + envelope_keys_from_master_key.cache_clear() + session_keys_from_master_key.cache_clear() + + +@pytest.fixture +def fips_mode_on(openssl_enforcing_fips: None, fips_mode: None) -> None: + return None diff --git a/tests/unit/proxy/mcp_server/outbound_credentials/test_bridge_credentials.py b/tests/unit/proxy/mcp_server/outbound_credentials/test_bridge_credentials.py index 47b6c3c2fa9..0ba19d13955 100644 --- a/tests/unit/proxy/mcp_server/outbound_credentials/test_bridge_credentials.py +++ b/tests/unit/proxy/mcp_server/outbound_credentials/test_bridge_credentials.py @@ -9,6 +9,7 @@ through the consumer; and no path leaks the upstream token in a repr. """ from datetime import datetime, timedelta, timezone +from typing import Final import pytest from pydantic import SecretStr @@ -318,3 +319,33 @@ def test_is_bridge_envelope_shaped_rejects_non_envelope_bearer(): assert is_bridge_envelope_shaped("Bearer sk-some-litellm-key") is False assert is_bridge_envelope_shaped("plain-upstream-token") is False assert is_bridge_envelope_shaped("") is False + + +def test_envelope_keys_without_fips_stay_the_scrypt_keys_of_the_master_key() -> None: + keys: Final = envelope_keys_from_master_key(_MASTER_KEY) + assert (keys.signing_key.get_secret_value(), keys.encryption_key.get_secret_value()) == ( + "34d3bb6ae2e03cddca3417209304e9603417b89258d3979d0569880b6fbd1bed", + "eebfbee8cbe767ee136ce3a9745b44de2b2ae35d1ea46dcf72c58b811fac3250", + ) + + +@pytest.mark.usefixtures("openssl_enforcing_fips") +def test_envelope_keys_on_a_fips_openssl_without_fips_mode_stay_the_scrypt_keys( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("LITELLM_SALT_KEY", raising=False) + keys: Final = envelope_keys_from_master_key(_MASTER_KEY) + assert (keys.signing_key.get_secret_value(), keys.encryption_key.get_secret_value()) == ( + "34d3bb6ae2e03cddca3417209304e9603417b89258d3979d0569880b6fbd1bed", + "eebfbee8cbe767ee136ce3a9745b44de2b2ae35d1ea46dcf72c58b811fac3250", + ) + + +@pytest.mark.usefixtures("fips_mode_on") +@pytest.mark.parametrize("salt_key", ["", "sk-salt-change-me", _MASTER_KEY, "sk-" + "5a" * 31]) +def test_envelope_keys_under_fips_refuse_a_salt_key_that_is_not_32_bytes_of_hex( + monkeypatch: pytest.MonkeyPatch, salt_key: str +) -> None: + monkeypatch.setenv("LITELLM_SALT_KEY", salt_key) + with pytest.raises(ValueError, match="LITELLM_SALT_KEY"): + envelope_keys_from_master_key(_MASTER_KEY) diff --git a/tests/unit/proxy/mcp_server/outbound_credentials/test_session_credentials.py b/tests/unit/proxy/mcp_server/outbound_credentials/test_session_credentials.py index f084d5c7d1d..795ded8528c 100644 --- a/tests/unit/proxy/mcp_server/outbound_credentials/test_session_credentials.py +++ b/tests/unit/proxy/mcp_server/outbound_credentials/test_session_credentials.py @@ -1,6 +1,8 @@ """Tests for the session-token KDF and the edge/token-endpoint resolvers.""" +import _hashlib from datetime import datetime, timedelta, timezone +from typing import Final import pytest from cryptography.hazmat.primitives import serialization @@ -230,3 +232,64 @@ def test_signing_config_error_detail_never_leaks_key_material(): ) assert isinstance(resolved, SessionSigningConfigError) assert pem.splitlines()[1] not in resolved.detail + + +def test_session_key_without_fips_stays_the_scrypt_key_of_the_master_key() -> None: + keys: Final = session_keys_from_master_key(MASTER_KEY) + assert keys.signing_key.get_secret_value() == "990dce331d97862e3af47815b707c42923fc760e6740d4a77a81583a206f339d" + + +@pytest.mark.usefixtures("openssl_enforcing_fips") +def test_session_key_on_a_fips_openssl_without_fips_mode_stays_the_scrypt_key(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("LITELLM_SALT_KEY", raising=False) + keys: Final = session_keys_from_master_key(MASTER_KEY) + assert keys.signing_key.get_secret_value() == "990dce331d97862e3af47815b707c42923fc760e6740d4a77a81583a206f339d" + + +def _fips_keys(master_key: str) -> tuple[str, str, str]: + envelope_keys_from_master_key.cache_clear() + session_keys_from_master_key.cache_clear() + envelope: Final = envelope_keys_from_master_key(master_key) + session: Final = session_keys_from_master_key(master_key) + return ( + envelope.signing_key.get_secret_value(), + envelope.encryption_key.get_secret_value(), + session.signing_key.get_secret_value(), + ) + + +_needs_shared_openssl: Final = pytest.mark.skipif( + not hasattr(_hashlib, "__file__"), reason="this Python links OpenSSL statically, out of ctypes reach" +) + + +@_needs_shared_openssl +@pytest.mark.usefixtures("fips_mode_on") +def test_keys_under_fips_come_from_the_salt_key_are_distinct_and_the_same_on_every_call( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("LITELLM_SALT_KEY", "sk-" + "5a" * 32) + first: Final = _fips_keys(MASTER_KEY) + assert first == _fips_keys(MASTER_KEY) + assert len(set(first)) == 3, first + assert first == ( + "ef694d415772aef09731c47709d4d9da47f7bbfcd6484836dea09c2bb387063b", + "d8a7cd70690fbf98c5581301b2e0460f2ad714f7c74b2af1b3dac3e0aa99db85", + "5e11e470bf9c838165b70d0b2b82a7ee5e2f00d9449eeb54f7c5bf9491b64354", + ) + + +@_needs_shared_openssl +@pytest.mark.usefixtures("fips_mode_on") +def test_keys_under_fips_all_change_when_the_master_key_rotates(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_SALT_KEY", "sk-" + "5a" * 32) + before: Final = _fips_keys(MASTER_KEY) + after: Final = _fips_keys("sk-a-different-master-key") + assert set(before).isdisjoint(after), (before, after) + + +@pytest.mark.usefixtures("fips_mode_on") +def test_session_key_under_fips_refuses_an_unset_salt_key(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("LITELLM_SALT_KEY", raising=False) + with pytest.raises(ValueError, match="LITELLM_SALT_KEY"): + session_keys_from_master_key(MASTER_KEY) From fa254b93467a35204729a1fea078b23d584ba71a Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Sat, 7 Nov 2026 14:53:07 -0800 Subject: [PATCH 08/39] fix(responses): surface mid-stream upstream errors that arrive without an event type (#50466) * fix(responses): surface mid-stream upstream errors that arrive without an event type Bedrock's native Responses route can end a stream with a bare {"error": {...}} chunk. It parsed as a typeless GenericEvent and crashed the iterator with AttributeError, so clients saw that instead of the upstream error. Such chunks are now read as error events, which raise the mapped upstream error and reach the client as response.failed. * refactor(responses): inline the untyped error chunk check to keep LIT001 at the base count --- litellm/responses/streaming_iterator.py | 11 ++- .../test_bedrock_gpt_responses_native_wire.py | 61 +++++++++++++++- .../test_streaming_iterator_error_events.py | 70 +++++++++++++++++++ 3 files changed, 138 insertions(+), 4 deletions(-) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 4273647567e..08b2c518632 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -486,9 +486,18 @@ class BaseResponsesAPIStreamingIterator: if _is_json_object(parsed_chunk): if self.responses_api_provider_config is None: raise ValueError("responses_api_provider_config is required to process live streaming chunks") + typed_chunk: Final = ( + parsed_chunk + if parsed_chunk.get("type") is not None or "error" not in parsed_chunk + else { + "sequence_number": SEQUENCE_NUMBER_STAMPED_ON_EMISSION, + **parsed_chunk, + "type": ResponsesAPIStreamEvents.ERROR.value, + } + ) openai_responses_api_chunk: Final = self.responses_api_provider_config.transform_streaming_response( model=self.model, - parsed_chunk=parsed_chunk, + parsed_chunk=typed_chunk, logging_obj=self.logging_obj, ) diff --git a/tests/integration/providers/test_bedrock_gpt_responses_native_wire.py b/tests/integration/providers/test_bedrock_gpt_responses_native_wire.py index 159097c80ee..a5cba28a48a 100644 --- a/tests/integration/providers/test_bedrock_gpt_responses_native_wire.py +++ b/tests/integration/providers/test_bedrock_gpt_responses_native_wire.py @@ -1,14 +1,15 @@ import base64 +import json import uuid from dataclasses import dataclass from typing import Final import openai -from integration._support.bedrock_runtime_peer import NATIVE_RESPONSES, answer, respond, target_of +from integration._support.bedrock_runtime_peer import NATIVE_RESPONSES, answer, marker_of, respond, target_of from integration._support.client import Gateway, Scenario, eventually from integration._support.database import read_rows -from integration._support.wire import Request, Wire, wire_server -from openai.types.responses import ResponseCompletedEvent, ResponseTextDeltaEvent +from integration._support.wire import Reply, Request, Wire, wire_server +from openai.types.responses import ResponseCompletedEvent, ResponseFailedEvent, ResponseTextDeltaEvent from pydantic import JsonValue, TypeAdapter from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_if_encrypted_with @@ -17,6 +18,7 @@ GPT: Final = "us.openai.gpt-5.6-sol" TOKEN: Final = "synthetic-bedrock-bearer" SALT: Final = "sk-integration-salt" _JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +UPSTREAM_OUTAGE: Final = "The server had an error while processing your request. Sorry about that!" @dataclass(frozen=True, slots=True) @@ -123,6 +125,59 @@ async def test_async_openai_sdk_responses_stream_is_served_by_the_native_respons assert _spend_row(completed.response.id) == _success_row(model) +def _stream_then_bare_error(request: Request) -> Reply: + marker: Final = marker_of(request) + created: Final[dict[str, JsonValue]] = { + "type": "response.created", + "sequence_number": 0, + "response": { + "id": f"resp_upstream_{marker}", + "object": "response", + "created_at": 1, + "status": "in_progress", + "model": GPT, + "output": [], + }, + } + delta: Final[dict[str, JsonValue]] = { + "type": "response.output_text.delta", + "sequence_number": 1, + "item_id": f"msg_{marker}", + "output_index": 0, + "content_index": 0, + "delta": answer(marker), + } + outage: Final[dict[str, JsonValue]] = { + "error": {"message": UPSTREAM_OUTAGE, "type": "server_error", "code": "service_unavailable"} + } + return Reply( + content_type="text/event-stream", + chunks=tuple(f"data: {json.dumps(event)}\n\n".encode() for event in (created, delta, outage)), + ) + + +async def test_mid_stream_upstream_error_without_an_event_type_reaches_the_client_as_response_failed( + gateway: Gateway, +) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_stream_then_bare_error) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire) + client: Final = openai.AsyncOpenAI(base_url=f"{gateway.client.base_url}/v1", api_key=gateway.key, max_retries=0) + stream: Final = await client.responses.create( + model=model, input=_prompt(marker), stream=True, extra_body={"cache": {"no-cache": True}} + ) + events: Final = [event async for event in stream] + assert [event.type for event in events] == [ + "response.created", + "response.output_text.delta", + "response.failed", + ], events + failed: Final = events[-1] + assert isinstance(failed, ResponseFailedEvent), failed + assert failed.response.error is not None and UPSTREAM_OUTAGE in failed.response.error.message, failed + _native_request(wire) + + def test_codex_agent_message_with_encrypted_content_reaches_the_native_responses_route_intact( gateway: Gateway, ) -> None: diff --git a/tests/unit/responses/test_streaming_iterator_error_events.py b/tests/unit/responses/test_streaming_iterator_error_events.py index 0acf5d9ce62..29cb0412422 100644 --- a/tests/unit/responses/test_streaming_iterator_error_events.py +++ b/tests/unit/responses/test_streaming_iterator_error_events.py @@ -959,3 +959,73 @@ async def test_responses_websocket_turn_failure_reports_the_deployment_it_was_se assert failure is not None _assert_names_the_azure_deployment(failure, reported_api_base) + + +_BEDROCK_SERVICE_UNAVAILABLE_CHUNK: Final = { + "error": { + "message": "The server had an error while processing your request. Sorry about that!", + "type": "server_error", + "code": "service_unavailable", + } +} + + +def _real_config_iterator(provider: str, model: str, events: tuple[Mapping[str, object], ...]): + payload: Final = b"".join(b"data: " + json.dumps(event).encode() + b"\n\n" for event in events) + + async def aiter_bytes(): + yield payload + + response: Final = Mock() + response.headers = {} + response.aiter_bytes = aiter_bytes + logging_obj: Final = Mock(spec=LiteLLMLoggingObj) + logging_obj.model_call_details = {"litellm_params": {}} + logging_obj.completion_start_time = None + config: Final = ProviderConfigManager.get_provider_responses_api_config( + provider=litellm.LlmProviders(provider), model=model + ) + assert config is not None + return ResponsesAPIStreamingIterator( + response=response, + model=model, + responses_api_provider_config=config, + logging_obj=logging_obj, + custom_llm_provider=provider, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("provider", "model"), + [ + ("bedrock", "us.openai.gpt-5.6-sol"), + ("openai", "gpt-5.5"), + ("volcengine", "doubao-seed-1-6"), + ("bigmodel", "glm-5"), + ], +) +async def test_untyped_mid_stream_error_chunk_raises_the_upstream_error(provider: str, model: str): + iterator: Final = _real_config_iterator( + provider, + model, + ( + { + "type": "response.output_text.delta", + "item_id": "msg_1", + "output_index": 0, + "content_index": 0, + "delta": "partial ", + "sequence_number": 1, + }, + _BEDROCK_SERVICE_UNAVAILABLE_CHUNK, + ), + ) + stream: Final = aiter(iterator) + assert (await anext(stream)).type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA + + with pytest.raises(MidStreamFallbackError) as exc_info: + await anext(stream) + assert "The server had an error while processing your request" in str(exc_info.value.original_exception) + assert exc_info.value.status_code >= 500 + assert exc_info.value.generated_content == "partial " From 0d0bf8f75f63a9c76ab75d121cdd6b7f498f2607 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 7 Nov 2026 15:08:23 -0800 Subject: [PATCH 09/39] test(integration): give each kill-burst its own prompts so cached control replies cannot stand in for late traffic (#50569) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../spend/test_normalized_error_long_message.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/integration/spend/test_normalized_error_long_message.py b/tests/integration/spend/test_normalized_error_long_message.py index 23200c0450e..23769391427 100644 --- a/tests/integration/spend/test_normalized_error_long_message.py +++ b/tests/integration/spend/test_normalized_error_long_message.py @@ -358,16 +358,17 @@ def _burst( *, rejected_model: str = CRAFTED_MODEL, ) -> tuple[_BurstResult, ...]: + nonce: Final = uuid.uuid4().hex crafted: Final = tuple( (f"crafted-{path}-{index}", path, _body(path, rejected_model, uuid.uuid4().hex, stream=index % 2 == 1), open_key) for path in (CHAT, MESSAGES, RESPONSES) for index in range(4) ) happy: Final = tuple( - (f"happy-{index}", CHAT, _body(CHAT, happy_model, f"happy-{index}"), happy_key) for index in range(8) + (f"happy-{index}", CHAT, _body(CHAT, happy_model, f"happy-{index}-{nonce}"), happy_key) for index in range(8) ) late: Final = tuple( - (f"late-{index}", CHAT, _body(CHAT, happy_model, f"late-{index}"), happy_key) for index in range(8) + (f"late-{index}", CHAT, _body(CHAT, happy_model, f"late-{index}-{nonce}"), happy_key) for index in range(8) ) with ( ThreadPoolExecutor(max_workers=28) as pool, @@ -427,6 +428,7 @@ def test_killing_one_worker_mid_burst_leaves_the_other_serving_crafted_and_happy open_key: Final = scenario.key() control: Final = _burst(client, model, key, open_key, lambda: None, rejected_model=BENIGN_288KB_MODEL) assert all(result.status == (400 if result.label.startswith("crafted") else 200) for result in control), control + wire.drain() workers: Final = owned.worker_pids() assert len(workers) == 2, workers victim: Final = min(workers) @@ -438,6 +440,8 @@ def test_killing_one_worker_mid_burst_leaves_the_other_serving_crafted_and_happy _assert_burst_rejections_fast(results, control) late: Final = tuple(result for result in results if result.label.startswith("late")) assert all(result.status == 200 for result in late), late + seen: Final = tuple(request.body.decode() for request in wire.drain()) + assert all(any(f"normalized error audit {result.label}-" in body for body in seen) for result in late), seen _assert_rows_land_exactly_once(results, "late") survivor: Final = tuple(pid for pid in owned.worker_pids() if pid != victim) assert survivor, "no worker left serving" From 83e2e5bf99bf443fa6b4723be6a49c39503b880d Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 7 Nov 2026 23:37:18 +0000 Subject: [PATCH 10/39] test(integration): budget proxy restart tests from calibrated boot time (#49761) Both restart tests boot two 2-worker proxies but capped the case at a fixed 180s, below even one calibrated 2-worker readiness budget. On a loaded runner each boot took 80-90s, so pytest-timeout fired mid-boot or mid-shutdown Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../test_azure_content_safety_dispatch_resilience.py | 4 +++- tests/integration/observability/test_azure_storage_chaos.py | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/integration/observability/test_azure_content_safety_dispatch_resilience.py b/tests/integration/observability/test_azure_content_safety_dispatch_resilience.py index 14eb687f69f..d184112f1ec 100644 --- a/tests/integration/observability/test_azure_content_safety_dispatch_resilience.py +++ b/tests/integration/observability/test_azure_content_safety_dispatch_resilience.py @@ -8,11 +8,13 @@ from typing import Final import httpx import pytest +from integration._support.budget import readiness_seconds from integration._support.client import Gateway, Scenario, eventually from integration._support.database import read_rows from integration._support.redis_process import owned_redis from integration._support.wire import Wire, wire_server from integration._support.ports import allocate_port +from integration._support.process import graceful_stop_seconds from integration.observability.azure_dispatch_support import ( ATTACK_MARKER, AzureBehavior, @@ -406,7 +408,7 @@ def test_x3_killing_one_worker_leaves_the_other_serving( _assert_spend(response.headers["x-litellm-call-id"], response.text) -@pytest.mark.timeout(180) +@pytest.mark.timeout(2 * readiness_seconds(2) + 2 * graceful_stop_seconds() + 120) def test_x4_proxy_restart_preserves_completed_spend_rows( gateway: Gateway, tmp_path: Path ) -> None: diff --git a/tests/integration/observability/test_azure_storage_chaos.py b/tests/integration/observability/test_azure_storage_chaos.py index acc2eb1439d..e78f1a4a1db 100644 --- a/tests/integration/observability/test_azure_storage_chaos.py +++ b/tests/integration/observability/test_azure_storage_chaos.py @@ -13,8 +13,9 @@ from _azure_storage_support import ( collect_files, ) from _s3_v2_support import matched_ids, mixed_burst, surface_reply +from integration._support.budget import readiness_seconds from integration._support.client import Gateway, JsonValue, eventually -from integration._support.process import group_members, owned_proxy_process +from integration._support.process import graceful_stop_seconds, group_members, owned_proxy_process from integration._support.tls import server_context, write_self_signed_cert from integration._support.wire import wire_server @@ -175,7 +176,7 @@ def test_killing_one_worker_keeps_the_other_serving_and_uploading(gateway: Gatew ) -@pytest.mark.timeout(180) +@pytest.mark.timeout(2 * readiness_seconds(WORKERS) + 2 * graceful_stop_seconds() + 120) def test_restarting_the_proxy_before_the_queue_flushes_bounds_the_loss_to_the_unflushed_queue_and_recovers( gateway: Gateway, tmp_path: Path ) -> None: From b83a1f0608835ecba4b199155e1a585711477840 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Sat, 7 Nov 2026 15:52:00 -0800 Subject: [PATCH 11/39] fix(otel): name the routed model on auto-router LLM spans (#50498) On an auto-router request the LLM span reported the name the caller asked for (the router or an alias of it) as gen_ai.request.model (OTel v2) and gen_ai.response.model (v1 and v2). Both now name the model group the router picked, read from the request's routing_decision. In v2 the span name and the vendor keys built from the request model (Langfuse, OpenInference, Langtrace, Weave) follow. Requests that were not auto-routed are unchanged. --- litellm/integrations/opentelemetry.py | 8 +- litellm/integrations/otel/model/metadata.py | 24 ++- litellm/integrations/otel/routing.py | 14 +- .../test_otel_auto_router_model.py | 180 ++++++++++++++++++ .../otel/test_otel_v2_sources_of_truth.py | 21 ++ tests/unit/integrations/test_opentelemetry.py | 29 +++ 6 files changed, 264 insertions(+), 12 deletions(-) create mode 100644 tests/integration/observability/test_otel_auto_router_model.py diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 5cd201a2f47..9bec220e073 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -34,7 +34,7 @@ from litellm.integrations.otel.model.db_endpoint import db_span_attributes from litellm.integrations.otel.model.metadata import flatten_metadata from litellm.integrations.otel.model.semconv import LiteLLM, Metric from litellm.integrations.otel.plumbing.otlp_tls import resolve_otlp_http_tls -from litellm.integrations.otel.routing import routing_decision_attributes +from litellm.integrations.otel.routing import routed_model_name, routing_decision_attributes from litellm.litellm_core_utils.core_helpers import iter_items, require_attributes from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params from litellm.litellm_core_utils.safe_json_dumps import safe_dumps @@ -2617,7 +2617,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): ) # The model used to generate the response. - response_model: Final = _response_field(response_obj, "model") + response_model: Final = routed_model_name( + _response_field(response_obj, "model"), + standard_logging_payload.get("model_group"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # kwargs is untyped + decision, + ) if response_model: self.safe_set_attribute( span=span, diff --git a/litellm/integrations/otel/model/metadata.py b/litellm/integrations/otel/model/metadata.py index 28d6ad54607..f9739a4ba15 100644 --- a/litellm/integrations/otel/model/metadata.py +++ b/litellm/integrations/otel/model/metadata.py @@ -29,9 +29,10 @@ asks for a *model group* (e.g. ``gpt-4o``) that routes to a concrete deployment * ``hidden_params.litellm_model_name`` — a secondary source for the dispatched model (populated only on some call paths, e.g. files). -So ``gen_ai.request.model`` is the *group* (falling back to the call model on the -SDK path, which has no group), and ``litellm.provider.model`` is the *dispatched* -model. They coincide on the SDK path, which is correct. +So ``gen_ai.request.model`` is the *group* (the group an auto router picked, when +one routed the call; the call model on the SDK path, which has no group), and +``litellm.provider.model`` is the *dispatched* model. They coincide on the SDK +path, which is correct. """ from __future__ import annotations @@ -52,6 +53,7 @@ from litellm.constants import ( from litellm.integrations.otel.model.semconv import PASSTHROUGH_OPERATION_KEY, GenAIOperation, resolve_operation from litellm.integrations.otel.model.trace_controls import TraceControls, caller_trace_controls from litellm.integrations.otel.model.utils import as_str, as_str_mapping, to_seconds +from litellm.integrations.otel.routing import routed_model_name from litellm.types.utils import InternalCallOrigin if TYPE_CHECKING: @@ -170,7 +172,8 @@ class RequestIdentity: class RequestContext: """The fully-resolved view of a closed request, parsed once from the payload. - ``request_model`` is the user-facing requested model and ``provider_model`` + ``request_model`` is the user-facing requested model (or the group an auto + router picked for it) and ``provider_model`` (on :attr:`identity`) is the model litellm dispatched to the provider; the two differ on the proxy (group vs. deployment) and coincide on the SDK path. """ @@ -194,12 +197,15 @@ class RequestContext: raw_response: Final = payload.get("response") response: Final = cast(Mapping[str, object], raw_response if isinstance(raw_response, dict) else {}) model_group: Final = as_str(payload.get("model_group")) or as_str(raw_meta.get("model_group")) + decision: Final = raw_meta.get("routing_decision") return cls( - # The user asked for the group; fall back to the call model on the SDK - # path, which has no group. Empty string (never None) so the span name - # builder and the mapper see a plain string. - request_model=model_group or as_str(payload.get("model")) or "", - response_model=as_str(response.get("model")), + # The user asked for the group, or an auto router picked one for it (the + # proxy restamps the response model to the requested name, so that gets + # the pick too); fall back to the call model on the SDK path, which has no + # group. Empty string (never None) so the span name builder and the mapper + # see a plain string. + request_model=routed_model_name(model_group, model_group, decision) or as_str(payload.get("model")) or "", + response_model=routed_model_name(as_str(response.get("model")), model_group, decision), model_group=model_group, model_id=as_str(payload.get("model_id")) or _model_info_id(raw_meta.get("model_info")), api_base=as_str(payload.get("api_base")) or as_str(hidden.get("api_base")), diff --git a/litellm/integrations/otel/routing.py b/litellm/integrations/otel/routing.py index 339775855be..1143e6d2dbe 100644 --- a/litellm/integrations/otel/routing.py +++ b/litellm/integrations/otel/routing.py @@ -1,8 +1,11 @@ from collections.abc import Mapping from types import MappingProxyType -from typing import Final, TypeAlias +from typing import Final, TypeAlias, TypeVar + +from litellm.litellm_core_utils.core_helpers import as_str_mapping RoutingAttributeValue: TypeAlias = str | int | float | bool +_Model = TypeVar("_Model") # rebind-ok: a TypeVar cannot be Final _ROUTING_SCALAR_FIELDS: Final = frozenset( { @@ -60,3 +63,12 @@ def routing_decision_attributes(decision: Mapping[str, object] | None) -> Mappin if isinstance(value := decision.get(key), (str, int, float, bool)) } ) + + +def routed_model_name(model: _Model, requested_model: object, decision: object) -> _Model | str: + """The model an auto router picked when ``model`` is only the name the caller requested, else ``model``.""" + fields: Final = as_str_mapping(decision) + routed: Final = fields.get("routed_model") if fields is not None else None + if model and model == requested_model and isinstance(routed, str) and routed: + return routed + return model diff --git a/tests/integration/observability/test_otel_auto_router_model.py b/tests/integration/observability/test_otel_auto_router_model.py new file mode 100644 index 00000000000..d056112daca --- /dev/null +++ b/tests/integration/observability/test_otel_auto_router_model.py @@ -0,0 +1,180 @@ +import json +import re +import uuid +from collections.abc import Iterator, Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +import pytest +import yaml +from integration._support.client import Gateway, eventually, gateway_from_environment +from integration._support.otlp_sink import SpanSinks, recorded_spans +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue + +pytestmark: Final = pytest.mark.timeout(300) + +MARKER: Final = re.compile(rb"autorouter-[0-9a-f]{32}") +TIERS: Final = ("SIMPLE", "MEDIUM", "COMPLEX", "REASONING") +USAGE: Final = {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6} + + +@dataclass(frozen=True, slots=True) +class AutoRouterRig: + proxy: Gateway + sink: str + router: str + alias: str + routed: str + + +def _upstream(request: Request) -> Reply: + found: Final = MARKER.search(request.body) + if found is None: + return Reply(body=json.dumps({"object": "list", "data": []}).encode()) + if request.target.endswith("/responses"): + output: Final = { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "ok", "annotations": []}], + } + return Reply( + body=json.dumps( + { + "id": "resp_" + found.group(0).decode(), + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-4o-mini-2024-07-18", + "output": [output], + "usage": {"input_tokens": 5, "output_tokens": 1, "total_tokens": 6}, + } + ).encode() + ) + head: Final = {"id": "chatcmpl-" + found.group(0).decode(), "created": 1, "model": "gpt-4o-mini-2024-07-18"} + if json.loads(request.body).get("stream") is not True: + message: Final = {"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"} + return Reply( + body=json.dumps({**head, "object": "chat.completion", "choices": [message], "usage": USAGE}).encode() + ) + chunks: Final = ( + {**head, "object": "chat.completion.chunk", "choices": [{"index": 0, "delta": {"content": "ok"}}]}, + { + **head, + "object": "chat.completion.chunk", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + "usage": USAGE, + }, + ) + return Reply( + content_type="text/event-stream", + chunks=(*(f"data: {json.dumps(chunk)}\n\n".encode() for chunk in chunks), b"data: [DONE]\n\n"), + ) + + +@pytest.fixture(scope="module") +def gateway(audit_sinks: SpanSinks) -> Iterator[Gateway]: + with gateway_from_environment() as base: + yield base + + +@pytest.fixture(scope="module", params=[False, True], ids=["otel-v1", "otel-v2"]) +def rig( + request: pytest.FixtureRequest, + gateway: Gateway, + audit_sinks: SpanSinks, + tmp_path_factory: pytest.TempPathFactory, +) -> Iterator[AutoRouterRig]: + otel_v2: Final = request.param is True + router: Final = "auto-router-" + uuid.uuid4().hex[:8] + alias: Final = "smart-" + uuid.uuid4().hex[:8] + routed: Final = "tier-model-" + uuid.uuid4().hex[:8] + directory: Final = tmp_path_factory.mktemp("auto-router-otel") + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + with wire_server(_upstream) as provider: + config["litellm_settings"] = {**config.get("litellm_settings", {}), "callbacks": ["otel"]} + config["router_settings"] = {**config.get("router_settings", {}), "model_group_alias": {alias: router}} + config["callback_settings"] = { + "otel": {"exporter": "http/json", "endpoint": audit_sinks.operator, "use_simple_processor": True} + } + config["model_list"] = [ + { + "model_name": routed, + "litellm_params": {"model": "openai/gpt-4o-mini", "api_base": provider.url, "api_key": "synthetic"}, + }, + { + "model_name": router, + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": {"tiers": dict.fromkeys(TIERS, routed)}, + }, + }, + ] + path: Final = directory / "auto_router_otel.yaml" + path.write_text(yaml.safe_dump(config)) + overrides: Final = { + "OTEL_EXPORTER": "http/json", + "OTEL_ENDPOINT": audit_sinks.operator, + **({"LITELLM_OTEL_V2": "1"} if otel_v2 else {}), + } + with owned_proxy( + gateway, + directory, + overrides, + config=path, + remove_environment=() if otel_v2 else ("LITELLM_OTEL_V2",), + ) as proxy: + yield AutoRouterRig(proxy=proxy, sink=audit_sinks.operator, router=router, alias=alias, routed=routed) + + +@pytest.mark.parametrize( + ("route", "stream", "via_alias"), + [ + ("/v1/chat/completions", False, False), + ("/v1/chat/completions", True, False), + ("/v1/messages", False, False), + ("/v1/responses", False, False), + ("/v1/chat/completions", False, True), + ], + ids=["chat", "chat-stream", "messages", "responses", "chat-via-alias"], +) +def test_auto_router_llm_span_names_the_routed_model_and_carries_the_decision( + rig: AutoRouterRig, route: str, stream: bool, via_alias: bool +) -> None: + prompt: Final = "hi autorouter-" + uuid.uuid4().hex + body: Final[dict[str, JsonValue]] = { + "model": rig.alias if via_alias else rig.router, + "stream": stream, + **({"input": prompt} if route == "/v1/responses" else {"messages": [{"role": "user", "content": prompt}]}), + **({"max_tokens": 16} if route == "/v1/messages" else {}), + } + start, _ = recorded_spans(rig.sink) + response: Final = rig.proxy.request("POST", route, body) + assert response.status_code == 200, response.text + call_id: Final = response.headers["x-litellm-call-id"] + + def llm_span() -> Mapping[str, JsonValue] | None: + return next( + ( + span["attributes"] + for span in recorded_spans(rig.sink, start)[1] + if span["attributes"].get("litellm.call_id") == call_id + and "litellm.routing.router_model_name" in span["attributes"] + ), + None, + ) + + attributes: Final = eventually(llm_span, lambda found: found is not None, seconds=60) + assert attributes is not None + shown: Final = json.dumps(attributes, sort_keys=True, default=str) + underlying: Final = frozenset({rig.routed, "gpt-4o-mini"}) + assert attributes["gen_ai.request.model"] in underlying, shown + assert attributes["gen_ai.response.model"] in underlying, shown + assert attributes["litellm.routing.router_model_name"] == rig.router, shown + assert attributes["litellm.routing.router_type"] == "complexity", shown + assert attributes["litellm.routing.routed_model"] == rig.routed, shown + assert attributes["litellm.routing.tier"] in TIERS, shown diff --git a/tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py b/tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py index c29da4607eb..3c95a85e9a8 100644 --- a/tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py +++ b/tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py @@ -539,6 +539,27 @@ def test_llm_call_adapter_extracts_all_fields(): assert data.identity.key_hash == "hsh" +@pytest.mark.parametrize( + ("model_group", "response_model", "expected"), + [ + ("auto-router", "auto-router", ("gpt-4o-group", "gpt-4o-group")), + ("smart", "smart", ("gpt-4o-group", "gpt-4o-group")), + ("auto-router", "gpt-4o-2024", ("gpt-4o-group", "gpt-4o-2024")), + (None, None, ("gpt-4o", None)), + ], + ids=["restamped-to-router", "restamped-to-alias", "provider-model", "no-group-no-response"], +) +def test_llm_call_adapter_names_the_routed_model_for_an_auto_router_call(model_group, response_model, expected): + data = LLMCallSpanData.from_standard_logging_payload( + _sample_payload( + model_group=model_group, + response={"id": "resp_1", "model": response_model, "choices": [{"finish_reason": "stop"}]}, + metadata={"routing_decision": {"routed_model": "gpt-4o-group"}}, + ) + ) + assert (data.request_model, data.response_model) == expected + + def test_llm_call_adapter_extracts_cache_tokens_from_usage_object(): payload = _sample_payload() payload["metadata"] = { diff --git a/tests/unit/integrations/test_opentelemetry.py b/tests/unit/integrations/test_opentelemetry.py index c4fd7ed51c7..4ef95d3a79d 100644 --- a/tests/unit/integrations/test_opentelemetry.py +++ b/tests/unit/integrations/test_opentelemetry.py @@ -372,6 +372,35 @@ def test_v1_routing_diagnostics_add_scalar_attributes_and_preserve_legacy_metada assert finished.attributes["metadata.routing_decision"] == str(decision) +@pytest.mark.parametrize( + ("decision", "expected"), + [({"router_model_name": "auto-router", "routed_model": "answer"}, "answer"), ("not-a-mapping", "auto-router")], + ids=["routed", "malformed-decision"], +) +def test_v1_auto_router_response_model_names_the_routed_model(decision: object, expected: str) -> None: + exporter: Final = InMemorySpanExporter() + provider: Final = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + otel: Final = OpenTelemetry(tracer_provider=provider) + with provider.get_tracer(__name__).start_as_current_span("inference") as span: + otel.set_attributes( + span, + { + "model": "gpt-4o-mini", + "standard_logging_object": { + "id": "routing-call", + "call_type": "acompletion", + "model_group": "auto-router", + "metadata": {"routing_decision": decision}, + }, + }, + {"model": "auto-router", "choices": [], "usage": {"prompt_tokens": 3}}, + ) + (finished,) = exporter.get_finished_spans() + attributes: Final = finished.attributes + assert (attributes["gen_ai.response.model"], attributes["gen_ai.usage.input_tokens"]) == (expected, 3) + + def test_v1_server_tool_calls_lead_the_completion_function_call_attributes() -> None: exporter: Final = InMemorySpanExporter() provider: Final = TracerProvider() From 51b49973c943e7ee01242f392151be30ff618aa8 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Sat, 7 Nov 2026 16:02:44 -0800 Subject: [PATCH 12/39] fix(proxy): hold DB_RECONCILE_LOCK around credential create, update and delete (#50551) * fix(proxy): hold MODEL_RECONCILE_LOCK around credential create, update and delete The config reload snapshots the credential table, awaits config rows, then prunes and upserts litellm.credential_list from that snapshot. A credential write landing inside that window was undone in memory until the next reload. The three endpoints now take the same lock the reload holds. Fixes #50273 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): take MODEL_RECONCILE_LOCK before the credential update reads its row Two PATCHes queued behind a reload read the same old row and the second overwrote the first. The relay now keeps enough bytes to spot a split after marker, and the new unit tests are fully typed. * test(credentials): annotate the reconcile lock test locals with Final * refactor(proxy): rename MODEL_RECONCILE_LOCK to DB_RECONCILE_LOCK The lock gates every db-to-memory reconcile and now the credential writes too, so the model-only name undersold what it protects. --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../common_utils/registry_read_through.py | 4 +- .../proxy/credential_endpoints/endpoints.py | 146 +++++++++--------- .../model_management_endpoints.py | 22 +-- litellm/proxy/proxy_server.py | 29 ++-- tests/integration/_support/database_relay.py | 40 ++++- .../test_credential_writes_survive_reload.py | 119 ++++++++++++++ .../test_registry_read_through.py | 14 +- .../credential_endpoints/test_endpoints.py | 115 +++++++++++++- .../test_model_management_endpoints.py | 12 +- 9 files changed, 383 insertions(+), 118 deletions(-) create mode 100644 tests/integration/management/test_credential_writes_survive_reload.py diff --git a/litellm/proxy/common_utils/registry_read_through.py b/litellm/proxy/common_utils/registry_read_through.py index 487c6feb68d..d58c58fa1a5 100644 --- a/litellm/proxy/common_utils/registry_read_through.py +++ b/litellm/proxy/common_utils/registry_read_through.py @@ -133,7 +133,7 @@ async def _resync_model_deployments(model_name: str) -> bool: rows: Final = await _find_rows_by_name_or_id(model_name, ModelRepository(prisma_client).table) if not rows: return False - async with proxy_server.MODEL_RECONCILE_LOCK: + async with proxy_server.DB_RECONCILE_LOCK: outcome: Final = await _install_model_rows(rows, _always) if outcome in ("no_router", "auto_router"): await _reload_every_model() @@ -150,7 +150,7 @@ async def resync_model_deployments_by_ids(model_ids: tuple[str, ...]) -> None: prisma_client: Final = proxy_server.prisma_client assert prisma_client is not None writer_table: Final = ModelRepository(WriterPinnedClient(prisma_client.db)).table - async with proxy_server.MODEL_RECONCILE_LOCK: + async with proxy_server.DB_RECONCILE_LOCK: rows: Final = await _find_rows_by_ids(model_ids, writer_table) _ = await _install_model_rows(_without_auto_routers(rows), _names_an_unloaded_credential) diff --git a/litellm/proxy/credential_endpoints/endpoints.py b/litellm/proxy/credential_endpoints/endpoints.py index 86e716f19e3..3026f2706ac 100644 --- a/litellm/proxy/credential_endpoints/endpoints.py +++ b/litellm/proxy/credential_endpoints/endpoints.py @@ -324,7 +324,7 @@ async def create_credential( Stores credential in DB. Reloads credentials in memory. """ - from litellm.proxy.proxy_server import prisma_client + from litellm.proxy.proxy_server import DB_RECONCILE_LOCK, prisma_client try: if prisma_client is None: @@ -365,21 +365,22 @@ async def create_credential( credentials_dict_jsonified: Final = cast( # cast-ok: deep-copies a model_dump, so keys are str "dict[str, object]", jsonify_object(credentials_dict) ) - try: - await CredentialsRepository(prisma_client).create( - data={ - **credentials_dict_jsonified, - "created_by": user_api_key_dict.user_id, - "updated_by": user_api_key_dict.user_id, - } - ) - except Exception as e: - if not is_unique_violation(e): - raise - raise HTTPException(status_code=409, detail=_credential_exists_detail(credential.credential_name)) + async with DB_RECONCILE_LOCK: + try: + await CredentialsRepository(prisma_client).create( + data={ + **credentials_dict_jsonified, + "created_by": user_api_key_dict.user_id, + "updated_by": user_api_key_dict.user_id, + } + ) + except Exception as e: + if not is_unique_violation(e): + raise + raise HTTPException(status_code=409, detail=_credential_exists_detail(credential.credential_name)) - ## ADD TO LITELLM ## - CredentialAccessor.upsert_credentials([processed_credential]) + ## ADD TO LITELLM ## + CredentialAccessor.upsert_credentials([processed_credential]) return {"success": True, "message": "Credential created successfully"} except Exception as e: @@ -865,7 +866,7 @@ async def delete_credential( """ [BETA] endpoint. This might change unexpectedly. """ - from litellm.proxy.proxy_server import prisma_client + from litellm.proxy.proxy_server import DB_RECONCILE_LOCK, prisma_client try: if prisma_client is None: @@ -876,14 +877,17 @@ async def delete_credential( _reject_non_admin_wif_fields( await named_credential_wif_fields(credential_name, prisma_client), user_api_key_dict ) - deleted: Final = await CredentialsRepository(prisma_client).delete_by_name(credential_name) - if deleted is None: - raise _not_found_unless_config_defined( - credential_name, "Credential not found. Got credential name: " + credential_name - ) + async with DB_RECONCILE_LOCK: + deleted: Final = await CredentialsRepository(prisma_client).delete_by_name(credential_name) + if deleted is None: + raise _not_found_unless_config_defined( + credential_name, "Credential not found. Got credential name: " + credential_name + ) - ## DELETE FROM LITELLM ## - litellm.credential_list = [cred for cred in litellm.credential_list if cred.credential_name != credential_name] + ## DELETE FROM LITELLM ## + litellm.credential_list = [ + cred for cred in litellm.credential_list if cred.credential_name != credential_name + ] await _purge_user_connections_for_credential(credential_name) return {"success": True, "message": "Credential deleted successfully"} except Exception as e: @@ -983,7 +987,7 @@ async def update_credential( """ [BETA] endpoint. This might change unexpectedly. """ - from litellm.proxy.proxy_server import prisma_client + from litellm.proxy.proxy_server import DB_RECONCILE_LOCK, prisma_client try: if credential.credential_name and credential.credential_name != credential_name: @@ -1007,53 +1011,55 @@ async def update_credential( detail={"error": CommonProxyErrors.db_not_connected_error.value}, ) credentials_repository: Final = CredentialsRepository(WriterPinnedClient(prisma_client.db)) - db_credential: Final = await credentials_repository.find_by_name(credential_name) - if db_credential is None: - raise _not_found_unless_config_defined(credential_name, "Credential not found in DB.") - _reject_non_admin_wif_fields(_stored_wif_fields(db_credential), user_api_key_dict) - decrypted_stored_credential: Final = decrypted_credential(db_credential) - decrypted_values: Final = _CREDENTIAL_DICT_ADAPTER.validate_python( - decrypted_stored_credential.credential_values - ) - incoming_values: Final = _without_masked_read_backs(requested_values, _stored_objects(decrypted_values)) - patch: Final = CredentialItem( - credential_name=credential_name, - display_name=( - requested_display_name if "display_name" in credential.model_fields_set else db_credential.display_name - ), - credential_info=_CREDENTIAL_DICT_ADAPTER.validate_python(credential.credential_info), - credential_values=_CREDENTIAL_DICT_ADAPTER.validate_python(incoming_values), - credential_values_to_delete=credential.credential_values_to_delete, - ) - stored_info: Final = _CREDENTIAL_DICT_ADAPTER.validate_python( - db_credential.credential_info # pyright: ignore[reportUnknownMemberType] # untyped credential field - ) - patch_info: Final = _CREDENTIAL_DICT_ADAPTER.validate_python( - patch.credential_info # pyright: ignore[reportUnknownMemberType] # untyped credential field - ) - effective_info: Final = {**stored_info, **patch_info} - normalized_patch: Final = ( - patch.model_copy( - update={"credential_values": _normalized_github_copilot_values(incoming_values, effective_info)} + async with DB_RECONCILE_LOCK: + db_credential: Final = await credentials_repository.find_by_name(credential_name) + if db_credential is None: + raise _not_found_unless_config_defined(credential_name, "Credential not found in DB.") + _reject_non_admin_wif_fields(_stored_wif_fields(db_credential), user_api_key_dict) + decrypted_stored_credential: Final = decrypted_credential(db_credential) + decrypted_values: Final = _CREDENTIAL_DICT_ADAPTER.validate_python( + decrypted_stored_credential.credential_values ) - if incoming_values.get(GITHUB_COPILOT_HOST_KEY) is not None - else patch - ) - merged_credential: Final = update_db_credential(db_credential, normalized_patch) - credential_object_jsonified: Final = cast( # cast-ok: deep-copies a model_dump, so keys are str - "dict[str, object]", jsonify_object(merged_credential.model_dump(exclude_none=True)) - ) - await credentials_repository.update_by_name( - credential_name, - data={ - **credential_object_jsonified, - "display_name": merged_credential.display_name, - "updated_by": user_api_key_dict.user_id, - }, - ) - - # Sync in-memory credential_list (skip if not in memory - e.g., proxy restarted) - _sync_in_memory_credential(normalized_patch, credential_name) + incoming_values: Final = _without_masked_read_backs(requested_values, _stored_objects(decrypted_values)) + patch: Final = CredentialItem( + credential_name=credential_name, + display_name=( + requested_display_name + if "display_name" in credential.model_fields_set + else db_credential.display_name + ), + credential_info=_CREDENTIAL_DICT_ADAPTER.validate_python(credential.credential_info), + credential_values=_CREDENTIAL_DICT_ADAPTER.validate_python(incoming_values), + credential_values_to_delete=credential.credential_values_to_delete, + ) + stored_info: Final = _CREDENTIAL_DICT_ADAPTER.validate_python( + db_credential.credential_info # pyright: ignore[reportUnknownMemberType] # untyped credential field + ) + patch_info: Final = _CREDENTIAL_DICT_ADAPTER.validate_python( + patch.credential_info # pyright: ignore[reportUnknownMemberType] # untyped credential field + ) + effective_info: Final = {**stored_info, **patch_info} + normalized_patch: Final = ( + patch.model_copy( + update={"credential_values": _normalized_github_copilot_values(incoming_values, effective_info)} + ) + if incoming_values.get(GITHUB_COPILOT_HOST_KEY) is not None + else patch + ) + merged_credential: Final = update_db_credential(db_credential, normalized_patch) + credential_object_jsonified: Final = cast( # cast-ok: deep-copies a model_dump, so keys are str + "dict[str, object]", jsonify_object(merged_credential.model_dump(exclude_none=True)) + ) + await credentials_repository.update_by_name( + credential_name, + data={ + **credential_object_jsonified, + "display_name": merged_credential.display_name, + "updated_by": user_api_key_dict.user_id, + }, + ) + # Sync in-memory credential_list (skip if not in memory - e.g., proxy restarted) + _sync_in_memory_credential(normalized_patch, credential_name) from litellm.llms.github_copilot.github_host import normalized_github_host diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index b3c1f48bf99..3c8f7d6d64f 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -2392,12 +2392,12 @@ async def delete_team_models( if deleted_model_ids: await publish_config_change(redis_cache=coordination_redis_cache(), object_type="litellm_proxymodeltable") - # Under MODEL_RECONCILE_LOCK, for the same reason as delete_model: the rows are + # Under DB_RECONCILE_LOCK, for the same reason as delete_model: the rows are # gone, but a reconcile holding a pre-delete snapshot would upsert these ids back # onto this pod. The lock orders the eviction after any in-flight reconcile. - from litellm.proxy.proxy_server import MODEL_RECONCILE_LOCK, proxy_config + from litellm.proxy.proxy_server import DB_RECONCILE_LOCK, proxy_config - async with MODEL_RECONCILE_LOCK: + async with DB_RECONCILE_LOCK: proxy_config.remove_auto_router_catalog_entries(frozenset(deleted_model_ids)) if llm_router is not None: for model_id in deleted_model_ids: @@ -2884,7 +2884,7 @@ async def delete_model( """ from litellm.proxy.proxy_server import ( - MODEL_RECONCILE_LOCK, + DB_RECONCILE_LOCK, general_settings_view, llm_router, premium_user, @@ -2945,13 +2945,13 @@ async def delete_model( ) ## DELETE FROM ROUTER ## - # Under MODEL_RECONCILE_LOCK. The db row is already gone, but a reconcile + # Under DB_RECONCILE_LOCK. The db row is already gone, but a reconcile # that snapshotted the db BEFORE that delete still lists this id as desired, # and its _add_deployment upserts the deployment straight back -- leaving # this pod serving a model the database no longer has, until the next # reconcile. Taking the lock orders this eviction after any such in-flight # reconcile's re-add, so the eviction is the last word. - async with MODEL_RECONCILE_LOCK: + async with DB_RECONCILE_LOCK: proxy_config.remove_auto_router_catalog_entries(frozenset({model_info.id})) if llm_router is not None: llm_router.apply(ChangeSet(delete=[model_info.id])) @@ -3892,7 +3892,7 @@ def reload_serving_verdict( Without it (no reconcile ran) every drop is reported, which is the safe direction. ``live_after`` is the router's serving state captured by the reload itself, while it - still held MODEL_RECONCILE_LOCK. Pass it whenever the caller has it: re-reading the + still held DB_RECONCILE_LOCK. Pass it whenever the caller has it: re-reading the router here instead means sampling it after the lock was released, where the NEXT reconcile's leading wipe (clear_cache un-serves every db model before reloading them) shows up as this reload having dropped them. Falling back to a fresh read is @@ -3970,13 +3970,13 @@ async def clear_cache() -> ReconcileOutcome: Returns what the reload saw (see ReconcileOutcome) so callers can pass it to raise_if_reload_degraded_serving. - Runs under MODEL_RECONCILE_LOCK for its whole extent, not just the reload at the + Runs under DB_RECONCILE_LOCK for its whole extent, not just the reload at the end, so the auto-router reset and the reload that rebuilds those routers are atomic to any other reconcile. The inner call is _add_deployment_locked because add_deployment would re-acquire the same non-reentrant lock and deadlock. """ from litellm.proxy.proxy_server import ( - MODEL_RECONCILE_LOCK, + DB_RECONCILE_LOCK, llm_router, prisma_client, proxy_config, @@ -3988,7 +3988,7 @@ async def clear_cache() -> ReconcileOutcome: verbose_proxy_logger.debug("llm_router or prisma_client is None, skipping cache clear") return ReconcileOutcome(still_desired=None, live_after=None) - async with MODEL_RECONCILE_LOCK: + async with DB_RECONCILE_LOCK: try: # Only clear DB models, preserve config models verbose_proxy_logger.debug("Clearing only DB models, preserving config models") @@ -4060,7 +4060,7 @@ async def clear_cache() -> ReconcileOutcome: llm_router.registry().plugins("switchyard").pop(model_name, None) # Reload only DB models. _add_deployment_locked, not add_deployment: this - # coroutine already holds MODEL_RECONCILE_LOCK and asyncio.Lock is not + # coroutine already holds DB_RECONCILE_LOCK and asyncio.Lock is not # reentrant, so the public wrapper would deadlock against itself. outcome: Final = await proxy_config.add_deployment_locked( prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index fe633238e20..8c92a058f43 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -3140,15 +3140,16 @@ experimental = False llm_router: Router | None = None llm_model_list: list | None = None heuristic_v1_tuning_baselines: Mapping[str, str] | None = None -# Serializes every model reconcile (ProxyConfig.add_deployment and clear_cache) so the -# read-modify-write of llm_router above is atomic. Without it, two concurrent model -# writes each reconcile the router against their OWN db snapshot, and the one holding -# the older snapshot evicts the deployment the newer one just added -- the db keeps the -# row, this pod stops serving it. Control-plane only (model create/update/delete and -# the config-sync tick), never on a completion path, so the serialization is free. -# Module-level rather than per-ProxyConfig because llm_router is a module global and a -# second ProxyConfig instance must not get its own independent lock over it. -MODEL_RECONCILE_LOCK: Final = asyncio.Lock() +# Serializes every db-to-memory reconcile (ProxyConfig.add_deployment and clear_cache) +# with every control-plane write to the state those reconciles rebuild: llm_router above +# and litellm.credential_list. Without it, two concurrent writes each reconcile against +# their OWN db snapshot, and the one holding the older snapshot evicts the deployment or +# credential the newer one just added -- the db keeps the row, this pod stops serving it. +# Control-plane only (model and credential create/update/delete and the config-sync +# tick), never on a completion path, so the serialization is free. Module-level rather +# than per-ProxyConfig because llm_router is a module global and a second ProxyConfig +# instance must not get its own independent lock over it. +DB_RECONCILE_LOCK: Final = asyncio.Lock() general_settings: dict = {} _GENERAL_SETTINGS_VIEW: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) @@ -8922,7 +8923,7 @@ class ProxyConfig: - Check if model id's in router already - If not, add to router - Serialized against every other model reconcile by MODEL_RECONCILE_LOCK, because + Serialized against every other model reconcile by DB_RECONCILE_LOCK, because the work below is a read-modify-write of the shared ``llm_router`` global: it reads the db into a snapshot and then makes the router match that snapshot. Two of those interleaving is not a lost update but an eviction -- the request whose @@ -8941,7 +8942,7 @@ class ProxyConfig: if not ui_settings_already_synced: await sync_ui_settings_to_general_settings(prisma_client) - async with MODEL_RECONCILE_LOCK: + async with DB_RECONCILE_LOCK: return await self.add_deployment_locked(prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj) async def add_deployment_locked( @@ -8949,7 +8950,7 @@ class ProxyConfig: prisma_client: PrismaClient, proxy_logging_obj: ProxyLogging, ) -> ReconcileOutcome: - """add_deployment's body, minus the locking. MODEL_RECONCILE_LOCK MUST already + """add_deployment's body, minus the locking. DB_RECONCILE_LOCK MUST already be held. Split out for the one caller that has to hold the lock across more than this reconcile -- clear_cache, which un-serves every db model before calling it and would deadlock on a re-acquire.""" @@ -9898,7 +9899,7 @@ class ProxyConfig: immediately, and by the periodic job in store_model_in_db-off deployments. Gated the same way as startup, so an admin who excluded search_tools from supported_db_objects opts out. - Serialized by MODEL_RECONCILE_LOCK for the reason add_deployment documents: the body is a + Serialized by DB_RECONCILE_LOCK for the reason add_deployment documents: the body is a read-modify-write of the shared ``llm_router`` global, so two of them interleaving lets the older snapshot's wholesale assignment land last and restore a tool the newer one deleted. The lock belongs here rather than in _init_search_tools_in_db, which _init_non_llm_objects_in_db @@ -9908,7 +9909,7 @@ class ProxyConfig: return if prisma_client is None: return - async with MODEL_RECONCILE_LOCK: + async with DB_RECONCILE_LOCK: await self._init_search_tools_in_db(prisma_client=prisma_client) @staticmethod diff --git a/tests/integration/_support/database_relay.py b/tests/integration/_support/database_relay.py index 2e9f2ec4a41..0ac1e1d2521 100644 --- a/tests/integration/_support/database_relay.py +++ b/tests/integration/_support/database_relay.py @@ -6,7 +6,6 @@ from contextlib import contextmanager from typing import Final from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit - from tests.integration._support.async_server import LoopbackServer, close_writer from tests.integration._support.ports import allocate_port @@ -74,16 +73,37 @@ class DatabaseRelay: class HeldStatementRelay: - def __init__(self, upstream_host: str, upstream_port: int, trigger: bytes) -> None: + """Holds the first client statement carrying ``trigger`` until ``release``. With ``after``, only a + trigger statement that follows a statement carrying ``after`` is held, so a test can pin the hold to + one point of a longer sequence. Starts armed unless ``armed`` is false, then ``arm`` starts it.""" + + def __init__( + self, + upstream_host: str, + upstream_port: int, + trigger: bytes, + *, + after: bytes | None = None, + armed: bool = True, + ) -> None: self.port: Final = allocate_port() self._upstream_host: Final = upstream_host self._upstream_port: Final = upstream_port self._trigger: Final = trigger + self._after: Final = after + self._marker_length: Final = max(len(trigger), len(after or b"")) self._server: Final = LoopbackServer(self.port, self._serve) self._loop: Final = self._server.loop self._released: Final = asyncio.Event() + self._armed: Final = threading.Event() + if armed: + self._armed.set() + self._after_seen: Final = threading.Event() self.held: Final = threading.Event() + def arm(self) -> None: + self._armed.set() + def release(self) -> None: if not self._loop.is_closed(): self._loop.call_soon_threadsafe(self._released.set) @@ -96,7 +116,13 @@ class HeldStatementRelay: self._server.stop() def _holds(self, window: bytes) -> bool: - return not self.held.is_set() and self._trigger in window + if not self._armed.is_set() or self.held.is_set(): + return False + if self._after is not None and not self._after_seen.is_set(): + if self._after in window: + self._after_seen.set() + return False + return self._trigger in window async def _serve(self, client_reader: asyncio.StreamReader, client_writer: asyncio.StreamWriter) -> None: server_reader, server_writer = await asyncio.open_connection(self._upstream_host, self._upstream_port) @@ -109,7 +135,7 @@ class HeldStatementRelay: if inspect and self._holds(window): self.held.set() await self._released.wait() - tail = window[-(len(self._trigger) - 1) :] + tail = window[-(self._marker_length - 1) :] writer.write(chunk) await writer.drain() except (ConnectionError, asyncio.IncompleteReadError): @@ -295,10 +321,12 @@ def database_relay(database_url: str, trigger: bytes) -> Generator[tuple[Databas @contextmanager -def held_statement_relay(database_url: str, trigger: bytes) -> Generator[tuple[HeldStatementRelay, str]]: +def held_statement_relay( + database_url: str, trigger: bytes, *, after: bytes | None = None, armed: bool = True +) -> Generator[tuple[HeldStatementRelay, str]]: parts: Final = urlsplit(database_url) assert parts.hostname is not None and parts.port is not None, database_url - relay: Final = HeldStatementRelay(parts.hostname, parts.port, trigger) + relay: Final = HeldStatementRelay(parts.hostname, parts.port, trigger, after=after, armed=armed) try: relay.start() yield relay, _relayed_url(database_url, relay.port) diff --git a/tests/integration/management/test_credential_writes_survive_reload.py b/tests/integration/management/test_credential_writes_survive_reload.py new file mode 100644 index 00000000000..9fb7d7eb681 --- /dev/null +++ b/tests/integration/management/test_credential_writes_survive_reload.py @@ -0,0 +1,119 @@ +import time +import uuid +from concurrent.futures import ThreadPoolExecutor, wait +from pathlib import Path +from typing import Final, Literal + +import pytest +import yaml +from pydantic import JsonValue + +from tests.integration._support.client import JSON_OBJECT, Gateway +from tests.integration._support.database import read_rows, scratch_database +from tests.integration._support.database_relay import held_statement_relay, plaintext_database_url +from tests.integration._support.process import owned_proxy + +RELOAD_INTERVAL_SECONDS: Final = 6 +CREDENTIAL_SNAPSHOT: Final = b'FROM "public"."LiteLLM_CredentialsTable"' +CONFIG_ROW_READ: Final = b"environment_variables" +CredentialWrite = Literal["create", "update", "delete"] + + +def _reloading_config(directory: Path) -> Path: + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["general_settings"] = { + **config.get("general_settings", {}), + "store_model_in_db": True, + "proxy_config_reload_interval_seconds": RELOAD_INTERVAL_SECONDS, + } + config["litellm_settings"] = { + name: value + for name, value in config.get("litellm_settings", {}).items() + if name not in ("cache", "cache_params", "enable_redis_auth_cache") + } + path: Final = directory / "credential-reload.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +def _credential_body(name: str, display_name: str) -> dict[str, JsonValue]: + return { + "credential_name": name, + "display_name": display_name, + "credential_values": {"api_key": "synthetic-credential"}, + "credential_info": {"custom_llm_provider": "openai"}, + } + + +def _write_credential(proxy: Gateway, write: CredentialWrite, name: str) -> int: + match write: + case "create": + return proxy.request("POST", "/credentials", _credential_body(name, "created")).status_code + case "update": + return proxy.request( + "PATCH", f"/credentials/{name}", {"display_name": "updated", "credential_info": {}} + ).status_code + case "delete": + return proxy.request("DELETE", f"/credentials/{name}").status_code + + +def _served_display_name(proxy: Gateway, name: str) -> JsonValue: + response: Final = proxy.request("GET", f"/credentials/by_name/{name}") + if response.status_code == 404: + return None + assert response.status_code == 200, response.text + return JSON_OBJECT.validate_json(response.content)["display_name"] + + +def _stored_display_name(name: str, database_url: str) -> JsonValue: + rows: Final = read_rows( + 'SELECT display_name FROM "LiteLLM_CredentialsTable" WHERE credential_name = %s', + (name,), + database_url=database_url, + ) + return rows[0]["display_name"] if rows else None + + +def _served_display_names_until_the_next_reload(proxy: Gateway, name: str) -> frozenset[JsonValue]: + deadline: Final = time.monotonic() + RELOAD_INTERVAL_SECONDS / 3 + observed: Final = tuple(_served_display_name(proxy, name) for _ in iter(lambda: time.monotonic() < deadline, False)) + return frozenset(observed) + + +@pytest.mark.timeout(300) +@pytest.mark.parametrize("write", ("create", "update", "delete")) +def test_credential_write_is_not_undone_by_an_in_flight_config_reload( + gateway: Gateway, tmp_path: Path, write: CredentialWrite +) -> None: + name: Final = f"credential-{uuid.uuid4().hex}" + expected: Final = {"create": "created", "update": "updated", "delete": None}[write] + with ( + scratch_database() as database_url, + held_statement_relay(database_url, CONFIG_ROW_READ, after=CREDENTIAL_SNAPSHOT, armed=False) as ( + relay, + relayed_url, + ), + ): + overrides: Final = { + "DATABASE_URL": plaintext_database_url(relayed_url), + "LITELLM_CONFIG_PARAM_CACHE_TTL_SECONDS": "0", + } + with owned_proxy( + gateway, + tmp_path, + overrides, + config=_reloading_config(tmp_path), + remove_environment=("REDIS_HOST", "REDIS_PORT", "REDIS_URL"), + workers=1, + ) as proxy: + if write != "create": + proxy.post("/credentials", _credential_body(name, "original")) + relay.arm() + assert relay.held.wait(timeout=RELOAD_INTERVAL_SECONDS * 3), "no config reload reached the database" + with ThreadPoolExecutor(max_workers=1) as pool: + writing: Final = pool.submit(_write_credential, proxy, write, name) + _ = wait((writing,), timeout=2) + relay.release() + assert writing.result(timeout=30) == 200 + assert _stored_display_name(name, database_url) == expected + assert _served_display_names_until_the_next_reload(proxy, name) == frozenset({expected}) diff --git a/tests/unit/proxy/common_utils/test_registry_read_through.py b/tests/unit/proxy/common_utils/test_registry_read_through.py index da7b1f497c9..6c9104bb711 100644 --- a/tests/unit/proxy/common_utils/test_registry_read_through.py +++ b/tests/unit/proxy/common_utils/test_registry_read_through.py @@ -433,7 +433,7 @@ async def test_resync_model_deployments_mutates_router_under_model_reconcile_loc lock_states: list[bool] = [] def record_add_deployment(db_models) -> None: - lock_states.append(proxy_server.MODEL_RECONCILE_LOCK.locked()) + lock_states.append(proxy_server.DB_RECONCILE_LOCK.locked()) monkeypatch.setattr(proxy_server, "prisma_client", prisma_client) monkeypatch.setattr(proxy_server, "store_model_in_db", True) @@ -443,7 +443,7 @@ async def test_resync_model_deployments_mutates_router_under_model_reconcile_loc assert await _resync_model_deployments("lock-scope-model") is True assert lock_states == [True] - assert not proxy_server.MODEL_RECONCILE_LOCK.locked() + assert not proxy_server.DB_RECONCILE_LOCK.locked() @pytest.mark.asyncio @@ -601,7 +601,7 @@ async def test_resync_model_deployments_by_ids_installs_only_the_published_rows( assert _served_ids(router.registry().deployments()) == {"primary-id", "m-1", "m-2"} assert _served_ids(proxy_server.llm_model_list) == {"primary-id", "m-1", "m-2"} - assert not proxy_server.MODEL_RECONCILE_LOCK.locked() + assert not proxy_server.DB_RECONCILE_LOCK.locked() @pytest.mark.asyncio @@ -619,7 +619,7 @@ async def test_resync_model_deployments_by_ids_leaves_the_router_alone_for_delet assert _served_ids(router.registry().deployments()) == {"primary-id"} assert proxy_server.llm_model_list is None - assert not proxy_server.MODEL_RECONCILE_LOCK.locked() + assert not proxy_server.DB_RECONCILE_LOCK.locked() @pytest.mark.asyncio @@ -1021,7 +1021,7 @@ async def test_unknown_model_name_read_through_reads_the_replica_without_waiting monkeypatch.setattr(proxy_server, "prisma_client", prisma_client) monkeypatch.setattr(proxy_server, "store_model_in_db", True) monkeypatch.setattr(proxy_server, "llm_router", MagicMock()) - monkeypatch.setattr(proxy_server, "MODEL_RECONCILE_LOCK", reload_in_progress) + monkeypatch.setattr(proxy_server, "DB_RECONCILE_LOCK", reload_in_progress) async with reload_in_progress: found: Final = await asyncio.wait_for(_resync_model_deployments("no-such-model"), timeout=2) @@ -1042,7 +1042,7 @@ async def test_resync_model_deployments_by_ids_installs_the_row_a_concurrent_rel router: Final = _router_serving("primary-id") _serve_db_models(monkeypatch, _Db(table), router) reload_in_progress: Final = asyncio.Lock() - monkeypatch.setattr(proxy_server, "MODEL_RECONCILE_LOCK", reload_in_progress) + monkeypatch.setattr(proxy_server, "DB_RECONCILE_LOCK", reload_in_progress) async with reload_in_progress: targeted_load: Final = asyncio.create_task(resync_model_deployments_by_ids(("m-1",))) @@ -1132,4 +1132,4 @@ async def test_resync_model_deployments_by_ids_installs_the_ordinary_rows_queued assert _served_ids(proxy_server.llm_model_list) == {"primary-id", "backup-id"} assert table.reads == 1 assert db.other_tables_read == [] - assert not proxy_server.MODEL_RECONCILE_LOCK.locked() + assert not proxy_server.DB_RECONCILE_LOCK.locked() diff --git a/tests/unit/proxy/credential_endpoints/test_endpoints.py b/tests/unit/proxy/credential_endpoints/test_endpoints.py index 10751def88d..f08fab25c57 100644 --- a/tests/unit/proxy/credential_endpoints/test_endpoints.py +++ b/tests/unit/proxy/credential_endpoints/test_endpoints.py @@ -1,6 +1,8 @@ """Tests for the credential management endpoints.""" +import asyncio import json +from collections.abc import Awaitable, Callable from contextlib import contextmanager from functools import reduce from types import SimpleNamespace @@ -19,11 +21,22 @@ import litellm from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.litellm_logging import get_masked_values from litellm.llms.github_copilot.per_user_auth import _SESSION_CACHE, _session_cache_key -from litellm.models.credentials import CredentialSource, UserConnectionFlowHandle +from litellm.models.credentials import ( + CreateCredentialItem, + CredentialSource, + UpdateCredentialItem, + UserConnectionFlowHandle, +) from litellm.proxy._types import LiteLLM_UserTable, LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper, encrypt_value_helper -from litellm.proxy.credential_endpoints.endpoints import _decode_flow_handle, get_llm_router +from litellm.proxy.credential_endpoints.endpoints import ( + _decode_flow_handle, + create_credential, + delete_credential, + get_llm_router, + update_credential, +) from litellm.proxy.proxy_server import app from litellm.types.utils import CredentialItem @@ -3345,3 +3358,101 @@ def test_get_credential_by_model_masks_the_deployment_credentials(monkeypatch): with pytest.raises(ProxyException) as missing: asyncio.run(get_credential_by_model(MagicMock(), MagicMock(), "no-such-id", caller)) assert missing.value.code == "404" + + +class TestCredentialWritesHoldTheReconcileLock: + """A config reload snapshots the credential table, awaits the config rows, then prunes and + upserts ``litellm.credential_list`` from that snapshot. A credential write that lands inside + that gap is undone in memory until the next reload, so each write has to queue behind + DB_RECONCILE_LOCK, the lock the reload already holds.""" + + @staticmethod + async def _assert_write_waits_for_the_lock( + monkeypatch: pytest.MonkeyPatch, call_endpoint: Callable[[], Awaitable[dict[str, object]]], write: str + ) -> dict[str, object]: + lock: Final = asyncio.Lock() + monkeypatch.setattr("litellm.proxy.proxy_server.DB_RECONCILE_LOCK", lock) + async with lock: + task: Final = asyncio.create_task(call_endpoint()) + for _ in range(50): + await asyncio.sleep(0) + assert not task.done(), ( + f"{write} did not wait for DB_RECONCILE_LOCK, so an in-flight config reload can undo it in memory" + ) + return await asyncio.wait_for(task, timeout=5) + + async def test_create_credential_waits_for_an_in_flight_reload( + self, monkeypatch: pytest.MonkeyPatch, credential_store: Callable[..., None] + ) -> None: + credential_store(create=AsyncMock(return_value=None)) + + async def call_endpoint() -> dict[str, object]: + return await create_credential( + request=MagicMock(spec=Request), + fastapi_response=MagicMock(), + credential=CreateCredentialItem( + credential_name="fresh", + credential_values={"api_key": "sk-new"}, + credential_info={"custom_llm_provider": "openai"}, + ), + user_api_key_dict=_as_admin(), + llm_router=None, + ) + + response: Final = await self._assert_write_waits_for_the_lock(monkeypatch, call_endpoint, "create") + + assert response["success"] is True + assert [credential.credential_name for credential in litellm.credential_list] == ["fresh"] + + async def test_update_credential_waits_for_an_in_flight_reload( + self, monkeypatch: pytest.MonkeyPatch, credential_store: Callable[..., None] + ) -> None: + stored: Final = CredentialItem( + credential_name="existing", + display_name="before", + credential_values={"api_key": "sk-old"}, + credential_info={"custom_llm_provider": "openai"}, + ) + credential_store( + in_memory=(stored,), + find_by_name=AsyncMock(return_value=stored), + update_by_name=AsyncMock(return_value=None), + ) + + async def call_endpoint() -> dict[str, object]: + return await update_credential( + request=MagicMock(spec=Request), + fastapi_response=MagicMock(), + credential=UpdateCredentialItem(display_name="after", credential_info={}), + credential_name="existing", + user_api_key_dict=_as_admin(), + llm_router=None, + ) + + response: Final = await self._assert_write_waits_for_the_lock(monkeypatch, call_endpoint, "update") + + assert response["success"] is True + assert [credential.display_name for credential in litellm.credential_list] == ["after"] + + async def test_delete_credential_waits_for_an_in_flight_reload( + self, monkeypatch: pytest.MonkeyPatch, credential_store: Callable[..., None] + ) -> None: + stored: Final = CredentialItem( + credential_name="doomed", + credential_values={"api_key": "sk-old"}, + credential_info={"custom_llm_provider": "openai"}, + ) + credential_store(in_memory=(stored,), delete_by_name=AsyncMock(return_value=MagicMock())) + + async def call_endpoint() -> dict[str, object]: + return await delete_credential( + request=MagicMock(spec=Request), + fastapi_response=MagicMock(), + credential_name="doomed", + user_api_key_dict=_as_admin(), + ) + + response: Final = await self._assert_write_waits_for_the_lock(monkeypatch, call_endpoint, "delete") + + assert response["success"] is True + assert litellm.credential_list == [] diff --git a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py index 2fdb77af8f9..ee16b68b750 100644 --- a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py @@ -1571,7 +1571,7 @@ def deleted_auto_router_catalog(monkeypatch): config = proxy_server.ProxyConfig() config.auto_router_db_catalog = build_auto_router_catalog(rows) monkeypatch.setattr(proxy_server, "proxy_config", config) - monkeypatch.setattr(proxy_server, "MODEL_RECONCILE_LOCK", asyncio.Lock()) + monkeypatch.setattr(proxy_server, "DB_RECONCILE_LOCK", asyncio.Lock()) monkeypatch.setattr(proxy_server, "llm_router", Router(model_list=[])) monkeypatch.setattr(proxy_server, "_license_check", SimpleNamespace(auto_router_capability_limit=lambda: 1)) monkeypatch.setattr(proxy_server, "heuristic_v1_tuning_baselines", {}) @@ -5959,7 +5959,7 @@ class TestConcurrentModelWritesDoNotEvictEachOther: @pytest.mark.asyncio async def test_reconciles_serialize_so_no_stale_snapshot_can_evict(self, monkeypatch): - """MODEL_RECONCILE_LOCK admits one reconcile at a time. + """DB_RECONCILE_LOCK admits one reconcile at a time. The fake body awaits, which is the whole point: without the lock the gather below parks all five inside the critical section at that await and observed depth goes @@ -6088,7 +6088,7 @@ class TestConcurrentModelWritesDoNotEvictEachOther: class TestDeleteEvictionsHoldTheReconcileLock: """A delete evicts from ``llm_router`` directly instead of reconciling, so it must - take MODEL_RECONCILE_LOCK to do it. + take DB_RECONCILE_LOCK to do it. The db row is gone by then, but a reconcile that snapshotted the db BEFORE the row was deleted still lists that id as desired, and its ``_add_deployment`` upserts the @@ -6102,7 +6102,7 @@ class TestDeleteEvictionsHoldTheReconcileLock: async def _assert_evicts_under_lock(monkeypatch, call_endpoint, model_id: str, config) -> None: """Run ``call_endpoint`` with the lock already held and assert it blocks. - Holding MODEL_RECONCILE_LOCK stands in for a reconcile that is mid-flight. If + Holding DB_RECONCILE_LOCK stands in for a reconcile that is mid-flight. If the eviction takes the lock it cannot run until we release; if it does not, it runs straight through and the deployment is evicted while the "reconcile" is still in its critical section -- exactly the interleaving that resurrects it. @@ -6115,7 +6115,7 @@ class TestDeleteEvictionsHoldTheReconcileLock: own, exactly as here. """ lock = asyncio.Lock() - monkeypatch.setattr("litellm.proxy.proxy_server.MODEL_RECONCILE_LOCK", lock) + monkeypatch.setattr("litellm.proxy.proxy_server.DB_RECONCILE_LOCK", lock) stale_catalog = config.auto_router_db_catalog async with lock: @@ -6124,7 +6124,7 @@ class TestDeleteEvictionsHoldTheReconcileLock: for _ in range(50): await asyncio.sleep(0) assert not task.done(), ( - f"deleting {model_id} did not wait for MODEL_RECONCILE_LOCK -- an " + f"deleting {model_id} did not wait for DB_RECONCILE_LOCK -- an " f"in-flight reconcile can resurrect the deployment it just evicted" ) config.auto_router_db_catalog = stale_catalog From c5ad400f0bb17b1bb6e21a838f10f38ce34fdad0 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Sat, 7 Nov 2026 16:02:59 -0800 Subject: [PATCH 13/39] fix(anthropic): refresh the WIF bearer token on the connection error retry (#50570) * Refresh the WIF bearer token on the connection error retry * Only refresh the bearer when the retry goes to the host the token was minted for --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/base_llm/auth/token_exchange.py | 34 ++++- litellm/llms/custom_httpx/http_handler.py | 32 +++- .../llms/custom_httpx/test_http_handler.py | 140 +++++++++++++++++- 3 files changed, 197 insertions(+), 9 deletions(-) diff --git a/litellm/llms/base_llm/auth/token_exchange.py b/litellm/llms/base_llm/auth/token_exchange.py index 5bf1f4dd0db..ce0e3c45639 100644 --- a/litellm/llms/base_llm/auth/token_exchange.py +++ b/litellm/llms/base_llm/auth/token_exchange.py @@ -9,6 +9,7 @@ public exception contract. import asyncio import hashlib +import hmac import json import re import threading @@ -538,10 +539,22 @@ class _Entry: """Single-flight state for one cache key; mutable by design, confined to the engine, and only ever mutated under the engine lock.""" - __slots__ = ("backoff_until", "done", "force_refresh", "in_flight", "last_error", "lifetime_seconds", "token") + __slots__ = ( + "backoff_until", + "done", + "force_refresh", + "in_flight", + "last_error", + "lifetime_seconds", + "previous_token", + "spec", + "token", + ) - def __init__(self, force_refresh: bool = False) -> None: + def __init__(self, spec: TokenExchangeSpec, force_refresh: bool = False) -> None: + self.spec: Final = spec self.token: MintedToken | None = None + self.previous_token: MintedToken | None = None self.lifetime_seconds: float | None = None self.in_flight: bool = False self.done: Final = threading.Event() @@ -562,7 +575,14 @@ class _Entry: self.in_flight = False self.done.set() + def issued(self, access_token: str) -> bool: + return any( + minted is not None and hmac.compare_digest(minted.access_token.get_secret_value(), access_token) + for minted in (self.token, self.previous_token) + ) + def _store(self, token: MintedToken, now: float) -> None: + self.previous_token = self.token self.token = token self.lifetime_seconds = None if token.expires_at is None else max(token.expires_at - now, 0.0) self.last_error = None @@ -684,11 +704,17 @@ class JwtBearerTokenExchangeEngine: async def aget_token(self, spec: TokenExchangeSpec) -> ExchangeResult: return await asyncio.to_thread(self.get_token, spec) + def spec_for_access_token(self, access_token: str) -> TokenExchangeSpec | None: + """Finds the identity a bearer was minted for, so resending an already built request can swap in + the current token instead of one that expired while the first attempt was in flight.""" + with self._lock: + return next((entry.spec for entry in self._entries.values() if entry.issued(access_token)), None) + def invalidate(self, spec: TokenExchangeSpec) -> None: key: Final = _cache_key(spec) with self._lock: if key in self._entries: - self._entries[key] = _Entry(force_refresh=True) + self._entries[key] = _Entry(spec, force_refresh=True) if self._shared_store is not None: self._shared_store.delete(key) @@ -699,7 +725,7 @@ class JwtBearerTokenExchangeEngine: return existing if len(self._entries) >= self._max_entries: self._evict_locked() - created: Final = _Entry() + created: Final = _Entry(spec) self._entries[key] = created return created diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 3a6fa5f51da..0cfdbc74227 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -44,6 +44,8 @@ from litellm.litellm_core_utils.logging_utils import track_llm_api_timing from litellm.litellm_core_utils.request_timeout_resolver import ( get_configured_request_timeout, ) +from litellm.llms.base_llm.auth.token_exchange import JwtBearerTokenExchangeEngine, default_token_exchange_engine +from litellm.llms.base_llm.auth.types import MintedToken from litellm.llms.custom_httpx.aiohttp_pool_diagnostics import build_pool_wait_trace_config from litellm.llms.custom_httpx.http2_connection_spread import AsyncHTTP2ConnectionSpread from litellm.types.llms.custom_http import * @@ -586,6 +588,28 @@ def _raise_masked_sync_error(e: httpx.HTTPStatusError, stream: bool) -> NoReturn raise MaskedHTTPStatusError(e, message=_text, text=_text) from None +def _same_origin(first: httpx.URL, second: httpx.URL) -> bool: + return (first.scheme, first.host, first.port) == (second.scheme, second.host, second.port) + + +async def _with_current_bearer( + headers: Mapping[str, str] | None, url: str, engine: JwtBearerTokenExchangeEngine +) -> Mapping[str, str] | None: + auth_key: Final = next((key for key in headers or () if key.lower() == "authorization"), None) + if headers is None or auth_key is None: + return headers + auth_value: Final = headers[auth_key] + if not auth_value.startswith("Bearer "): + return headers + spec: Final = engine.spec_for_access_token(auth_value.removeprefix("Bearer ")) + if spec is None or not _same_origin(httpx.URL(spec.token_url), httpx.URL(url)): + return headers + current: Final = await engine.aget_token(spec) + if not isinstance(current, MintedToken): + return headers + return MappingProxyType({**headers, auth_key: f"Bearer {current.access_token.get_secret_value()}"}) + + async def _raise_masked_async_error(e: httpx.HTTPStatusError, stream: bool) -> NoReturn: """Raise a MaskedHTTPStatusError for async HTTP handlers.""" if stream: @@ -639,7 +663,7 @@ class HTTPResponseLimitError(ValueError): class MaskedHTTPStatusError(httpx.HTTPStatusError): - def __init__(self, original_error, message: str | None = None, text: str | None = None): + def __init__(self, original_error: httpx.HTTPStatusError, message: str | None = None, text: str | None = None): # Create a new error with the masked URL masked_url: Final = mask_sensitive_info(str(original_error.request.url)) # Mask the original exception message too (it contains the full URL) @@ -702,12 +726,14 @@ class AsyncHTTPHandler: transport: httpx.AsyncBaseTransport | None = None, follow_redirects: bool = True, http2: bool = False, + token_exchange_engine: JwtBearerTokenExchangeEngine = default_token_exchange_engine, ): self.timeout = timeout self.event_hooks = event_hooks self.ssl_verify = ssl_verify self.shared_session = shared_session self.transport = transport + self.token_exchange_engine = token_exchange_engine self.follow_redirects = follow_redirects self.http2 = http2 self._owns_client = True @@ -1037,7 +1063,7 @@ class AsyncHTTPHandler: data=retry_data, json=json, params=params, - headers=headers, + headers=await _with_current_bearer(headers, url, self.token_exchange_engine), timeout=timeout, content=retry_content, ) @@ -1193,7 +1219,7 @@ class AsyncHTTPHandler: data=request_data, json=json, params=params, - headers=headers, + headers=await _with_current_bearer(headers, url, self.token_exchange_engine), content=request_content, ) response: Final = await client.send(req, stream=stream) diff --git a/tests/unit/llms/custom_httpx/test_http_handler.py b/tests/unit/llms/custom_httpx/test_http_handler.py index 3e583bf8dff..0de8efa2662 100644 --- a/tests/unit/llms/custom_httpx/test_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_http_handler.py @@ -26,6 +26,8 @@ from yarl import URL import litellm from litellm.caching.llm_caching_handler import LLMClientCache from litellm.llms.custom_httpx.aiohttp_transport import LiteLLMAiohttpTransport +from litellm.llms.base_llm.auth.token_exchange import JwtBearerTokenExchangeEngine, default_token_exchange_engine +from litellm.llms.base_llm.auth.types import MintedToken, TokenExchangeSpec from litellm.llms.custom_httpx.http_handler import ( _CLIENT_REFCOUNT_WHEN_HANDLER_IS_SOLE_REFERRER, AsyncHTTPHandler, @@ -1513,9 +1515,14 @@ async def test_finalizer_on_live_loop_disposes_foreign_loop_session_without_sche class _RetryClientHandler(AsyncHTTPHandler): - def __init__(self, first: httpx.AsyncClient, retry: httpx.AsyncClient) -> None: + def __init__( + self, + first: httpx.AsyncClient, + retry: httpx.AsyncClient, + token_exchange_engine: JwtBearerTokenExchangeEngine = default_token_exchange_engine, + ) -> None: self._retry_client: Final = retry - super().__init__() + super().__init__(token_exchange_engine=token_exchange_engine) self.client = first def create_client( @@ -1552,6 +1559,135 @@ async def test_connection_error_retry_forwards_content(method: str): await handler.close() +class _WifClock: + def __init__(self) -> None: + self.now = 1_000.0 # mutable-ok: fake clock advanced by the test + + def __call__(self) -> float: + return self.now + + +class _CountingIssuer: + def __init__(self) -> None: + self.mints = 0 # mutable-ok: counts tokens minted so each one is distinct + + def post(self, url: str, *, content: bytes, headers: Mapping[str, str], timeout: float) -> httpx.Response: + self.mints += 1 + return httpx.Response( + 200, json={"access_token": f"wif-token-{self.mints}", "expires_in": 600, "token_type": "Bearer"} + ) + + +_WIF_SPEC: Final = TokenExchangeSpec( + token_url="https://api.anthropic.com/v1/oauth/token", + assertion_ref="oidc/env/ANTHROPIC_IDENTITY_TOKEN", + assertion_field="assertion", + static_body={"grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer", "federation_rule_id": "fdrl_1"}, + body_encoding="json", + request_headers={}, + cache_key_identity=("fdrl_1", "org-1", "", ""), + timeout_seconds=2.0, +) + + +def _minted_bearer(engine: JwtBearerTokenExchangeEngine) -> str: + minted: Final = engine.get_token(_WIF_SPEC) + assert isinstance(minted, MintedToken) + return minted.access_token.get_secret_value() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("method", ["post", "put", "patch", "delete"]) +@pytest.mark.parametrize("refreshed_by_another_request", [False, True]) +async def test_connection_error_retry_sends_current_wif_token(method: str, refreshed_by_another_request: bool) -> None: + clock: Final = _WifClock() + engine: Final = JwtBearerTokenExchangeEngine( + poster=_CountingIssuer(), + assertion_reader=lambda ref: "identity-jwt", + clock=clock, + refresh_executor=ThreadPoolExecutor(max_workers=1), + ) + first_token: Final = _minted_bearer(engine) + sent: Final[list[str]] = [] # mutable-ok: async closure capture buffer + + async def expire_then_drop(request: httpx.Request) -> httpx.Response: + sent.append(request.headers["authorization"]) + clock.now += 601 + if refreshed_by_another_request: + _minted_bearer(engine) + raise httpx.ConnectError("connection dropped", request=request) + + async def capture_and_succeed(request: httpx.Request) -> httpx.Response: + sent.append(request.headers["authorization"]) + return httpx.Response(200, request=request) + + first: Final = httpx.AsyncClient(transport=httpx.MockTransport(expire_then_drop)) + retry: Final = httpx.AsyncClient(transport=httpx.MockTransport(capture_and_succeed)) + async with first, retry: + handler: Final = _RetryClientHandler(first=first, retry=retry, token_exchange_engine=engine) + await getattr(handler, method)( + "https://api.anthropic.com/v1/messages", + headers={"authorization": f"Bearer {first_token}", "x-trace": "kept"}, + content=b"{}", + ) + await handler.close() + + assert sent == [f"Bearer {first_token}", f"Bearer {_minted_bearer(engine)}"] + assert first_token != _minted_bearer(engine) + + +@pytest.mark.asyncio +async def test_connection_error_retry_keeps_bearer_the_engine_did_not_mint() -> None: + engine: Final = JwtBearerTokenExchangeEngine( + poster=_CountingIssuer(), assertion_reader=lambda ref: "identity-jwt", clock=_WifClock() + ) + _minted_bearer(engine) + sent: Final[list[str]] = [] # mutable-ok: async closure capture buffer + + async def raise_connection_error(request: httpx.Request) -> httpx.Response: + raise httpx.ConnectError("connection dropped", request=request) + + async def capture_and_succeed(request: httpx.Request) -> httpx.Response: + sent.append(request.headers["authorization"]) + return httpx.Response(200, request=request) + + first: Final = httpx.AsyncClient(transport=httpx.MockTransport(raise_connection_error)) + retry: Final = httpx.AsyncClient(transport=httpx.MockTransport(capture_and_succeed)) + async with first, retry: + handler: Final = _RetryClientHandler(first=first, retry=retry, token_exchange_engine=engine) + await handler.post("https://api.anthropic.com/v1/messages", headers={"Authorization": "Bearer sk-static"}) + await handler.close() + + assert sent == ["Bearer sk-static"] + + +@pytest.mark.asyncio +async def test_connection_error_retry_keeps_wif_token_sent_to_another_host() -> None: + clock: Final = _WifClock() + engine: Final = JwtBearerTokenExchangeEngine( + poster=_CountingIssuer(), assertion_reader=lambda ref: "identity-jwt", clock=clock + ) + first_token: Final = _minted_bearer(engine) + sent: Final[list[str]] = [] # mutable-ok: async closure capture buffer + + async def expire_then_drop(request: httpx.Request) -> httpx.Response: + clock.now += 601 + raise httpx.ConnectError("connection dropped", request=request) + + async def capture_and_succeed(request: httpx.Request) -> httpx.Response: + sent.append(request.headers["authorization"]) + return httpx.Response(200, request=request) + + first: Final = httpx.AsyncClient(transport=httpx.MockTransport(expire_then_drop)) + retry: Final = httpx.AsyncClient(transport=httpx.MockTransport(capture_and_succeed)) + async with first, retry: + handler: Final = _RetryClientHandler(first=first, retry=retry, token_exchange_engine=engine) + await handler.post("https://elsewhere.example/v1/messages", headers={"authorization": f"Bearer {first_token}"}) + await handler.close() + + assert sent == [f"Bearer {first_token}"] + + @pytest.mark.asyncio @pytest.mark.parametrize("method", ["post", "patch"]) async def test_disconnect_after_send_is_not_replayed(method: str) -> None: From 7617e66acfe1f3f9bb2984646c7da7c2f7d9db1b Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sun, 8 Nov 2026 00:03:40 +0000 Subject: [PATCH 14/39] test(translation): expect encrypted reasoning in message responses (#50576) Co-authored-by: mateo <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../translation/messages/bases/azure.py | 176 +++++++++--------- .../messages/bases/bedrock_mantle.py | 84 ++++++++- 2 files changed, 172 insertions(+), 88 deletions(-) diff --git a/tests/integration/translation/messages/bases/azure.py b/tests/integration/translation/messages/bases/azure.py index b731f7b7760..cbe03515b0e 100644 --- a/tests/integration/translation/messages/bases/azure.py +++ b/tests/integration/translation/messages/bases/azure.py @@ -323,6 +323,24 @@ GPT_6_LUNA_CONTENT_FILTERS: Final[JsonValue] = [ }, ] +GPT_6_LUNA_ENCRYPTED_CONTENT: Final = ( + "gAAAAABq0CO7N77UmN8oIxFPRrP2rvE9LuqnrGGHsFzckseHLpNB24KIlg0OKnfemaCgYarrVksxeJBKgoiQ79xkF1NR9RI8-82w" + "4OGNB0ViGNYNA7nf5kOqOTrMh1_gHecIN0Evxn8I0meiqOds7h1biKfkiZlm19b-9r_Y2K--KXONVKFEbe8kP54_ka9KCxQ3_5Xn" + "L4IRfRS7lsLfoMjORKnapkVSnMjueRVVOprw_897JJypeT61mzzBYM_18T6k68OE70JndMsr4GNRq4c_Lf_rSRoHg7_iFrdHsBdv" + "MfN79dfp_PeLu5MZVRgcag-qM9aKUAvzhTD2-EutMw1C1ZNabU2S5Qa4RXq2OiU_iZljJI6JhpnJ9jT9g_M3W_0mklmFU2HBhQ6O" + "csIFmPePVcSaQlmofKgvq2ZThSHyFpLsT15ftBL2ZMZ3OWeodDnkfRr2fY5NDM7v7U1TLcAx7xCnP88W03kyb3SuJGSsbPnYTGbN" + "ZN7ye3SBJRkCMpRgUfguSYdC46mmMjPdvdkLnTWHZ23GOoDG5A2lffUJZ4WXNgPrMdG2FNThtHT0otHeDC75ahpa6i5xtqIsCGiF" + "RVH_iSxhItArxFXoc7e6oBU9YxsgkUkyXW_zz6h5h5E6DoJz4QckOAdL8E3SwL1nu6ovpS3_QgeiHa5RewKs6WclFCxeiwiyXO6b" + "fjhl_VtOA8KyWqkO-rt4T0RneV_bZHav6KoVanyCX-ZsAYj1WIhn4xEOXh6RdBjZJJB-yDw5cq6zdXkmN1hxs_nmA7hUemHkPclb" + "2lGF41N28IwMpcXq1rx2jrw2XleLnqFVKwgCLysinS9erU2vRN1CmmDQrGl7C5lIOrfSVCpB2W_p3BZWNDKwAfF4i8uXCxy1_5-E" + "jJ08CJXXYVchceOreHYNYxAXhVTHhJArQqVWQgTwJl9e_HhyA0OFZHojB54BjlkMr7QJMm9VE9LIkOJgRgIrtUM2ULIy1tZ-Ezuz" + "Ey9HYx5NuUKe_12ExKuUVrQmaJ2HRHvj_pPcmSk6lOVrN5VvVBMj_KHd3MfkbUWVeVfzIsgWc149TexK6FPtvHGPxf_VQBdyA4Jl" + "77dPp671wxH7puzy184LV1IMdxq5eY1JG4QdCOluIDkkqbw_qYEiKGNXImGnp9LsXCcYa6JivYmSAw_vgZ5YhO98L49aLEI6nT3H" + "WRWuzY3H39aCkFdA8Y0DttzwXYvhz8UuenFgoGvIk0W7WSOLBxi6pSefdczpRizWLMm10zlrYVFvdMzUPEx7OyDUhSXag61gYcsm" + "6L5mcXGb2cojThdHEXcFZg36bQYjQVJzu_ASlvN-Z9AFNajIIyw8qKx3Q8pRVg3HCpQcSyRiDhicvT_Gp8d0X7VhoAhG_-LPg4AO" + "z7U985JqP0wB2dB4sYhRgMZpQ29x7wgvgjNtsyqbCA==" +) + GPT_6_LUNA_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", @@ -347,27 +365,7 @@ GPT_6_LUNA_TEST_CASE: Final = TranslationTestCase( "id": "rs_0534b0c3898d6d1b006ad023bb09b8819590793fe299bffdcf", "type": "reasoning", "content": [], - "encrypted_content": ( - "gAAAAABq0CO7N77UmN8oIxFPRrP2rvE9LuqnrGGHsFzckseHLpNB24KIlg0OKnfemaCgYarrVksxeJBK" - "goiQ79xkF1NR9RI8-82w4OGNB0ViGNYNA7nf5kOqOTrMh1_gHecIN0Evxn8I0meiqOds7h1biKfkiZlm" - "19b-9r_Y2K--KXONVKFEbe8kP54_ka9KCxQ3_5XnL4IRfRS7lsLfoMjORKnapkVSnMjueRVVOprw_897" - "JJypeT61mzzBYM_18T6k68OE70JndMsr4GNRq4c_Lf_rSRoHg7_iFrdHsBdvMfN79dfp_PeLu5MZVRgc" - "ag-qM9aKUAvzhTD2-EutMw1C1ZNabU2S5Qa4RXq2OiU_iZljJI6JhpnJ9jT9g_M3W_0mklmFU2HBhQ6O" - "csIFmPePVcSaQlmofKgvq2ZThSHyFpLsT15ftBL2ZMZ3OWeodDnkfRr2fY5NDM7v7U1TLcAx7xCnP88W" - "03kyb3SuJGSsbPnYTGbNZN7ye3SBJRkCMpRgUfguSYdC46mmMjPdvdkLnTWHZ23GOoDG5A2lffUJZ4WX" - "NgPrMdG2FNThtHT0otHeDC75ahpa6i5xtqIsCGiFRVH_iSxhItArxFXoc7e6oBU9YxsgkUkyXW_zz6h5" - "h5E6DoJz4QckOAdL8E3SwL1nu6ovpS3_QgeiHa5RewKs6WclFCxeiwiyXO6bfjhl_VtOA8KyWqkO-rt4" - "T0RneV_bZHav6KoVanyCX-ZsAYj1WIhn4xEOXh6RdBjZJJB-yDw5cq6zdXkmN1hxs_nmA7hUemHkPclb" - "2lGF41N28IwMpcXq1rx2jrw2XleLnqFVKwgCLysinS9erU2vRN1CmmDQrGl7C5lIOrfSVCpB2W_p3BZW" - "NDKwAfF4i8uXCxy1_5-EjJ08CJXXYVchceOreHYNYxAXhVTHhJArQqVWQgTwJl9e_HhyA0OFZHojB54B" - "jlkMr7QJMm9VE9LIkOJgRgIrtUM2ULIy1tZ-EzuzEy9HYx5NuUKe_12ExKuUVrQmaJ2HRHvj_pPcmSk6" - "lOVrN5VvVBMj_KHd3MfkbUWVeVfzIsgWc149TexK6FPtvHGPxf_VQBdyA4Jl77dPp671wxH7puzy184L" - "V1IMdxq5eY1JG4QdCOluIDkkqbw_qYEiKGNXImGnp9LsXCcYa6JivYmSAw_vgZ5YhO98L49aLEI6nT3H" - "WRWuzY3H39aCkFdA8Y0DttzwXYvhz8UuenFgoGvIk0W7WSOLBxi6pSefdczpRizWLMm10zlrYVFvdMzU" - "PEx7OyDUhSXag61gYcsm6L5mcXGb2cojThdHEXcFZg36bQYjQVJzu_ASlvN-Z9AFNajIIyw8qKx3Q8pR" - "Vg3HCpQcSyRiDhicvT_Gp8d0X7VhoAhG_-LPg4AOz7U985JqP0wB2dB4sYhRgMZpQ29x7wgvgjNtsyqb" - "CA==" - ), + "encrypted_content": GPT_6_LUNA_ENCRYPTED_CONTENT, "summary": [], }, { @@ -411,6 +409,10 @@ GPT_6_LUNA_TEST_CASE: Final = TranslationTestCase( "model": "azure/gpt-6-luna", "usage": {"input_tokens": 309, "output_tokens": 114}, "content": [ + { + "type": "redacted_thinking", + "data": "litellm_encrypted_reasoning:" + GPT_6_LUNA_ENCRYPTED_CONTENT, + }, { "type": "text", "text": ( @@ -457,6 +459,25 @@ GPT_6_1_SOL_CONTENT_FILTERS: Final[JsonValue] = [ }, ] +GPT_6_1_SOL_ENCRYPTED_CONTENT: Final = ( + "gAAAAABq0CO0QoINZ4oOVkvDvinFAJAMUBmzqqpXLoppdlyP4fFQsNqPOh-L08mQt8yf4OUtOwzk3E6Q7CLHieco0s_tfuYHRlTU" + "0Blr62KF_C77RScOjKYa6p4E2fjdwO44Y2HIiGfIo_GnUpkfwW1EiVZmj6L8PO0biPqihvK3KJ-GNKB_R4-lLyKXrrvVN-985glS" + "dHg-_fRrmyVGBvVxPFyERlD-WPm8fPujqMmXnUcAv3N-wUoIEccd7PfJGqBs52UzllpH27mNCI71C_qbc2nZ97jZgSabTvAZKF8Q" + "9zSJzWA_Gs72iQqd0X55F5vD0-xrMuVdXFTYLV-y-S_1rN8tY1bEbdJWr7Dl84NvNWCYqA5byZgsMxAVyjADWXcVaagrZhNtQ1ol" + "II2oePJL2WyrKrOs4d7ZuGFW5BcFCDS5zxk6MwckKbLkSi3HM34z-AhIOjr6SNc5qcR8yur4Phm9pSJQqFscqQwtmEgbAzXqC9U5" + "rwQ_c11N7o6Xc2bnoRXxeXQvKyGCPZWOqQr8AffHBCCOmeKjbPMV0ITek7HyRiOWg_Pjq-_xlVCoUGM7gIq5nq3HG1SqH7Yr6qev" + "5Arq24UALiBbr0NAtmq3p-YGRgia3oDnSrnygIX8AbqhnGrHz1kjBbwGr6fXF2XruSKuL96bQghfWGx6nhsbIDH4FI3akERb6Rh1" + "B9XIWsWw4Zzo6-tg6DnjzfaJteZqIY4qS5mHSD9stXEVqu_CfnGh7EdOlZlnkG6kIQ3m6oAo2ZG9E58m32X-KEqZDjD4yUgWss9y" + "7cnFbYeEzntQneytoBpaTNXtSEsM1fUoz73Ol7FzqPI0CKIsp3UGje2K7LqAwkV54JrhqCfxkD5HmPJnyEOnP0dODvyqD3CK-cbk" + "oPoszI2Ba0dk11re5kF985he-Fjm7dXDNhQX93__nj4awDCtnX8_Ke5ld7O3sA1ou7WHhgfGo-Jr4ZcGL5JWflPbI9oPpGOOvLsS" + "-e8CUjTcTRb_eA2jApeGU2nAdW42U-CnXeScaR5eJE1RoXTFXIP6MMSpxqpTRWYBfhIDyPEzwTveki-WOWgJtyNJ5W3GJxaEZla2" + "uJ5_cdiD4-HgRy1UwxIJODUh0F2hKC6niDVIZqoZ6Z4nb4OUvJP00_nGBA5-IL31Ij8I8U740RAZvnOsMOIsKelsJioM8Ii4MdPi" + "O2FYSdKjijEJmXLDdbYkQEXeqwfKGC7trXBOyRwO5AF6mLR9Yf8Bu_eiKbuf5DT9sw88P4JtfPi5LLIso2Ki44UnxePraO7vinqg" + "7H7oOi-1gw7BbDaKH7tJs3vUOaxl6mA0BNTg5cDt6TtFZMpFgKNRyanWzSwGWQGYlYbfUGVUnM_tjtgoBNFJAQz1s8fd-qOl1NQD" + "ggjlryl6TJk3RSiNYKyJ3fmpE10LM3LUtN3AoFMDFbEEBstDuVg0zzvwe64zaKosn7PIh_Ym-y7QlMdcvUbsqO-T6pfsq8iqEz2a" + "nv3yujHkAXIaa3292lbAdFIq80ycQSJbWNd2MtkYQHdWtWmQQoqMX1PdyJGp6N3GG45A4A==" +) + GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", @@ -480,28 +501,7 @@ GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase( "id": "rs_0a4a896afe94ce65006ad023b19f8481938d51de793f83feed", "type": "reasoning", "content": [], - "encrypted_content": ( - "gAAAAABq0CO0QoINZ4oOVkvDvinFAJAMUBmzqqpXLoppdlyP4fFQsNqPOh-L08mQt8yf4OUtOwzk3E6Q" - "7CLHieco0s_tfuYHRlTU0Blr62KF_C77RScOjKYa6p4E2fjdwO44Y2HIiGfIo_GnUpkfwW1EiVZmj6L8" - "PO0biPqihvK3KJ-GNKB_R4-lLyKXrrvVN-985glSdHg-_fRrmyVGBvVxPFyERlD-WPm8fPujqMmXnUcA" - "v3N-wUoIEccd7PfJGqBs52UzllpH27mNCI71C_qbc2nZ97jZgSabTvAZKF8Q9zSJzWA_Gs72iQqd0X55" - "F5vD0-xrMuVdXFTYLV-y-S_1rN8tY1bEbdJWr7Dl84NvNWCYqA5byZgsMxAVyjADWXcVaagrZhNtQ1ol" - "II2oePJL2WyrKrOs4d7ZuGFW5BcFCDS5zxk6MwckKbLkSi3HM34z-AhIOjr6SNc5qcR8yur4Phm9pSJQ" - "qFscqQwtmEgbAzXqC9U5rwQ_c11N7o6Xc2bnoRXxeXQvKyGCPZWOqQr8AffHBCCOmeKjbPMV0ITek7Hy" - "RiOWg_Pjq-_xlVCoUGM7gIq5nq3HG1SqH7Yr6qev5Arq24UALiBbr0NAtmq3p-YGRgia3oDnSrnygIX8" - "AbqhnGrHz1kjBbwGr6fXF2XruSKuL96bQghfWGx6nhsbIDH4FI3akERb6Rh1B9XIWsWw4Zzo6-tg6Dnj" - "zfaJteZqIY4qS5mHSD9stXEVqu_CfnGh7EdOlZlnkG6kIQ3m6oAo2ZG9E58m32X-KEqZDjD4yUgWss9y" - "7cnFbYeEzntQneytoBpaTNXtSEsM1fUoz73Ol7FzqPI0CKIsp3UGje2K7LqAwkV54JrhqCfxkD5HmPJn" - "yEOnP0dODvyqD3CK-cbkoPoszI2Ba0dk11re5kF985he-Fjm7dXDNhQX93__nj4awDCtnX8_Ke5ld7O3" - "sA1ou7WHhgfGo-Jr4ZcGL5JWflPbI9oPpGOOvLsS-e8CUjTcTRb_eA2jApeGU2nAdW42U-CnXeScaR5e" - "JE1RoXTFXIP6MMSpxqpTRWYBfhIDyPEzwTveki-WOWgJtyNJ5W3GJxaEZla2uJ5_cdiD4-HgRy1UwxIJ" - "ODUh0F2hKC6niDVIZqoZ6Z4nb4OUvJP00_nGBA5-IL31Ij8I8U740RAZvnOsMOIsKelsJioM8Ii4MdPi" - "O2FYSdKjijEJmXLDdbYkQEXeqwfKGC7trXBOyRwO5AF6mLR9Yf8Bu_eiKbuf5DT9sw88P4JtfPi5LLIs" - "o2Ki44UnxePraO7vinqg7H7oOi-1gw7BbDaKH7tJs3vUOaxl6mA0BNTg5cDt6TtFZMpFgKNRyanWzSwG" - "WQGYlYbfUGVUnM_tjtgoBNFJAQz1s8fd-qOl1NQDggjlryl6TJk3RSiNYKyJ3fmpE10LM3LUtN3AoFMD" - "FbEEBstDuVg0zzvwe64zaKosn7PIh_Ym-y7QlMdcvUbsqO-T6pfsq8iqEz2anv3yujHkAXIaa3292lbA" - "dFIq80ycQSJbWNd2MtkYQHdWtWmQQoqMX1PdyJGp6N3GG45A4A==" - ), + "encrypted_content": GPT_6_1_SOL_ENCRYPTED_CONTENT, "summary": [], }, { @@ -545,6 +545,10 @@ GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase( "model": "azure/gpt-6.1-sol", "usage": {"input_tokens": 309, "output_tokens": 137}, "content": [ + { + "type": "redacted_thinking", + "data": "litellm_encrypted_reasoning:" + GPT_6_1_SOL_ENCRYPTED_CONTENT, + }, { "type": "text", "text": ( @@ -592,6 +596,25 @@ GPT_6_ASTRA_CONTENT_FILTERS: Final[JsonValue] = [ }, ] +GPT_6_ASTRA_ENCRYPTED_CONTENT: Final = ( + "gAAAAABq0CPHdct5jJ3BolLC7QmXutcBsSZ0YzHiI8z2uK07jqwLifVhEROdF12ygdCVRtcrLrHPyHShpVbh41ULxn2L3SyVYLHb" + "bkFMcwbzfky_kqZdUe_1D-PLxsVyI1KFMPLg0Fb9K-N9u_ElvkFcXXsmAhRQCcB4SE9TMe_5tKBHiZbqvOI112wig7cc57rDbvM0" + "mEoKyFn3WgprP_ewHeCeP6DSEvVGDEMjUd_TIXAHH7y2Ae3y0WRW7WUag_wAiO7xzAH13mh4S6eKeBTKh62krmYeUH3wETO8BF1W" + "nkuZ_86lS0zWLBzeLJJnJiXKXVquKatJ6HvO3m6FkPF0tBXhdc8YYtTF1B_WBMECARKcw4n3mPL3R6J2jpHe05Eo7W7uZkcY5E-q" + "mE_cqUK8YQXTXx-mR339iuAvqSe0N2Rgl9Ey1l7z0MBATIV0c6J4wRav6Di5aAxEEXkgDVk1jF9LZSHomcGjh2B2UWFiAZEELRDD" + "7ovpP3c7B05h0QvwazXlZnskJf-xYZEHAl2CNZk0iqagFs6EtNrFguM5W2AH6dsuaGjNGmdWM_kt4JqIUeScZLNgMty8qTOHFzd7" + "tiBpuLkQgcnHzOigWb2vY_8ahy_zfzM2lVL-JQxP9SB7Gq1m49Ln-byC1O_kPyqEFXELhhQ8peXQY5do2R2T-3I0WLOhDhP5rruL" + "sL66vNHAAMjka6WmjI2goiy-JF_nVCplJd791Wqa3yf7Q7ksfasnSKtjR7-cgTtMcXX7n_5joMnqf4_LsqSq6gHdeKw4VTeT8wGB" + "Uo3MHLkgi8PnAIphdb3PKgifSjstE_oH4_ZbmlyCsHaFYq2aoy-S85ndlrR_PAHBtU8II-Ud_42EPzS-Etdt5d1Qjjri3KTyPjPO" + "7fAm8aPFr1k3GbjwASPMQ7cUs7yLhPmI-TCm5U-zgK1pIALviiU2o4IM7kdH0-Dg1eyyZN8hsFGSHaUB7G_tGB0fj0H0GSztH8lt" + "NjTK_276IOh8Jv5csxjjmlcMOHVBmDCkkE_DlVjvNQtWQS9QBt9jd1gFSzaOhXVkKZJK3vTIaq8JxTvYtgLc8WM5pRHbnQr71TfW" + "B-eLEYgN-aX5PueKkXGrGauDpKkzVJyNZ6mZ_dVXwl1p5IiNNJ1xzhmZK2GVDFWt0nfOBYyxmCDfIxvQL8YAhXXAi1Hrjal82c68" + "lw7UBauyMojDOcbH7EDG9KNbVV-kmkhCW3aH0QCIALzHqO_cukXDqnYX-iVNuxWpW5KUAJdg9cfgwX35O246HxDoSpoIYQ1GooqI" + "WSImYrlNQajPHpJcpxnWeSgKOSYIAYCAr70MouYhUg-lsId7n2pr4gspPf1ZUrBhAG3h4n9qSguKTlpnAYlln_2MzgVVN1cH5sop" + "vgJJLr20HKS1cfu9lB-CHDF5h74z7q3ydZJljn8dQLBqL3csr0ovOZkTjP21rOjPhdsedg4CLKnylgk5nTewbQz99lKxKPhfYK0-" + "XxiDdE6aKvobQUoRupGNHv-gMLqJjehMUynaC6XzBwJ63pPNhH3AlCj3zSte0Mj1FkPQQIY2PjhMB49CmXBV8Z9YJv0=" +) + GPT_6_ASTRA_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", @@ -616,28 +639,7 @@ GPT_6_ASTRA_TEST_CASE: Final = TranslationTestCase( "id": "rs_019af305501e4822006ad023c49ce4819590da413b9879e06b", "type": "reasoning", "content": [], - "encrypted_content": ( - "gAAAAABq0CPHdct5jJ3BolLC7QmXutcBsSZ0YzHiI8z2uK07jqwLifVhEROdF12ygdCVRtcrLrHPyHSh" - "pVbh41ULxn2L3SyVYLHbbkFMcwbzfky_kqZdUe_1D-PLxsVyI1KFMPLg0Fb9K-N9u_ElvkFcXXsmAhRQ" - "CcB4SE9TMe_5tKBHiZbqvOI112wig7cc57rDbvM0mEoKyFn3WgprP_ewHeCeP6DSEvVGDEMjUd_TIXAH" - "H7y2Ae3y0WRW7WUag_wAiO7xzAH13mh4S6eKeBTKh62krmYeUH3wETO8BF1WnkuZ_86lS0zWLBzeLJJn" - "JiXKXVquKatJ6HvO3m6FkPF0tBXhdc8YYtTF1B_WBMECARKcw4n3mPL3R6J2jpHe05Eo7W7uZkcY5E-q" - "mE_cqUK8YQXTXx-mR339iuAvqSe0N2Rgl9Ey1l7z0MBATIV0c6J4wRav6Di5aAxEEXkgDVk1jF9LZSHo" - "mcGjh2B2UWFiAZEELRDD7ovpP3c7B05h0QvwazXlZnskJf-xYZEHAl2CNZk0iqagFs6EtNrFguM5W2AH" - "6dsuaGjNGmdWM_kt4JqIUeScZLNgMty8qTOHFzd7tiBpuLkQgcnHzOigWb2vY_8ahy_zfzM2lVL-JQxP" - "9SB7Gq1m49Ln-byC1O_kPyqEFXELhhQ8peXQY5do2R2T-3I0WLOhDhP5rruLsL66vNHAAMjka6WmjI2g" - "oiy-JF_nVCplJd791Wqa3yf7Q7ksfasnSKtjR7-cgTtMcXX7n_5joMnqf4_LsqSq6gHdeKw4VTeT8wGB" - "Uo3MHLkgi8PnAIphdb3PKgifSjstE_oH4_ZbmlyCsHaFYq2aoy-S85ndlrR_PAHBtU8II-Ud_42EPzS-" - "Etdt5d1Qjjri3KTyPjPO7fAm8aPFr1k3GbjwASPMQ7cUs7yLhPmI-TCm5U-zgK1pIALviiU2o4IM7kdH" - "0-Dg1eyyZN8hsFGSHaUB7G_tGB0fj0H0GSztH8ltNjTK_276IOh8Jv5csxjjmlcMOHVBmDCkkE_DlVjv" - "NQtWQS9QBt9jd1gFSzaOhXVkKZJK3vTIaq8JxTvYtgLc8WM5pRHbnQr71TfWB-eLEYgN-aX5PueKkXGr" - "GauDpKkzVJyNZ6mZ_dVXwl1p5IiNNJ1xzhmZK2GVDFWt0nfOBYyxmCDfIxvQL8YAhXXAi1Hrjal82c68" - "lw7UBauyMojDOcbH7EDG9KNbVV-kmkhCW3aH0QCIALzHqO_cukXDqnYX-iVNuxWpW5KUAJdg9cfgwX35" - "O246HxDoSpoIYQ1GooqIWSImYrlNQajPHpJcpxnWeSgKOSYIAYCAr70MouYhUg-lsId7n2pr4gspPf1Z" - "UrBhAG3h4n9qSguKTlpnAYlln_2MzgVVN1cH5sopvgJJLr20HKS1cfu9lB-CHDF5h74z7q3ydZJljn8d" - "QLBqL3csr0ovOZkTjP21rOjPhdsedg4CLKnylgk5nTewbQz99lKxKPhfYK0-XxiDdE6aKvobQUoRupGN" - "Hv-gMLqJjehMUynaC6XzBwJ63pPNhH3AlCj3zSte0Mj1FkPQQIY2PjhMB49CmXBV8Z9YJv0=" - ), + "encrypted_content": GPT_6_ASTRA_ENCRYPTED_CONTENT, "summary": [], }, { @@ -681,6 +683,10 @@ GPT_6_ASTRA_TEST_CASE: Final = TranslationTestCase( "model": "azure/gpt-6-astra", "usage": {"input_tokens": 309, "output_tokens": 138}, "content": [ + { + "type": "redacted_thinking", + "data": "litellm_encrypted_reasoning:" + GPT_6_ASTRA_ENCRYPTED_CONTENT, + }, { "type": "text", "text": ( @@ -728,6 +734,22 @@ GPT_5_6_TERRA_CONTENT_FILTERS: Final[JsonValue] = [ }, ] +GPT_5_6_TERRA_ENCRYPTED_CONTENT: Final = ( + "gAAAAABq0CPQDJzIdOV-2rO0aGYCUhKcJifPRv4ZkBaIrKZEARo0rk8CqAe8IV0c2kyBwOFe4M70GMM-HrR1L3hIUxan9dcDB5_S" + "7e45yDaxFMiKpbr20izrXNktdwBQfvYOKoMMd07fslVRR-GHOEtz8JvU98v581sxXgVTxNAWGSl_Kb22g2Fds02t1-1g2hT304vs" + "i1sBxbA6P2b-NFNQ8N3NEfX6FoWi7UF0nH69-Dg3sCMb4dpZ0gGZaZSHwvzXXHESEuiMv1nJSX18tfrFIt29gmw3Mq3sUxZ3eALk" + "PZ54yDUlfYXl6sm2MHXT2xn3l5x6jlG4X8hEqgdfHI76otxmfsXvgtbXCADCes87f00bkWatqWgq27oluRYsY02xDMhtrT4h4C4s" + "PIpV7PA9tdhS6Lo3znsQiOEQvMqnSS3dH1NKnZsAiyjNwVZVZa0opIzI5fpNCGjBmMv_4hB3jBRgenG-Ku75OcwPp6HXMej9Mcvp" + "LIKtXb8FMKS5wqiZzYBLp-F25hzOHI0QVl0pGFL5ya9mWU4hkQYeu6HuH6ZjSujzyy-8ZmlBjnRZSQp8mN4t34u2ofk4RtyuZKJk" + "gjJfN2m5MxBTT3AccMalmbiEa7lJv7GV9kLcqR0GT2Rzd6UMX6qPG92VdfjIgCj1Rzh1IWrJvMloKkMj5qB3nN2dABqfCtjBleSt" + "NoTjO2VDensXf11ceG9-KKWeCjfrG0ONgTVXPjTTUPHR0BxtoixnRxIYBZFb8sKdYAPI_QINjby2SVhhQH7UtookBf7N7OqmIJKb" + "zR4CLshKnvthH4fJQAZdsfoEPR9I943uBpZi4HnF8mQPF61Aiw9B3bh3USBc9_Oh2rJL044B6qYVD3QKKGydzcplRxQWA31m-4Zn" + "9eVxn-xOtx4Tvpdh6BhlFPKX9p2W0mSnb7kjpzX51Ec0J2AsTNUOonaaMFt82MJDELTGcAb3tsAkPzzNK0M_zjQ43CCH5Cu1yV2W" + "hbUzxCu4UM0spAIuzpAZv6JGmW1wF4RtAgLfJjc2LxoxcS9MJbLOUPAYdRlwBTj_IL9PIDeNlGVRgVQYL-9xnNKiWD8Iyu2vj8Ao" + "slM41wVTEXvOAYyEs25Eyo9ia2km8909d42DjFi2_ngA5ofZI89KQHtCzGwfCf_VTLbpMmrOB4NDFVTUYQlAMBvuSsXLXJJp0QPl" + "n2UBHhZ3erIcQ1D990SFRdx1uY47-E2FnCCPljyhJrWHnFxTlGbuEY4i5vPOweqYzj-uBbLVeDbcfjCLjSLFgM4Fa5Zg" +) + GPT_5_6_TERRA_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", @@ -752,25 +774,7 @@ GPT_5_6_TERRA_TEST_CASE: Final = TranslationTestCase( "id": "rs_0d178c5d2401a624006ad023cfe304819486bdf040ec55eab7", "type": "reasoning", "content": [], - "encrypted_content": ( - "gAAAAABq0CPQDJzIdOV-2rO0aGYCUhKcJifPRv4ZkBaIrKZEARo0rk8CqAe8IV0c2kyBwOFe4M70GMM-" - "HrR1L3hIUxan9dcDB5_S7e45yDaxFMiKpbr20izrXNktdwBQfvYOKoMMd07fslVRR-GHOEtz8JvU98v5" - "81sxXgVTxNAWGSl_Kb22g2Fds02t1-1g2hT304vsi1sBxbA6P2b-NFNQ8N3NEfX6FoWi7UF0nH69-Dg3" - "sCMb4dpZ0gGZaZSHwvzXXHESEuiMv1nJSX18tfrFIt29gmw3Mq3sUxZ3eALkPZ54yDUlfYXl6sm2MHXT" - "2xn3l5x6jlG4X8hEqgdfHI76otxmfsXvgtbXCADCes87f00bkWatqWgq27oluRYsY02xDMhtrT4h4C4s" - "PIpV7PA9tdhS6Lo3znsQiOEQvMqnSS3dH1NKnZsAiyjNwVZVZa0opIzI5fpNCGjBmMv_4hB3jBRgenG-" - "Ku75OcwPp6HXMej9McvpLIKtXb8FMKS5wqiZzYBLp-F25hzOHI0QVl0pGFL5ya9mWU4hkQYeu6HuH6Zj" - "Sujzyy-8ZmlBjnRZSQp8mN4t34u2ofk4RtyuZKJkgjJfN2m5MxBTT3AccMalmbiEa7lJv7GV9kLcqR0G" - "T2Rzd6UMX6qPG92VdfjIgCj1Rzh1IWrJvMloKkMj5qB3nN2dABqfCtjBleStNoTjO2VDensXf11ceG9-" - "KKWeCjfrG0ONgTVXPjTTUPHR0BxtoixnRxIYBZFb8sKdYAPI_QINjby2SVhhQH7UtookBf7N7OqmIJKb" - "zR4CLshKnvthH4fJQAZdsfoEPR9I943uBpZi4HnF8mQPF61Aiw9B3bh3USBc9_Oh2rJL044B6qYVD3QK" - "KGydzcplRxQWA31m-4Zn9eVxn-xOtx4Tvpdh6BhlFPKX9p2W0mSnb7kjpzX51Ec0J2AsTNUOonaaMFt8" - "2MJDELTGcAb3tsAkPzzNK0M_zjQ43CCH5Cu1yV2WhbUzxCu4UM0spAIuzpAZv6JGmW1wF4RtAgLfJjc2" - "LxoxcS9MJbLOUPAYdRlwBTj_IL9PIDeNlGVRgVQYL-9xnNKiWD8Iyu2vj8AoslM41wVTEXvOAYyEs25E" - "yo9ia2km8909d42DjFi2_ngA5ofZI89KQHtCzGwfCf_VTLbpMmrOB4NDFVTUYQlAMBvuSsXLXJJp0QPl" - "n2UBHhZ3erIcQ1D990SFRdx1uY47-E2FnCCPljyhJrWHnFxTlGbuEY4i5vPOweqYzj-uBbLVeDbcfjCL" - "jSLFgM4Fa5Zg" - ), + "encrypted_content": GPT_5_6_TERRA_ENCRYPTED_CONTENT, "summary": [], }, { @@ -814,6 +818,10 @@ GPT_5_6_TERRA_TEST_CASE: Final = TranslationTestCase( "model": "azure/gpt-5.6-terra", "usage": {"input_tokens": 309, "output_tokens": 92}, "content": [ + { + "type": "redacted_thinking", + "data": "litellm_encrypted_reasoning:" + GPT_5_6_TERRA_ENCRYPTED_CONTENT, + }, { "type": "text", "text": ( diff --git a/tests/integration/translation/messages/bases/bedrock_mantle.py b/tests/integration/translation/messages/bases/bedrock_mantle.py index 7524bf8f7ea..065c8553231 100644 --- a/tests/integration/translation/messages/bases/bedrock_mantle.py +++ b/tests/integration/translation/messages/bases/bedrock_mantle.py @@ -467,6 +467,22 @@ BEDROCK_MANTLE_OPENAI_RESPONSES_MESSAGES_RESPONSE: Final[Mapping[str, JsonValue] } +GPT_6_1_SOL_ENCRYPTED_CONTENT: Final = ( + "rsn_rru6BQHgUc/VWS5vRIAlHwIFeB+Lw/sASAOaRTS5iOdUyDiwGqyLb6EJDqxBCdFOQvGfAF8AAQAVYXdzLWNyeXB0by1wdWJs" + "aWMta2V5AERBaTJyR044dmNxUGt3TndhZkNmeTBpOEFUWlBHSGN4L2s3U1E0UGpyUEp0bUdlZVhXVDREVy9tS2FPdStQUFd6OGc9" + "PQABABFhd3Mta21zLWhpZXJhcmNoeQAKYmVybS1zdG9yZQBcJ5O6OH7BzhGiCKEzUwANq0bcBtwE7aXPWBjYzgTyexPQO0PepR8j" + "RQ3zVYRhZpm6K2TOk2lvrp72PZ5N5jG3KGRkWT2Y2jsL9pnkWwE5kN16+aY2OKOY9ZlzQukCAAAQACDHXld1blcGog/Y0exPb0n+" + "8Vo0dT1X9pL401/Nv3/uQlT6BC0wwixOAufgfaY+ev////8AAAABAAAAAAAAAAAAAAABAAABrRdgMGsGAKubmJ7dCj1A9UQgYVFb" + "1O6+NB+Aom8IGUy3Y6E3c63eWNkWXxN8APSjWFwq012VWZ9Q11DzJmGSwM8gtV25ckYezPfNn46VmSj2idcLs2TxTSxP89NAOfSM" + "zWRnW6Dr3lSlx72uQKBrHWYPyAskoHTw4E6CfnFhXotCshAmT5dgImjniqdLFu4CWh2xpQcmYkFbxXakTBsK80TEI8fpJVjGYQbo" + "mlrZTwAb8bdyEp0/05xVeMVYC47F0tBY5zFOeZ0Nk0ps2PFteD1HQ5GEt7MRMb+VTGGSbq0wmjY7pvdC0f31cs+0GhEZ1N/3707c" + "GjuMevEMn17YYxJ0oelPOatqdAeSSxBGkEKLmzos0tDcisqwWU/94DL2hmTJWacHamboEN74xZ+KwOjKEoyEr0HOL7n5+B9Oojl7" + "MPdWgBnyGUwCsB3BH8t+h5MwMtMJY451PvO4hgl3qK2cN4WfgoYHRvWM2+rj2iJqYBXFPLo5yqWd3XIIimuTGltinK/0sr1uC/AM" + "Q2Qa0qEB0J4Zxla83H7HCEBg4e8wde2qlXZCG9AxVpPQVd1LijTvn7sPIJgYsmSMwKYAZzBlAjBvywiP8Kq5WqQ+kknoQv+bn4Ot" + "wISL+sXhXE+9JK6hhjn02Xd9iSkC/h6IGxAc22UCMQCBBpc3UEvXQSZAxGDy1xUr3JoG4X3sj2Y4yB3SHvBVcxmAWoVo+QAUVMNu" + "lgqXbwU=" +) + GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", @@ -492,7 +508,7 @@ GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase( "output": [ { "content": [], - "encrypted_content": "rsn_rru6BQHgUc/VWS5vRIAlHwIFeB+Lw/sASAOaRTS5iOdUyDiwGqyLb6EJDqxBCdFOQvGfAF8AAQAVYXdzLWNyeXB0by1wdWJsaWMta2V5AERBaTJyR044dmNxUGt3TndhZkNmeTBpOEFUWlBHSGN4L2s3U1E0UGpyUEp0bUdlZVhXVDREVy9tS2FPdStQUFd6OGc9PQABABFhd3Mta21zLWhpZXJhcmNoeQAKYmVybS1zdG9yZQBcJ5O6OH7BzhGiCKEzUwANq0bcBtwE7aXPWBjYzgTyexPQO0PepR8jRQ3zVYRhZpm6K2TOk2lvrp72PZ5N5jG3KGRkWT2Y2jsL9pnkWwE5kN16+aY2OKOY9ZlzQukCAAAQACDHXld1blcGog/Y0exPb0n+8Vo0dT1X9pL401/Nv3/uQlT6BC0wwixOAufgfaY+ev////8AAAABAAAAAAAAAAAAAAABAAABrRdgMGsGAKubmJ7dCj1A9UQgYVFb1O6+NB+Aom8IGUy3Y6E3c63eWNkWXxN8APSjWFwq012VWZ9Q11DzJmGSwM8gtV25ckYezPfNn46VmSj2idcLs2TxTSxP89NAOfSMzWRnW6Dr3lSlx72uQKBrHWYPyAskoHTw4E6CfnFhXotCshAmT5dgImjniqdLFu4CWh2xpQcmYkFbxXakTBsK80TEI8fpJVjGYQbomlrZTwAb8bdyEp0/05xVeMVYC47F0tBY5zFOeZ0Nk0ps2PFteD1HQ5GEt7MRMb+VTGGSbq0wmjY7pvdC0f31cs+0GhEZ1N/3707cGjuMevEMn17YYxJ0oelPOatqdAeSSxBGkEKLmzos0tDcisqwWU/94DL2hmTJWacHamboEN74xZ+KwOjKEoyEr0HOL7n5+B9Oojl7MPdWgBnyGUwCsB3BH8t+h5MwMtMJY451PvO4hgl3qK2cN4WfgoYHRvWM2+rj2iJqYBXFPLo5yqWd3XIIimuTGltinK/0sr1uC/AMQ2Qa0qEB0J4Zxla83H7HCEBg4e8wde2qlXZCG9AxVpPQVd1LijTvn7sPIJgYsmSMwKYAZzBlAjBvywiP8Kq5WqQ+kknoQv+bn4OtwISL+sXhXE+9JK6hhjn02Xd9iSkC/h6IGxAc22UCMQCBBpc3UEvXQSZAxGDy1xUr3JoG4X3sj2Y4yB3SHvBVcxmAWoVo+QAUVMNulgqXbwU=", + "encrypted_content": GPT_6_1_SOL_ENCRYPTED_CONTENT, "id": "rs_187e1b0bc4995fc9b7fd2796a1889637", "summary": [], "type": "reasoning", @@ -536,6 +552,10 @@ GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase( "model": "bedrock_mantle/openai.gpt-6.1-sol", "usage": {"input_tokens": 309, "output_tokens": 141}, "content": [ + { + "type": "redacted_thinking", + "data": "litellm_encrypted_reasoning:" + GPT_6_1_SOL_ENCRYPTED_CONTENT, + }, { "type": "text", "text": "I’ll read `shop/validation.py` first to check how it handles zero, negative, and " @@ -551,6 +571,21 @@ GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase( }, ) +GPT_6_LUNA_ENCRYPTED_CONTENT: Final = ( + "rsn_rru6BQHgUc/VWS5vRIAlHwIFeJWGzSslRfiCKfPUzG9PfxThM9nB8cV14wc1iKvcbztDAF8AAQAVYXdzLWNyeXB0by1wdWJs" + "aWMta2V5AERBb2ZiS1dnQ24waEpudTVSZGtlLyt3emJKY2VIUXlCeU9rWGd6enJDYU5ydXZYMEFmV1NwK01vVEUwZzhHWnpWb0E9" + "PQABABFhd3Mta21zLWhpZXJhcmNoeQAKYmVybS1zdG9yZQBcJhapxvTAKRUzTqbKGUoTlXJqSZjSljHkimXkxwTyexPQO0PepR8j" + "RQ3zVYQcb1ucdG8HiPjq86SsgccbqbIEmYllOUwDws5V0DtxxRnqoVEhkF3h+EG4I5uuB2kCAAAQABDKdaE231TqbLUInCwHGBD9" + "+U9x7lXzuoVQNhj0KATZ6U5+Ki+ZDt6KLjrNb/LM2v////8AAAABAAAAAAAAAAAAAAABAAABdglH8KRxssIDHD4kcMUX0rX0yogE" + "2V0AOj4vmUzgN+tpKEUhA0/BNphpi42nYRZK61EgLzZk42sWn6sJhR5/WKHHn/7N7FppZkt6kFCDTyhRfPimhQT0S7NiACE4511F" + "HXE/WbRGd2/3uuLOx6PfMFlxuKgJHYQuMEttkYRZz1lwzMrtQTyTtghKKz0QaNtp7umhKHlaFVetqVsKqLD7JpKyBxjbe5VojoKm" + "MQEY3dExNYgvCnOBPb8uLhZCcBemLq1ZQzinbMRyRO1bhQmfGsx4GgFbuA4cyJlyzHe+Af4VqcRM94bELBMXiU64UveyqRkWBdoc" + "sPfPFpVWa+Hnvl37fgn4jBTnhG1xqNuoVCXX2nV/GAMYHdPwsKweIhxg9I8EcrsXOcTWv26hwDRpmdoHHtJkSqKSbudxEs1F5kr5" + "ELytEFS2OWJwwsMvDfBAlC4boaYtxUBG5sREazNwDNRkhFWHb8MwsXBpOqwOBtu1L2n953WUP2I2tHE6WcZ3DpoD2dP3pwBnMGUC" + "MQDonP31z1HPHjm2mGf7SINbrdFZ5xl5ztMBj3g8hJmIpdWtN/W5dJkc7kfNSErvVrQCMDGAL4pNuVt19O4BfjAmLKnNyz5x/z1F" + "PLGJDz2j+9b0Y0FYBIqafypg88wJ6XSc0Q==" +) + GPT_6_LUNA_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", @@ -578,7 +613,7 @@ GPT_6_LUNA_TEST_CASE: Final = TranslationTestCase( "output": [ { "content": [], - "encrypted_content": "rsn_rru6BQHgUc/VWS5vRIAlHwIFeJWGzSslRfiCKfPUzG9PfxThM9nB8cV14wc1iKvcbztDAF8AAQAVYXdzLWNyeXB0by1wdWJsaWMta2V5AERBb2ZiS1dnQ24waEpudTVSZGtlLyt3emJKY2VIUXlCeU9rWGd6enJDYU5ydXZYMEFmV1NwK01vVEUwZzhHWnpWb0E9PQABABFhd3Mta21zLWhpZXJhcmNoeQAKYmVybS1zdG9yZQBcJhapxvTAKRUzTqbKGUoTlXJqSZjSljHkimXkxwTyexPQO0PepR8jRQ3zVYQcb1ucdG8HiPjq86SsgccbqbIEmYllOUwDws5V0DtxxRnqoVEhkF3h+EG4I5uuB2kCAAAQABDKdaE231TqbLUInCwHGBD9+U9x7lXzuoVQNhj0KATZ6U5+Ki+ZDt6KLjrNb/LM2v////8AAAABAAAAAAAAAAAAAAABAAABdglH8KRxssIDHD4kcMUX0rX0yogE2V0AOj4vmUzgN+tpKEUhA0/BNphpi42nYRZK61EgLzZk42sWn6sJhR5/WKHHn/7N7FppZkt6kFCDTyhRfPimhQT0S7NiACE4511FHXE/WbRGd2/3uuLOx6PfMFlxuKgJHYQuMEttkYRZz1lwzMrtQTyTtghKKz0QaNtp7umhKHlaFVetqVsKqLD7JpKyBxjbe5VojoKmMQEY3dExNYgvCnOBPb8uLhZCcBemLq1ZQzinbMRyRO1bhQmfGsx4GgFbuA4cyJlyzHe+Af4VqcRM94bELBMXiU64UveyqRkWBdocsPfPFpVWa+Hnvl37fgn4jBTnhG1xqNuoVCXX2nV/GAMYHdPwsKweIhxg9I8EcrsXOcTWv26hwDRpmdoHHtJkSqKSbudxEs1F5kr5ELytEFS2OWJwwsMvDfBAlC4boaYtxUBG5sREazNwDNRkhFWHb8MwsXBpOqwOBtu1L2n953WUP2I2tHE6WcZ3DpoD2dP3pwBnMGUCMQDonP31z1HPHjm2mGf7SINbrdFZ5xl5ztMBj3g8hJmIpdWtN/W5dJkc7kfNSErvVrQCMDGAL4pNuVt19O4BfjAmLKnNyz5x/z1FPLGJDz2j+9b0Y0FYBIqafypg88wJ6XSc0Q==", + "encrypted_content": GPT_6_LUNA_ENCRYPTED_CONTENT, "id": "rs_8de42923e05d5521b0dbd93c1adcea91", "summary": [], "type": "reasoning", @@ -622,6 +657,10 @@ GPT_6_LUNA_TEST_CASE: Final = TranslationTestCase( "model": "bedrock_mantle/openai.gpt-6-luna", "usage": {"input_tokens": 309, "output_tokens": 119}, "content": [ + { + "type": "redacted_thinking", + "data": "litellm_encrypted_reasoning:" + GPT_6_LUNA_ENCRYPTED_CONTENT, + }, { "type": "text", "text": "I’ll read `shop/validation.py` first to check the behavior for zero, negative, and " @@ -637,6 +676,21 @@ GPT_6_LUNA_TEST_CASE: Final = TranslationTestCase( }, ) +GPT_6_ASTRA_ENCRYPTED_CONTENT: Final = ( + "rsn_rru6BQHgUc/VWS5vRIAlHwIFeK6rtxWPF9TjmgBuzPaPZhcqwiqUtYeqNCEyD/fgEZAfAF8AAQAVYXdzLWNyeXB0by1wdWJs" + "aWMta2V5AERBaUE5eUozT3MzTE1rRFhLWGRuWUFYRjdZU2Q3OStjT3V5LzBIRUNnZXBUWDloQXZncVppTC9qVUc3Y2h3dlQ5QVE9" + "PQABABFhd3Mta21zLWhpZXJhcmNoeQAKYmVybS1zdG9yZQBcuTxMV80kCzRfLH6n87syGDT8XBcjqEaE8tDuSQTyexPQO0PepR8j" + "RQ3zVYTdphRKCdq3CJr491CVE1gdVwCTcbqsK/1k7shfoHg93E2fCZXYKq0tnNLGBKhnl90CAAAQAHNHMusCy4BS4gUSPEf8T3Dw" + "PC4nfHsYIw/8I2M8N4Z4dxH0a/6ULM8Zaf5qgZInLv////8AAAABAAAAAAAAAAAAAAABAAABmFno3HYlCh8skcb8d7DvpK045XzP" + "/rPTF9J1TXQEj+Oam1ybdDFFN5bbUrMUhvhaAKoUdhp0TPWFBQnQD7rM1QSBtZY60vbMXdscVeDkmAPh7MsZ6OLAY1olCWPsid7O" + "qATdHKm+S6UfnqnvNLJQ0Hp12YYilLOhW5+BLnmqdm04wkU2kvH4G4vxgRK0hFGMGaoKlCIKRvAmwre2DGXdWx85E9jsq2j1ta1X" + "T6/2eCr7zyXn8+tRI69tNwFVRTcaHu8DvSS1LBzdO7ZwNwUr6PbksO44r/sXeNstFU4u3ZQnUhWz8JNym0ydaia57zXQG3/pl8rJ" + "aA3IshV+Gm+Ie7N3qySlcIEonqN+cYhQN669G4oBuvCKARc2LFdRB1OXhErskZ8c2c3NA9Xt/dXBbclKY9i/I3Usp46t46mALqz7" + "rA2ft/cGLtX3gNoPPEKL2+i1ZDa0EB1Y7agcESa5pYLKgajhUDbaxupsVQ93wZiCcDTr7SlEfMrBkNFGijc3kpdn1avRFEzVC3H/" + "Ys1cPPYJ93hOUYO42b/RibFpBlkLnKtNf//v81kAZzBlAjAdmUn/WQ5cOGyMLQF2CQxoUrAO9HhKULmwznaURUbjukpBRw8v6Pwf" + "lSA+mmU8iv0CMQCASBvNjVVmEd4/s4jH27TFub4sikcdi7+LR5HeitoRhyEJIfB41mRciDxhY80hp6s=" +) + GPT_6_ASTRA_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", @@ -664,7 +718,7 @@ GPT_6_ASTRA_TEST_CASE: Final = TranslationTestCase( "output": [ { "content": [], - "encrypted_content": "rsn_rru6BQHgUc/VWS5vRIAlHwIFeK6rtxWPF9TjmgBuzPaPZhcqwiqUtYeqNCEyD/fgEZAfAF8AAQAVYXdzLWNyeXB0by1wdWJsaWMta2V5AERBaUE5eUozT3MzTE1rRFhLWGRuWUFYRjdZU2Q3OStjT3V5LzBIRUNnZXBUWDloQXZncVppTC9qVUc3Y2h3dlQ5QVE9PQABABFhd3Mta21zLWhpZXJhcmNoeQAKYmVybS1zdG9yZQBcuTxMV80kCzRfLH6n87syGDT8XBcjqEaE8tDuSQTyexPQO0PepR8jRQ3zVYTdphRKCdq3CJr491CVE1gdVwCTcbqsK/1k7shfoHg93E2fCZXYKq0tnNLGBKhnl90CAAAQAHNHMusCy4BS4gUSPEf8T3DwPC4nfHsYIw/8I2M8N4Z4dxH0a/6ULM8Zaf5qgZInLv////8AAAABAAAAAAAAAAAAAAABAAABmFno3HYlCh8skcb8d7DvpK045XzP/rPTF9J1TXQEj+Oam1ybdDFFN5bbUrMUhvhaAKoUdhp0TPWFBQnQD7rM1QSBtZY60vbMXdscVeDkmAPh7MsZ6OLAY1olCWPsid7OqATdHKm+S6UfnqnvNLJQ0Hp12YYilLOhW5+BLnmqdm04wkU2kvH4G4vxgRK0hFGMGaoKlCIKRvAmwre2DGXdWx85E9jsq2j1ta1XT6/2eCr7zyXn8+tRI69tNwFVRTcaHu8DvSS1LBzdO7ZwNwUr6PbksO44r/sXeNstFU4u3ZQnUhWz8JNym0ydaia57zXQG3/pl8rJaA3IshV+Gm+Ie7N3qySlcIEonqN+cYhQN669G4oBuvCKARc2LFdRB1OXhErskZ8c2c3NA9Xt/dXBbclKY9i/I3Usp46t46mALqz7rA2ft/cGLtX3gNoPPEKL2+i1ZDa0EB1Y7agcESa5pYLKgajhUDbaxupsVQ93wZiCcDTr7SlEfMrBkNFGijc3kpdn1avRFEzVC3H/Ys1cPPYJ93hOUYO42b/RibFpBlkLnKtNf//v81kAZzBlAjAdmUn/WQ5cOGyMLQF2CQxoUrAO9HhKULmwznaURUbjukpBRw8v6PwflSA+mmU8iv0CMQCASBvNjVVmEd4/s4jH27TFub4sikcdi7+LR5HeitoRhyEJIfB41mRciDxhY80hp6s=", + "encrypted_content": GPT_6_ASTRA_ENCRYPTED_CONTENT, "id": "rs_e6237911ea3e5775acee304296484fab", "summary": [], "type": "reasoning", @@ -708,6 +762,10 @@ GPT_6_ASTRA_TEST_CASE: Final = TranslationTestCase( "model": "bedrock_mantle/openai.gpt-6-astra", "usage": {"input_tokens": 309, "output_tokens": 138}, "content": [ + { + "type": "redacted_thinking", + "data": "litellm_encrypted_reasoning:" + GPT_6_ASTRA_ENCRYPTED_CONTENT, + }, { "type": "text", "text": "I’ll read `shop/validation.py` first to check how it handles zero, negative, and " @@ -723,6 +781,20 @@ GPT_6_ASTRA_TEST_CASE: Final = TranslationTestCase( }, ) +GPT_5_6_TERRA_ENCRYPTED_CONTENT: Final = ( + "rsn_rru6BQHgUc/VWS5vRIAlHwIFeCIDnqsrWiO+hx6u0oBst+OnJIA0RIm1eTWC5ORgkFFNAF8AAQAVYXdzLWNyeXB0by1wdWJs" + "aWMta2V5AERBMEZqYXRRMkhMSEw1clFlcythQm0wc1Bid2hBWHJhN3ZZOFdGenRwOXlsd0NqSkxBOGxWd2loU2dleHhmTFN0Snc9" + "PQABABFhd3Mta21zLWhpZXJhcmNoeQAKYmVybS1zdG9yZQBcWxvahbwUWb5zEZ4mn/Ri1V5Z4KCuYAzvlyiI2ATyexPQO0PepR8j" + "RQ3zVYTwOcBo7nb2ZZfI77C2nwtaEjD275c1LF19dmrCUZ7VPKIYQCMlismkx7f2X1LE4xUCAAAQAPc8v4U/p6fjgu2ZpZgPQUfp" + "PsrIs2TWrckQcNPOhJmhKYDB5gthEp93h1Hf2+ad3P////8AAAABAAAAAAAAAAAAAAABAAABHw7y8mi4YLItPYN81mZNbuQhK0FT" + "G04aQuvQ0966UQxFIMfbeJe+qtyO7uHmyKoz58NNb0ud67KhOPoip6wfSxVcw2RTBCAljj+gkLciZdZlGDiLyutdaHfCRh6StNZD" + "tXG0xPgbn0mN93xnE0hDa05mjcNsNckDaS2KkTMzoXF0tqWoi+Vl0fMSK3CF/49QKyGXYsDeOPB0b8XghpfF1PvMqMI/Ua9Sd98l" + "999mYu71TaBKvgRS9yLWBXg9YcmfZx2hulOUkK2a9aCQ5aN661B0MTdykIm+CokaKogZCv6e/MXncHPPtBeqtuQtsggedinoW/qD" + "09HU+rLUysC+vbHT65YqoVL9wW42XZa79V5QBFYIQL787eVy54qbPxrryWKuTOsN9xRFqaSOv1L10ABnMGUCMFnJHyhVr8ujzWTw" + "nSQCuS+zKrvPgXQfEePxJOR/KFHa7CUXZ8kBuEoW0kVxx3biLgIxAK+8PazVOuepKMy5cuKKsJChOnvQ1Npz7Gpmrx0CXO7X2sIN" + "1WJTlhdNHEclLuwoSQ==" +) + GPT_5_6_TERRA_TEST_CASE: Final = TranslationTestCase( scenario="basic", litellm_endpoint="/v1/messages", @@ -750,7 +822,7 @@ GPT_5_6_TERRA_TEST_CASE: Final = TranslationTestCase( "output": [ { "content": [], - "encrypted_content": "rsn_rru6BQHgUc/VWS5vRIAlHwIFeCIDnqsrWiO+hx6u0oBst+OnJIA0RIm1eTWC5ORgkFFNAF8AAQAVYXdzLWNyeXB0by1wdWJsaWMta2V5AERBMEZqYXRRMkhMSEw1clFlcythQm0wc1Bid2hBWHJhN3ZZOFdGenRwOXlsd0NqSkxBOGxWd2loU2dleHhmTFN0Snc9PQABABFhd3Mta21zLWhpZXJhcmNoeQAKYmVybS1zdG9yZQBcWxvahbwUWb5zEZ4mn/Ri1V5Z4KCuYAzvlyiI2ATyexPQO0PepR8jRQ3zVYTwOcBo7nb2ZZfI77C2nwtaEjD275c1LF19dmrCUZ7VPKIYQCMlismkx7f2X1LE4xUCAAAQAPc8v4U/p6fjgu2ZpZgPQUfpPsrIs2TWrckQcNPOhJmhKYDB5gthEp93h1Hf2+ad3P////8AAAABAAAAAAAAAAAAAAABAAABHw7y8mi4YLItPYN81mZNbuQhK0FTG04aQuvQ0966UQxFIMfbeJe+qtyO7uHmyKoz58NNb0ud67KhOPoip6wfSxVcw2RTBCAljj+gkLciZdZlGDiLyutdaHfCRh6StNZDtXG0xPgbn0mN93xnE0hDa05mjcNsNckDaS2KkTMzoXF0tqWoi+Vl0fMSK3CF/49QKyGXYsDeOPB0b8XghpfF1PvMqMI/Ua9Sd98l999mYu71TaBKvgRS9yLWBXg9YcmfZx2hulOUkK2a9aCQ5aN661B0MTdykIm+CokaKogZCv6e/MXncHPPtBeqtuQtsggedinoW/qD09HU+rLUysC+vbHT65YqoVL9wW42XZa79V5QBFYIQL787eVy54qbPxrryWKuTOsN9xRFqaSOv1L10ABnMGUCMFnJHyhVr8ujzWTwnSQCuS+zKrvPgXQfEePxJOR/KFHa7CUXZ8kBuEoW0kVxx3biLgIxAK+8PazVOuepKMy5cuKKsJChOnvQ1Npz7Gpmrx0CXO7X2sIN1WJTlhdNHEclLuwoSQ==", + "encrypted_content": GPT_5_6_TERRA_ENCRYPTED_CONTENT, "id": "rs_a4e6e9898f225c88b4d948b6488bfacc", "summary": [], "type": "reasoning", @@ -795,6 +867,10 @@ GPT_5_6_TERRA_TEST_CASE: Final = TranslationTestCase( "model": "bedrock_mantle/openai.gpt-5.6-terra", "usage": {"input_tokens": 309, "output_tokens": 109}, "content": [ + { + "type": "redacted_thinking", + "data": "litellm_encrypted_reasoning:" + GPT_5_6_TERRA_ENCRYPTED_CONTENT, + }, { "type": "text", "text": "I’ll read `shop/validation.py` first to verify the current handling of zero, " From c22dc253168e36d9921ba05b697a00edd3f7677d Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 7 Nov 2026 16:42:38 -0800 Subject: [PATCH 15/39] refactor(ui): clear hardcoded English from dashboard JSX (#50575) * refactor(ui): clear hardcoded English from dashboard JSX Move the 283 strings the planned no-literal-string guard flags so main starts at zero: - missed UI text (Switchyard router form, Create User custom ID field, advanced settings placeholders) and our own fallbacks ("unknown", seconds suffixes) onto t() with new en.json keys - product names into src/i18n/untranslated/productNames.ts and code samples, sample values and JSON placeholders into src/i18n/untranslated/codeSamples.ts - values that never display (filter ids, enum fallbacks, query keys, CSS, ids) onto literal-union types or module constants The rendered English is byte-identical. * refactor(ui): put Bedrock policy chips and test panel error on t() The six Bedrock policy chips and the compliance test panel's error word are our own display labels, not server values, so they go through t() instead of module constants. Rendered English is unchanged. --- .../_components/AgentIdentityFields.tsx | 3 +- .../_components/KillSwitchFormFields.tsx | 5 +- .../_components/APIReferenceView.tsx | 7 +- .../caching/_components/cache_health.tsx | 4 +- .../_components/EvaluationSettingsModal.tsx | 3 +- .../_components/GuardrailReadOnlyDetails.tsx | 4 +- .../_components/GuardrailTestPanel.tsx | 3 +- .../_components/TeamGuardrailsTab.tsx | 5 +- .../_components/add_guardrail_form.tsx | 3 +- .../ContentCategoryConfiguration.tsx | 13 +- .../content_filter/KeywordTable.tsx | 2 +- .../custom_code/CustomCodeModal.tsx | 5 +- .../_components/guardrail_garden_card.tsx | 3 +- .../_components/guardrail_garden_detail.tsx | 6 +- .../ToolPermissionRulesEditor.tsx | 7 +- .../app/(dashboard)/hooks/models/useModels.ts | 2 +- .../_components/IdJagFormFields.tsx | 9 +- .../_components/MCPSubmissionsTab.tsx | 4 +- .../_components/OAuthFormFields.tsx | 3 +- .../_components/TokenExchangeFormFields.tsx | 5 +- .../_components/UpstreamTokenHeaderField.tsx | 3 +- .../mcp-servers/_components/mcp_connect.tsx | 14 +- .../mcp-servers/_components/mcp_discovery.tsx | 12 +- .../_components/mcp_server_edit.tsx | 3 +- .../mcp-servers/_components/mcp_servers.tsx | 6 +- .../memory/_components/MemoryEditModal.tsx | 9 +- .../_components/ModelInsightsView.tsx | 3 +- .../components/chat_ui/A2AMetrics.tsx | 3 +- .../components/chat_ui/SessionManagement.tsx | 11 +- .../components/systemOneUI/SystemOneUI.tsx | 3 +- .../components/systemOneUI/lib/formPayload.ts | 2 +- .../_components/pipeline_flow_builder.tsx | 10 +- .../_components/PromptTableColumns.tsx | 8 +- .../_components/prompt_editor_view/index.tsx | 4 +- .../_components/RequestTagMappings.tsx | 10 +- .../skills/_components/add_plugin_form.tsx | 5 +- .../EntityUsage/SpendByProvider.tsx | 4 +- .../_components/components/UsagePageView.tsx | 4 +- .../DefaultUserSettingsForm.tsx | 8 +- .../_components/view_users/UsersTable.tsx | 3 +- .../_components/VectorStoreForm.tsx | 3 +- .../_components/vector_store_info.tsx | 7 +- .../(dashboard)/workflows/WorkflowRuns.tsx | 3 +- .../src/app/login/LoginPage.tsx | 5 +- .../src/app/mcp/oauth/callback/page.tsx | 3 +- .../src/app/onboarding/OnboardingFormBody.tsx | 3 +- .../src/components/AIHub/ModelHubTable.tsx | 2 - .../src/components/CreateUserButton.tsx | 11 +- .../src/components/ModelInfoEditForm.tsx | 5 +- .../src/components/Navbar/ViewSwitcher.tsx | 3 +- .../AdminSettings/CyberArk/CyberArk.tsx | 3 +- .../HashicorpVault/HashicorpVault.tsx | 3 +- .../MCPToolSearchSettings.tsx | 3 +- .../PluginSettings/PluginSettings.tsx | 3 +- .../UISettings/PageVisibilitySettings.tsx | 4 +- .../AdminSettings/UISettings/UISettings.tsx | 3 +- .../SidebarAccountMenu/SidebarAccountMenu.tsx | 3 +- ui/litellm-dashboard/src/components/Teams.tsx | 3 +- .../src/components/TeamsPage/TeamsTable.tsx | 3 +- .../components/ThemeToggle/ThemeToggle.tsx | 4 +- .../src/components/UIAccessControlForm.tsx | 5 +- .../UsagePage/components/KeyActivityPanel.tsx | 2 +- .../VirtualKeysPage/VirtualKeysTable.tsx | 3 +- .../src/components/activity_metrics.tsx | 4 +- .../add_model/AutoRouterClassifierTabs.tsx | 25 ++-- .../add_model/AutoRouterSimpleSetup.tsx | 4 +- .../add_model/CustomDimensionRows.tsx | 4 +- .../add_model/ForecastClassifierConfig.tsx | 4 +- .../add_model/OpeningPromptEditor.tsx | 3 +- .../components/add_model/ReminderMarkers.tsx | 5 +- .../add_model/SwitchyardConfigFields.tsx | 132 +++++++++++------- .../add_model/SwitchyardRouterForm.tsx | 13 +- .../add_model/advanced_settings.tsx | 15 +- .../add_model/auto_router_connection_test.tsx | 2 +- .../src/components/add_pass_through.tsx | 3 +- .../components/bulk_create_users_button.tsx | 5 +- .../src/components/chat/ChatMessages.tsx | 52 +++---- .../src/components/chat/ChatModelPicker.tsx | 2 +- .../src/components/chat/ChatShell.tsx | 4 +- .../components/chat_ui/MCPEventsDisplay.tsx | 4 +- .../src/components/cloudzero_export_modal.tsx | 3 +- .../common_components/LoadingScreen.tsx | 3 +- .../common_components/MemberTable.tsx | 3 +- .../common_components/ModelAliasManager.tsx | 3 +- .../PassThroughGuardrailsSection.tsx | 23 +-- .../PremiumLoggingSettings.tsx | 5 +- .../edit_auto_router_modal.tsx | 3 +- .../src/components/email_settings.tsx | 4 +- .../components/logs/audit/AuditLogsTable.tsx | 6 +- .../logs/detail/LogDetailContent.tsx | 6 +- .../logs/detail/LogDetailsDrawer.tsx | 10 +- .../logs/detail/RealtimePrettyView.tsx | 2 +- .../BedrockGuardrailDetails.tsx | 19 +-- .../GuardrailViewer/ContentFilterDetails.tsx | 12 +- .../HealthChecksTableColumns.tsx | 3 +- .../components/model_group_alias_settings.tsx | 3 +- .../src/components/model_info_view.tsx | 3 +- .../src/components/moyai/MoyaiLanding.tsx | 5 +- .../organisms/create_key_button.tsx | 38 ++--- .../src/components/pass_through_info.tsx | 3 +- .../publicModelHub/ModelInspector.tsx | 5 +- .../routing_groups/RoutingGroupModal.tsx | 9 +- .../src/components/settings.tsx | 9 +- .../components/shared/CreatedKeyDisplay.tsx | 15 +- .../DataTable/DataTableFilterDrawer.tsx | 6 +- .../src/components/shared/SidebarBrand.tsx | 3 +- .../shared/charts/StackedUsageChart.tsx | 5 +- .../components/shared/search/SearchBox.tsx | 4 +- .../src/components/team/EditMembership.tsx | 3 +- .../src/components/team/LoggingSettings.tsx | 5 +- .../src/components/team/MyUserTab.tsx | 6 +- .../src/components/team/TeamInfo.tsx | 7 +- .../components/templates/key_edit_view.tsx | 7 +- .../src/components/ui/chart.tsx | 6 +- ui/litellm-dashboard/src/i18n/locales/en.json | 91 ++++++++++++ .../src/i18n/untranslated/codeSamples.ts | 95 +++++++++++++ .../src/i18n/untranslated/productNames.ts | 28 ++++ 117 files changed, 746 insertions(+), 304 deletions(-) create mode 100644 ui/litellm-dashboard/src/i18n/untranslated/codeSamples.ts create mode 100644 ui/litellm-dashboard/src/i18n/untranslated/productNames.ts diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentIdentityFields.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentIdentityFields.tsx index d4af3ff3833..00c8fce98d1 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentIdentityFields.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/AgentIdentityFields.tsx @@ -8,6 +8,7 @@ import { Input } from "@/components/ui/input"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { AgentFormField, type AgentFormValues } from "./AgentFormKit"; import { entraTenantFromIssuer, IDENTITY_UUID_PATTERN } from "./agent_identity"; +import { codeSamples } from "@/i18n/untranslated/codeSamples"; const PROVIDER_OPTIONS: { value: string; labelKey: ParseKeys }[] = [ { value: "none", labelKey: "agents.identityFields.providers.none" }, @@ -206,7 +207,7 @@ export const AgentIdentityFields = ({ accessToken }: { accessToken: string | nul ref={ref} value={typeof value === "string" ? value : ""} onChange={onChange} - placeholder="Agent.Invoke" + placeholder={codeSamples.agents.identityScope} /> )} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/KillSwitchFormFields.tsx b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/KillSwitchFormFields.tsx index 2b9a0edb70d..6abeb5d18ea 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/KillSwitchFormFields.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/agents/_components/KillSwitchFormFields.tsx @@ -11,6 +11,7 @@ import { PasswordInput } from "@/components/shared/PasswordInput"; import i18n from "@/i18n/i18n"; import { AgentFormField, AgentFormValues, labelWithHint } from "./AgentFormKit"; import { KILL_SWITCH_AUTH_TYPES, KILL_SWITCH_METHODS, validateKillSwitchBody } from "./kill_switch_config"; +import { codeSamples } from "@/i18n/untranslated/codeSamples"; const KeyValueFieldArray = ({ name, @@ -135,7 +136,7 @@ const KillSwitchAuthFields = () => { ( {...control} ref={ref} rows={4} - placeholder='{"reason": "manual kill switch"}' + placeholder={codeSamples.agents.killSwitchPayloadJson} value={typeof value === "string" ? value : ""} onChange={onChange} /> diff --git a/ui/litellm-dashboard/src/app/(dashboard)/api-reference/_components/APIReferenceView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/api-reference/_components/APIReferenceView.tsx index 7f6f7ea1d17..f55dc2e4c82 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/api-reference/_components/APIReferenceView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/api-reference/_components/APIReferenceView.tsx @@ -4,6 +4,7 @@ import { useTranslation } from "react-i18next"; import CodeBlock from "@/components/CodeBlock"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import DocLink from "./DocLink"; +import { productNames } from "@/i18n/untranslated/productNames"; interface ApiRefProps { proxySettings: { @@ -36,13 +37,13 @@ const APIReferenceView: React.FC = ({ proxySettings }) => { - OpenAI Python SDK + {productNames.openAIPythonSdk} - LlamaIndex + {productNames.llamaIndex} - Langchain Py + {productNames.langchainPython} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_health.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_health.tsx index 382438d9efe..db532fafefe 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_health.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_health.tsx @@ -5,6 +5,8 @@ import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { CheckCircle2, XCircle, ClipboardCopy } from "lucide-react"; import { ResponseTimeIndicator } from "./response_time_indicator"; +const UNHEALTHY_STATUS = "unhealthy"; + // Helper function to deep-parse a JSON string if possible const deepParse = (input: any) => { let parsed = input; @@ -179,7 +181,7 @@ const HealthCheckDetails: React.FC<{ response: any }> = ({ response }) => { className={`text-sm font-medium ${response?.status === "healthy" ? "text-success" : "text-destructive"}`} > {t("caching.health.cacheStatus")} - {response?.status || "unhealthy"} + {response?.status || UNHEALTHY_STATUS}

diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/EvaluationSettingsModal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/EvaluationSettingsModal.tsx index 12ff10bc0c5..0c3cb62a57f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/EvaluationSettingsModal.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/EvaluationSettingsModal.tsx @@ -14,6 +14,7 @@ import { DialogTitle, } from "@/components/ui/dialog"; import { Textarea } from "@/components/ui/textarea"; +import { codeSamples } from "@/i18n/untranslated/codeSamples"; const DEFAULT_PROMPT = `Evaluate whether this guardrail's decision was correct. Analyze the user input, the guardrail action taken, and determine if it was appropriate. @@ -130,7 +131,7 @@ export function EvaluationSettingsModal({ -

response_format: json_schema

+

{codeSamples.guardrails.evaluationResponseFormat}