diff --git a/litellm/__init__.py b/litellm/__init__.py index 41e3fa34323..2cff21bf29a 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -710,6 +710,7 @@ transcribe_models: Set = set() gigachat_models: Set = set() llamagate_models: Set = set() coralbricks_models: Set[str] = set() +aiand_models: Set[str] = set() reducto_models: Set = set() bedrock_mantle_models: Set = set() @@ -1022,6 +1023,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje llamagate_models.add(key) elif value.get("litellm_provider") == "coralbricks": coralbricks_models.add(key) + elif value.get("litellm_provider") == "aiand": + aiand_models.add(key) elif value.get("litellm_provider") == "reducto": reducto_models.add(key) elif value.get("litellm_provider") == "bedrock_mantle": @@ -1159,6 +1162,7 @@ model_list = list( | reducto_models | bedrock_mantle_models | coralbricks_models + | aiand_models | set(clarifai_models) ) @@ -1283,6 +1287,7 @@ def _build_models_by_provider() -> dict: "gigachat": gigachat_models, "llamagate": llamagate_models, "coralbricks": coralbricks_models, + "aiand": aiand_models, "reducto": reducto_models, "bedrock_mantle": bedrock_mantle_models, } diff --git a/litellm/constants.py b/litellm/constants.py index 8cc51d0ab5e..0d6dfb5c0d4 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1035,6 +1035,7 @@ openai_compatible_endpoints: Final[list[str]] = [ "https://api-inference.modelscope.cn/v1", "https://api.moonshot.ai/v1", "https://api.publicai.co/v1", + "https://api.aiand.com/v1", "https://api.synthetic.new/openai/v1", "https://serverless.tensormesh.ai/v1", "https://api.stima.tech/v1", @@ -1100,6 +1101,7 @@ openai_compatible_providers: Final[list[str]] = [ "novita", "meta_llama", "publicai", # PublicAI - JSON-configured provider + "aiand", # ai& - JSON-configured provider "synthetic", # Synthetic - JSON-configured provider "tensormesh", # Tensormesh - JSON-configured provider "apertis", # Apertis - JSON-configured provider diff --git a/litellm/llms/anthropic/count_tokens/token_counter.py b/litellm/llms/anthropic/count_tokens/token_counter.py index 0764fb41941..368bb929eeb 100644 --- a/litellm/llms/anthropic/count_tokens/token_counter.py +++ b/litellm/llms/anthropic/count_tokens/token_counter.py @@ -27,6 +27,9 @@ anthropic_count_tokens_handler: Final = AnthropicCountTokensHandler() class AnthropicTokenCounter(BaseTokenCounter): """Token counter implementation for Anthropic provider using the CountTokens API.""" + def __init__(self, client_oauth_token: str | None = None) -> None: + self.client_oauth_token: Final = client_oauth_token + def should_use_token_counting_api( self, custom_llm_provider: str | None = None, @@ -95,12 +98,17 @@ class AnthropicTokenCounter(BaseTokenCounter): api_base: Final = litellm_params.get("api_base") try: - auth_header: Final = await AnthropicModelInfo.aget_auth_header( + configured_auth_header: Final = await AnthropicModelInfo.aget_auth_header( api_key=litellm_params.get("api_key") or litellm.anthropic_key or litellm.api_key, api_base=api_base, litellm_params=litellm_params, allow_workload_identity=True, ) + auth_header: Final = ( + configured_auth_header + if configured_auth_header is not None or self.client_oauth_token is None + else AnthropicModelInfo.get_auth_header(api_key=self.client_oauth_token, api_base=api_base) + ) if auth_header is None: return TokenCountResponse( total_tokens=0, diff --git a/litellm/llms/bedrock/chat/native_output_fallback.py b/litellm/llms/bedrock/chat/native_output_fallback.py index 1b8ce5fcda4..6892cd37beb 100644 --- a/litellm/llms/bedrock/chat/native_output_fallback.py +++ b/litellm/llms/bedrock/chat/native_output_fallback.py @@ -1,11 +1,14 @@ from collections.abc import Callable, Coroutine +from functools import partial from typing import Final, TypeAlias from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.bedrock.common_utils import ( NativeOutputFallback, is_bedrock_native_output_rejection, + is_bedrock_strict_tools_rejection, warn_bedrock_native_output_rejected, + warn_bedrock_strict_tools_dropped, ) from litellm.types.utils import ModelResponse from litellm.utils import CustomStreamWrapper @@ -14,12 +17,13 @@ NativeOutputResponse: TypeAlias = ModelResponse | CustomStreamWrapper NativeOutputAttemptResult: TypeAlias = NativeOutputResponse | Coroutine[object, object, NativeOutputResponse] NativeOutputAttempt: TypeAlias = Callable[[NativeOutputFallback | None], NativeOutputAttemptResult] NativeOutputFallbackFactory: TypeAlias = Callable[[], NativeOutputFallback | None] +_RetryFor: TypeAlias = Callable[[BaseLLMException], NativeOutputFallback | None] -def _fallback_for( - error: BaseLLMException, fallback: NativeOutputFallbackFactory, model: str +def _native_output_retry( + error: BaseLLMException, fallback: NativeOutputFallbackFactory | None, model: str ) -> NativeOutputFallback | None: - if not is_bedrock_native_output_rejection(error.status_code, error.message): + if fallback is None or not is_bedrock_native_output_rejection(error.status_code, error.message): return None retry: Final = fallback() if retry is not None: @@ -27,16 +31,38 @@ def _fallback_for( return retry +def _strict_tools_retry( + error: BaseLLMException, fallback: NativeOutputFallbackFactory | None, model: str +) -> NativeOutputFallback | None: + if fallback is None or not is_bedrock_strict_tools_rejection(error.status_code, error.message): + return None + retry: Final = fallback() + if retry is not None: + warn_bedrock_strict_tools_dropped(model) + return retry + + +def _fallback_for( + error: BaseLLMException, + fallback: NativeOutputFallbackFactory | None, + strict_tools_fallback: NativeOutputFallbackFactory | None, + model: str, +) -> NativeOutputFallback | None: + native_output_retry: Final = _native_output_retry(error, fallback, model) + if native_output_retry is not None: + return native_output_retry + return _strict_tools_retry(error, strict_tools_fallback, model) + + async def _await_with_native_output_fallback( first_attempt: Coroutine[object, object, NativeOutputResponse], attempt: NativeOutputAttempt, - fallback: NativeOutputFallbackFactory, - model: str, + retry_for: _RetryFor, ) -> NativeOutputResponse: try: return await first_attempt except BaseLLMException as error: - retry: Final = _fallback_for(error, fallback, model) + retry: Final = retry_for(error) if retry is None: raise second_attempt: Final = attempt(retry) @@ -47,16 +73,20 @@ def call_with_native_output_fallback( attempt: NativeOutputAttempt, fallback: NativeOutputFallbackFactory | None, model: str, + strict_tools_fallback: NativeOutputFallbackFactory | None = None, ) -> NativeOutputAttemptResult: - if fallback is None: + if fallback is None and strict_tools_fallback is None: return attempt(None) + retry_for: Final = partial( + _fallback_for, fallback=fallback, strict_tools_fallback=strict_tools_fallback, model=model + ) try: first_attempt: Final = attempt(None) except BaseLLMException as error: - retry: Final = _fallback_for(error, fallback, model) + retry: Final = retry_for(error) if retry is None: raise return attempt(retry) if isinstance(first_attempt, Coroutine): - return _await_with_native_output_fallback(first_attempt, attempt, fallback, model) + return _await_with_native_output_fallback(first_attempt, attempt, retry_for) return first_attempt diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 28888f1a23a..fde995c7907 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -362,6 +362,60 @@ class NativeOutputFallback: litellm_params: Mapping[str, object] +_BEDROCK_STRICT_TOOLS_REJECTION: Final = re.compile( + r"Too many strict tools \(\d+\)" + r"|Schemas contains too many (?:optional parameters|parameters with union types)" + r"|[Cc]ompiled grammar (?:is too large|size)" + r"|tools\.\d+\.custom(?:\.strict)?: " + r"|Grammar compilation timed out" +) +_TOOLS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...]) +_TOOL_SPEC_ADAPTER: Final = TypeAdapter(Mapping[str, object]) + + +def is_bedrock_strict_tools_rejection(status_code: int, error_text: str) -> bool: + return status_code == 400 and _BEDROCK_STRICT_TOOLS_REJECTION.search(error_text) is not None + + +def warn_bedrock_strict_tools_dropped(model: str) -> None: + verbose_logger.warning( + "Bedrock rejected the strict tool schemas for model=%s, so litellm sent the request again without `strict` " + "because drop_params is set.", + model, + ) + + +def _without_strict(tool: Mapping[str, object]) -> Mapping[str, object]: + spec_key: Final = next((key for key in ("function", "toolSpec") if key in tool), None) + if spec_key is None: + return tool + spec: Final = _TOOL_SPEC_ADAPTER.validate_python(tool[spec_key]) + return {**tool, spec_key: {key: value for key, value in spec.items() if key != "strict"}} + + +def _sends_strict(tool: Mapping[str, object], supports_strict_tools: bool) -> bool: + if "toolSpec" in tool: + return "strict" in _TOOL_SPEC_ADAPTER.validate_python(tool["toolSpec"]) + if "function" not in tool: + return False + return supports_strict_tools and bool(_TOOL_SPEC_ADAPTER.validate_python(tool["function"]).get("strict")) + + +def bedrock_strict_tools_fallback( + model: str, optional_params: Mapping[str, object], litellm_params: Mapping[str, object] +) -> NativeOutputFallback | None: + tools: Final = _TOOLS_ADAPTER.validate_python(optional_params.get("tools", ())) + supports_strict_tools: Final = bedrock_converse_supports_strict_tools( + bedrock_capability_model(model, litellm_params.get("base_model")) + ) + if not any(_sends_strict(tool, supports_strict_tools) for tool in tools): + return None + return NativeOutputFallback( + optional_params={**optional_params, "tools": [_without_strict(tool) for tool in tools]}, + litellm_params=litellm_params, + ) + + def requests_output_config_format(optional_params: Mapping[str, object]) -> bool: match optional_params: case {"output_config": {"format": object()}}: diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index 115d914d3ac..0034ed2b9d4 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -107,6 +107,12 @@ "api_key_env": "AIHUBMIX_API_KEY", "api_base_env": "AIHUBMIX_API_BASE" }, + "aiand": { + "base_url": "https://api.aiand.com/v1", + "api_key_env": "AIAND_API_KEY", + "api_base_env": "AIAND_API_BASE", + "supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/messages"] + }, "crusoe": { "base_url": "https://api.inference.crusoecloud.com/v1", "api_key_env": "CRUSOE_API_KEY", diff --git a/litellm/main.py b/litellm/main.py index 9f412e1f24b..39a0a0cd3c1 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -123,6 +123,7 @@ from litellm.llms.bedrock.common_utils import ( BedrockModelInfo, NativeOutputFallback, bedrock_route_for_request, + bedrock_strict_tools_fallback, deployment_model_info, without_bedrock_native_output_opt_in, without_bedrock_route_prefix, @@ -206,7 +207,12 @@ from litellm.utils import ( from ._logging import verbose_logger from .caching.caching import disable_cache, enable_cache, update_cache -from .litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params, safe_deep_copy +from .litellm_core_utils.core_helpers import ( + as_str_mapping, + drops_unsupported_params, + normalize_drop_params, + safe_deep_copy, +) from .litellm_core_utils.fallback_utils import ( async_completion_with_fallbacks, completion_with_fallbacks, @@ -4378,6 +4384,11 @@ def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResu else None ), converse_model, + strict_tools_fallback=( + partial(bedrock_strict_tools_fallback, converse_model, converse_params, dict(litellm_params)) + if drops_unsupported_params(litellm_params.get("drop_params")) + else None + ), ) elif bedrock_route == "converse_like": model = model.replace("converse_like/", "") diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 5774e699ad7..8207c3122f9 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -37504,7 +37504,6 @@ "input_cost_per_token": 5e-08, "input_cost_per_token_batches": 2.5e-08, "input_cost_per_token_flex": 2.5e-08, - "input_cost_per_token_priority": 2.5e-06, "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, @@ -37552,7 +37551,6 @@ "deprecation_date": "2026-12-11", "input_cost_per_token": 5e-08, "input_cost_per_token_batches": 2.5e-08, - "input_cost_per_token_priority": 2.5e-06, "input_cost_per_token_flex": 2.5e-08, "litellm_provider": "openai", "max_input_tokens": 272000, @@ -69827,6 +69825,201 @@ "supports_reasoning": true, "supports_vision": true }, + "aiand/deepseek-ai/deepseek-v4-flash": { + "cache_read_input_token_cost": 8e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "aiand", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "aiand/deepseek-ai/deepseek-v4-pro": { + "cache_read_input_token_cost": 2.5e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "aiand", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "aiand/deepseek-ai/deepseek-v4.1-flash": { + "cache_read_input_token_cost": 2e-08, + "input_cost_per_token": 3e-07, + "litellm_provider": "aiand", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 6e-07, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "aiand/google/gemma-4-31b-it": { + "cache_read_input_token_cost": 5e-08, + "input_cost_per_token": 2e-07, + "litellm_provider": "aiand", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "aiand/moonshotai/kimi-k2.7-code": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 7.5e-07, + "litellm_provider": "aiand", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.5e-06, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "aiand/moonshotai/kimi-k3": { + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 3e-06, + "litellm_provider": "aiand", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.25e-05, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "aiand/motif-technologies/motif-3": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 5e-07, + "litellm_provider": "aiand", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 2e-06, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "aiand/openai/gpt-oss-120b": { + "cache_read_input_token_cost": 8e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "aiand", + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 6e-07, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "aiand/qwen/qwen3.6-27b": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 3.2e-07, + "litellm_provider": "aiand", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 3.2e-06, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "aiand/qwen/qwen3.8-27b": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 4e-07, + "litellm_provider": "aiand", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 3e-06, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "aiand/zai-org/glm-5.2": { + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "aiand", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "aiand/zai-org/glm-5.3": { + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "aiand", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "aiand/zai-org/glm-5.3-flash": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "aiand", + "max_input_tokens": 1048550, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, "bespoke/nimble-latest": { "input_cost_per_token": 0.0, "litellm_provider": "bespoke", diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index 9ba6ed342ea..1ab03fb3fdc 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -189,6 +189,32 @@ "generateContent": false } }, + "aiand": { + "display_name": "ai& (`aiand`)", + "url": "https://docs.aiand.com/api/chat-completions/", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } + }, "amazon_nova": { "display_name": "Amazon Nova (`amazon_nova`)", "url": "https://docs.litellm.ai/docs/providers/amazon_nova", diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 13a1844a6de..1d2045eb743 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -50,7 +50,10 @@ from litellm.proxy.common_utils.openai_error_payload import ( with_litellm_call_id, ) from litellm.proxy.common_utils.token_count_settings import count_source_fields -from litellm.proxy.litellm_pre_call_utils import should_auto_drop_params_for_agentic_cli +from litellm.proxy.litellm_pre_call_utils import ( + forwarded_anthropic_oauth_token, + should_auto_drop_params_for_agentic_cli, +) from litellm.router import Router router: Final = APIRouter() @@ -412,6 +415,7 @@ async def count_tokens( request=token_request, call_endpoint=True, messages_format="anthropic", + client_oauth_token=forwarded_anthropic_oauth_token(request.headers), **_count_tokens_body_options(data, auto_drop_params), ) except UnsupportedContentTypeError as e: diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 1e7cdc44c70..314f14b6279 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -363,6 +363,8 @@ _UNTRUSTED_ROOT_CONTROL_FIELDS: Final = ( "fallback_depth", "_target_order", "attempted_targets", + "_excluded_deployment_ids", + "_retry_skipped_deployment_ids", "proxy_server_request", "standard_logging_object", "secret_fields", @@ -444,6 +446,7 @@ _UNTRUSTED_METADATA_CONTROL_FIELDS: Final = ( "_pipeline_managed_guardrails", "client_disconnected", "error_information", + "_failover_excluded_ids", PRE_CALL_EXECUTED_GUARDRAILS_KEY, ) @@ -1412,12 +1415,34 @@ def _tenant_service_name(user_api_key_dict: UserAPIKeyAuth) -> str | None: ) +def litellm_auth_header_name(headers: Headers) -> str: + if "x-litellm-api-key" in headers: + return "x-litellm-api-key" + if "authorization" in headers: + return "authorization" + return "x-api-key" + + +def forwarded_anthropic_oauth_token(headers: Headers) -> str | None: + from litellm.llms.anthropic.common_utils import is_anthropic_oauth_key + + forwarded: Final = clean_headers(headers, authenticated_with_header=litellm_auth_header_name(headers)) + return next( + ( + value.removeprefix("Bearer ") + for header, value in forwarded.items() + if header.lower() == "authorization" and is_anthropic_oauth_key(value) + ), + None, + ) + + def clean_headers( headers: Headers, litellm_key_header_name: str | None = None, forward_llm_provider_auth_headers: bool = False, authenticated_with_header: str | None = None, -) -> dict: +) -> dict[str, str]: """ Removes litellm api key from headers @@ -2510,26 +2535,13 @@ async def add_litellm_data_to_request( forward_llm_auth = general_settings.get("forward_llm_provider_auth_headers", False) if not forward_llm_auth: forward_llm_auth = getattr(litellm, "forward_llm_provider_auth_headers", False) - # Determine which header was used for authentication - # This enables forwarding provider keys (e.g., x-api-key) when they weren't used for LiteLLM auth - authenticated_with_header = None - if "x-litellm-api-key" in request.headers: - # If x-litellm-api-key is present, it was used for auth - authenticated_with_header = "x-litellm-api-key" - elif "authorization" in request.headers: - # Authorization header was used for auth - authenticated_with_header = "authorization" - else: - # x-api-key or another header was used for auth - authenticated_with_header = "x-api-key" - _headers: Final[dict[str, str]] = clean_headers( request.headers, litellm_key_header_name=( general_settings.get("litellm_key_header_name") if general_settings is not None else None ), forward_llm_provider_auth_headers=forward_llm_auth, - authenticated_with_header=authenticated_with_header, + authenticated_with_header=litellm_auth_header_name(request.headers), ) _strip_untrusted_request_header_controls( _headers, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 3b36ea5afce..04f84f1f7b5 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -14628,6 +14628,7 @@ async def run_thread( # ) # async def get_available_routes(user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth)): from litellm.litellm_core_utils.tool_fields import DropPolicy, drop_policy +from litellm.llms.anthropic.count_tokens.token_counter import AnthropicTokenCounter from litellm.llms.base_llm.base_utils import BaseTokenCounter from litellm.llms.openai.responses.count_tokens.token_counter import OpenAITokenCounter from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient @@ -14649,6 +14650,14 @@ from litellm.responses.litellm_completion_transformation.count_request import ( ) +def _with_client_oauth_token( + counter: BaseTokenCounter | None, client_oauth_token: str | None +) -> BaseTokenCounter | None: + if client_oauth_token is None or not isinstance(counter, AnthropicTokenCounter): + return counter + return AnthropicTokenCounter(client_oauth_token=client_oauth_token) + + def _get_provider_token_counter( deployment: dict, model_to_use: str ) -> tuple[BaseTokenCounter | None, str | None, str | None]: @@ -15264,6 +15273,7 @@ async def count_request_tokens( context_management: JsonValue = None, request_litellm_params: Mapping[str, JsonValue] = MappingProxyType({}), responses_request: Mapping[str, object] | None = None, + client_oauth_token: str | None = None, ) -> TokenCountResponse: prompt: Final = request.prompt messages: Final = request.messages @@ -15321,7 +15331,8 @@ async def count_request_tokens( custom_llm_provider: str | None = None if call_endpoint is True and deployment is not None: # Auto-route to the correct provider based on model - provider_counter, _model, custom_llm_provider = _get_provider_token_counter(deployment, model_to_use) + routed_counter, _model, custom_llm_provider = _get_provider_token_counter(deployment, model_to_use) + provider_counter = _with_client_oauth_token(routed_counter, client_oauth_token) if _model is not None: model_to_use = _model diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index d96f2976508..0dd96c856f8 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -1,4 +1,32 @@ [ + { + "provider": "AIAND", + "provider_display_name": "ai&", + "litellm_provider": "aiand", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://api.aiand.com/v1", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "aiand/openai/gpt-oss-120b" + }, { "provider": "SCALEDOWN", "provider_display_name": "ScaleDown", diff --git a/litellm/router.py b/litellm/router.py index c8153b596d6..e53bab4659e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -212,6 +212,7 @@ from litellm.router_utils.clientside_credential_handler import ( ) from litellm.router_utils.common_utils import ( MID_STREAM_CONTINUATION_KWARG, + as_retry_skipped_deployment_ids, deployments_the_caller_may_use, filter_team_based_models, filter_web_search_deployments, @@ -238,11 +239,12 @@ from litellm.router_utils.cooldown_handlers import ( async_get_cooldown_deployments_with_debug_info, get_cooldown_deployment_info, get_cooldown_deployments, - is_advisor_orchestration_failure, is_background_response_poll_not_found, is_caller_scoped_auth_failure, is_caller_timeout_408, + is_failure_outside_selected_deployment, is_video_lookup_not_found, + mark_deployment_selection_failure, set_cooldown_deployments, ) from litellm.router_utils.deployment_pricing import deployment_pricing_models @@ -684,10 +686,6 @@ def _configured_model_info(deployment: Mapping[str, object]) -> Mapping[str, obj return _MODEL_INFO_ADAPTER.validate_python(deployment.get("model_info") or {}) -def _as_retry_skipped_deployment_ids(value: object) -> tuple[str, ...]: - return tuple(item for item in value if isinstance(item, str)) if isinstance(value, tuple) else () - - def _silent_experiment_targets(silent_model: object) -> tuple[str, ...]: if silent_model is None: return () @@ -1519,6 +1517,7 @@ class Router: self.allowed_fails = allowed_fails else: self.allowed_fails = litellm.allowed_fails + self.explicit_allowed_fails = allowed_fails self.cooldown_time = cooldown_time or DEFAULT_COOLDOWN_TIME_SECONDS self.cooldown_cache = CooldownCache(cache=self.cache, default_cooldown_time=self.cooldown_time) self.disable_cooldowns = disable_cooldowns @@ -8940,7 +8939,7 @@ class Router: return () if litellm.should_retry(status_code): return () - already_skipped_ids: Final = _as_retry_skipped_deployment_ids(already_skipped) + already_skipped_ids: Final = as_retry_skipped_deployment_ids(already_skipped) skipped: Final = tuple(sorted(frozenset((*already_skipped_ids, failed_deployment_id)))) verbose_router_logger.debug( "Retry skips deployments that already answered %s to this request: %s", status_code, skipped @@ -9579,10 +9578,10 @@ class Router: try: exception: Final = kwargs.get("exception", None) - if is_advisor_orchestration_failure(exception): + if is_failure_outside_selected_deployment(exception): verbose_router_logger.debug( "Router: Exiting 'deployment_callback_on_failure' without cooldown. " - "Failure originated from advisor orchestration, not the selected deployment." + "Failure came from advisor orchestration or deployment selection, not the selected deployment." ) return False @@ -15184,7 +15183,7 @@ class Router: ## RETRY SKIP ## -> drop deployments that already refused this request with a ## non-retryable status, unless that leaves nothing, so the caller still gets ## the provider's own error instead of a no-deployments error. - _retry_skipped_deployment_ids: Final = _as_retry_skipped_deployment_ids( + _retry_skipped_deployment_ids: Final = as_retry_skipped_deployment_ids( request_kwargs.pop("_retry_skipped_deployment_ids", None) if request_kwargs else None ) healthy_deployments = ( @@ -15473,6 +15472,7 @@ class Router: phase_event(_DEPLOYMENT_SELECTED_EVENT, pick_attributes) return deployment except Exception as e: + mark_deployment_selection_failure(e) traceback_exception: Final = traceback.format_exc() # if router rejects call -> log to langfuse/otel/etc. if request_kwargs is not None: @@ -15641,6 +15641,7 @@ class Router: return deployment except Exception as e: + mark_deployment_selection_failure(e) traceback_exception: Final = traceback.format_exc() if request_kwargs is not None: logging_obj: Final = request_kwargs.get("litellm_logging_obj", None) @@ -16414,7 +16415,7 @@ class Router: ) ## RETRY SKIP ## -> see async counterpart in async_get_healthy_deployments. - _retry_skipped_deployment_ids: Final = _as_retry_skipped_deployment_ids( + _retry_skipped_deployment_ids: Final = as_retry_skipped_deployment_ids( request_kwargs.pop("_retry_skipped_deployment_ids", None) if request_kwargs else None ) healthy_deployments = ( diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index a00d84babf3..91b9e5cb114 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -4,7 +4,7 @@ from collections.abc import Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Final, TypeVar -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError if TYPE_CHECKING: from litellm.types.llms.openai import OpenAIFileObject @@ -38,6 +38,7 @@ _OBJECT_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[object, object]]] = TypeAdapt _DEPLOYMENT_ROWS_ADAPTER: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter( tuple[Mapping[str, object], ...] ) +_DEPLOYMENT_IDS_ADAPTER: Final = TypeAdapter(tuple[str, ...]) def with_admin_api_key_secret_reference(deployment: Deployment) -> Deployment: @@ -57,6 +58,31 @@ def without_router_only_kwargs(kwargs: Mapping[str, _V]) -> dict[str, _V]: return {key: value for key, value in kwargs.items() if key not in ROUTER_ONLY_CALL_KWARGS} +def as_retry_skipped_deployment_ids(value: object) -> tuple[str, ...]: + if not isinstance(value, tuple): + return () + items: Final[tuple[object, ...]] = value + return tuple(item for item in items if isinstance(item, str)) + + +def as_excluded_deployment_ids(value: object) -> tuple[str, ...]: + if value is None: + return () + try: + return _DEPLOYMENT_IDS_ADAPTER.validate_python(value) + except ValidationError: + return () + + +def deployment_ids_failed_in_request(request_kwargs: Mapping[str, object]) -> frozenset[str]: + return frozenset( + ( + *as_excluded_deployment_ids(request_kwargs.get("_excluded_deployment_ids")), + *as_retry_skipped_deployment_ids(request_kwargs.get("_retry_skipped_deployment_ids")), + ) + ) + + def request_user_api_key_auth(request_kwargs: Mapping[str, object]) -> object: """The caller's auth object, from whichever metadata bucket this surface writes to.""" metadata_value: Final = request_kwargs.get("metadata") diff --git a/litellm/router_utils/cooldown_handlers.py b/litellm/router_utils/cooldown_handlers.py index 330500789a4..e1baf66b201 100644 --- a/litellm/router_utils/cooldown_handlers.py +++ b/litellm/router_utils/cooldown_handlers.py @@ -57,6 +57,7 @@ else: Span = Any _ADVISOR_ORCHESTRATION_FAILURE_ATTR: Final = "_litellm_advisor_orchestration_failure" +_DEPLOYMENT_SELECTION_FAILURE_ATTR: Final = "_litellm_deployment_selection_failure" _CREDENTIAL_VALUES_ADAPTER: Final = TypeAdapter(Mapping[str, object]) @@ -79,6 +80,16 @@ def is_advisor_orchestration_failure(exception: BaseException | None) -> bool: return bool(getattr(exception, _ADVISOR_ORCHESTRATION_FAILURE_ATTR, False)) +def mark_deployment_selection_failure(exception: BaseException) -> None: + setattr(exception, _DEPLOYMENT_SELECTION_FAILURE_ATTR, True) + + +def is_failure_outside_selected_deployment(exception: BaseException | None) -> bool: + return is_advisor_orchestration_failure(exception) or bool( + getattr(exception, _DEPLOYMENT_SELECTION_FAILURE_ATTR, False) + ) + + _BACKGROUND_RESPONSE_POLL_CALL_ORIGINS: Final[frozenset[InternalCallOrigin]] = frozenset( {BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, BACKGROUND_RESPONSE_STATUS_POLL_CALL_ORIGIN} ) @@ -781,14 +792,17 @@ def _is_allowed_fails_set_on_router( litellm_router_instance: LitellmRouter, ) -> bool: """ - Check if Router.allowed_fails is set or is Non-default Value + Check if Router.allowed_fails was passed to the Router or is a Non-default Value Returns: - - True if Router.allowed_fails is set or is Non-default Value - - False if Router.allowed_fails is None or is Default Value + - True if allowed_fails was passed to the Router, even when it equals the default + - True if Router.allowed_fails is a Non-default Value + - False if Router.allowed_fails is None, or is the Default Value and was not passed """ if litellm_router_instance.allowed_fails is None: return False + if litellm_router_instance.explicit_allowed_fails is not None: + return True if litellm_router_instance.allowed_fails != litellm.allowed_fails: return True return False diff --git a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py index 8693eaa5151..daf7d6bb345 100644 --- a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py @@ -31,6 +31,7 @@ from litellm.caching.dual_cache import DualCache from litellm.constants import SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger, Span from litellm.responses.utils import ResponsesAPIRequestUtils +from litellm.router_utils.common_utils import deployment_ids_failed_in_request from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import CallTypes @@ -429,6 +430,9 @@ class DeploymentAffinityCheck(CustomLogger): session_affinity_active: Final = ( enable_session_id or self._get_marker_session_affinity_ttl(request_kwargs=request_kwargs) is not None ) + failed_deployment_ids: Final = deployment_ids_failed_in_request( + cast(Mapping[str, object], request_kwargs) # cast-ok: request kwargs keys are param names + ) user_key: Final = ( self.get_user_key_from_request_kwargs(request_kwargs=request_kwargs) if (session_affinity_active or enable_user_key) @@ -458,7 +462,7 @@ class DeploymentAffinityCheck(CustomLogger): elif isinstance(session_cache_result, str): session_model_id = session_cache_result - if session_model_id: + if session_model_id and session_model_id not in failed_deployment_ids: session_deployment: Final = self._find_deployment_by_model_id( healthy_deployments=typed_healthy_deployments, model_id=session_model_id, @@ -493,7 +497,7 @@ class DeploymentAffinityCheck(CustomLogger): # Backwards / safety: allow raw string values. model_id = cache_result - if not model_id: + if not model_id or model_id in failed_deployment_ids: return typed_healthy_deployments deployment = self._find_deployment_by_model_id( diff --git a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py index 5d47bf63a38..806c3db7601 100644 --- a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py @@ -51,6 +51,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( strip_encrypted_reasoning_from_messages, ) from litellm.responses.utils import ResponsesAPIRequestUtils +from litellm.router_utils.common_utils import as_excluded_deployment_ids, as_retry_skipped_deployment_ids from litellm.types.llms.openai import AllMessageValues from litellm.types.router import Deployment from litellm.utils import get_order_filtered_deployments @@ -182,6 +183,14 @@ class EncryptedContentAffinityCheck(CustomLogger): None, ) + @staticmethod + def _deployment_model_id(deployment: Mapping[str, object]) -> str | None: + model_info: Final = deployment.get("model_info") + if not isinstance(model_info, Mapping): + return None + deployment_id: Final = cast(Mapping[str, object], model_info).get("id") # cast-ok: narrowed by isinstance + return None if deployment_id is None else str(deployment_id) + @staticmethod def _find_deployment_by_model_id(healthy_deployments: list[dict], model_id: str) -> dict | None: for deployment in healthy_deployments: @@ -345,7 +354,8 @@ class EncryptedContentAffinityCheck(CustomLogger): typed_healthy_deployments: Final = cast(list[dict[str, object]], healthy_deployments) if not self._is_enabled_for_model_group(model): return typed_healthy_deployments - target_order: Final = _REQUEST_KWARGS_ADAPTER.validate_python(request_kwargs).get("_target_order") + typed_request_kwargs: Final = _REQUEST_KWARGS_ADAPTER.validate_python(request_kwargs) + target_order: Final = typed_request_kwargs.get("_target_order") candidates: Final = ( get_order_filtered_deployments(typed_healthy_deployments, target_order=target_order) if isinstance(target_order, int) @@ -375,8 +385,31 @@ class EncryptedContentAffinityCheck(CustomLogger): model_id, ) + previous_response_id: Final = typed_request_kwargs.get("previous_response_id") + previous_response_origin: Final = ResponsesAPIRequestUtils.get_model_id_from_response_id( + None if previous_response_id is None else str(previous_response_id) + ) + previous_response_binds_origin: Final = previous_response_origin == model_id + excluded_ids: Final[frozenset[str]] = ( + frozenset() + if previous_response_binds_origin + else frozenset(as_excluded_deployment_ids(typed_request_kwargs.get("_excluded_deployment_ids"))) + ) + retry_skipped_ids: Final[frozenset[str]] = ( + frozenset() + if previous_response_binds_origin + else frozenset(as_retry_skipped_deployment_ids(typed_request_kwargs.get("_retry_skipped_deployment_ids"))) + ) + not_excluded_candidates: Final = [ + candidate for candidate in candidates if self._deployment_model_id(candidate) not in excluded_ids + ] + untried_candidates: Final = [ + candidate + for candidate in not_excluded_candidates + if self._deployment_model_id(candidate) not in retry_skipped_ids + ] or not_excluded_candidates deployment: Final = self._find_deployment_by_model_id( - healthy_deployments=candidates, + healthy_deployments=untried_candidates, model_id=model_id, ) if deployment is not None: @@ -392,7 +425,7 @@ class EncryptedContentAffinityCheck(CustomLogger): # Follow-up switched model_name (LIT-2531): pin by Azure resource instead. boundary_matches, _originating = self._find_deployments_on_same_encryption_boundary( - healthy_deployments=candidates, + healthy_deployments=untried_candidates, model_id=model_id, ) if boundary_matches: diff --git a/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py b/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py index bb3475cdcbe..c04bab4d360 100644 --- a/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py +++ b/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py @@ -19,6 +19,7 @@ from litellm.integrations.anthropic_cache_control_hook import ( from litellm.integrations.custom_logger import CustomLogger, Span from litellm.litellm_core_utils.token_counter import offload_token_count from litellm.llms.bedrock.common_utils import bedrock_cache_point_ttl_supported, bedrock_capability_model +from litellm.router_utils.common_utils import deployment_ids_failed_in_request from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam from litellm.types.utils import CallTypes, StandardLoggingPayload from litellm.utils import ( @@ -152,7 +153,9 @@ class PromptCachingDeploymentCheck(CustomLogger): request_tools, ), ) - if model_id_dict is not None: + if model_id_dict is not None and model_id_dict["model_id"] not in deployment_ids_failed_in_request( + request_data or {} + ): model_id: Final = model_id_dict["model_id"] for deployment in healthy_deployments: if deployment["model_info"]["id"] == model_id: diff --git a/litellm/types/utils.py b/litellm/types/utils.py index b7ed96c4a29..71f80e07aea 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4574,6 +4574,7 @@ class LlmProviders(str, Enum): POE = "poe" CHUTES = "chutes" NEOSANTARA = "neosantara" + AIAND = "aiand" PARASAIL = "parasail" XIAOMI_MIMO = "xiaomi_mimo" TENSORMESH = "tensormesh" diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 5774e699ad7..8207c3122f9 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -37504,7 +37504,6 @@ "input_cost_per_token": 5e-08, "input_cost_per_token_batches": 2.5e-08, "input_cost_per_token_flex": 2.5e-08, - "input_cost_per_token_priority": 2.5e-06, "litellm_provider": "openai", "max_input_tokens": 272000, "max_output_tokens": 128000, @@ -37552,7 +37551,6 @@ "deprecation_date": "2026-12-11", "input_cost_per_token": 5e-08, "input_cost_per_token_batches": 2.5e-08, - "input_cost_per_token_priority": 2.5e-06, "input_cost_per_token_flex": 2.5e-08, "litellm_provider": "openai", "max_input_tokens": 272000, @@ -69827,6 +69825,201 @@ "supports_reasoning": true, "supports_vision": true }, + "aiand/deepseek-ai/deepseek-v4-flash": { + "cache_read_input_token_cost": 8e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "aiand", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "aiand/deepseek-ai/deepseek-v4-pro": { + "cache_read_input_token_cost": 2.5e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "aiand", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "aiand/deepseek-ai/deepseek-v4.1-flash": { + "cache_read_input_token_cost": 2e-08, + "input_cost_per_token": 3e-07, + "litellm_provider": "aiand", + "max_input_tokens": 1048576, + "max_output_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 6e-07, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "aiand/google/gemma-4-31b-it": { + "cache_read_input_token_cost": 5e-08, + "input_cost_per_token": 2e-07, + "litellm_provider": "aiand", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "aiand/moonshotai/kimi-k2.7-code": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 7.5e-07, + "litellm_provider": "aiand", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3.5e-06, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "aiand/moonshotai/kimi-k3": { + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 3e-06, + "litellm_provider": "aiand", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 1.25e-05, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "aiand/motif-technologies/motif-3": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 5e-07, + "litellm_provider": "aiand", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 2e-06, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "aiand/openai/gpt-oss-120b": { + "cache_read_input_token_cost": 8e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "aiand", + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 6e-07, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "aiand/qwen/qwen3.6-27b": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 3.2e-07, + "litellm_provider": "aiand", + "max_input_tokens": 262144, + "max_output_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 3.2e-06, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "aiand/qwen/qwen3.8-27b": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 4e-07, + "litellm_provider": "aiand", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 3e-06, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "aiand/zai-org/glm-5.2": { + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "aiand", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "aiand/zai-org/glm-5.3": { + "cache_read_input_token_cost": 3e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "aiand", + "max_input_tokens": 1048576, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "aiand/zai-org/glm-5.3-flash": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "aiand", + "max_input_tokens": 1048550, + "max_output_tokens": 131072, + "mode": "chat", + "output_cost_per_token": 5e-07, + "source": "https://api.aiand.com/v1/models", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, "bespoke/nimble-latest": { "input_cost_per_token": 0.0, "litellm_provider": "bespoke", diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 9ba6ed342ea..1ab03fb3fdc 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -189,6 +189,32 @@ "generateContent": false } }, + "aiand": { + "display_name": "ai& (`aiand`)", + "url": "https://docs.aiand.com/api/chat-completions/", + "endpoints": { + "chat_completions": true, + "responses": true, + "text_completion": true, + "embeddings": false, + "image_generations": false, + "image_edits": false, + "audio_transcriptions": false, + "audio_speech": false, + "rerank": false, + "ocr": false, + "search": false, + "moderations": false, + "realtime": false, + "video_generations": false, + "batches": false, + "messages": true, + "interactions": true, + "systemone": false, + "a2a": true, + "generateContent": true + } + }, "amazon_nova": { "display_name": "Amazon Nova (`amazon_nova`)", "url": "https://docs.litellm.ai/docs/providers/amazon_nova", diff --git a/tests/integration/compatibility/test_compressed_request_bodies.py b/tests/integration/compatibility/test_compressed_request_bodies.py new file mode 100644 index 00000000000..ef2490add40 --- /dev/null +++ b/tests/integration/compatibility/test_compressed_request_bodies.py @@ -0,0 +1,392 @@ +from __future__ import annotations + +import gzip +import json +import os +import secrets +import threading +import uuid +from collections.abc import Callable, Iterator, Mapping +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from functools import partial +from pathlib import Path +from types import MappingProxyType +from typing import Final + +import httpx +import pytest +import yaml +from integration._support.budget import case_timeout_seconds +from integration._support.client import ( + JSON_OBJECT, + Gateway, + eventually, + gateway_from_environment, + list_value, + object_value, + string_value, +) +from integration._support.process import graceful_stop_seconds, owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue + +MIB: Final = 1024 * 1024 +CHAT: Final = "/v1/chat/completions" +CHAT_MODEL: Final = "compressed-chat" +OTHER_MODEL: Final = "compressed-chat-other" +ANTHROPIC_MODEL: Final = "claude-haiku-5-5" +GATED_PREFIX: Final = "gated-" +ORIGIN: Final = "http://browser.example" +MIDDLEWARE_HEADERS: Final = ( + "access-control-allow-origin", + "x-frame-options", + "content-security-policy", + "x-content-type-options", +) +ZSTD_MAGIC: Final = b"\x28\xb5\x2f\xfd" +ZSTD_SINGLE_SEGMENT_FOUR_BYTE_SIZE: Final = b"\xa0" +ZSTD_MAX_BLOCK: Final = 128 * 1024 +NO_HEADERS: Final[Mapping[str, str]] = MappingProxyType({}) + +pytestmark: Final = [ + pytest.mark.timeout(2 * graceful_stop_seconds() + 120), + pytest.mark.xdist_group("compressed-request-bodies"), +] + + +def zstd_frame(data: bytes) -> bytes: + blocks: Final = tuple(data[start : start + ZSTD_MAX_BLOCK] for start in range(0, len(data), ZSTD_MAX_BLOCK)) + framed: Final = blocks or (b"",) + return ( + ZSTD_MAGIC + + ZSTD_SINGLE_SEGMENT_FOUR_BYTE_SIZE + + len(data).to_bytes(4, "little") + + b"".join( + (len(block) << 3 | (index == len(framed) - 1)).to_bytes(3, "little") + block + for index, block in enumerate(framed) + ) + ) + + +def gzip_body(data: bytes) -> bytes: + return gzip.compress(data, mtime=0) + + +def as_sent(data: bytes) -> bytes: + return data + + +@dataclass(frozen=True, slots=True) +class Rig: + gateway: Gateway + upstream: Wire + gate: threading.Event + limited_key: str + + def calls(self) -> tuple[Request, ...]: + return tuple(request for request in self.upstream.drain() if request.method == "POST") + + +def _chat_completion() -> bytes: + return json.dumps( + { + "id": f"chatcmpl-{uuid.uuid4().hex}", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "scripted"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}, + } + ).encode() + + +def _stream_frame(identity: str, delta: Mapping[str, JsonValue], finish_reason: str | None) -> bytes: + chunk: Final = { + "id": identity, + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "delta": dict(delta), "finish_reason": finish_reason}], + } + return b"data: " + json.dumps(chunk).encode() + b"\n\n" + + +def _chat_stream(gate: threading.Event | None) -> Reply: + identity: Final = f"chatcmpl-{uuid.uuid4().hex}" + return Reply( + content_type="text/event-stream", + chunks=( + _stream_frame(identity, {"role": "assistant", "content": "streamed "}, None), + _stream_frame(identity, {"content": "answer"}, "stop"), + b"data: [DONE]\n\n", + ), + gate_after_first=gate, + ) + + +def _anthropic_message() -> bytes: + return json.dumps( + { + "id": f"msg_{uuid.uuid4().hex}", + "type": "message", + "role": "assistant", + "model": ANTHROPIC_MODEL, + "content": [{"type": "text", "text": "scripted"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 3, "output_tokens": 2}, + } + ).encode() + + +def _respond(gate: threading.Event, request: Request) -> Reply: + if request.method == "GET": + return Reply(body=b'{"object":"list","data":[]}') + if request.target == "/v1/messages": + return Reply(body=_anthropic_message()) + body: Final = JSON_OBJECT.validate_json(request.body) + if body.get("stream") is not True: + return Reply(body=_chat_completion()) + text: Final = string_value(object_value(list_value(body["messages"])[0])["content"]) + return _chat_stream(gate if text.startswith(GATED_PREFIX) else None) + + +def _deployment(name: str, upstream: Wire) -> Mapping[str, JsonValue]: + return { + "model_name": name, + "litellm_params": {"model": "openai/gpt-4o-mini", "api_base": f"{upstream.url}/v1", "api_key": "sk-fixture"}, + } + + +def _config(directory: Path, upstream: Wire) -> Path: + base: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + configuration: Final = { + **base, + "model_list": [_deployment(CHAT_MODEL, upstream), _deployment(OTHER_MODEL, upstream)], + "general_settings": { + **object_value(base.get("general_settings", {})), + "max_request_size_mb": 1, + "enable_claude_code_gateway": True, + }, + "router_settings": {**object_value(base.get("router_settings", {})), "num_retries": 0}, + } + path: Final = directory / "compressed-request-bodies.yaml" + path.write_text(yaml.safe_dump(configuration)) + return path + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + assert os.environ.get("LITELLM_LICENSE"), ( + "LITELLM_LICENSE must be set: max_request_size_mb is an enterprise feature" + ) + gate: Final = threading.Event() + directory: Final = tmp_path_factory.mktemp("compressed-request-bodies") + with gateway_from_environment() as root, wire_server(partial(_respond, gate)) as upstream: + overrides: Final = { + "ANTHROPIC_API_BASE": upstream.url, + "ANTHROPIC_API_KEY": "sk-ant-synthetic", + "MAX_DECOMPRESSED_REQUEST_BODIES_IN_FLIGHT_MB": "1", + } + with ( + owned_proxy_process(root, directory, overrides, config=_config(directory, upstream), workers=1) as owned, + owned.gateway.scenario() as scenario, + ): + yield Rig(owned.gateway, upstream, gate, scenario.key(models=[CHAT_MODEL])) + + +@pytest.fixture(autouse=True) +def empty_upstream_log(rig: Rig) -> None: + rig.upstream.drain() + + +def _headers(key: str, coding: str | None, extra: Mapping[str, str] = NO_HEADERS) -> Mapping[str, str]: + return { + "Authorization": f"Bearer {key}", + "content-type": "application/json", + **({"content-encoding": coding} if coding is not None else {}), + **extra, + } + + +def _send( + rig: Rig, + data: bytes, + coding: str | None, + *, + key: str | None = None, + extra: Mapping[str, str] = NO_HEADERS, +) -> httpx.Response: + return rig.gateway.client.post( + CHAT, + content=data, + headers=_headers(rig.gateway.key if key is None else key, coding, extra), + timeout=case_timeout_seconds(), + ) + + +def _chat_body(model: str, text: str, *, stream: bool = False) -> bytes: + return json.dumps( + { + "model": model, + "messages": [{"role": "user", "content": text}], + "stream": stream, + "cache": {"no-cache": True}, + } + ).encode() + + +def _error_type(response: httpx.Response) -> JsonValue: + return object_value(JSON_OBJECT.validate_json(response.content)["error"])["type"] + + +def _answer(response: httpx.Response) -> JsonValue: + choice: Final = object_value(list_value(JSON_OBJECT.validate_json(response.content)["choices"])[0]) + return object_value(choice["message"])["content"] + + +def _stream(gateway: Gateway, data: bytes, first_text: threading.Event) -> tuple[int, str]: + with ( + httpx.Client(base_url=gateway.client.base_url, timeout=case_timeout_seconds(), trust_env=False) as client, + client.stream("POST", CHAT, content=data, headers=_headers(gateway.key, "gzip")) as response, + ): + pieces: Final = response.iter_text() + first: Final = next(pieces, "") + first_text.set() + return response.status_code, first + "".join(pieces) + + +CODINGS: Final = ( + pytest.param("zstd", zstd_frame, id="zstd"), + pytest.param("gzip", gzip_body, id="gzip"), + pytest.param("gzip", as_sent, id="plain-labeled-gzip"), +) + + +@pytest.mark.parametrize(("coding", "encode"), CODINGS) +def test_a_compressed_chat_body_reaches_the_provider_as_its_plain_twin( + rig: Rig, coding: str, encode: Callable[[bytes], bytes] +) -> None: + body: Final = _chat_body(CHAT_MODEL, f"twin {uuid.uuid4().hex}") + plain: Final = _send(rig, body, None) + compressed: Final = _send(rig, encode(body), coding) + assert plain.status_code == 200, plain.text + assert compressed.status_code == 200, compressed.text + assert _answer(compressed) == _answer(plain) + assert compressed.headers["x-litellm-response-cost"] == plain.headers["x-litellm-response-cost"] + calls: Final = rig.calls() + assert len(calls) == 2, [call.target for call in calls] + assert JSON_OBJECT.validate_json(calls[1].body) == JSON_OBJECT.validate_json(calls[0].body) + assert "content-encoding" not in calls[1].headers, calls[1].headers + + +def test_auth_reads_the_model_from_a_compressed_body(rig: Rig) -> None: + refused: Final = _send(rig, zstd_frame(_chat_body(OTHER_MODEL, "outside the key")), "zstd", key=rig.limited_key) + assert refused.status_code == 403, refused.text + assert _error_type(refused) == "key_model_access_denied", refused.text + assert rig.calls() == () + served: Final = _send(rig, zstd_frame(_chat_body(CHAT_MODEL, "inside the key")), "zstd", key=rig.limited_key) + assert served.status_code == 200, served.text + assert len(rig.calls()) == 1 + + +@pytest.mark.parametrize( + ("coding", "encode"), + (pytest.param("zstd", zstd_frame, id="zstd"), pytest.param("gzip", as_sent, id="plain-labeled-gzip")), +) +def test_a_compressed_pass_through_body_reaches_the_vendor_decoded( + rig: Rig, coding: str, encode: Callable[[bytes], bytes] +) -> None: + body: Final = json.dumps( + { + "model": ANTHROPIC_MODEL, + "max_tokens": 8, + "messages": [{"role": "user", "content": f"pass through {uuid.uuid4().hex}"}], + } + ).encode() + response: Final = rig.gateway.client.post( + "/anthropic/v1/messages", + content=encode(body), + headers={ + "x-litellm-api-key": rig.gateway.key, + "anthropic-version": "2023-06-01", + "content-type": "application/json", + "content-encoding": coding, + }, + timeout=case_timeout_seconds(), + ) + assert response.status_code == 200, response.text + calls: Final = rig.calls() + assert [call.target for call in calls] == ["/v1/messages"] + assert JSON_OBJECT.validate_json(calls[0].body) == JSON_OBJECT.validate_json(body) + assert "content-encoding" not in calls[0].headers, calls[0].headers + + +def test_a_gzip_bomb_gets_a_413_with_the_headers_other_errors_carry(rig: Rig) -> None: + origin: Final = {"origin": ORIGIN} + rejected: Final = _send(rig, gzip_body(b"\0" * 800_000), "gzip", extra=origin) + unauthorized: Final = _send( + rig, _chat_body(CHAT_MODEL, "unknown key"), None, key=f"sk-unknown-{uuid.uuid4().hex}", extra=origin + ) + assert rejected.status_code == 413, rejected.text + assert unauthorized.status_code == 401, unauthorized.text + assert unauthorized.headers.get("access-control-allow-origin") is not None, dict(unauthorized.headers) + assert {name: rejected.headers.get(name) for name in MIDDLEWARE_HEADERS} == { + name: unauthorized.headers.get(name) for name in MIDDLEWARE_HEADERS + } + assert rig.calls() == () + + +def test_the_operator_size_limit_holds_a_compressed_body_to_its_plain_twins_limit(rig: Rig) -> None: + over: Final = _chat_body(CHAT_MODEL, secrets.token_hex(600_000)) + compressed_over: Final = gzip_body(over) + assert len(compressed_over) < MIB < len(over) + plain: Final = _send(rig, over, None) + rejected: Final = _send(rig, compressed_over, "gzip") + served: Final = _send(rig, gzip_body(_chat_body(CHAT_MODEL, secrets.token_hex(300_000))), "gzip") + assert plain.status_code == 413, plain.text + assert rejected.status_code == plain.status_code, rejected.text + assert served.status_code == 200, served.text + assert len(rig.calls()) == 1 + + +def test_the_decoded_body_budget_turns_away_a_second_body_until_the_stream_ends(rig: Rig) -> None: + held: Final = gzip_body(_chat_body(CHAT_MODEL, GATED_PREFIX + secrets.token_hex(300_000), stream=True)) + second: Final = gzip_body(_chat_body(CHAT_MODEL, secrets.token_hex(300_000))) + keyless: Final = _send(rig, second, "gzip", key=f"sk-unknown-{uuid.uuid4().hex}") + assert keyless.status_code == 401, keyless.text + first_text: Final = threading.Event() + with ThreadPoolExecutor(1) as pool: + streaming: Final = pool.submit(_stream, rig.gateway, held, first_text) + try: + eventually(lambda: first_text.is_set() or streaming.done(), bool, seconds=case_timeout_seconds()) + reached: Final = rig.calls() + turned_away: Final = _send(rig, second, "gzip") + finally: + rig.gate.set() + status, streamed = streaming.result(timeout=case_timeout_seconds()) + assert len(reached) == 1, streamed + assert turned_away.status_code == 503, turned_away.text + assert _error_type(turned_away) == "overloaded_error", turned_away.text + assert turned_away.headers.get("retry-after") == "1", dict(turned_away.headers) + assert status == 200, streamed + assert "streamed " in streamed and "answer" in streamed and "data: [DONE]" in streamed, streamed + assert rig.calls() == () + after_the_stream: Final = _send(rig, second, "gzip") + assert after_the_stream.status_code == 200, after_the_stream.text + assert len(rig.calls()) == 1 + + +def test_a_claude_code_otlp_export_keeps_a_two_member_gzip_body(rig: Rig) -> None: + response: Final = rig.gateway.client.post( + "/claude_code_gateway/v1/metrics", + content=gzip_body(b"first member") + gzip_body(b"second member"), + headers={ + "Authorization": f"Bearer {rig.gateway.key}", + "content-type": "application/x-protobuf", + "content-encoding": "gzip", + }, + timeout=case_timeout_seconds(), + ) + assert response.status_code == 200, response.text diff --git a/tests/integration/providers/_api_base_url_join_support.py b/tests/integration/providers/_api_base_url_join_support.py index 44bd6ec4dca..5711cc77309 100644 --- a/tests/integration/providers/_api_base_url_join_support.py +++ b/tests/integration/providers/_api_base_url_join_support.py @@ -13,6 +13,7 @@ from pydantic import JsonValue COUNTED_TOKENS: Final = 7 RELEVANCE: Final = 0.75 +AZURE_DEPLOYMENT: Final = "gpt-url-join" _USAGE: Final = {"input_tokens": 3, "output_tokens": 2} @@ -146,12 +147,20 @@ def _cohere_rerank(request: Request) -> Reply: ) +def _titan_embedding(_: Request) -> Reply: + return _json({"embedding": list(VECTOR), "inputTextTokenCount": 2}) + + _BY_PREFIX: Final[Mapping[str, Callable[[Request], Reply]]] = MappingProxyType( { "an": _anthropic, "bi": lambda request: _json(anthropic_message(marker_of(request))), "bc": _bedrock_converse, + "be": _titan_embedding, "gm": _gemini, + "vx": _gemini, + "fd": _anthropic, + "az": _openai_like(AZURE_DEPLOYMENT), "mi": _openai_like("mistral-small-latest"), "db": _openai_like("databricks-meta-llama-3-3-70b-instruct"), "co": _cohere_rerank, diff --git a/tests/integration/providers/test_api_base_url_join_host_owned_proxy.py b/tests/integration/providers/test_api_base_url_join_host_owned_proxy.py index 4a075138305..687c15283f4 100644 --- a/tests/integration/providers/test_api_base_url_join_host_owned_proxy.py +++ b/tests/integration/providers/test_api_base_url_join_host_owned_proxy.py @@ -1,6 +1,7 @@ import os from collections.abc import Callable, Iterator from pathlib import Path +from types import MappingProxyType from typing import Final import pytest @@ -8,6 +9,7 @@ import yaml from integration._support.client import JSON_OBJECT, Gateway, gateway_from_environment from integration._support.forward_proxy import ForwardProxy, refusing_forward_proxy from integration._support.process import graceful_stop_seconds, owned_proxy +from integration._support.vertex import service_account_json from integration.providers._api_base_trailing_slash_support import KEY, new_marker from pydantic import JsonValue @@ -17,6 +19,12 @@ _RERANK: Final = "host-rerank" _AZURE_CLAUDE: Final = "host-azure-claude" _AZURE_COUNT: Final = "host-azure-count" _EMPTY_BASE: Final = "host-empty-base" +_VERTEX_HOST: Final = "vertex-gateway" +_VERTEX_PROJECT: Final = "url-join-project" +_VERTEX_LOCATION: Final = "us-east5" +_VERTEX_CLAUDE_BASES: Final = MappingProxyType( + {"host-vertex-claude": f"http://{_VERTEX_HOST}", "host-vertex-claude-slash": f"http://{_VERTEX_HOST}/"} +) _SCRUBBED_PREFIXES: Final = ("ANTHROPIC_", "AZURE_AI_", "INFINITY_") _PROXY_VARIABLES: Final = frozenset({"HTTPS_PROXY", "HTTP_PROXY", "ALL_PROXY", "NO_PROXY"}) _load_yaml: Final[Callable[[str], object]] = yaml.safe_load @@ -28,7 +36,20 @@ def _deployment(name: str, model: str, api_base: str) -> dict[str, JsonValue]: return {"model_name": name, "litellm_params": {"model": model, "api_base": api_base, "api_key": KEY}} -def _config(directory: Path) -> Path: +def _vertex_claude_deployment(name: str, api_base: str, token_url: str) -> dict[str, JsonValue]: + return { + "model_name": name, + "litellm_params": { + "model": "vertex_ai/claude-haiku-5-5", + "api_base": api_base, + "vertex_project": _VERTEX_PROJECT, + "vertex_location": _VERTEX_LOCATION, + "vertex_credentials": service_account_json(_VERTEX_PROJECT, token_url), + }, + } + + +def _config(directory: Path, token_url: str) -> Path: base: Final = JSON_OBJECT.validate_python(_load_yaml(Path("tests/integration/proxy_config.yaml").read_text())) configuration: Final = { **base, @@ -38,6 +59,7 @@ def _config(directory: Path) -> Path: _deployment(_AZURE_CLAUDE, "azure_ai/claude-haiku-5-5", f"http://{_AZURE_HOST}/anthropic"), _deployment(_AZURE_COUNT, "azure_ai/claude-haiku-5-5", f"http://{_AZURE_HOST}"), _deployment(_EMPTY_BASE, "anthropic/claude-haiku-5-5", ""), + *(_vertex_claude_deployment(name, base, token_url) for name, base in _VERTEX_CLAUDE_BASES.items()), ], "router_settings": {"disable_cooldowns": True, "num_retries": 0}, } @@ -72,7 +94,7 @@ def isolated(forward: ForwardProxy, tmp_path_factory: pytest.TempPathFactory) -> "NO_PROXY": "127.0.0.1,localhost", "PROXY_BATCH_POLLING_ENABLED": "false", }, - config=_config(directory), + config=_config(directory, shared.upstream_url.rstrip("/")), remove_environment=_scrubbed(), ) as candidate: yield candidate @@ -138,3 +160,14 @@ def test_azure_claude_count_tokens_on_a_bare_host_starting_with_anthropic_adds_t def test_empty_api_base_falls_back_to_the_provider_default_host(isolated: Gateway, forward: ForwardProxy) -> None: targets: Final = _targets_after(forward, lambda: _chat(isolated, _EMPTY_BASE)) assert targets == {"api.anthropic.com:443"} + + +@pytest.mark.parametrize("model", tuple(_VERTEX_CLAUDE_BASES), ids=("no-slash", "slash")) +def test_vertex_partner_claude_on_a_bare_host_reaches_one_single_slash_url( + isolated: Gateway, forward: ForwardProxy, model: str +) -> None: + targets: Final = _targets_after(forward, lambda: _chat(isolated, model)) + assert targets == { + f"http://{_VERTEX_HOST}/v1/projects/{_VERTEX_PROJECT}/locations/{_VERTEX_LOCATION}" + "/publishers/anthropic/models/claude-haiku-5-5:rawPredict" + } diff --git a/tests/integration/providers/test_api_base_url_join_wire.py b/tests/integration/providers/test_api_base_url_join_wire.py index 42e53ff9f95..74f8dde3cf7 100644 --- a/tests/integration/providers/test_api_base_url_join_wire.py +++ b/tests/integration/providers/test_api_base_url_join_wire.py @@ -7,6 +7,7 @@ import openai import pytest from integration._support.client import Gateway, Scenario, eventually, list_value, object_value, string_value from integration._support.database import read_rows +from integration._support.vertex import service_account_json from integration._support.wire import Reply, Request, wire_server from integration.providers._api_base_trailing_slash_support import ( KEY, @@ -22,7 +23,7 @@ from integration.providers._api_base_trailing_slash_support import ( reply_text, request_json, ) -from integration.providers._api_base_url_join_support import COUNTED_TOKENS, RELEVANCE, join_peer +from integration.providers._api_base_url_join_support import AZURE_DEPLOYMENT, COUNTED_TOKENS, RELEVANCE, join_peer from pydantic import JsonValue _HAIKU: Final = "anthropic.claude-haiku-5-5" @@ -49,10 +50,20 @@ _RESPONSES_ROWS: Final = (_ANTHROPIC, _GEMINI) _EMBEDDING_ROWS: Final = ( Row("mistral/mistral-embed", "/mi/v1", "/embeddings"), Row("gemini/gemini-embedding-2", "/gm/v1beta", "/models/gemini-embedding-2:batchEmbedContents"), + Row("bedrock/amazon.titan-embed-text-v2:0", "/be", "/model/amazon.titan-embed-text-v2%3A0/invoke"), ) _TRANSCRIPTION: Final = Row("mistral/voxtral-mini-latest", "/mi/v1", "/audio/transcriptions") -_FULL_ROUTE_SUFFIXES: Final = ("", "/", "?beta=true") -_FULL_ROUTE_IDS: Final = ("exact", "slash", "query") +_FULL_ROUTE_SUFFIXES: Final = (("", ""), ("/", ""), ("?beta=true", "?beta=true"), ("/?beta=true", "?beta=true")) +_FULL_ROUTE_IDS: Final = ("exact", "slash", "query", "slash-query") +_VERTEX_PROJECT: Final = "url-join-project" +_VERTEX_LOCATION: Final = "us-east5" +_VERTEX_GEMINI_PATH: Final = ( + f"/vx/v1/projects/{_VERTEX_PROJECT}/locations/{_VERTEX_LOCATION}/publishers/google/models/gemini-3.8-flash" +) +_AZURE_V1_BASES: Final = ("/az", "/az/", "/az/openai/v1", "/az/openai/v1/") +_AZURE_V1_IDS: Final = ("host", "host-slash", "route", "route-slash") +_AZURE_ANTHROPIC_BASES: Final = ("/fd/anthropic", "/fd/anthropic/v1/messages") +_AZURE_ANTHROPIC_IDS: Final = ("anthropic-root", "messages-route") def _ids(rows: tuple[Row, ...]) -> tuple[str, ...]: @@ -65,6 +76,17 @@ def _deploy(scenario: Scenario, row: Row, api_base: str) -> str: return scenario.model(model=row.model, api_base=api_base, api_key=KEY) +def _deploy_vertex(gateway: Gateway, scenario: Scenario, model: str, api_base: str) -> str: + return scenario.model( + model=model, + api_base=api_base, + api_key=None, + vertex_project=_VERTEX_PROJECT, + vertex_location=_VERTEX_LOCATION, + vertex_credentials=service_account_json(_VERTEX_PROJECT, gateway.upstream_url.rstrip("/")), + ) + + def _openai(gateway: Gateway) -> openai.OpenAI: return openai.OpenAI(base_url=str(gateway.client.base_url) + "/v1", api_key=gateway.key, max_retries=0) @@ -80,6 +102,13 @@ def _chat(gateway: Gateway, model: str, marker: str) -> str | None: return completion.choices[0].message.content +def _streamed_chat(gateway: Gateway, model: str, marker: str) -> str: + stream: Final = _openai(gateway).chat.completions.create( + model=model, messages=[{"role": "user", "content": marker}], stream=True + ) + return "".join(chunk.choices[0].delta.content or "" for chunk in stream if chunk.choices) + + def _message_text(gateway: Gateway, model: str, marker: str) -> list[str]: message: Final = _anthropic(gateway).messages.create( model=model, max_tokens=16, messages=[{"role": "user", "content": marker}] @@ -270,22 +299,74 @@ def test_cache_hit_on_a_slash_base_skips_the_upstream_and_logs_its_own_row(gatew assert "//" not in urlsplit(string_value(hit["api_base"])).path, hit -@pytest.mark.parametrize("suffix", _FULL_ROUTE_SUFFIXES, ids=_FULL_ROUTE_IDS) -def test_chat_base_that_already_names_the_route_keeps_its_path_and_query(gateway: Gateway, suffix: str) -> None: +@pytest.mark.parametrize(("suffix", "sent"), _FULL_ROUTE_SUFFIXES, ids=_FULL_ROUTE_IDS) +def test_chat_base_that_already_names_the_route_keeps_its_path_and_query( + gateway: Gateway, suffix: str, sent: str +) -> None: marker: Final = new_marker() with wire_server(join_peer) as wire, gateway.scenario() as scenario: model: Final = _deploy(scenario, _ANTHROPIC, f"{wire.url}/an/v1/messages{suffix}") assert _chat(gateway, model, marker) == reply_text(marker) - assert_one_request(wire.drain(), f"/an/v1/messages{suffix.rstrip('/')}", marker) + assert_one_request(wire.drain(), f"/an/v1/messages{sent}", marker) -@pytest.mark.parametrize("suffix", _FULL_ROUTE_SUFFIXES, ids=_FULL_ROUTE_IDS) -def test_messages_base_that_already_names_the_route_keeps_its_path_and_query(gateway: Gateway, suffix: str) -> None: +@pytest.mark.parametrize(("suffix", "sent"), _FULL_ROUTE_SUFFIXES, ids=_FULL_ROUTE_IDS) +def test_messages_base_that_already_names_the_route_keeps_its_path_and_query( + gateway: Gateway, suffix: str, sent: str +) -> None: marker: Final = new_marker() with wire_server(join_peer) as wire, gateway.scenario() as scenario: model: Final = _deploy(scenario, _ANTHROPIC, f"{wire.url}/an/v1/messages{suffix}") assert _message_text(gateway, model, marker) == [reply_text(marker)] - assert_one_request(wire.drain(), f"/an/v1/messages{suffix.rstrip('/')}", marker) + assert_one_request(wire.drain(), f"/an/v1/messages{sent}", marker) + + +@pytest.mark.parametrize("trailing", TRAILING, ids=TRAILING_IDS) +def test_vertex_model_url_base_calls_its_method_with_no_slash_before_the_colon(gateway: Gateway, trailing: str) -> None: + marker: Final = new_marker() + with wire_server(join_peer) as wire, gateway.scenario() as scenario: + model: Final = _deploy_vertex( + gateway, scenario, "vertex_ai/gemini-3.8-flash", f"{wire.url}{_VERTEX_GEMINI_PATH}{trailing}" + ) + assert _chat(gateway, model, marker) == reply_text(marker) + assert_one_request(wire.drain(), f"{_VERTEX_GEMINI_PATH}:generateContent", marker, keyed=False) + + +@pytest.mark.parametrize("trailing", TRAILING, ids=TRAILING_IDS) +def test_vertex_model_url_base_streams_with_no_slash_before_the_colon(gateway: Gateway, trailing: str) -> None: + marker: Final = new_marker() + with wire_server(join_peer) as wire, gateway.scenario() as scenario: + model: Final = _deploy_vertex( + gateway, scenario, "vertex_ai/gemini-3.8-flash", f"{wire.url}{_VERTEX_GEMINI_PATH}{trailing}" + ) + assert _streamed_chat(gateway, model, marker) == reply_text(marker) + assert_one_request(wire.drain(), f"{_VERTEX_GEMINI_PATH}:streamGenerateContent?alt=sse", marker, keyed=False) + + +@pytest.mark.parametrize("base", _AZURE_V1_BASES, ids=_AZURE_V1_IDS) +def test_azure_v1_chat_reaches_one_openai_v1_route(gateway: Gateway, base: str) -> None: + marker: Final = new_marker() + with wire_server(join_peer) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=f"azure/{AZURE_DEPLOYMENT}", api_base=f"{wire.url}{base}", api_key=KEY, api_version="v1" + ) + assert _chat(gateway, model, marker) == reply_text(marker) + assert_one_request(wire.drain(), "/az/openai/v1/chat/completions", marker) + + +@pytest.mark.parametrize("trailing", TRAILING, ids=TRAILING_IDS) +@pytest.mark.parametrize("base", _AZURE_ANTHROPIC_BASES, ids=_AZURE_ANTHROPIC_IDS) +def test_azure_ai_count_tokens_reaches_one_count_tokens_route(gateway: Gateway, base: str, trailing: str) -> None: + marker: Final = new_marker() + with wire_server(join_peer) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model="azure_ai/claude-haiku-5-5", api_base=f"{wire.url}{base}{trailing}", api_key=KEY + ) + response: Final = gateway.request( + "POST", "/v1/messages/count_tokens", {"model": model, "messages": [{"role": "user", "content": marker}]} + ) + assert body_of(response)["input_tokens"] == COUNTED_TOKENS + assert_one_request(wire.drain(), "/fd/anthropic/v1/messages/count_tokens", marker) @pytest.mark.parametrize("suffix", ("", "/", "?tier=a"), ids=("exact", "slash", "query")) diff --git a/tests/integration/providers/test_count_tokens_error_shapes_wire.py b/tests/integration/providers/test_count_tokens_error_shapes_wire.py index a6c3a6dc62f..59114e4aced 100644 --- a/tests/integration/providers/test_count_tokens_error_shapes_wire.py +++ b/tests/integration/providers/test_count_tokens_error_shapes_wire.py @@ -1,5 +1,6 @@ import json -from collections.abc import Iterator +from collections.abc import Callable, Iterator +from types import MappingProxyType from typing import Final import anthropic @@ -8,6 +9,7 @@ import pytest from integration._support.client import Gateway, gateway_from_environment from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process from integration._support.wire import Reply, Request, wire_server +from litellm.types.llms.anthropic import ANTHROPIC_OAUTH_BETA_HEADER from pydantic import JsonValue, TypeAdapter pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120) @@ -26,6 +28,14 @@ _PROVIDER_KEY_VARIABLES: Final = ( "AZURE_TENANT_ID", "OPENAI_API_KEY", ) +_ENVIRONMENT_KEYS: Final = MappingProxyType( + { + "ANTHROPIC_API_KEY": "synthetic-anthropic-env-key", + "AZURE_API_KEY": "synthetic-azure-env-key", + "OPENAI_API_KEY": "synthetic-openai-env-key", + } +) +_SUBSCRIPTION_TOKEN: Final = "sk-ant-oat01-synthetic-subscription-token" @pytest.fixture(scope="module") @@ -44,14 +54,14 @@ def keyless_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[OwnedPro @pytest.fixture(scope="module") -def azure_environment_key_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[OwnedProxy]: +def environment_key_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[OwnedProxy]: with ( gateway_from_environment() as gateway, owned_proxy_process( gateway, - tmp_path_factory.mktemp("count-tokens-azure-environment-key"), - {"AZURE_API_KEY": "synthetic-azure-env-key"}, - remove_environment=tuple(name for name in _PROVIDER_KEY_VARIABLES if name != "AZURE_API_KEY"), + tmp_path_factory.mktemp("count-tokens-environment-keys"), + _ENVIRONMENT_KEYS, + remove_environment=tuple(name for name in _PROVIDER_KEY_VARIABLES if name not in _ENVIRONMENT_KEYS), workers=1, ) as owned, ): @@ -147,9 +157,9 @@ def test_anthropic_count_without_a_provider_credential_is_an_anthropic_401( def test_azure_ai_anthropic_count_sends_the_azure_environment_key_as_x_api_key( - azure_environment_key_proxy: OwnedProxy, + environment_key_proxy: OwnedProxy, ) -> None: - gateway: Final = azure_environment_key_proxy.gateway + gateway: Final = environment_key_proxy.gateway def count(request: Request) -> Reply: assert request.headers.get("x-api-key") == "synthetic-azure-env-key", request.headers @@ -257,3 +267,122 @@ def test_google_count_tokens_errors_use_the_google_envelope(keyless_proxy: Owned assert _JSON_OBJECT.validate_json(malformed.content) == { "error": {"code": 400, "message": "Invalid Google countTokens request body", "status": "INVALID_ARGUMENT"} } + + +def _counted(tokens: int) -> Reply: + return Reply(body=json.dumps({"object": "response.input_tokens", "input_tokens": tokens}).encode()) + + +def _anthropic_count(model: str) -> tuple[str, dict[str, JsonValue]]: + return "/v1/messages/count_tokens", {"model": model, "messages": _MESSAGES} + + +def _responses_count(model: str) -> tuple[str, dict[str, JsonValue]]: + return "/v1/responses/input_tokens", {"model": model, "input": "Count this synthetic request."} + + +@pytest.mark.parametrize( + ("provider_model", "base_path", "count_request", "header", "credential"), + ( + ("anthropic/claude-sonnet-5-5", "", _anthropic_count, "x-api-key", "synthetic-anthropic-env-key"), + ("openai/gpt-4o-mini", "/v1", _responses_count, "authorization", "Bearer synthetic-openai-env-key"), + ), + ids=("anthropic", "openai"), +) +def test_count_uses_a_provider_key_set_only_in_the_proxy_environment( + environment_key_proxy: OwnedProxy, + provider_model: str, + base_path: str, + count_request: Callable[[str], tuple[str, dict[str, JsonValue]]], + header: str, + credential: str, +) -> None: + gateway: Final = environment_key_proxy.gateway + with wire_server(lambda _request: _counted(47)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model=provider_model, api_base=f"{wire.url}{base_path}", api_key=None, num_retries=0 + ) + path, body = count_request(model) + counted: Final = gateway.request("POST", path, body) + assert counted.status_code == 200, counted.text + assert _JSON_OBJECT.validate_json(counted.content)["input_tokens"] == 47, counted.text + received: Final = wire.drain() + assert [request.headers.get(header) for request in received] == [credential], received + + +def _google_count(model: str) -> tuple[str, dict[str, JsonValue]]: + return f"/v1beta/models/{model}:countTokens", { + "contents": [{"role": "user", "parts": [{"text": "Count this synthetic request."}]}] + } + + +def _utils_count(model: str) -> tuple[str, dict[str, JsonValue]]: + return "/utils/token_counter?call_endpoint=true", {"model": model, "messages": _MESSAGES} + + +@pytest.mark.parametrize( + ("count_request", "google_status"), + ((_google_count, "UNAUTHENTICATED"), (_utils_count, None)), + ids=("google", "utils"), +) +def test_count_entrypoints_report_a_missing_provider_key_in_their_own_shape( + keyless_proxy: OwnedProxy, + count_request: Callable[[str], tuple[str, dict[str, JsonValue]]], + google_status: str | None, +) -> None: + gateway: Final = keyless_proxy.gateway + with wire_server(_never_called) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model="anthropic/claude-sonnet-5-5", api_base=wire.url, api_key=None, num_retries=0 + ) + path, body = count_request(model) + rejected: Final = gateway.request("POST", path, body) + assert rejected.status_code == 401, rejected.text + error: Final = _JSON_OBJECT.validate_json(rejected.content)["error"] + assert isinstance(error, dict), rejected.text + assert f"No Anthropic API key is set for model {model}" in str(error["message"]), rejected.text + if google_status is not None: + assert (error["code"], error["status"]) == (401, google_status), rejected.text + assert wire.drain() == () + + +def test_anthropic_count_uses_the_callers_subscription_token_when_the_deployment_has_no_key( + keyless_proxy: OwnedProxy, +) -> None: + gateway: Final = keyless_proxy.gateway + with wire_server(lambda _request: _counted(41)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model="anthropic/claude-sonnet-5-5", api_base=wire.url, api_key=None, num_retries=0 + ) + counted: Final = gateway.request( + "POST", + "/v1/messages/count_tokens", + {"model": model, "messages": _MESSAGES}, + headers={"Authorization": f"Bearer {_SUBSCRIPTION_TOKEN}", "x-litellm-api-key": f"Bearer {gateway.key}"}, + ) + assert counted.status_code == 200, counted.text + assert _JSON_OBJECT.validate_json(counted.content)["input_tokens"] == 41, counted.text + (received,) = wire.drain() + assert received.headers.get("authorization") == f"Bearer {_SUBSCRIPTION_TOKEN}", received.headers + assert "x-api-key" not in received.headers, received.headers + assert ANTHROPIC_OAUTH_BETA_HEADER in received.headers.get("anthropic-beta", ""), received.headers + + +def test_anthropic_count_keeps_the_deployments_key_over_the_callers_subscription_token( + keyless_proxy: OwnedProxy, +) -> None: + gateway: Final = keyless_proxy.gateway + with wire_server(lambda _request: _counted(43)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model="anthropic/claude-sonnet-5-5", api_base=wire.url, api_key="synthetic-deployment-key", num_retries=0 + ) + counted: Final = gateway.request( + "POST", + "/v1/messages/count_tokens", + {"model": model, "messages": _MESSAGES}, + headers={"Authorization": f"Bearer {_SUBSCRIPTION_TOKEN}", "x-litellm-api-key": f"Bearer {gateway.key}"}, + ) + assert counted.status_code == 200, counted.text + (received,) = wire.drain() + assert received.headers.get("x-api-key") == "synthetic-deployment-key", received.headers + assert _SUBSCRIPTION_TOKEN not in json.dumps(dict(received.headers)), received.headers diff --git a/tests/integration/security/test_files_batches_access.py b/tests/integration/security/test_files_batches_access.py index 7b6fd4652cf..119055c64c7 100644 --- a/tests/integration/security/test_files_batches_access.py +++ b/tests/integration/security/test_files_batches_access.py @@ -48,6 +48,7 @@ VERTEX_ACCESS_TOKEN: Final = "ya29.scripted-delete-token" AWS_ACCESS_KEY_ID: Final = "AKIAIOSFODNN7EXAMPLE" AWS_SECRET_ACCESS_KEY: Final = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" AWS_REGION: Final = "us-east-1" +S3_OBJECT_BODY: Final = b'{"recordId":"r1","modelOutput":{"stop_reason":"end_turn"}}\n' _ForbiddenSurface = Literal[ "upload_sdk_extra_body", "upload_form", @@ -708,6 +709,86 @@ def test_non_admin_cannot_delete_a_raw_gcs_uri_before_vertex_auth_or_storage(gat assert token_requests == (), "non-admin raw GCS delete reached the Vertex token endpoint" +def test_non_admin_still_reads_an_ordinary_provider_file_id(gateway: Gateway) -> None: + with wire_server(_openai_backend()) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model="openai/gpt-4o-mini", api_base=wire.url + "/v1", api_key="provider-key-" + uuid.uuid4().hex[:8] + ) + _wait_until_every_worker_serves(gateway, model) + member: Final = _member(scenario, model) + file_id: Final = "file-" + uuid.uuid4().hex + + read: Final = gateway.request("GET", f"/v1/files/{file_id}/content", key=member.key, params={"model": model}) + assert read.status_code == 200, read.text + assert read.content == b"file-content\n", read.content + assert [ + (request.method, urlsplit(request.target).path) for request in _drained_other_than_model_list_probes(wire) + ] == [("GET", f"/v1/files/{file_id}/content")] + + +@pytest.mark.parametrize( + ("method", "route_suffix", "verb"), + [("GET", "/content", "retrieved"), ("DELETE", "", "deleted")], + ids=["content", "delete"], +) +def test_non_admin_cannot_reach_vertex_with_a_twice_encoded_raw_gcs_uri( + gateway: Gateway, method: Literal["GET", "DELETE"], route_suffix: str, verb: str +) -> None: + with ( + wire_server(_vertex_token_backend) as token_wire, + gateway.scenario() as scenario, + ): + model: Final = scenario.model( + model=VERTEX_MODEL, + api_key=None, + vertex_project=VERTEX_PROJECT, + vertex_location=VERTEX_LOCATION, + vertex_credentials=_vertex_credentials(token_wire.url), + gcs_bucket_name=BUCKET, + ) + _wait_until_every_worker_serves(gateway, model) + caller_user: Final = scenario.user(user_role="internal_user") + caller_key: Final = scenario.key(user_id=caller_user, models=[model]) + object_uri: Final = f"gs://{BUCKET}/litellm-vertex-files/litellm-vertex-files-obj.jsonl" + twice_encoded: Final = quote(quote(object_uri, safe=""), safe="") + token_wire.drain() + + denied: Final = gateway.request( + method, f"/v1/files/{twice_encoded}{route_suffix}", key=caller_key, params={"model": model} + ) + token_requests: Final = token_wire.drain() + assert denied.status_code == 403, f"{denied.text}; token requests={token_requests}" + assert _error_message(denied, 403) == ( + f"Raw cloud storage file ids can only be {verb} by a proxy admin key. " + "Use the LiteLLM managed file id returned when the file was created." + ), denied.text + assert token_requests == (), f"non-admin twice-encoded raw GCS {method} reached the Vertex token endpoint" + + +def _assert_signed_empty_body_s3_request(request: Request, method: Literal["GET", "DELETE"], wire_url: str) -> None: + assert request.method == method, request.method + request_date: Final = request.headers["x-amz-date"] + content_hash: Final = hashlib.sha256(b"").hexdigest() + assert request.headers["x-amz-content-sha256"] == content_hash, request.headers + aws_request: Final = AWSRequest( + method=method, + url=wire_url + request.target, + headers={ + "host": request.headers["host"], + "x-amz-date": request_date, + "x-amz-content-sha256": content_hash, + }, + ) + aws_request.context["timestamp"] = request_date + auth: Final = S3SigV4Auth(Credentials(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY), "s3", AWS_REGION) + signature: Final = auth.signature(auth.string_to_sign(aws_request, auth.canonical_request(aws_request)), aws_request) + expected_authorization: Final = ( + f"AWS4-HMAC-SHA256 Credential={AWS_ACCESS_KEY_ID}/{request_date[:8]}/{AWS_REGION}/s3/aws4_request, " + f"SignedHeaders={auth.signed_headers(auth.headers_to_sign(aws_request))}, Signature={signature}" + ) + assert request.headers["authorization"] == expected_authorization, request.headers["authorization"] + + def test_proxy_admin_deletes_a_raw_s3_uri_with_one_signed_delete(gateway: Gateway) -> None: with wire_server(_s3_backend()) as wire, gateway.scenario() as scenario: model: Final = _bedrock_model(scenario, wire) @@ -721,29 +802,30 @@ def test_proxy_admin_deletes_a_raw_s3_uri_with_one_signed_delete(gateway: Gatewa assert parsed.deleted is True and parsed.id == object_uri, deleted.text (request,) = _drained_other_than_model_list_probes(wire) assert request.target == f"/{BUCKET}/litellm-bedrock-files-obj.jsonl", request.target - request_date: Final = request.headers["x-amz-date"] - content_hash: Final = hashlib.sha256(b"").hexdigest() - assert request.headers["x-amz-content-sha256"] == content_hash, request.headers - aws_request: Final = AWSRequest( - method="DELETE", - url=wire.url + request.target, - headers={ - "host": request.headers["host"], - "x-amz-date": request_date, - "x-amz-content-sha256": content_hash, - }, + _assert_signed_empty_body_s3_request(request, "DELETE", wire.url) + + +def _s3_object_backend(request: Request) -> Reply: + if request.method == "GET": + return Reply(body=S3_OBJECT_BODY, content_type="application/octet-stream") + return _json_reply({"error": {"message": f"unscripted {request.method} {request.target}"}}, 404) + + +def test_proxy_admin_reads_a_raw_s3_uri_with_one_signed_get(gateway: Gateway) -> None: + with wire_server(_s3_object_backend) as wire, gateway.scenario() as scenario: + model: Final = _bedrock_model(scenario, wire) + _wait_until_every_worker_serves(gateway, model) + object_key: Final = f"litellm-batch-outputs/{uuid.uuid4().hex}.jsonl.out" + object_uri: Final = f"s3://{BUCKET}/{object_key}" + + read: Final = gateway.request( + "GET", "/v1/files/" + quote(object_uri, safe="") + "/content", params={"model": model} ) - aws_request.context["timestamp"] = request_date - auth: Final = S3SigV4Auth(Credentials(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY), "s3", AWS_REGION) - signature: Final = auth.signature( - auth.string_to_sign(aws_request, auth.canonical_request(aws_request)), aws_request - ) - expected_authorization: Final = ( - f"AWS4-HMAC-SHA256 Credential={AWS_ACCESS_KEY_ID}/{request_date[:8]}/{AWS_REGION}/s3/aws4_request, " - f"SignedHeaders={auth.signed_headers(auth.headers_to_sign(aws_request))}, Signature={signature}" - ) - assert request.headers["authorization"] == expected_authorization, request.headers["authorization"] - assert request.method == "DELETE", request.method + storage_requests: Final = _drained_other_than_model_list_probes(wire) + assert read.status_code == 200, f"{read.text}; storage requests={storage_requests}" + assert read.content == S3_OBJECT_BODY, read.content + assert [request.target for request in storage_requests] == [f"/{BUCKET}/{object_key}"], storage_requests + _assert_signed_empty_body_s3_request(storage_requests[0], "GET", wire.url) def _granting_vertex_token_backend(request: Request) -> Reply: diff --git a/tests/integration/spend/test_background_response_billed_once.py b/tests/integration/spend/test_background_response_billed_once.py index 1b7fc975217..b207e46204a 100644 --- a/tests/integration/spend/test_background_response_billed_once.py +++ b/tests/integration/spend/test_background_response_billed_once.py @@ -642,6 +642,15 @@ def _key_budget(scenario: Scenario, budget_owner: str, model_budget: JsonValue) return {} +def _assert_the_next_create_exceeds_the_model_budget(scenario: Scenario, key: str, model: str) -> None: + refused: Final = eventually( + lambda: _next_create(scenario, key, model), + lambda response: response.status_code != 200, + seconds=SPEND_LANDING_SECONDS, + ) + assert f"exceeded budget for model={model}" in refused.text, refused.text + + @pytest.mark.timeout(240) @pytest.mark.parametrize("budget_owner", ["key", "team", "user", "budget_tier"]) def test_a_billed_background_response_counts_toward_its_creators_project_and_model_budget( @@ -663,12 +672,24 @@ def test_a_billed_background_response_counts_toward_its_creators_project_and_mod lambda spends: spends[0] > 0 and costs(spends[0])(spends[1]), seconds=SPEND_LANDING_SECONDS, ) - refused: Final = eventually( - lambda: _next_create(scenario, key, upstream.model), - lambda response: response.status_code != 200, - seconds=SPEND_LANDING_SECONDS, + _assert_the_next_create_exceeds_the_model_budget(scenario, key, upstream.model) + + +@pytest.mark.timeout(240) +def test_a_billed_background_response_on_a_team_model_counts_toward_the_budget_of_the_model_name_it_was_created_with( + gateway: Gateway, +) -> None: + with gateway.scenario() as scenario: + team_id: Final = scenario.team() + upstream: Final = scripted_upstream(scenario, background_routes("completed"), model_info={"team_id": team_id}) + key: Final = scenario.key( + team_id=team_id, model_max_budget={upstream.model: {"max_budget": PRICE / 2, "budget_duration": "1d"}} ) - assert f"exceeded budget for model={upstream.model}" in refused.text, refused.text + response_id: Final = create_background_response(scenario, key, upstream.model) + read: Final = read_response(gateway, key, response_id) + assert (read,) in (settling_reads("completed", 1), free_reads("completed", 1)) + eventually(lambda: key_spend(key), lambda spend: spend > 0, seconds=SPEND_LANDING_SECONDS) + _assert_the_next_create_exceeds_the_model_budget(scenario, key, upstream.model) @pytest.mark.timeout(240) diff --git a/tests/integration/spend/test_failed_dispatch_tokens.py b/tests/integration/spend/test_failed_dispatch_tokens.py index b92606ebc88..c9875be562c 100644 --- a/tests/integration/spend/test_failed_dispatch_tokens.py +++ b/tests/integration/spend/test_failed_dispatch_tokens.py @@ -167,3 +167,51 @@ def test_provider_500_failure_rows_count_deepseek_v4_prompt_tokens_with_deepseek short: Final = _failure_row_prompt_tokens(gateway, model, key, SHORT_TEXT) assert wire.drain() assert_counted_with_deepseek(chinese - short, chinese_minus_short) + + +COMPACTION_INSTRUCTIONS: Final[JsonValue] = {"role": "system", "content": "answer briefly"} +COMPACTED_HISTORY: Final[tuple[JsonValue, ...]] = ( + {"role": "user", "content": "a long turn " * 200}, + {"role": "assistant", "content": "a long reply " * 200}, +) +COMPACTION_ITEM: Final[JsonValue] = { + "type": "compaction", + "encrypted_content": json.dumps({"type": "compaction", "content": "the cat is named Biscuit"}), +} +TURN_AFTER_COMPACTION: Final[JsonValue] = {"role": "user", "content": "what is the cat's name?"} + + +def _failed_responses_prompt_tokens(gateway: Gateway, model: str, key: str, items: list[JsonValue]) -> int: + failed: Final = gateway.request("POST", "/v1/responses", {"model": model, "input": items}, key=key) + assert failed.status_code == 500 and "synthetic provider outage" in failed.text, failed.text + rows: Final = eventually( + lambda: read_rows( + 'SELECT status, prompt_tokens FROM "LiteLLM_SpendLogs" WHERE request_id=%s', + (failed.headers["x-litellm-call-id"],), + ), + lambda values: len(values) == 1, + seconds=70, + ) + assert rows[0]["status"] == "failure", rows + prompt_tokens: Final = rows[0]["prompt_tokens"] + assert isinstance(prompt_tokens, int) and prompt_tokens > 0, rows + return prompt_tokens + + +def test_a_failed_responses_request_counts_its_input_from_the_newest_compaction_item(gateway: Gateway) -> None: + with wire_server(_provider_500) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(api_base=wire.url + "/v1", num_retries=0) + key: Final = scenario.key(models=[model]) + resent: Final = _failed_responses_prompt_tokens( + gateway, model, key, [COMPACTION_INSTRUCTIONS, *COMPACTED_HISTORY, COMPACTION_ITEM, TURN_AFTER_COMPACTION] + ) + compacted: Final = _failed_responses_prompt_tokens( + gateway, model, key, [COMPACTION_INSTRUCTIONS, COMPACTION_ITEM, TURN_AFTER_COMPACTION] + ) + uncompacted: Final = _failed_responses_prompt_tokens( + gateway, model, key, [COMPACTION_INSTRUCTIONS, *COMPACTED_HISTORY, TURN_AFTER_COMPACTION] + ) + assert len(wire.drain()) == 3 + + assert resent == compacted + assert resent < uncompacted diff --git a/tests/integration/spend/test_streamed_usage_estimate.py b/tests/integration/spend/test_streamed_usage_estimate.py new file mode 100644 index 00000000000..c7bcdf4401a --- /dev/null +++ b/tests/integration/spend/test_streamed_usage_estimate.py @@ -0,0 +1,73 @@ +import json +import uuid +from hashlib import sha256 +from typing import Final + +from integration._support.client import Gateway, eventually +from integration._support.database import read_rows +from integration._support.deepseek_tokens import ( + CHINESE_TEXT, + SHORT_TEXT, + assert_counted_with_deepseek, + chinese_minus_short, +) +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue + +_DEEPSEEK_V4: Final = "DeepSeek-V4-Flash-0731" + + +def _data_frame(frame: dict[str, JsonValue]) -> bytes: + return b"data: " + json.dumps(frame).encode() + b"\n\n" + + +def _stream_without_usage(request: Request) -> Reply: + assert request.method == "POST" + chunk: Final[dict[str, JsonValue]] = { + "id": f"chatcmpl-{uuid.uuid4().hex}", + "object": "chat.completion.chunk", + "created": 1, + "model": _DEEPSEEK_V4, + } + return Reply( + content_type="text/event-stream", + chunks=( + _data_frame({**chunk, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "OK"}}]}), + _data_frame({**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}), + b"data: [DONE]\n\n", + ), + ) + + +def _streamed_prompt_tokens(gateway: Gateway, key: str, model: str, text: str) -> int: + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": text}], "stream": True}, + key=key, + ) + assert response.status_code == 200, response.text + assert response.text.rstrip().endswith("data: [DONE]"), response.text + rows: Final = eventually( + lambda: read_rows( + 'SELECT status, prompt_tokens FROM "LiteLLM_SpendLogs" WHERE api_key=%s', + (sha256(key.encode()).hexdigest(),), + ), + lambda values: len(values) == 1, + seconds=70, + ) + assert rows[0]["status"] == "success", rows + prompt_tokens: Final = rows[0]["prompt_tokens"] + assert isinstance(prompt_tokens, int), rows + return prompt_tokens + + +def test_a_deepseek_v4_stream_without_usage_bills_prompt_tokens_counted_with_deepseeks_tokenizer( + gateway: Gateway, +) -> None: + with wire_server(_stream_without_usage) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"azure_ai/{_DEEPSEEK_V4}", api_base=wire.url, num_retries=0) + chinese: Final = _streamed_prompt_tokens(gateway, scenario.key(models=[model]), model, CHINESE_TEXT) + short: Final = _streamed_prompt_tokens(gateway, scenario.key(models=[model]), model, SHORT_TEXT) + assert len(wire.drain()) == 2 + assert_counted_with_deepseek(chinese - short, chinese_minus_short) diff --git a/tests/unit/llms/anthropic/test_count_tokens_oauth.py b/tests/unit/llms/anthropic/test_count_tokens_oauth.py index 16ec513916c..b8a7c001b37 100644 --- a/tests/unit/llms/anthropic/test_count_tokens_oauth.py +++ b/tests/unit/llms/anthropic/test_count_tokens_oauth.py @@ -221,6 +221,45 @@ class TestCountTokensUsesWorkloadIdentity: assert result.total_tokens == 13 assert route.calls.last.request.headers["x-api-key"] == FAKE_REGULAR_KEY + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("deployment_key", "expected_credentials"), + [ + (None, {"authorization": f"Bearer {FAKE_OAUTH_TOKEN}", "x-api-key": None}), + (FAKE_REGULAR_KEY, {"authorization": None, "x-api-key": FAKE_REGULAR_KEY}), + ], + ids=["keyless-deployment-uses-the-callers-token", "deployment-key-wins-over-the-callers-token"], + ) + async def test_a_callers_subscription_token_counts_only_when_no_key_is_configured( + self, + monkeypatch: pytest.MonkeyPatch, + httpx_transport_clients: None, + deployment_key: str | None, + expected_credentials: dict[str, str | None], + ) -> None: + for name in ("ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_API_BASE", "ANTHROPIC_BASE_URL"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setattr(litellm, "anthropic_key", None) + monkeypatch.setattr(litellm, "api_key", None) + + with respx.mock(assert_all_called=True) as router: + route = router.post("https://api.anthropic.com/v1/messages/count_tokens").mock( + return_value=httpx.Response(200, json={"input_tokens": 17}) + ) + result = await AnthropicTokenCounter(client_oauth_token=FAKE_OAUTH_TOKEN).count_tokens( + model_to_use="claude-sonnet-4-5", + messages=[{"role": "user", "content": "hi"}], + contents=None, + deployment={"litellm_params": {"model": "anthropic/claude-sonnet-4-5", "api_key": deployment_key}}, + request_model="claude-sonnet-4-5", + ) + + assert result is not None + assert result.total_tokens == 17 + sent = route.calls.last.request.headers + assert {name: sent.get(name) for name in expected_credentials} == expected_credentials + assert (ANTHROPIC_OAUTH_BETA_HEADER in sent["anthropic-beta"].split(",")) is (deployment_key is None) + @pytest.mark.asyncio async def test_a_failed_mint_degrades_like_an_anthropic_error(self, monkeypatch): from litellm.llms.anthropic.count_tokens import token_counter as token_counter_module diff --git a/tests/unit/llms/bedrock/chat/test_native_output_fallback.py b/tests/unit/llms/bedrock/chat/test_native_output_fallback.py index 301fdaeb245..9ee8c14f5d6 100644 --- a/tests/unit/llms/bedrock/chat/test_native_output_fallback.py +++ b/tests/unit/llms/bedrock/chat/test_native_output_fallback.py @@ -121,3 +121,105 @@ def test_a_rejection_logs_the_retry_and_advises_opting_out_only_when_native_outp assert _MODEL in caplog.text assert ("supports_bedrock_native_structured_output: false" in caplog.text) is advises_opting_out + + +_STRICT_FALLBACK: Final = NativeOutputFallback(optional_params={"tools": "without_strict"}, litellm_params={}) +# Bedrock's 400s for strict tools, seen live on us.anthropic.claude-sonnet-4-6 from 2026-10-20 to 2026-10-28 and quoted +# in issue #34388, each answered 200 once the request dropped strict +_STRICT_TOOLS_REJECTIONS: Final = tuple( + BedrockError(status_code=400, message=f'{{"message":"The model returned the following errors: {text}"}}') + for text in ( + "Too many strict tools (21). The maximum number of strict tools supported is 20.", + "Schemas contains too many optional parameters (27), which would make grammar compilation inefficient.", + "Schemas contains too many parameters with union types (18 parameters with type arrays or anyOf). " + "This causes exponential compilation cost.", + "The compiled grammar is too large, which would cause performance issues.", + "tools.0.custom: Invalid schema: Schema type is missing for schema: {'description': 'tag'}", + "tools.3.custom: For 'integer' type, property 'minimum' is not supported", + "Compiled grammar size (410.4MB) exceeds maximum allowed size (300MB)", + "tools.1.custom.strict: Extra inputs are not permitted", + "Grammar compilation timed out.", + ) +) +_TOO_MANY_STRICT_TOOLS: Final = _STRICT_TOOLS_REJECTIONS[0] +# Bedrock's 400s for requests with a strict tool that fail for other reasons, seen live on +# us.anthropic.claude-sonnet-4-6 on 2026-10-21, which echo tool names and tool call ids +_OTHER_400S_WITH_A_STRICT_TOOL: Final = tuple( + BedrockError(status_code=400, message=f'{{"message":"{text}"}}') + for text in ( + "The tool check_grammar is already defined at toolConfig.tools.1.", + "Expected toolResult blocks at messages.0.content for the following Ids: call_grammar_9", + "tool_use ids were found without tool_result blocks immediately after: call_grammar_1", + ) +) + + +def _call_with_strict_tools_fallback( + bedrock: ScriptedBedrock, fallback: NativeOutputFallback | None, strict_tools_fallback: NativeOutputFallback | None +) -> object: + return _resolve( + call_with_native_output_fallback( + bedrock, + None if fallback is None else lambda: fallback, + _MODEL, + strict_tools_fallback=None if strict_tools_fallback is None else lambda: strict_tools_fallback, + ) + ) + + +@_MODES +@pytest.mark.parametrize("rejection", _STRICT_TOOLS_REJECTIONS, ids=lambda error: error.message[60:100]) +def test_a_strict_tools_rejection_is_sent_again_once_with_the_strict_tools_fallback( + asynchronous: bool, rejection: BedrockError +) -> None: + bedrock: Final = ScriptedBedrock(errors=(rejection, None), asynchronous=asynchronous) + + response: Final = _call_with_strict_tools_fallback(bedrock, None, _STRICT_FALLBACK) + + assert isinstance(response, ModelResponse) and response.id == "answer-2" + assert bedrock.sent == (None, _STRICT_FALLBACK) + + +@_MODES +@pytest.mark.parametrize( + ("error", "strict_tools_fallback"), + [ + (_OTHER_400, _STRICT_FALLBACK), + *((error, _STRICT_FALLBACK) for error in _OTHER_400S_WITH_A_STRICT_TOOL), + (BedrockError(status_code=500, message=_TOO_MANY_STRICT_TOOLS.message), _STRICT_FALLBACK), + (_TOO_MANY_STRICT_TOOLS, None), + ], + ids=[ + "other_400", + "duplicate_tool_name", + "missing_tool_result", + "orphan_tool_use", + "server_error", + "no_strict_tools_fallback", + ], +) +def test_no_second_request_without_both_a_strict_tools_rejection_and_a_strict_tools_fallback( + asynchronous: bool, error: BedrockError, strict_tools_fallback: NativeOutputFallback | None +) -> None: + bedrock: Final = ScriptedBedrock(errors=(error,), asynchronous=asynchronous) + + with pytest.raises(BedrockError) as raised: + _call_with_strict_tools_fallback(bedrock, None, strict_tools_fallback) + + assert raised.value is error + assert bedrock.sent == (None,) + + +@pytest.mark.parametrize( + ("error", "retried_with"), + [(_REJECTION, _FALLBACK), (_TOO_MANY_STRICT_TOOLS, _STRICT_FALLBACK)], + ids=["native_output_rejection", "strict_tools_rejection"], +) +def test_each_rejection_is_sent_again_with_its_own_fallback_when_both_are_set( + error: BedrockError, retried_with: NativeOutputFallback +) -> None: + bedrock: Final = ScriptedBedrock(errors=(error, None), asynchronous=False) + + _call_with_strict_tools_fallback(bedrock, _FALLBACK, _STRICT_FALLBACK) + + assert bedrock.sent == (None, retried_with) diff --git a/tests/unit/llms/bedrock/test_bedrock_common_utils.py b/tests/unit/llms/bedrock/test_bedrock_common_utils.py index 7bde1805e18..dd89d9c46b6 100644 --- a/tests/unit/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/unit/llms/bedrock/test_bedrock_common_utils.py @@ -6,7 +6,9 @@ from typing import Final from litellm.llms.bedrock.common_utils import( AmazonBedrockGlobalConfig, BedrockModelInfo, + NativeOutputFallback, bedrock_native_output_is_opt_in, + bedrock_strict_tools_fallback, deployment_model_info, extract_model_name_from_bedrock_arn, get_bedrock_base_model, @@ -2150,3 +2152,80 @@ def test_bedrock_response_stream_shape_is_unavailable_when_botocore_describes_th assert shape is None (logged,) = warning.call_args_list assert "only-in-the-description" not in logged.args[0] % logged.args[1:] + + +def test_bedrock_strict_tools_fallback_drops_only_strict_and_keeps_everything_else(): + strict_tool: Final = { + "type": "function", + "cache_control": {"type": "ephemeral"}, + "function": {"name": "lookup", "description": "Look up", "strict": True, "parameters": {"type": "object"}}, + } + plain_tool: Final = {"type": "function", "function": {"name": "plain", "parameters": {"type": "object"}}} + anthropic_tool: Final = {"name": "native", "input_schema": {"type": "object"}} + bedrock_tool: Final = {"toolSpec": {"name": "spec", "strict": True, "inputSchema": {"json": {"type": "object"}}}} + litellm_params: Final = {"drop_params": True} + + fallback: Final = bedrock_strict_tools_fallback( + "us.anthropic.claude-sonnet-4-6", + {"tools": [strict_tool, plain_tool, anthropic_tool, bedrock_tool], "maxTokens": 16, "stream": True}, + litellm_params, + ) + + assert fallback == NativeOutputFallback( + optional_params={ + "tools": [ + {**strict_tool, "function": {"name": "lookup", "description": "Look up", "parameters": {"type": "object"}}}, + plain_tool, + anthropic_tool, + {"toolSpec": {"name": "spec", "inputSchema": {"json": {"type": "object"}}}}, + ], + "maxTokens": 16, + "stream": True, + }, + litellm_params=litellm_params, + ) + assert strict_tool["function"]["strict"] is True + assert bedrock_tool["toolSpec"]["strict"] is True + + +@pytest.mark.parametrize( + "optional_params", + [ + {"maxTokens": 16}, + {"tools": [{"type": "function", "function": {"name": "plain", "parameters": {}}}]}, + {"tools": [{"toolSpec": {"name": "spec", "inputSchema": {"json": {}}}}]}, + ], + ids=["no_tools", "no_strict_tool", "no_strict_tool_spec"], +) +def test_bedrock_strict_tools_fallback_is_none_without_a_strict_tool(optional_params: dict[str, object]) -> None: + assert bedrock_strict_tools_fallback("us.anthropic.claude-sonnet-4-6", optional_params, {}) is None + + +_CLAUDE_PROFILE_ARN: Final = "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/claude" + + +@pytest.mark.parametrize( + ("model", "litellm_params"), + [ + ("us.anthropic.claude-sonnet-4-6", {}), + ("us.amazon.nova-pro-v1:0", {}), + (_CLAUDE_PROFILE_ARN, {"base_model": "bedrock/us.anthropic.claude-sonnet-4-6"}), + (_CLAUDE_PROFILE_ARN, {}), + ], + ids=["claude", "nova", "claude_profile_with_base_model", "profile_without_base_model"], +) +@pytest.mark.parametrize("strict", [True, False]) +def test_bedrock_strict_tools_fallback_retries_only_when_the_request_sent_strict( + model: str, litellm_params: dict[str, object], strict: bool +) -> None: + from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig + + tools: Final = [{"type": "function", "function": {"name": "lookup", "strict": strict, "parameters": {}}}] + request: Final = AmazonConverseConfig().transform_request( + model, [{"role": "user", "content": "hi"}], {"tools": tools}, {**litellm_params}, {} + ) + sent_strict: Final = any("strict" in tool["toolSpec"] for tool in request["toolConfig"]["tools"]) + + fallback: Final = bedrock_strict_tools_fallback(model, {"tools": tools}, litellm_params) + + assert (fallback is not None) == sent_strict diff --git a/tests/unit/llms/openai_like/test_aiand_provider.py b/tests/unit/llms/openai_like/test_aiand_provider.py new file mode 100644 index 00000000000..804f3a3b6eb --- /dev/null +++ b/tests/unit/llms/openai_like/test_aiand_provider.py @@ -0,0 +1,145 @@ +import json +from typing import Final + +import httpx +import pytest +import respx + +import litellm +from litellm.caching.llm_caching_handler import LLMClientCache +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.llms.openai_like.json_loader import JSONProviderRegistry +from litellm.proxy.auth.model_checks import get_known_models_from_wildcard +from litellm.utils import ProviderConfigManager + +AIAND_API_BASE = "https://api.aiand.com/v1" + + +def test_aiand_chat_provider(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("AIAND_API_KEY", "test-key") + + provider_config = JSONProviderRegistry.get("aiand") + assert provider_config is not None + assert litellm.LlmProviders.AIAND.value == "aiand" + assert "aiand" in litellm.openai_compatible_providers + assert provider_config.base_url == AIAND_API_BASE + assert provider_config.api_key_env == "AIAND_API_KEY" + assert provider_config.api_base_env == "AIAND_API_BASE" + + model, provider, api_key, api_base = get_llm_provider("aiand/openai/gpt-oss-120b") + assert model == "openai/gpt-oss-120b" + assert provider == "aiand" + assert api_key == "test-key" + assert api_base == AIAND_API_BASE + + +def test_aiand_completion_request(respx_mock: respx.MockRouter): + messages = [{"role": "user", "content": "Hello"}] + route = respx_mock.post(f"{AIAND_API_BASE}/chat/completions").mock( + return_value=httpx.Response( + 200, + json={ + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1, + "model": "openai/gpt-oss-120b", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Hello"}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 1, + "completion_tokens": 1, + "total_tokens": 2, + }, + }, + ) + ) + + litellm.completion( + model="aiand/openai/gpt-oss-120b", + api_key="test-key", + messages=messages, + ) + + assert route.called + request = route.calls.last.request + assert str(request.url) == f"{AIAND_API_BASE}/chat/completions" + assert request.headers["Authorization"] == "Bearer test-key" + request_body = json.loads(request.read()) + assert request_body["model"] == "openai/gpt-oss-120b" + assert request_body["messages"] == messages + + +def test_aiand_responses_provider(): + config = ProviderConfigManager.get_provider_responses_api_config( + provider="aiand", + model="aiand/openai/gpt-oss-120b", + ) + + assert config is not None + assert config.custom_llm_provider == "aiand" + assert config.get_complete_url(api_base=None, litellm_params={}) == f"{AIAND_API_BASE}/responses" + + +def _aiand_catalog() -> frozenset[str]: + return frozenset(name for name, info in litellm.model_cost.items() if info.get("litellm_provider") == "aiand") + + +def test_aiand_wildcard_lists_its_priced_models(): + catalog: Final = _aiand_catalog() + + assert catalog + assert frozenset(get_known_models_from_wildcard("aiand/*")) == catalog + + +def test_aiand_calls_are_billed_from_their_cost_rows(): + catalog: Final = _aiand_catalog() + assert catalog + + for model in sorted(catalog): + row: Final = litellm.model_cost[model] + prompt_cost, completion_cost = litellm.cost_per_token(model=model, prompt_tokens=1000, completion_tokens=500) + + assert prompt_cost == pytest.approx(1000 * row["input_cost_per_token"]) + assert completion_cost == pytest.approx(500 * row["output_cost_per_token"]) + assert prompt_cost > 0 and completion_cost > 0 + + +@pytest.mark.asyncio +async def test_aiand_anthropic_messages_reach_ai_and_with_the_thinking_budget(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + thinking: Final = {"type": "enabled", "budget_tokens": 1024} + with respx.mock() as upstream: + route: Final = upstream.post(f"{AIAND_API_BASE}/messages").respond( + 200, + json={ + "id": "msg_aiand", + "type": "message", + "role": "assistant", + "model": "zai-org/glm-5.3", + "content": [{"type": "thinking", "thinking": "A greeting", "signature": ""}, {"type": "text", "text": "Hello"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 4, "output_tokens": 3}, + }, + ) + response: Final = await litellm.anthropic.messages.acreate( + model="aiand/zai-org/glm-5.3", + messages=[{"role": "user", "content": "Say hello"}], + max_tokens=2048, + thinking=thinking, + api_key="aiand-test-key", + ) + + request: Final = route.calls.last.request + body: Final = json.loads(request.content) + assert route.call_count == 1 + assert body["model"] == "zai-org/glm-5.3" + assert body["thinking"] == thinking + assert "reasoning_effort" not in body + assert [block["type"] for block in response["content"]] == ["thinking", "text"] diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index a44373b87c8..a32c40e5a43 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -53,6 +53,7 @@ from litellm.proxy.litellm_pre_call_utils import ( add_provider_specific_headers_to_request, check_if_token_is_service_account, clean_headers, + forwarded_anthropic_oauth_token, get_dynamic_logging_metadata, get_metadata_variable_name, move_guardrails_to_metadata, @@ -1393,6 +1394,8 @@ async def test_add_litellm_data_to_request_strips_string_encoded_admin_injection ("fallback_depth", True), ("_target_order", 2), ("attempted_targets", ["forged"]), + ("_excluded_deployment_ids", ["pinned-deployment"]), + ("_retry_skipped_deployment_ids", ["pinned-deployment"]), ], ) async def test_add_litellm_data_to_request_strips_forged_fallback_hop_state( @@ -1477,6 +1480,7 @@ async def test_add_litellm_data_to_request_strips_user_control_fields(): "_managed_background_response": True, "_guardrail_pipelines": [{"name": "spoofed"}], "_pipeline_managed_guardrails": ["evaded"], + "_failover_excluded_ids": ["pinned-deployment"], "safe_user_metadata": "kept", } data = { @@ -1537,6 +1541,7 @@ async def test_add_litellm_data_to_request_strips_user_control_fields(): "_managed_background_response", "_guardrail_pipelines", "_pipeline_managed_guardrails", + "_failover_excluded_ids", } assert "litellm_metadata" not in updated for stripped_key in stripped_keys: @@ -5962,6 +5967,25 @@ def test_clean_headers_strips_x_api_key_when_byok_disabled(): assert "x-api-key" not in result +@pytest.mark.parametrize( + ("headers", "expected_token"), + [ + ( + {"authorization": "Bearer sk-ant-oat01-caller", "x-litellm-api-key": "Bearer sk-proxy"}, + "sk-ant-oat01-caller", + ), + ({"authorization": "Bearer sk-ant-oat01-caller"}, None), + ({"authorization": "Bearer sk-ant-api03-caller", "x-litellm-api-key": "Bearer sk-proxy"}, None), + ({"x-litellm-api-key": "Bearer sk-proxy"}, None), + ], + ids=["oauth-beside-proxy-key", "oauth-used-as-proxy-key", "non-oauth-authorization", "no-authorization"], +) +def test_forwarded_anthropic_oauth_token_follows_chats_forwarding_rule( + headers: dict[str, str], expected_token: str | None +) -> None: + assert forwarded_anthropic_oauth_token(Headers(headers)) == expected_token + + def test_clean_headers_strips_x_api_key_when_byok_enabled_but_x_api_key_was_auth_header(): """ Anti-replay regression: even when forward_llm_provider_auth_headers=True, diff --git a/tests/unit/proxy/test_proxy_token_counter.py b/tests/unit/proxy/test_proxy_token_counter.py index fa0c9eda02d..ebe3c7407d8 100644 --- a/tests/unit/proxy/test_proxy_token_counter.py +++ b/tests/unit/proxy/test_proxy_token_counter.py @@ -364,6 +364,7 @@ async def test_anthropic_messages_count_tokens_endpoint(): output_format=None, context_management=None, request_litellm_params=None, + client_oauth_token=None, ): assert call_endpoint == True, "Should be called with call_endpoint=True for Anthropic endpoint" assert request.model == "claude-3-sonnet-20240229" @@ -444,6 +445,7 @@ async def test_anthropic_messages_count_tokens_with_non_anthropic_model(): output_format=None, context_management=None, request_litellm_params=None, + client_oauth_token=None, ): assert call_endpoint == True, "Should be called with call_endpoint=True for Anthropic endpoint" assert request.model == "gpt-4" @@ -1262,6 +1264,7 @@ async def test_anthropic_endpoint_returns_anthropic_error_format(): output_format=None, context_management=None, request_litellm_params=None, + client_oauth_token=None, ): raise ProxyException( message='{"detail":{"message":"Input is too long for requested model."}}', @@ -1319,6 +1322,7 @@ async def test_anthropic_endpoint_403_permission_error_format(): output_format=None, context_management=None, request_litellm_params=None, + client_oauth_token=None, ): raise ProxyException( message='{"Message":"Bearer Token has expired"}', @@ -1376,6 +1380,7 @@ async def test_anthropic_endpoint_429_rate_limit_error_format(): output_format=None, context_management=None, request_litellm_params=None, + client_oauth_token=None, ): raise ProxyException( message="Rate limit exceeded", diff --git a/tests/unit/router_utils/pre_call_checks/test_deployment_affinity_check.py b/tests/unit/router_utils/pre_call_checks/test_deployment_affinity_check.py index d8bc4c45ab8..fa75f980cc0 100644 --- a/tests/unit/router_utils/pre_call_checks/test_deployment_affinity_check.py +++ b/tests/unit/router_utils/pre_call_checks/test_deployment_affinity_check.py @@ -9,6 +9,7 @@ import pytest import litellm from litellm.caching.dual_cache import DualCache +from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.router_utils.pre_call_checks.deployment_affinity_check import ( DeploymentAffinityCheck, ) @@ -1209,3 +1210,92 @@ async def test_virtual_key_hash_wins_over_user_id_for_affinity(): parent_otel_span=None, ) assert other_key_same_user == healthy_deployments + + +class _ForbiddenError(Exception): + status_code: Final = 403 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("pre_call_check", "metadata"), + [ + ("session_affinity", {"session_id": "sess-retry"}), + ("deployment_affinity", {"user_api_key_hash": "hashed-key-retry"}), + ], +) +async def test_retry_reaches_healthy_sibling_when_affinity_pins_deployment_that_refused( + pre_call_check: str, metadata: dict[str, str] +): + router: Final = litellm.Router( + model_list=[ + { + "model_name": "test-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": "bad", + "mock_response": _ForbiddenError("A refuses"), + "weight": 1, + }, + "model_info": {"id": "A"}, + }, + { + "model_name": "test-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": "good", + "mock_response": "ok from B", + "weight": 0, + }, + "model_info": {"id": "B"}, + }, + ], + routing_strategy="simple-shuffle", + num_retries=1, + retry_after=0, + disable_cooldowns=True, + optional_pre_call_checks=[pre_call_check], + ) + + responses: Final = [ + await router.acompletion( + model="test-model", + messages=[{"role": "user", "content": "hi"}], + metadata=dict(metadata), + ) + for _ in range(2) + ] + + assert [response._hidden_params["model_id"] for response in responses] == ["B", "B"] + assert [response._hidden_params["additional_headers"]["x-litellm-attempted-retries"] for response in responses] == [ + 1, + 1, + ] + + +@pytest.mark.asyncio +async def test_previous_response_id_pin_holds_even_after_its_deployment_failed(): + callback: Final = DeploymentAffinityCheck( + cache=DualCache(), + ttl_seconds=60, + enable_user_key_affinity=True, + enable_responses_api_affinity=True, + ) + healthy_deployments: Final = [ + {"model_name": "gpt-4o", "litellm_params": {"model": "gpt-4o"}, "model_info": {"id": "A"}}, + {"model_name": "gpt-4o", "litellm_params": {"model": "gpt-4o"}, "model_info": {"id": "B"}}, + ] + + assert await callback.async_filter_deployments( + model="gpt-4o", + healthy_deployments=healthy_deployments, + messages=None, + request_kwargs={ + "previous_response_id": ResponsesAPIRequestUtils._build_responses_api_response_id( + custom_llm_provider="openai", model_id="A", response_id="resp_upstream" + ), + "_excluded_deployment_ids": ["A"], + "metadata": {"user_api_key_hash": "hashed-key"}, + }, + parent_otel_span=None, + ) == [healthy_deployments[0]] diff --git a/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py b/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py index 87fbf4433a1..a3745c7b23f 100644 --- a/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py +++ b/tests/unit/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py @@ -16,6 +16,7 @@ The mechanism works without any cache and supports two encoding strategies: """ import time +from typing import Final from unittest.mock import AsyncMock, patch import pytest @@ -2606,3 +2607,111 @@ async def test_affinity_strips_for_team_and_pattern_routes(): assert result is sibling_pool assert not request_kwargs["input"][1].get("encrypted_content") + + +def _region_deployment(region: str) -> dict[str, object]: + return { + "model_name": "gpt-5.4", + "litellm_params": { + "model": "openai/gpt-5.4", + "api_base": f"https://{region}.example.com/v1", + "api_key": f"key-{region}", + }, + "model_info": {"id": region}, + } + + +def _reasoning_input_minted_by(origin: str) -> list[dict[str, object]]: + return [ + { + "type": "reasoning", + "id": ResponsesAPIRequestUtils._build_encrypted_item_id(origin, "rs_test"), + "encrypted_content": ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id("gAAAAA-blob", origin), + "summary": [{"type": "summary_text", "text": "scattering"}], + }, + {"role": "user", "content": "and sunsets?"}, + ] + + +def _previous_response_id_from(origin: str) -> str: + return ResponsesAPIRequestUtils._build_responses_api_response_id("openai", origin, "resp_test") + + +_ORIGIN_FAILED_THIS_REQUEST: Final = pytest.mark.parametrize( + "failed_in_request", + [{"_excluded_deployment_ids": ["region-a"]}, {"_retry_skipped_deployment_ids": ("region-a",)}], +) + + +@pytest.mark.asyncio +@_ORIGIN_FAILED_THIS_REQUEST +@pytest.mark.parametrize("continuing", [{}, {"previous_response_id": _previous_response_id_from("region-b")}]) +async def test_affinity_strips_and_dispatches_when_origin_already_failed_this_request( + failed_in_request: dict[str, object], continuing: dict[str, object] +): + from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( + EncryptedContentAffinityCheck, + ) + + router: Final = litellm.Router(model_list=[_region_deployment("region-a"), _region_deployment("region-b")]) + group: Final = router.model_list + reasoning_input: Final = _reasoning_input_minted_by("region-a") + + result: Final = await EncryptedContentAffinityCheck(router=router).async_filter_deployments( + model="gpt-5.4", + healthy_deployments=group, + messages=None, + request_kwargs={"litellm_metadata": {}, "input": reasoning_input, **failed_in_request, **continuing}, + ) + + assert result is group + assert reasoning_input[0] == {"type": "reasoning", "summary": [{"type": "summary_text", "text": "scattering"}]} + + +@pytest.mark.asyncio +@_ORIGIN_FAILED_THIS_REQUEST +async def test_affinity_keeps_origin_and_reasoning_when_previous_response_id_binds_the_failed_origin( + failed_in_request: dict[str, object], +): + from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( + EncryptedContentAffinityCheck, + ) + + pool: Final = (_region_deployment("region-a"), _region_deployment("region-b")) + reasoning_input: Final = _reasoning_input_minted_by("region-a") + original_reasoning: Final = dict(reasoning_input[0]) + + assert await EncryptedContentAffinityCheck(router=litellm.Router(model_list=list(pool))).async_filter_deployments( + model="gpt-5.4", + healthy_deployments=list(pool), + messages=None, + request_kwargs={ + "litellm_metadata": {}, + "input": reasoning_input, + "previous_response_id": _previous_response_id_from("region-a"), + **failed_in_request, + }, + ) == [pool[0]] + assert reasoning_input[0] == original_reasoning + + +@pytest.mark.asyncio +@pytest.mark.parametrize("regions", [("region-a",), ("region-a", "region-b")]) +async def test_affinity_keeps_reasoning_when_every_deployment_already_refused_the_request(regions: tuple[str, ...]): + from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import ( + EncryptedContentAffinityCheck, + ) + + router: Final = litellm.Router(model_list=[_region_deployment(region) for region in regions]) + reasoning_input: Final = _reasoning_input_minted_by("region-a") + original_reasoning: Final = dict(reasoning_input[0]) + + result: Final = await EncryptedContentAffinityCheck(router=router).async_filter_deployments( + model="gpt-5.4", + healthy_deployments=router.model_list, + messages=None, + request_kwargs={"litellm_metadata": {}, "input": reasoning_input, "_retry_skipped_deployment_ids": regions}, + ) + + assert [deployment["model_info"]["id"] for deployment in result] == ["region-a"] + assert reasoning_input[0] == original_reasoning diff --git a/tests/unit/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py b/tests/unit/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py index 8d893160780..9487e4d6321 100644 --- a/tests/unit/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py +++ b/tests/unit/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py @@ -313,6 +313,27 @@ async def test_async_filter_deployments_does_not_pin_when_target_order_is_set(): assert filtered == deployments +@pytest.mark.asyncio +@pytest.mark.parametrize( + "failed_in_request", + [{"_excluded_deployment_ids": ["dep-2"]}, {"_retry_skipped_deployment_ids": ("dep-2",)}], +) +async def test_async_filter_deployments_does_not_pin_to_a_deployment_that_failed_this_request( + failed_in_request: dict[str, object], +): + cache: Final = DualCache() + check: Final = PromptCachingDeploymentCheck(cache=cache) + messages: Final = _messages(word_count=5000) + await PromptCachingCache(cache=cache).async_add_model_id(model_id="dep-2", messages=messages, tools=None) + + assert await check.async_filter_deployments( + model=MODEL_GROUP_ALIAS, + healthy_deployments=_deployments("anthropic/claude-opus-4-6", "anthropic/claude-opus-4-6"), + messages=messages, + request_kwargs=failed_in_request, + ) == _deployments("anthropic/claude-opus-4-6", "anthropic/claude-opus-4-6") + + @pytest.mark.asyncio async def test_async_filter_deployments_narrows_for_group_whose_model_minimum_is_lower(): """ diff --git a/tests/unit/router_utils/test_cooldown_handlers.py b/tests/unit/router_utils/test_cooldown_handlers.py index bab6d257579..4299b61c703 100644 --- a/tests/unit/router_utils/test_cooldown_handlers.py +++ b/tests/unit/router_utils/test_cooldown_handlers.py @@ -1,10 +1,12 @@ import asyncio import copy +import datetime import importlib import itertools import random import time -from collections.abc import Mapping +import uuid +from collections.abc import Mapping, Sequence from typing import Final from unittest.mock import MagicMock, patch @@ -682,6 +684,45 @@ class TestDeploymentLevelAllowedFails: ) +def _gpt_4_group_with_a_sibling() -> Sequence[Mapping[str, object]]: + return ( + {"model_name": "gpt-4", "litellm_params": {"model": "openai/gpt-4"}, "model_info": {"id": "primary"}}, + {"model_name": "gpt-4", "litellm_params": {"model": "openai/gpt-4"}, "model_info": {"id": "secondary"}}, + ) + + +def _primary_cooldown_decisions_on_repeated_503s(router: Router) -> tuple[bool, ...]: + exception: Final = litellm.ServiceUnavailableError("down", "openai", "gpt-4") + return tuple( + _should_cooldown_deployment( + litellm_router_instance=router, + deployment="primary", + exception_status=503, + original_exception=exception, + ) + for _ in range(litellm.allowed_fails + 1) + ) + + +@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") +class TestRouterLevelAllowedFailsEqualToDefault: + def test_allowed_fails_equal_to_the_default_cools_down_after_that_many_fails(self): + router: Final = _make_router(model_list=list(_gpt_4_group_with_a_sibling()), allowed_fails=litellm.allowed_fails) + + assert _primary_cooldown_decisions_on_repeated_503s(router) == (False,) * litellm.allowed_fails + (True,) + + def test_unset_allowed_fails_keeps_the_rate_based_default(self): + router: Final = _make_router(model_list=list(_gpt_4_group_with_a_sibling())) + + assert _primary_cooldown_decisions_on_repeated_503s(router) == (False,) * (litellm.allowed_fails + 1) + + def test_default_allowed_fails_saved_through_update_settings_keeps_the_rate_based_default(self): + router: Final = _make_router(model_list=list(_gpt_4_group_with_a_sibling())) + router.update_settings(allowed_fails=litellm.allowed_fails) + + assert _primary_cooldown_decisions_on_repeated_503s(router) == (False,) * (litellm.allowed_fails + 1) + + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") class TestDeploymentLevelAllowedFailsPolicyByExceptionType: def test_rate_limit_error_triggers_cooldown_with_zero_threshold(self): @@ -2931,6 +2972,104 @@ async def test_bad_request_error_does_not_cool_down_a_deployment( assert tuple(cooldown_deployments) == () +async def _complete_like_the_proxy(router: Router, model: str) -> str: + logging_obj, request = litellm.utils.function_setup( + original_function="acompletion", + rules_obj=litellm.utils.Rules(), + start_time=datetime.datetime(2026, 1, 1), + model=model, + messages=[{"role": "user", "content": "hi"}], + litellm_call_id=str(uuid.uuid4()), + ) + response: Final = await router.acompletion(**request, litellm_logging_obj=logging_obj) + model_id: Final = response._hidden_params["model_id"] + assert isinstance(model_id, str) + return model_id + + +_CHAT_COMPLETION_OK: Final = { + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, +} + + +def _openai_deployment(model_name: str, host: str, deployment_id: str) -> Mapping[str, object]: + return { + "model_name": model_name, + "litellm_params": {"model": "openai/gpt-4o", "api_key": "k", "api_base": f"https://{host}/v1"}, + "model_info": {"id": deployment_id}, + } + + +@pytest.mark.asyncio +async def test_fallback_deployment_stays_up_while_the_primary_is_in_cooldown( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://primary.test/v1/chat/completions").mock( + return_value=httpx.Response(503, json={"error": {"message": "down", "type": "server_error"}}) + ) + backup_route: Final = respx_mock.post("https://backup.test/v1/chat/completions").mock( + return_value=httpx.Response(200, json=_CHAT_COMPLETION_OK) + ) + router: Final = litellm.Router( + model_list=[ + _openai_deployment("primary", "primary.test", "primary-dep"), + _openai_deployment("backup", "backup.test", "backup-dep"), + ], + fallbacks=[{"primary": ["backup"]}], + allowed_fails=litellm.allowed_fails, + num_retries=0, + ) + + served_by: Final = [await _complete_like_the_proxy(router, "primary") for _ in range(20)] + cooldown_deployments: Final = await async_get_cooldown_deployments( + litellm_router_instance=router, + parent_otel_span=None, + ) + + assert served_by == ["backup-dep"] * 20 + assert len(backup_route.calls) == 20 + assert tuple(cooldown_deployments) == ("primary-dep",) + + +@pytest.mark.asyncio +async def test_fallback_group_stays_up_after_a_bad_key_cools_the_primary_under_the_rate_based_default( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://primary.test/v1/chat/completions").mock( + return_value=httpx.Response(401, json={"error": {"message": "bad key", "type": "invalid_request_error"}}) + ) + respx_mock.post("https://backup.test/v1/chat/completions").mock( + return_value=httpx.Response(200, json=_CHAT_COMPLETION_OK) + ) + router: Final = litellm.Router( + model_list=[ + _openai_deployment("primary", "primary.test", "primary-dep"), + _openai_deployment("backup", "backup.test", "backup-1"), + _openai_deployment("backup", "backup.test", "backup-2"), + ], + fallbacks=[{"primary": ["backup"]}], + num_retries=0, + ) + + served_by: Final = [await _complete_like_the_proxy(router, "primary") for _ in range(12)] + cooldown_deployments: Final = await async_get_cooldown_deployments( + litellm_router_instance=router, + parent_otel_span=None, + ) + + assert set(served_by) <= {"backup-1", "backup-2"} + assert tuple(cooldown_deployments) == ("primary-dep",) + + TIMED_OUT_DEPLOYMENT: Final = { "model_name": "test-model", "litellm_params": {"model": "openai/test-model", "api_key": "test-key"}, diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 70b268bf2a2..f12129af1fc 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -14,7 +14,7 @@ from dataclasses import dataclass from datetime import datetime from importlib import import_module from pathlib import Path -from typing import Final +from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -44,12 +44,13 @@ from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kw from litellm.litellm_core_utils.get_litellm_params import stored_control_options from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt -from litellm.llms.custom_httpx.http_handler import HTTPHandler +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.types.litellm_params import ControlOptions from litellm.types.llms.openai import AllMessageValues, HttpxBinaryResponseContent from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.utils import Delta, ModelResponseStream, StandardCallbackDynamicParams, StreamingChoices, Usage from tests.unit.llms.bedrock.slow_upstream import ( + CONVERSE_ANSWER, STREAM_TIMEOUT_SECONDS, UPSTREAM_FIRST_BYTE_SECONDS, slow_upstream_sync_client, @@ -9950,3 +9951,126 @@ async def test_converse_fallback_prices_the_current_attempt( else: client.close() configuration.reset_rust_configuration() + + +_LOOKUP_FUNCTION: Final = { + "name": "lookup_record", + "description": "Look up a record", + "parameters": { + "type": "object", + "properties": {"query": {"type": "string"}}, + "required": ["query"], + "additionalProperties": False, + }, +} +_STRICT_LOOKUP_TOOL: Final = {"type": "function", "function": {**_LOOKUP_FUNCTION, "strict": True}} +# Bedrock's 400 for a request past Claude's strict tool limits, seen live on us.anthropic.claude-sonnet-4-6 on 2026-10-20 +_TOO_MANY_STRICT_TOOLS: Final = ( + "The model returned the following errors: Too many strict tools (21). The maximum number of strict tools " + "supported is 20. Try reducing the number of tools marked as strict." +) + + +@dataclass(slots=True) +class _StrictToolsLimitRejectionFirst: + stream: Literal["off", "fake", "real"] + sent: tuple[tuple[str, dict[str, object]], ...] = () + + def __call__(self, request: httpx.Request) -> httpx.Response: + self.sent = (*self.sent, (request.url.path.rsplit("/", 1)[-1], json.loads(request.content))) + if len(self.sent) == 1: + return httpx.Response(400, json={"message": _TOO_MANY_STRICT_TOOLS}) + return httpx.Response(200, content=b"" if self.stream == "real" else CONVERSE_ANSWER) + + +async def _converse_with_a_strict_tools_limit_rejection_first( + asynchronous: bool, + stream: Literal["off", "fake", "real"], + drop_params: bool | None, + tool: Mapping[str, object] = _STRICT_LOOKUP_TOOL, +) -> tuple[tuple[tuple[str, dict[str, object]], ...], litellm.BadRequestError | None]: + respond: Final = _StrictToolsLimitRejectionFirst(stream) + kwargs: Final = { + "model": "bedrock/us.anthropic.claude-sonnet-4-6", + "messages": [{"role": "user", "content": "Find invoice 42"}], + "tools": [tool], + "aws_access_key_id": "AKIASTRICTTOOLS", + "aws_secret_access_key": "strict-tools-secret", + "aws_region_name": "us-east-1", + "num_retries": 0, + "drop_params": drop_params, + "stream": stream != "off", + "fake_stream": stream == "fake", + } + try: + chunks: Final = ( + await _acompletion_chunks(AsyncHTTPHandler(transport=httpx.MockTransport(respond)), kwargs, stream) + if asynchronous + else _completion_chunks( + HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))), kwargs, stream + ) + ) + except litellm.BadRequestError as error: + return respond.sent, error + assert chunks + return respond.sent, None + + +async def _acompletion_chunks( + client: AsyncHTTPHandler, kwargs: Mapping[str, object], stream: Literal["off", "fake", "real"] +) -> tuple[object, ...]: + response: Final = await litellm.acompletion(client=client, **kwargs) + return tuple([chunk async for chunk in response]) if stream == "fake" else (response,) + + +def _completion_chunks( + client: HTTPHandler, kwargs: Mapping[str, object], stream: Literal["off", "fake", "real"] +) -> tuple[object, ...]: + response: Final = litellm.completion(client=client, **kwargs) + return tuple(response) if stream == "fake" else (response,) + + +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("stream", ["off", "fake", "real"], ids=["completion", "fake_stream", "stream"]) +async def test_bedrock_strict_tools_limit_rejection_is_sent_again_without_strict_under_drop_params( + asynchronous: bool, stream: Literal["off", "fake", "real"], local_model_cost_map: None +) -> None: + sent, error = await _converse_with_a_strict_tools_limit_rejection_first(asynchronous, stream, drop_params=True) + + assert error is None + (first_path, first_body), (second_path, second_body) = sent + first_tool_spec: Final = first_body["toolConfig"]["tools"][0]["toolSpec"] + assert first_tool_spec["strict"] is True + assert first_path == second_path == ("converse-stream" if stream == "real" else "converse") + assert second_body == { + **first_body, + "toolConfig": { + **first_body["toolConfig"], + "tools": [{"toolSpec": {key: value for key, value in first_tool_spec.items() if key != "strict"}}], + }, + } + + +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +async def test_bedrock_strict_tools_limit_rejection_reaches_the_caller_without_drop_params( + asynchronous: bool, local_model_cost_map: None +) -> None: + sent, error = await _converse_with_a_strict_tools_limit_rejection_first(asynchronous, stream="off", drop_params=None) + + assert error is not None and "Too many strict tools" in str(error) + assert len(sent) == 1 + + +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +async def test_bedrock_strict_tools_limit_rejection_is_not_sent_again_when_no_tool_sent_strict( + asynchronous: bool, local_model_cost_map: None +) -> None: + sent, error = await _converse_with_a_strict_tools_limit_rejection_first( + asynchronous, + stream="off", + drop_params=True, + tool={"type": "function", "function": {**_LOOKUP_FUNCTION, "strict": False}}, + ) + + assert error is not None and "Too many strict tools" in str(error) + assert len(sent) == 1 diff --git a/tests/unit/test_model_prices_and_context_window.py b/tests/unit/test_model_prices_and_context_window.py index 1f84be7710f..08a39b94c12 100644 --- a/tests/unit/test_model_prices_and_context_window.py +++ b/tests/unit/test_model_prices_and_context_window.py @@ -2,7 +2,7 @@ from __future__ import annotations import json import re -from collections.abc import Mapping +from collections.abc import Iterator, Mapping from pathlib import Path from types import MappingProxyType from typing import Final @@ -224,6 +224,50 @@ def test_dated_variants_carry_base_alias_service_tier_pricing(prices: dict): ) +PRIORITY_SUFFIX: Final = "_priority" +MIN_PRIORITY_MARKUP: Final = 1.0 +MAX_PRIORITY_MARKUP: Final = 5.0 + + +def _rate(value: JsonValue | None) -> float | None: + return float(value) if isinstance(value, (int, float)) and not isinstance(value, bool) else None + + +def _priority_markup(entry: Mapping[str, JsonValue], key: str) -> float | None: + if not key.endswith(PRIORITY_SUFFIX): + return None + priority: Final = _rate(entry[key]) + standard: Final = _rate(entry.get(key.removesuffix(PRIORITY_SUFFIX))) + if priority is None or standard is None or standard <= 0: + return None + return priority / standard + + +def priority_markups(prices: Mapping[str, JsonValue]) -> Iterator[tuple[str, str, float]]: + for name, entry in prices.items(): + if not isinstance(entry, dict): + continue + for key in entry: + if (markup := _priority_markup(entry, key)) is not None: + yield name, key, markup + + +@pytest.mark.parametrize("path", (PRICES_PATH, BACKUP_PRICES_PATH), ids=("main", "backup")) +def test_priority_rates_stay_within_a_premium_of_their_standard_rate(path: Path) -> None: + prices: Final = TypeAdapter(dict[str, JsonValue]).validate_json(path.read_bytes()) + markups: Final = tuple(priority_markups(prices)) + assert markups + overpriced: Final = tuple( + f"{name}.{key} is {markup:.1f}x its standard rate" + for name, key, markup in markups + if not MIN_PRIORITY_MARKUP <= markup <= MAX_PRIORITY_MARKUP + ) + assert overpriced == (), ( + "a priority rate is a premium on the standard rate, so one below it or far above it was likely copied from " + "another model; drop the key when the provider sells no priority tier for the model:\n" + "\n".join(overpriced) + ) + + # Azure Foundry GPT-6.1 Sol announcement, 2026-09-29: https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-6-1-sol-in-microsoft-foundry-advanced-intelligence-optimized-for/4560811 AZURE_GPT_6_1_SOL_REGIONAL_PREMIUM: Final = MappingProxyType({"us": 1.1, "eu": 1.2, "apac": 1.2}) diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index d9b655a0aff..30d675e130d 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -21239,9 +21239,9 @@ def test_router_retry_skip_stamp_feeds_deployment_ids_to_skip_on_retry( ], ) def test_router_as_retry_skipped_deployment_ids_keeps_only_a_tuple_of_strings(value, expected): - from litellm.router import _as_retry_skipped_deployment_ids + from litellm.router_utils.common_utils import as_retry_skipped_deployment_ids - assert _as_retry_skipped_deployment_ids(value) == expected + assert as_retry_skipped_deployment_ids(value) == expected @pytest.mark.parametrize( diff --git a/tests/unit/test_router_weighted_failover.py b/tests/unit/test_router_weighted_failover.py index 9cb38b00e1e..8b7fc81dbde 100644 --- a/tests/unit/test_router_weighted_failover.py +++ b/tests/unit/test_router_weighted_failover.py @@ -8,13 +8,14 @@ cross-group fallback runs. """ from collections import Counter -from typing import Optional +from typing import Final, Optional from unittest.mock import AsyncMock, patch import pytest import litellm from litellm import Router +from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo from litellm.utils import get_excluded_filtered_deployments @@ -944,3 +945,77 @@ async def test_failover_falls_through_to_external_fallback_when_remaining_in_coo ) assert response._hidden_params["model_id"] == "fallback" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("pre_call_check", "metadata"), + [ + ("session_affinity", {"session_id": "sess-1"}), + ("deployment_affinity", {"user_api_key_hash": "hashed-key-1"}), + ], +) +async def test_failover_reaches_healthy_sibling_when_affinity_pins_failed_deployment( + pre_call_check: str, metadata: dict[str, str] +): + router: Final = Router( + model_list=[ + { + "model_name": "test-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": "bad", + "mock_response": Exception("A down"), + "weight": 1, + }, + "model_info": {"id": "A"}, + }, + { + "model_name": "test-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": "good", + "mock_response": "ok from B", + "weight": 0, + }, + "model_info": {"id": "B"}, + }, + { + "model_name": "fallback-model", + "litellm_params": { + "model": "gpt-4o", + "api_key": "good", + "mock_response": "ok from fallback", + }, + "model_info": {"id": "fallback"}, + }, + ], + routing_strategy="simple-shuffle", + num_retries=0, + disable_cooldowns=True, + enable_weighted_failover=True, + optional_pre_call_checks=[pre_call_check], + fallbacks=[{"test-model": ["fallback-model"]}], + ) + + while_a_is_down: Final = await router.acompletion( + model="test-model", + messages=[{"role": "user", "content": "hi"}], + metadata=dict(metadata), + ) + router.upsert_deployment( + Deployment( + model_name="test-model", + litellm_params=LiteLLM_Params(model="gpt-4o", api_key="bad", mock_response="ok from A", weight=1), + model_info=ModelInfo(id="A"), + ) + ) + after_a_recovers: Final = await router.acompletion( + model="test-model", + messages=[{"role": "user", "content": "hi"}], + metadata=dict(metadata), + ) + + assert while_a_is_down._hidden_params["model_id"] == "B" + assert while_a_is_down._hidden_params["additional_headers"]["x-litellm-attempted-fallbacks"] == 1 + assert after_a_recovers._hidden_params["model_id"] == "A"