Merge remote-tracking branch 'origin/main' into litellm_responses_comparison_response_id_decode

This commit is contained in:
mateo-berri 2026-10-28 04:23:45 -07:00
commit ee414f9b05
45 changed files with 2659 additions and 104 deletions

View file

@ -710,6 +710,7 @@ transcribe_models: Set = set()
gigachat_models: Set = set()
llamagate_models: Set = set()
coralbricks_models: Set[str] = set()
aiand_models: Set[str] = set()
reducto_models: Set = set()
bedrock_mantle_models: Set = set()
@ -1022,6 +1023,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
llamagate_models.add(key)
elif value.get("litellm_provider") == "coralbricks":
coralbricks_models.add(key)
elif value.get("litellm_provider") == "aiand":
aiand_models.add(key)
elif value.get("litellm_provider") == "reducto":
reducto_models.add(key)
elif value.get("litellm_provider") == "bedrock_mantle":
@ -1159,6 +1162,7 @@ model_list = list(
| reducto_models
| bedrock_mantle_models
| coralbricks_models
| aiand_models
| set(clarifai_models)
)
@ -1283,6 +1287,7 @@ def _build_models_by_provider() -> dict:
"gigachat": gigachat_models,
"llamagate": llamagate_models,
"coralbricks": coralbricks_models,
"aiand": aiand_models,
"reducto": reducto_models,
"bedrock_mantle": bedrock_mantle_models,
}

View file

@ -1035,6 +1035,7 @@ openai_compatible_endpoints: Final[list[str]] = [
"https://api-inference.modelscope.cn/v1",
"https://api.moonshot.ai/v1",
"https://api.publicai.co/v1",
"https://api.aiand.com/v1",
"https://api.synthetic.new/openai/v1",
"https://serverless.tensormesh.ai/v1",
"https://api.stima.tech/v1",
@ -1100,6 +1101,7 @@ openai_compatible_providers: Final[list[str]] = [
"novita",
"meta_llama",
"publicai", # PublicAI - JSON-configured provider
"aiand", # ai& - JSON-configured provider
"synthetic", # Synthetic - JSON-configured provider
"tensormesh", # Tensormesh - JSON-configured provider
"apertis", # Apertis - JSON-configured provider

View file

@ -27,6 +27,9 @@ anthropic_count_tokens_handler: Final = AnthropicCountTokensHandler()
class AnthropicTokenCounter(BaseTokenCounter):
"""Token counter implementation for Anthropic provider using the CountTokens API."""
def __init__(self, client_oauth_token: str | None = None) -> None:
self.client_oauth_token: Final = client_oauth_token
def should_use_token_counting_api(
self,
custom_llm_provider: str | None = None,
@ -95,12 +98,17 @@ class AnthropicTokenCounter(BaseTokenCounter):
api_base: Final = litellm_params.get("api_base")
try:
auth_header: Final = await AnthropicModelInfo.aget_auth_header(
configured_auth_header: Final = await AnthropicModelInfo.aget_auth_header(
api_key=litellm_params.get("api_key") or litellm.anthropic_key or litellm.api_key,
api_base=api_base,
litellm_params=litellm_params,
allow_workload_identity=True,
)
auth_header: Final = (
configured_auth_header
if configured_auth_header is not None or self.client_oauth_token is None
else AnthropicModelInfo.get_auth_header(api_key=self.client_oauth_token, api_base=api_base)
)
if auth_header is None:
return TokenCountResponse(
total_tokens=0,

View file

@ -1,11 +1,14 @@
from collections.abc import Callable, Coroutine
from functools import partial
from typing import Final, TypeAlias
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.bedrock.common_utils import (
NativeOutputFallback,
is_bedrock_native_output_rejection,
is_bedrock_strict_tools_rejection,
warn_bedrock_native_output_rejected,
warn_bedrock_strict_tools_dropped,
)
from litellm.types.utils import ModelResponse
from litellm.utils import CustomStreamWrapper
@ -14,12 +17,13 @@ NativeOutputResponse: TypeAlias = ModelResponse | CustomStreamWrapper
NativeOutputAttemptResult: TypeAlias = NativeOutputResponse | Coroutine[object, object, NativeOutputResponse]
NativeOutputAttempt: TypeAlias = Callable[[NativeOutputFallback | None], NativeOutputAttemptResult]
NativeOutputFallbackFactory: TypeAlias = Callable[[], NativeOutputFallback | None]
_RetryFor: TypeAlias = Callable[[BaseLLMException], NativeOutputFallback | None]
def _fallback_for(
error: BaseLLMException, fallback: NativeOutputFallbackFactory, model: str
def _native_output_retry(
error: BaseLLMException, fallback: NativeOutputFallbackFactory | None, model: str
) -> NativeOutputFallback | None:
if not is_bedrock_native_output_rejection(error.status_code, error.message):
if fallback is None or not is_bedrock_native_output_rejection(error.status_code, error.message):
return None
retry: Final = fallback()
if retry is not None:
@ -27,16 +31,38 @@ def _fallback_for(
return retry
def _strict_tools_retry(
error: BaseLLMException, fallback: NativeOutputFallbackFactory | None, model: str
) -> NativeOutputFallback | None:
if fallback is None or not is_bedrock_strict_tools_rejection(error.status_code, error.message):
return None
retry: Final = fallback()
if retry is not None:
warn_bedrock_strict_tools_dropped(model)
return retry
def _fallback_for(
error: BaseLLMException,
fallback: NativeOutputFallbackFactory | None,
strict_tools_fallback: NativeOutputFallbackFactory | None,
model: str,
) -> NativeOutputFallback | None:
native_output_retry: Final = _native_output_retry(error, fallback, model)
if native_output_retry is not None:
return native_output_retry
return _strict_tools_retry(error, strict_tools_fallback, model)
async def _await_with_native_output_fallback(
first_attempt: Coroutine[object, object, NativeOutputResponse],
attempt: NativeOutputAttempt,
fallback: NativeOutputFallbackFactory,
model: str,
retry_for: _RetryFor,
) -> NativeOutputResponse:
try:
return await first_attempt
except BaseLLMException as error:
retry: Final = _fallback_for(error, fallback, model)
retry: Final = retry_for(error)
if retry is None:
raise
second_attempt: Final = attempt(retry)
@ -47,16 +73,20 @@ def call_with_native_output_fallback(
attempt: NativeOutputAttempt,
fallback: NativeOutputFallbackFactory | None,
model: str,
strict_tools_fallback: NativeOutputFallbackFactory | None = None,
) -> NativeOutputAttemptResult:
if fallback is None:
if fallback is None and strict_tools_fallback is None:
return attempt(None)
retry_for: Final = partial(
_fallback_for, fallback=fallback, strict_tools_fallback=strict_tools_fallback, model=model
)
try:
first_attempt: Final = attempt(None)
except BaseLLMException as error:
retry: Final = _fallback_for(error, fallback, model)
retry: Final = retry_for(error)
if retry is None:
raise
return attempt(retry)
if isinstance(first_attempt, Coroutine):
return _await_with_native_output_fallback(first_attempt, attempt, fallback, model)
return _await_with_native_output_fallback(first_attempt, attempt, retry_for)
return first_attempt

View file

@ -362,6 +362,60 @@ class NativeOutputFallback:
litellm_params: Mapping[str, object]
_BEDROCK_STRICT_TOOLS_REJECTION: Final = re.compile(
r"Too many strict tools \(\d+\)"
r"|Schemas contains too many (?:optional parameters|parameters with union types)"
r"|[Cc]ompiled grammar (?:is too large|size)"
r"|tools\.\d+\.custom(?:\.strict)?: "
r"|Grammar compilation timed out"
)
_TOOLS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...])
_TOOL_SPEC_ADAPTER: Final = TypeAdapter(Mapping[str, object])
def is_bedrock_strict_tools_rejection(status_code: int, error_text: str) -> bool:
return status_code == 400 and _BEDROCK_STRICT_TOOLS_REJECTION.search(error_text) is not None
def warn_bedrock_strict_tools_dropped(model: str) -> None:
verbose_logger.warning(
"Bedrock rejected the strict tool schemas for model=%s, so litellm sent the request again without `strict` "
"because drop_params is set.",
model,
)
def _without_strict(tool: Mapping[str, object]) -> Mapping[str, object]:
spec_key: Final = next((key for key in ("function", "toolSpec") if key in tool), None)
if spec_key is None:
return tool
spec: Final = _TOOL_SPEC_ADAPTER.validate_python(tool[spec_key])
return {**tool, spec_key: {key: value for key, value in spec.items() if key != "strict"}}
def _sends_strict(tool: Mapping[str, object], supports_strict_tools: bool) -> bool:
if "toolSpec" in tool:
return "strict" in _TOOL_SPEC_ADAPTER.validate_python(tool["toolSpec"])
if "function" not in tool:
return False
return supports_strict_tools and bool(_TOOL_SPEC_ADAPTER.validate_python(tool["function"]).get("strict"))
def bedrock_strict_tools_fallback(
model: str, optional_params: Mapping[str, object], litellm_params: Mapping[str, object]
) -> NativeOutputFallback | None:
tools: Final = _TOOLS_ADAPTER.validate_python(optional_params.get("tools", ()))
supports_strict_tools: Final = bedrock_converse_supports_strict_tools(
bedrock_capability_model(model, litellm_params.get("base_model"))
)
if not any(_sends_strict(tool, supports_strict_tools) for tool in tools):
return None
return NativeOutputFallback(
optional_params={**optional_params, "tools": [_without_strict(tool) for tool in tools]},
litellm_params=litellm_params,
)
def requests_output_config_format(optional_params: Mapping[str, object]) -> bool:
match optional_params:
case {"output_config": {"format": object()}}:

View file

@ -107,6 +107,12 @@
"api_key_env": "AIHUBMIX_API_KEY",
"api_base_env": "AIHUBMIX_API_BASE"
},
"aiand": {
"base_url": "https://api.aiand.com/v1",
"api_key_env": "AIAND_API_KEY",
"api_base_env": "AIAND_API_BASE",
"supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/messages"]
},
"crusoe": {
"base_url": "https://api.inference.crusoecloud.com/v1",
"api_key_env": "CRUSOE_API_KEY",

View file

@ -123,6 +123,7 @@ from litellm.llms.bedrock.common_utils import (
BedrockModelInfo,
NativeOutputFallback,
bedrock_route_for_request,
bedrock_strict_tools_fallback,
deployment_model_info,
without_bedrock_native_output_opt_in,
without_bedrock_route_prefix,
@ -206,7 +207,12 @@ from litellm.utils import (
from ._logging import verbose_logger
from .caching.caching import disable_cache, enable_cache, update_cache
from .litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params, safe_deep_copy
from .litellm_core_utils.core_helpers import (
as_str_mapping,
drops_unsupported_params,
normalize_drop_params,
safe_deep_copy,
)
from .litellm_core_utils.fallback_utils import (
async_completion_with_fallbacks,
completion_with_fallbacks,
@ -4378,6 +4384,11 @@ def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResu
else None
),
converse_model,
strict_tools_fallback=(
partial(bedrock_strict_tools_fallback, converse_model, converse_params, dict(litellm_params))
if drops_unsupported_params(litellm_params.get("drop_params"))
else None
),
)
elif bedrock_route == "converse_like":
model = model.replace("converse_like/", "")

View file

@ -37504,7 +37504,6 @@
"input_cost_per_token": 5e-08,
"input_cost_per_token_batches": 2.5e-08,
"input_cost_per_token_flex": 2.5e-08,
"input_cost_per_token_priority": 2.5e-06,
"litellm_provider": "openai",
"max_input_tokens": 272000,
"max_output_tokens": 128000,
@ -37552,7 +37551,6 @@
"deprecation_date": "2026-12-11",
"input_cost_per_token": 5e-08,
"input_cost_per_token_batches": 2.5e-08,
"input_cost_per_token_priority": 2.5e-06,
"input_cost_per_token_flex": 2.5e-08,
"litellm_provider": "openai",
"max_input_tokens": 272000,
@ -69827,6 +69825,201 @@
"supports_reasoning": true,
"supports_vision": true
},
"aiand/deepseek-ai/deepseek-v4-flash": {
"cache_read_input_token_cost": 8e-08,
"input_cost_per_token": 1.5e-07,
"litellm_provider": "aiand",
"max_input_tokens": 1048576,
"max_output_tokens": 384000,
"mode": "chat",
"output_cost_per_token": 2.5e-07,
"source": "https://api.aiand.com/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": false
},
"aiand/deepseek-ai/deepseek-v4-pro": {
"cache_read_input_token_cost": 2.5e-07,
"input_cost_per_token": 1e-06,
"litellm_provider": "aiand",
"max_input_tokens": 1048576,
"max_output_tokens": 384000,
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"source": "https://api.aiand.com/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": false
},
"aiand/deepseek-ai/deepseek-v4.1-flash": {
"cache_read_input_token_cost": 2e-08,
"input_cost_per_token": 3e-07,
"litellm_provider": "aiand",
"max_input_tokens": 1048576,
"max_output_tokens": 384000,
"mode": "chat",
"output_cost_per_token": 6e-07,
"source": "https://api.aiand.com/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
},
"aiand/google/gemma-4-31b-it": {
"cache_read_input_token_cost": 5e-08,
"input_cost_per_token": 2e-07,
"litellm_provider": "aiand",
"max_input_tokens": 262144,
"max_output_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 5e-07,
"source": "https://api.aiand.com/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
},
"aiand/moonshotai/kimi-k2.7-code": {
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 7.5e-07,
"litellm_provider": "aiand",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 3.5e-06,
"source": "https://api.aiand.com/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
},
"aiand/moonshotai/kimi-k3": {
"cache_read_input_token_cost": 5e-07,
"input_cost_per_token": 3e-06,
"litellm_provider": "aiand",
"max_input_tokens": 1048576,
"max_output_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 1.25e-05,
"source": "https://api.aiand.com/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
},
"aiand/motif-technologies/motif-3": {
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 5e-07,
"litellm_provider": "aiand",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 2e-06,
"source": "https://api.aiand.com/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": false
},
"aiand/openai/gpt-oss-120b": {
"cache_read_input_token_cost": 8e-08,
"input_cost_per_token": 1.5e-07,
"litellm_provider": "aiand",
"max_input_tokens": 131072,
"max_output_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 6e-07,
"source": "https://api.aiand.com/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": false
},
"aiand/qwen/qwen3.6-27b": {
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 3.2e-07,
"litellm_provider": "aiand",
"max_input_tokens": 262144,
"max_output_tokens": 65536,
"mode": "chat",
"output_cost_per_token": 3.2e-06,
"source": "https://api.aiand.com/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
},
"aiand/qwen/qwen3.8-27b": {
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 4e-07,
"litellm_provider": "aiand",
"max_input_tokens": 262144,
"max_output_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 3e-06,
"source": "https://api.aiand.com/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
},
"aiand/zai-org/glm-5.2": {
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 1e-06,
"litellm_provider": "aiand",
"max_input_tokens": 1048576,
"max_output_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 4e-06,
"source": "https://api.aiand.com/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": false
},
"aiand/zai-org/glm-5.3": {
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 1e-06,
"litellm_provider": "aiand",
"max_input_tokens": 1048576,
"max_output_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 4e-06,
"source": "https://api.aiand.com/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": false
},
"aiand/zai-org/glm-5.3-flash": {
"cache_read_input_token_cost": 3e-08,
"input_cost_per_token": 1.5e-07,
"litellm_provider": "aiand",
"max_input_tokens": 1048550,
"max_output_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 5e-07,
"source": "https://api.aiand.com/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
},
"bespoke/nimble-latest": {
"input_cost_per_token": 0.0,
"litellm_provider": "bespoke",

View file

@ -189,6 +189,32 @@
"generateContent": false
}
},
"aiand": {
"display_name": "ai& (`aiand`)",
"url": "https://docs.aiand.com/api/chat-completions/",
"endpoints": {
"chat_completions": true,
"responses": true,
"text_completion": true,
"embeddings": false,
"image_generations": false,
"image_edits": false,
"audio_transcriptions": false,
"audio_speech": false,
"rerank": false,
"ocr": false,
"search": false,
"moderations": false,
"realtime": false,
"video_generations": false,
"batches": false,
"messages": true,
"interactions": true,
"systemone": false,
"a2a": true,
"generateContent": true
}
},
"amazon_nova": {
"display_name": "Amazon Nova (`amazon_nova`)",
"url": "https://docs.litellm.ai/docs/providers/amazon_nova",

View file

@ -50,7 +50,10 @@ from litellm.proxy.common_utils.openai_error_payload import (
with_litellm_call_id,
)
from litellm.proxy.common_utils.token_count_settings import count_source_fields
from litellm.proxy.litellm_pre_call_utils import should_auto_drop_params_for_agentic_cli
from litellm.proxy.litellm_pre_call_utils import (
forwarded_anthropic_oauth_token,
should_auto_drop_params_for_agentic_cli,
)
from litellm.router import Router
router: Final = APIRouter()
@ -412,6 +415,7 @@ async def count_tokens(
request=token_request,
call_endpoint=True,
messages_format="anthropic",
client_oauth_token=forwarded_anthropic_oauth_token(request.headers),
**_count_tokens_body_options(data, auto_drop_params),
)
except UnsupportedContentTypeError as e:

View file

@ -363,6 +363,8 @@ _UNTRUSTED_ROOT_CONTROL_FIELDS: Final = (
"fallback_depth",
"_target_order",
"attempted_targets",
"_excluded_deployment_ids",
"_retry_skipped_deployment_ids",
"proxy_server_request",
"standard_logging_object",
"secret_fields",
@ -444,6 +446,7 @@ _UNTRUSTED_METADATA_CONTROL_FIELDS: Final = (
"_pipeline_managed_guardrails",
"client_disconnected",
"error_information",
"_failover_excluded_ids",
PRE_CALL_EXECUTED_GUARDRAILS_KEY,
)
@ -1412,12 +1415,34 @@ def _tenant_service_name(user_api_key_dict: UserAPIKeyAuth) -> str | None:
)
def litellm_auth_header_name(headers: Headers) -> str:
if "x-litellm-api-key" in headers:
return "x-litellm-api-key"
if "authorization" in headers:
return "authorization"
return "x-api-key"
def forwarded_anthropic_oauth_token(headers: Headers) -> str | None:
from litellm.llms.anthropic.common_utils import is_anthropic_oauth_key
forwarded: Final = clean_headers(headers, authenticated_with_header=litellm_auth_header_name(headers))
return next(
(
value.removeprefix("Bearer ")
for header, value in forwarded.items()
if header.lower() == "authorization" and is_anthropic_oauth_key(value)
),
None,
)
def clean_headers(
headers: Headers,
litellm_key_header_name: str | None = None,
forward_llm_provider_auth_headers: bool = False,
authenticated_with_header: str | None = None,
) -> dict:
) -> dict[str, str]:
"""
Removes litellm api key from headers
@ -2510,26 +2535,13 @@ async def add_litellm_data_to_request(
forward_llm_auth = general_settings.get("forward_llm_provider_auth_headers", False)
if not forward_llm_auth:
forward_llm_auth = getattr(litellm, "forward_llm_provider_auth_headers", False)
# Determine which header was used for authentication
# This enables forwarding provider keys (e.g., x-api-key) when they weren't used for LiteLLM auth
authenticated_with_header = None
if "x-litellm-api-key" in request.headers:
# If x-litellm-api-key is present, it was used for auth
authenticated_with_header = "x-litellm-api-key"
elif "authorization" in request.headers:
# Authorization header was used for auth
authenticated_with_header = "authorization"
else:
# x-api-key or another header was used for auth
authenticated_with_header = "x-api-key"
_headers: Final[dict[str, str]] = clean_headers(
request.headers,
litellm_key_header_name=(
general_settings.get("litellm_key_header_name") if general_settings is not None else None
),
forward_llm_provider_auth_headers=forward_llm_auth,
authenticated_with_header=authenticated_with_header,
authenticated_with_header=litellm_auth_header_name(request.headers),
)
_strip_untrusted_request_header_controls(
_headers,

View file

@ -14628,6 +14628,7 @@ async def run_thread(
# )
# async def get_available_routes(user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth)):
from litellm.litellm_core_utils.tool_fields import DropPolicy, drop_policy
from litellm.llms.anthropic.count_tokens.token_counter import AnthropicTokenCounter
from litellm.llms.base_llm.base_utils import BaseTokenCounter
from litellm.llms.openai.responses.count_tokens.token_counter import OpenAITokenCounter
from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient
@ -14649,6 +14650,14 @@ from litellm.responses.litellm_completion_transformation.count_request import (
)
def _with_client_oauth_token(
counter: BaseTokenCounter | None, client_oauth_token: str | None
) -> BaseTokenCounter | None:
if client_oauth_token is None or not isinstance(counter, AnthropicTokenCounter):
return counter
return AnthropicTokenCounter(client_oauth_token=client_oauth_token)
def _get_provider_token_counter(
deployment: dict, model_to_use: str
) -> tuple[BaseTokenCounter | None, str | None, str | None]:
@ -15264,6 +15273,7 @@ async def count_request_tokens(
context_management: JsonValue = None,
request_litellm_params: Mapping[str, JsonValue] = MappingProxyType({}),
responses_request: Mapping[str, object] | None = None,
client_oauth_token: str | None = None,
) -> TokenCountResponse:
prompt: Final = request.prompt
messages: Final = request.messages
@ -15321,7 +15331,8 @@ async def count_request_tokens(
custom_llm_provider: str | None = None
if call_endpoint is True and deployment is not None:
# Auto-route to the correct provider based on model
provider_counter, _model, custom_llm_provider = _get_provider_token_counter(deployment, model_to_use)
routed_counter, _model, custom_llm_provider = _get_provider_token_counter(deployment, model_to_use)
provider_counter = _with_client_oauth_token(routed_counter, client_oauth_token)
if _model is not None:
model_to_use = _model

View file

@ -1,4 +1,32 @@
[
{
"provider": "AIAND",
"provider_display_name": "ai&",
"litellm_provider": "aiand",
"credential_fields": [
{
"key": "api_base",
"label": "API Base",
"placeholder": "https://api.aiand.com/v1",
"tooltip": null,
"required": false,
"field_type": "text",
"options": null,
"default_value": null
},
{
"key": "api_key",
"label": "API Key",
"placeholder": null,
"tooltip": null,
"required": true,
"field_type": "password",
"options": null,
"default_value": null
}
],
"default_model_placeholder": "aiand/openai/gpt-oss-120b"
},
{
"provider": "SCALEDOWN",
"provider_display_name": "ScaleDown",

View file

@ -212,6 +212,7 @@ from litellm.router_utils.clientside_credential_handler import (
)
from litellm.router_utils.common_utils import (
MID_STREAM_CONTINUATION_KWARG,
as_retry_skipped_deployment_ids,
deployments_the_caller_may_use,
filter_team_based_models,
filter_web_search_deployments,
@ -238,11 +239,12 @@ from litellm.router_utils.cooldown_handlers import (
async_get_cooldown_deployments_with_debug_info,
get_cooldown_deployment_info,
get_cooldown_deployments,
is_advisor_orchestration_failure,
is_background_response_poll_not_found,
is_caller_scoped_auth_failure,
is_caller_timeout_408,
is_failure_outside_selected_deployment,
is_video_lookup_not_found,
mark_deployment_selection_failure,
set_cooldown_deployments,
)
from litellm.router_utils.deployment_pricing import deployment_pricing_models
@ -684,10 +686,6 @@ def _configured_model_info(deployment: Mapping[str, object]) -> Mapping[str, obj
return _MODEL_INFO_ADAPTER.validate_python(deployment.get("model_info") or {})
def _as_retry_skipped_deployment_ids(value: object) -> tuple[str, ...]:
return tuple(item for item in value if isinstance(item, str)) if isinstance(value, tuple) else ()
def _silent_experiment_targets(silent_model: object) -> tuple[str, ...]:
if silent_model is None:
return ()
@ -1519,6 +1517,7 @@ class Router:
self.allowed_fails = allowed_fails
else:
self.allowed_fails = litellm.allowed_fails
self.explicit_allowed_fails = allowed_fails
self.cooldown_time = cooldown_time or DEFAULT_COOLDOWN_TIME_SECONDS
self.cooldown_cache = CooldownCache(cache=self.cache, default_cooldown_time=self.cooldown_time)
self.disable_cooldowns = disable_cooldowns
@ -8940,7 +8939,7 @@ class Router:
return ()
if litellm.should_retry(status_code):
return ()
already_skipped_ids: Final = _as_retry_skipped_deployment_ids(already_skipped)
already_skipped_ids: Final = as_retry_skipped_deployment_ids(already_skipped)
skipped: Final = tuple(sorted(frozenset((*already_skipped_ids, failed_deployment_id))))
verbose_router_logger.debug(
"Retry skips deployments that already answered %s to this request: %s", status_code, skipped
@ -9579,10 +9578,10 @@ class Router:
try:
exception: Final = kwargs.get("exception", None)
if is_advisor_orchestration_failure(exception):
if is_failure_outside_selected_deployment(exception):
verbose_router_logger.debug(
"Router: Exiting 'deployment_callback_on_failure' without cooldown. "
"Failure originated from advisor orchestration, not the selected deployment."
"Failure came from advisor orchestration or deployment selection, not the selected deployment."
)
return False
@ -15184,7 +15183,7 @@ class Router:
## RETRY SKIP ## -> drop deployments that already refused this request with a
## non-retryable status, unless that leaves nothing, so the caller still gets
## the provider's own error instead of a no-deployments error.
_retry_skipped_deployment_ids: Final = _as_retry_skipped_deployment_ids(
_retry_skipped_deployment_ids: Final = as_retry_skipped_deployment_ids(
request_kwargs.pop("_retry_skipped_deployment_ids", None) if request_kwargs else None
)
healthy_deployments = (
@ -15473,6 +15472,7 @@ class Router:
phase_event(_DEPLOYMENT_SELECTED_EVENT, pick_attributes)
return deployment
except Exception as e:
mark_deployment_selection_failure(e)
traceback_exception: Final = traceback.format_exc()
# if router rejects call -> log to langfuse/otel/etc.
if request_kwargs is not None:
@ -15641,6 +15641,7 @@ class Router:
return deployment
except Exception as e:
mark_deployment_selection_failure(e)
traceback_exception: Final = traceback.format_exc()
if request_kwargs is not None:
logging_obj: Final = request_kwargs.get("litellm_logging_obj", None)
@ -16414,7 +16415,7 @@ class Router:
)
## RETRY SKIP ## -> see async counterpart in async_get_healthy_deployments.
_retry_skipped_deployment_ids: Final = _as_retry_skipped_deployment_ids(
_retry_skipped_deployment_ids: Final = as_retry_skipped_deployment_ids(
request_kwargs.pop("_retry_skipped_deployment_ids", None) if request_kwargs else None
)
healthy_deployments = (

View file

@ -4,7 +4,7 @@ from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, TypeVar
from pydantic import TypeAdapter
from pydantic import TypeAdapter, ValidationError
if TYPE_CHECKING:
from litellm.types.llms.openai import OpenAIFileObject
@ -38,6 +38,7 @@ _OBJECT_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[object, object]]] = TypeAdapt
_DEPLOYMENT_ROWS_ADAPTER: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter(
tuple[Mapping[str, object], ...]
)
_DEPLOYMENT_IDS_ADAPTER: Final = TypeAdapter(tuple[str, ...])
def with_admin_api_key_secret_reference(deployment: Deployment) -> Deployment:
@ -57,6 +58,31 @@ def without_router_only_kwargs(kwargs: Mapping[str, _V]) -> dict[str, _V]:
return {key: value for key, value in kwargs.items() if key not in ROUTER_ONLY_CALL_KWARGS}
def as_retry_skipped_deployment_ids(value: object) -> tuple[str, ...]:
if not isinstance(value, tuple):
return ()
items: Final[tuple[object, ...]] = value
return tuple(item for item in items if isinstance(item, str))
def as_excluded_deployment_ids(value: object) -> tuple[str, ...]:
if value is None:
return ()
try:
return _DEPLOYMENT_IDS_ADAPTER.validate_python(value)
except ValidationError:
return ()
def deployment_ids_failed_in_request(request_kwargs: Mapping[str, object]) -> frozenset[str]:
return frozenset(
(
*as_excluded_deployment_ids(request_kwargs.get("_excluded_deployment_ids")),
*as_retry_skipped_deployment_ids(request_kwargs.get("_retry_skipped_deployment_ids")),
)
)
def request_user_api_key_auth(request_kwargs: Mapping[str, object]) -> object:
"""The caller's auth object, from whichever metadata bucket this surface writes to."""
metadata_value: Final = request_kwargs.get("metadata")

View file

@ -57,6 +57,7 @@ else:
Span = Any
_ADVISOR_ORCHESTRATION_FAILURE_ATTR: Final = "_litellm_advisor_orchestration_failure"
_DEPLOYMENT_SELECTION_FAILURE_ATTR: Final = "_litellm_deployment_selection_failure"
_CREDENTIAL_VALUES_ADAPTER: Final = TypeAdapter(Mapping[str, object])
@ -79,6 +80,16 @@ def is_advisor_orchestration_failure(exception: BaseException | None) -> bool:
return bool(getattr(exception, _ADVISOR_ORCHESTRATION_FAILURE_ATTR, False))
def mark_deployment_selection_failure(exception: BaseException) -> None:
setattr(exception, _DEPLOYMENT_SELECTION_FAILURE_ATTR, True)
def is_failure_outside_selected_deployment(exception: BaseException | None) -> bool:
return is_advisor_orchestration_failure(exception) or bool(
getattr(exception, _DEPLOYMENT_SELECTION_FAILURE_ATTR, False)
)
_BACKGROUND_RESPONSE_POLL_CALL_ORIGINS: Final[frozenset[InternalCallOrigin]] = frozenset(
{BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, BACKGROUND_RESPONSE_STATUS_POLL_CALL_ORIGIN}
)
@ -781,14 +792,17 @@ def _is_allowed_fails_set_on_router(
litellm_router_instance: LitellmRouter,
) -> bool:
"""
Check if Router.allowed_fails is set or is Non-default Value
Check if Router.allowed_fails was passed to the Router or is a Non-default Value
Returns:
- True if Router.allowed_fails is set or is Non-default Value
- False if Router.allowed_fails is None or is Default Value
- True if allowed_fails was passed to the Router, even when it equals the default
- True if Router.allowed_fails is a Non-default Value
- False if Router.allowed_fails is None, or is the Default Value and was not passed
"""
if litellm_router_instance.allowed_fails is None:
return False
if litellm_router_instance.explicit_allowed_fails is not None:
return True
if litellm_router_instance.allowed_fails != litellm.allowed_fails:
return True
return False

View file

@ -31,6 +31,7 @@ from litellm.caching.dual_cache import DualCache
from litellm.constants import SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY
from litellm.integrations.custom_logger import CustomLogger, Span
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.router_utils.common_utils import deployment_ids_failed_in_request
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import CallTypes
@ -429,6 +430,9 @@ class DeploymentAffinityCheck(CustomLogger):
session_affinity_active: Final = (
enable_session_id or self._get_marker_session_affinity_ttl(request_kwargs=request_kwargs) is not None
)
failed_deployment_ids: Final = deployment_ids_failed_in_request(
cast(Mapping[str, object], request_kwargs) # cast-ok: request kwargs keys are param names
)
user_key: Final = (
self.get_user_key_from_request_kwargs(request_kwargs=request_kwargs)
if (session_affinity_active or enable_user_key)
@ -458,7 +462,7 @@ class DeploymentAffinityCheck(CustomLogger):
elif isinstance(session_cache_result, str):
session_model_id = session_cache_result
if session_model_id:
if session_model_id and session_model_id not in failed_deployment_ids:
session_deployment: Final = self._find_deployment_by_model_id(
healthy_deployments=typed_healthy_deployments,
model_id=session_model_id,
@ -493,7 +497,7 @@ class DeploymentAffinityCheck(CustomLogger):
# Backwards / safety: allow raw string values.
model_id = cache_result
if not model_id:
if not model_id or model_id in failed_deployment_ids:
return typed_healthy_deployments
deployment = self._find_deployment_by_model_id(

View file

@ -51,6 +51,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
strip_encrypted_reasoning_from_messages,
)
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.router_utils.common_utils import as_excluded_deployment_ids, as_retry_skipped_deployment_ids
from litellm.types.llms.openai import AllMessageValues
from litellm.types.router import Deployment
from litellm.utils import get_order_filtered_deployments
@ -182,6 +183,14 @@ class EncryptedContentAffinityCheck(CustomLogger):
None,
)
@staticmethod
def _deployment_model_id(deployment: Mapping[str, object]) -> str | None:
model_info: Final = deployment.get("model_info")
if not isinstance(model_info, Mapping):
return None
deployment_id: Final = cast(Mapping[str, object], model_info).get("id") # cast-ok: narrowed by isinstance
return None if deployment_id is None else str(deployment_id)
@staticmethod
def _find_deployment_by_model_id(healthy_deployments: list[dict], model_id: str) -> dict | None:
for deployment in healthy_deployments:
@ -345,7 +354,8 @@ class EncryptedContentAffinityCheck(CustomLogger):
typed_healthy_deployments: Final = cast(list[dict[str, object]], healthy_deployments)
if not self._is_enabled_for_model_group(model):
return typed_healthy_deployments
target_order: Final = _REQUEST_KWARGS_ADAPTER.validate_python(request_kwargs).get("_target_order")
typed_request_kwargs: Final = _REQUEST_KWARGS_ADAPTER.validate_python(request_kwargs)
target_order: Final = typed_request_kwargs.get("_target_order")
candidates: Final = (
get_order_filtered_deployments(typed_healthy_deployments, target_order=target_order)
if isinstance(target_order, int)
@ -375,8 +385,31 @@ class EncryptedContentAffinityCheck(CustomLogger):
model_id,
)
previous_response_id: Final = typed_request_kwargs.get("previous_response_id")
previous_response_origin: Final = ResponsesAPIRequestUtils.get_model_id_from_response_id(
None if previous_response_id is None else str(previous_response_id)
)
previous_response_binds_origin: Final = previous_response_origin == model_id
excluded_ids: Final[frozenset[str]] = (
frozenset()
if previous_response_binds_origin
else frozenset(as_excluded_deployment_ids(typed_request_kwargs.get("_excluded_deployment_ids")))
)
retry_skipped_ids: Final[frozenset[str]] = (
frozenset()
if previous_response_binds_origin
else frozenset(as_retry_skipped_deployment_ids(typed_request_kwargs.get("_retry_skipped_deployment_ids")))
)
not_excluded_candidates: Final = [
candidate for candidate in candidates if self._deployment_model_id(candidate) not in excluded_ids
]
untried_candidates: Final = [
candidate
for candidate in not_excluded_candidates
if self._deployment_model_id(candidate) not in retry_skipped_ids
] or not_excluded_candidates
deployment: Final = self._find_deployment_by_model_id(
healthy_deployments=candidates,
healthy_deployments=untried_candidates,
model_id=model_id,
)
if deployment is not None:
@ -392,7 +425,7 @@ class EncryptedContentAffinityCheck(CustomLogger):
# Follow-up switched model_name (LIT-2531): pin by Azure resource instead.
boundary_matches, _originating = self._find_deployments_on_same_encryption_boundary(
healthy_deployments=candidates,
healthy_deployments=untried_candidates,
model_id=model_id,
)
if boundary_matches:

View file

@ -19,6 +19,7 @@ from litellm.integrations.anthropic_cache_control_hook import (
from litellm.integrations.custom_logger import CustomLogger, Span
from litellm.litellm_core_utils.token_counter import offload_token_count
from litellm.llms.bedrock.common_utils import bedrock_cache_point_ttl_supported, bedrock_capability_model
from litellm.router_utils.common_utils import deployment_ids_failed_in_request
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam
from litellm.types.utils import CallTypes, StandardLoggingPayload
from litellm.utils import (
@ -152,7 +153,9 @@ class PromptCachingDeploymentCheck(CustomLogger):
request_tools,
),
)
if model_id_dict is not None:
if model_id_dict is not None and model_id_dict["model_id"] not in deployment_ids_failed_in_request(
request_data or {}
):
model_id: Final = model_id_dict["model_id"]
for deployment in healthy_deployments:
if deployment["model_info"]["id"] == model_id:

View file

@ -4574,6 +4574,7 @@ class LlmProviders(str, Enum):
POE = "poe"
CHUTES = "chutes"
NEOSANTARA = "neosantara"
AIAND = "aiand"
PARASAIL = "parasail"
XIAOMI_MIMO = "xiaomi_mimo"
TENSORMESH = "tensormesh"

View file

@ -37504,7 +37504,6 @@
"input_cost_per_token": 5e-08,
"input_cost_per_token_batches": 2.5e-08,
"input_cost_per_token_flex": 2.5e-08,
"input_cost_per_token_priority": 2.5e-06,
"litellm_provider": "openai",
"max_input_tokens": 272000,
"max_output_tokens": 128000,
@ -37552,7 +37551,6 @@
"deprecation_date": "2026-12-11",
"input_cost_per_token": 5e-08,
"input_cost_per_token_batches": 2.5e-08,
"input_cost_per_token_priority": 2.5e-06,
"input_cost_per_token_flex": 2.5e-08,
"litellm_provider": "openai",
"max_input_tokens": 272000,
@ -69827,6 +69825,201 @@
"supports_reasoning": true,
"supports_vision": true
},
"aiand/deepseek-ai/deepseek-v4-flash": {
"cache_read_input_token_cost": 8e-08,
"input_cost_per_token": 1.5e-07,
"litellm_provider": "aiand",
"max_input_tokens": 1048576,
"max_output_tokens": 384000,
"mode": "chat",
"output_cost_per_token": 2.5e-07,
"source": "https://api.aiand.com/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": false
},
"aiand/deepseek-ai/deepseek-v4-pro": {
"cache_read_input_token_cost": 2.5e-07,
"input_cost_per_token": 1e-06,
"litellm_provider": "aiand",
"max_input_tokens": 1048576,
"max_output_tokens": 384000,
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"source": "https://api.aiand.com/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": false
},
"aiand/deepseek-ai/deepseek-v4.1-flash": {
"cache_read_input_token_cost": 2e-08,
"input_cost_per_token": 3e-07,
"litellm_provider": "aiand",
"max_input_tokens": 1048576,
"max_output_tokens": 384000,
"mode": "chat",
"output_cost_per_token": 6e-07,
"source": "https://api.aiand.com/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
},
"aiand/google/gemma-4-31b-it": {
"cache_read_input_token_cost": 5e-08,
"input_cost_per_token": 2e-07,
"litellm_provider": "aiand",
"max_input_tokens": 262144,
"max_output_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 5e-07,
"source": "https://api.aiand.com/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
},
"aiand/moonshotai/kimi-k2.7-code": {
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 7.5e-07,
"litellm_provider": "aiand",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 3.5e-06,
"source": "https://api.aiand.com/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
},
"aiand/moonshotai/kimi-k3": {
"cache_read_input_token_cost": 5e-07,
"input_cost_per_token": 3e-06,
"litellm_provider": "aiand",
"max_input_tokens": 1048576,
"max_output_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 1.25e-05,
"source": "https://api.aiand.com/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
},
"aiand/motif-technologies/motif-3": {
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 5e-07,
"litellm_provider": "aiand",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 2e-06,
"source": "https://api.aiand.com/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": false
},
"aiand/openai/gpt-oss-120b": {
"cache_read_input_token_cost": 8e-08,
"input_cost_per_token": 1.5e-07,
"litellm_provider": "aiand",
"max_input_tokens": 131072,
"max_output_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 6e-07,
"source": "https://api.aiand.com/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": false
},
"aiand/qwen/qwen3.6-27b": {
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 3.2e-07,
"litellm_provider": "aiand",
"max_input_tokens": 262144,
"max_output_tokens": 65536,
"mode": "chat",
"output_cost_per_token": 3.2e-06,
"source": "https://api.aiand.com/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
},
"aiand/qwen/qwen3.8-27b": {
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 4e-07,
"litellm_provider": "aiand",
"max_input_tokens": 262144,
"max_output_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 3e-06,
"source": "https://api.aiand.com/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
},
"aiand/zai-org/glm-5.2": {
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 1e-06,
"litellm_provider": "aiand",
"max_input_tokens": 1048576,
"max_output_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 4e-06,
"source": "https://api.aiand.com/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": false
},
"aiand/zai-org/glm-5.3": {
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 1e-06,
"litellm_provider": "aiand",
"max_input_tokens": 1048576,
"max_output_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 4e-06,
"source": "https://api.aiand.com/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": false
},
"aiand/zai-org/glm-5.3-flash": {
"cache_read_input_token_cost": 3e-08,
"input_cost_per_token": 1.5e-07,
"litellm_provider": "aiand",
"max_input_tokens": 1048550,
"max_output_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 5e-07,
"source": "https://api.aiand.com/v1/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true
},
"bespoke/nimble-latest": {
"input_cost_per_token": 0.0,
"litellm_provider": "bespoke",

View file

@ -189,6 +189,32 @@
"generateContent": false
}
},
"aiand": {
"display_name": "ai& (`aiand`)",
"url": "https://docs.aiand.com/api/chat-completions/",
"endpoints": {
"chat_completions": true,
"responses": true,
"text_completion": true,
"embeddings": false,
"image_generations": false,
"image_edits": false,
"audio_transcriptions": false,
"audio_speech": false,
"rerank": false,
"ocr": false,
"search": false,
"moderations": false,
"realtime": false,
"video_generations": false,
"batches": false,
"messages": true,
"interactions": true,
"systemone": false,
"a2a": true,
"generateContent": true
}
},
"amazon_nova": {
"display_name": "Amazon Nova (`amazon_nova`)",
"url": "https://docs.litellm.ai/docs/providers/amazon_nova",

View file

@ -0,0 +1,392 @@
from __future__ import annotations
import gzip
import json
import os
import secrets
import threading
import uuid
from collections.abc import Callable, Iterator, Mapping
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from functools import partial
from pathlib import Path
from types import MappingProxyType
from typing import Final
import httpx
import pytest
import yaml
from integration._support.budget import case_timeout_seconds
from integration._support.client import (
JSON_OBJECT,
Gateway,
eventually,
gateway_from_environment,
list_value,
object_value,
string_value,
)
from integration._support.process import graceful_stop_seconds, owned_proxy_process
from integration._support.wire import Reply, Request, Wire, wire_server
from pydantic import JsonValue
MIB: Final = 1024 * 1024
CHAT: Final = "/v1/chat/completions"
CHAT_MODEL: Final = "compressed-chat"
OTHER_MODEL: Final = "compressed-chat-other"
ANTHROPIC_MODEL: Final = "claude-haiku-5-5"
GATED_PREFIX: Final = "gated-"
ORIGIN: Final = "http://browser.example"
MIDDLEWARE_HEADERS: Final = (
"access-control-allow-origin",
"x-frame-options",
"content-security-policy",
"x-content-type-options",
)
ZSTD_MAGIC: Final = b"\x28\xb5\x2f\xfd"
ZSTD_SINGLE_SEGMENT_FOUR_BYTE_SIZE: Final = b"\xa0"
ZSTD_MAX_BLOCK: Final = 128 * 1024
NO_HEADERS: Final[Mapping[str, str]] = MappingProxyType({})
pytestmark: Final = [
pytest.mark.timeout(2 * graceful_stop_seconds() + 120),
pytest.mark.xdist_group("compressed-request-bodies"),
]
def zstd_frame(data: bytes) -> bytes:
blocks: Final = tuple(data[start : start + ZSTD_MAX_BLOCK] for start in range(0, len(data), ZSTD_MAX_BLOCK))
framed: Final = blocks or (b"",)
return (
ZSTD_MAGIC
+ ZSTD_SINGLE_SEGMENT_FOUR_BYTE_SIZE
+ len(data).to_bytes(4, "little")
+ b"".join(
(len(block) << 3 | (index == len(framed) - 1)).to_bytes(3, "little") + block
for index, block in enumerate(framed)
)
)
def gzip_body(data: bytes) -> bytes:
return gzip.compress(data, mtime=0)
def as_sent(data: bytes) -> bytes:
return data
@dataclass(frozen=True, slots=True)
class Rig:
gateway: Gateway
upstream: Wire
gate: threading.Event
limited_key: str
def calls(self) -> tuple[Request, ...]:
return tuple(request for request in self.upstream.drain() if request.method == "POST")
def _chat_completion() -> bytes:
return json.dumps(
{
"id": f"chatcmpl-{uuid.uuid4().hex}",
"object": "chat.completion",
"created": 1,
"model": "gpt-4o-mini",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "scripted"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5},
}
).encode()
def _stream_frame(identity: str, delta: Mapping[str, JsonValue], finish_reason: str | None) -> bytes:
chunk: Final = {
"id": identity,
"object": "chat.completion.chunk",
"created": 1,
"model": "gpt-4o-mini",
"choices": [{"index": 0, "delta": dict(delta), "finish_reason": finish_reason}],
}
return b"data: " + json.dumps(chunk).encode() + b"\n\n"
def _chat_stream(gate: threading.Event | None) -> Reply:
identity: Final = f"chatcmpl-{uuid.uuid4().hex}"
return Reply(
content_type="text/event-stream",
chunks=(
_stream_frame(identity, {"role": "assistant", "content": "streamed "}, None),
_stream_frame(identity, {"content": "answer"}, "stop"),
b"data: [DONE]\n\n",
),
gate_after_first=gate,
)
def _anthropic_message() -> bytes:
return json.dumps(
{
"id": f"msg_{uuid.uuid4().hex}",
"type": "message",
"role": "assistant",
"model": ANTHROPIC_MODEL,
"content": [{"type": "text", "text": "scripted"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 3, "output_tokens": 2},
}
).encode()
def _respond(gate: threading.Event, request: Request) -> Reply:
if request.method == "GET":
return Reply(body=b'{"object":"list","data":[]}')
if request.target == "/v1/messages":
return Reply(body=_anthropic_message())
body: Final = JSON_OBJECT.validate_json(request.body)
if body.get("stream") is not True:
return Reply(body=_chat_completion())
text: Final = string_value(object_value(list_value(body["messages"])[0])["content"])
return _chat_stream(gate if text.startswith(GATED_PREFIX) else None)
def _deployment(name: str, upstream: Wire) -> Mapping[str, JsonValue]:
return {
"model_name": name,
"litellm_params": {"model": "openai/gpt-4o-mini", "api_base": f"{upstream.url}/v1", "api_key": "sk-fixture"},
}
def _config(directory: Path, upstream: Wire) -> Path:
base: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
configuration: Final = {
**base,
"model_list": [_deployment(CHAT_MODEL, upstream), _deployment(OTHER_MODEL, upstream)],
"general_settings": {
**object_value(base.get("general_settings", {})),
"max_request_size_mb": 1,
"enable_claude_code_gateway": True,
},
"router_settings": {**object_value(base.get("router_settings", {})), "num_retries": 0},
}
path: Final = directory / "compressed-request-bodies.yaml"
path.write_text(yaml.safe_dump(configuration))
return path
@pytest.fixture(scope="module")
def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]:
assert os.environ.get("LITELLM_LICENSE"), (
"LITELLM_LICENSE must be set: max_request_size_mb is an enterprise feature"
)
gate: Final = threading.Event()
directory: Final = tmp_path_factory.mktemp("compressed-request-bodies")
with gateway_from_environment() as root, wire_server(partial(_respond, gate)) as upstream:
overrides: Final = {
"ANTHROPIC_API_BASE": upstream.url,
"ANTHROPIC_API_KEY": "sk-ant-synthetic",
"MAX_DECOMPRESSED_REQUEST_BODIES_IN_FLIGHT_MB": "1",
}
with (
owned_proxy_process(root, directory, overrides, config=_config(directory, upstream), workers=1) as owned,
owned.gateway.scenario() as scenario,
):
yield Rig(owned.gateway, upstream, gate, scenario.key(models=[CHAT_MODEL]))
@pytest.fixture(autouse=True)
def empty_upstream_log(rig: Rig) -> None:
rig.upstream.drain()
def _headers(key: str, coding: str | None, extra: Mapping[str, str] = NO_HEADERS) -> Mapping[str, str]:
return {
"Authorization": f"Bearer {key}",
"content-type": "application/json",
**({"content-encoding": coding} if coding is not None else {}),
**extra,
}
def _send(
rig: Rig,
data: bytes,
coding: str | None,
*,
key: str | None = None,
extra: Mapping[str, str] = NO_HEADERS,
) -> httpx.Response:
return rig.gateway.client.post(
CHAT,
content=data,
headers=_headers(rig.gateway.key if key is None else key, coding, extra),
timeout=case_timeout_seconds(),
)
def _chat_body(model: str, text: str, *, stream: bool = False) -> bytes:
return json.dumps(
{
"model": model,
"messages": [{"role": "user", "content": text}],
"stream": stream,
"cache": {"no-cache": True},
}
).encode()
def _error_type(response: httpx.Response) -> JsonValue:
return object_value(JSON_OBJECT.validate_json(response.content)["error"])["type"]
def _answer(response: httpx.Response) -> JsonValue:
choice: Final = object_value(list_value(JSON_OBJECT.validate_json(response.content)["choices"])[0])
return object_value(choice["message"])["content"]
def _stream(gateway: Gateway, data: bytes, first_text: threading.Event) -> tuple[int, str]:
with (
httpx.Client(base_url=gateway.client.base_url, timeout=case_timeout_seconds(), trust_env=False) as client,
client.stream("POST", CHAT, content=data, headers=_headers(gateway.key, "gzip")) as response,
):
pieces: Final = response.iter_text()
first: Final = next(pieces, "")
first_text.set()
return response.status_code, first + "".join(pieces)
CODINGS: Final = (
pytest.param("zstd", zstd_frame, id="zstd"),
pytest.param("gzip", gzip_body, id="gzip"),
pytest.param("gzip", as_sent, id="plain-labeled-gzip"),
)
@pytest.mark.parametrize(("coding", "encode"), CODINGS)
def test_a_compressed_chat_body_reaches_the_provider_as_its_plain_twin(
rig: Rig, coding: str, encode: Callable[[bytes], bytes]
) -> None:
body: Final = _chat_body(CHAT_MODEL, f"twin {uuid.uuid4().hex}")
plain: Final = _send(rig, body, None)
compressed: Final = _send(rig, encode(body), coding)
assert plain.status_code == 200, plain.text
assert compressed.status_code == 200, compressed.text
assert _answer(compressed) == _answer(plain)
assert compressed.headers["x-litellm-response-cost"] == plain.headers["x-litellm-response-cost"]
calls: Final = rig.calls()
assert len(calls) == 2, [call.target for call in calls]
assert JSON_OBJECT.validate_json(calls[1].body) == JSON_OBJECT.validate_json(calls[0].body)
assert "content-encoding" not in calls[1].headers, calls[1].headers
def test_auth_reads_the_model_from_a_compressed_body(rig: Rig) -> None:
refused: Final = _send(rig, zstd_frame(_chat_body(OTHER_MODEL, "outside the key")), "zstd", key=rig.limited_key)
assert refused.status_code == 403, refused.text
assert _error_type(refused) == "key_model_access_denied", refused.text
assert rig.calls() == ()
served: Final = _send(rig, zstd_frame(_chat_body(CHAT_MODEL, "inside the key")), "zstd", key=rig.limited_key)
assert served.status_code == 200, served.text
assert len(rig.calls()) == 1
@pytest.mark.parametrize(
("coding", "encode"),
(pytest.param("zstd", zstd_frame, id="zstd"), pytest.param("gzip", as_sent, id="plain-labeled-gzip")),
)
def test_a_compressed_pass_through_body_reaches_the_vendor_decoded(
rig: Rig, coding: str, encode: Callable[[bytes], bytes]
) -> None:
body: Final = json.dumps(
{
"model": ANTHROPIC_MODEL,
"max_tokens": 8,
"messages": [{"role": "user", "content": f"pass through {uuid.uuid4().hex}"}],
}
).encode()
response: Final = rig.gateway.client.post(
"/anthropic/v1/messages",
content=encode(body),
headers={
"x-litellm-api-key": rig.gateway.key,
"anthropic-version": "2023-06-01",
"content-type": "application/json",
"content-encoding": coding,
},
timeout=case_timeout_seconds(),
)
assert response.status_code == 200, response.text
calls: Final = rig.calls()
assert [call.target for call in calls] == ["/v1/messages"]
assert JSON_OBJECT.validate_json(calls[0].body) == JSON_OBJECT.validate_json(body)
assert "content-encoding" not in calls[0].headers, calls[0].headers
def test_a_gzip_bomb_gets_a_413_with_the_headers_other_errors_carry(rig: Rig) -> None:
origin: Final = {"origin": ORIGIN}
rejected: Final = _send(rig, gzip_body(b"\0" * 800_000), "gzip", extra=origin)
unauthorized: Final = _send(
rig, _chat_body(CHAT_MODEL, "unknown key"), None, key=f"sk-unknown-{uuid.uuid4().hex}", extra=origin
)
assert rejected.status_code == 413, rejected.text
assert unauthorized.status_code == 401, unauthorized.text
assert unauthorized.headers.get("access-control-allow-origin") is not None, dict(unauthorized.headers)
assert {name: rejected.headers.get(name) for name in MIDDLEWARE_HEADERS} == {
name: unauthorized.headers.get(name) for name in MIDDLEWARE_HEADERS
}
assert rig.calls() == ()
def test_the_operator_size_limit_holds_a_compressed_body_to_its_plain_twins_limit(rig: Rig) -> None:
over: Final = _chat_body(CHAT_MODEL, secrets.token_hex(600_000))
compressed_over: Final = gzip_body(over)
assert len(compressed_over) < MIB < len(over)
plain: Final = _send(rig, over, None)
rejected: Final = _send(rig, compressed_over, "gzip")
served: Final = _send(rig, gzip_body(_chat_body(CHAT_MODEL, secrets.token_hex(300_000))), "gzip")
assert plain.status_code == 413, plain.text
assert rejected.status_code == plain.status_code, rejected.text
assert served.status_code == 200, served.text
assert len(rig.calls()) == 1
def test_the_decoded_body_budget_turns_away_a_second_body_until_the_stream_ends(rig: Rig) -> None:
held: Final = gzip_body(_chat_body(CHAT_MODEL, GATED_PREFIX + secrets.token_hex(300_000), stream=True))
second: Final = gzip_body(_chat_body(CHAT_MODEL, secrets.token_hex(300_000)))
keyless: Final = _send(rig, second, "gzip", key=f"sk-unknown-{uuid.uuid4().hex}")
assert keyless.status_code == 401, keyless.text
first_text: Final = threading.Event()
with ThreadPoolExecutor(1) as pool:
streaming: Final = pool.submit(_stream, rig.gateway, held, first_text)
try:
eventually(lambda: first_text.is_set() or streaming.done(), bool, seconds=case_timeout_seconds())
reached: Final = rig.calls()
turned_away: Final = _send(rig, second, "gzip")
finally:
rig.gate.set()
status, streamed = streaming.result(timeout=case_timeout_seconds())
assert len(reached) == 1, streamed
assert turned_away.status_code == 503, turned_away.text
assert _error_type(turned_away) == "overloaded_error", turned_away.text
assert turned_away.headers.get("retry-after") == "1", dict(turned_away.headers)
assert status == 200, streamed
assert "streamed " in streamed and "answer" in streamed and "data: [DONE]" in streamed, streamed
assert rig.calls() == ()
after_the_stream: Final = _send(rig, second, "gzip")
assert after_the_stream.status_code == 200, after_the_stream.text
assert len(rig.calls()) == 1
def test_a_claude_code_otlp_export_keeps_a_two_member_gzip_body(rig: Rig) -> None:
response: Final = rig.gateway.client.post(
"/claude_code_gateway/v1/metrics",
content=gzip_body(b"first member") + gzip_body(b"second member"),
headers={
"Authorization": f"Bearer {rig.gateway.key}",
"content-type": "application/x-protobuf",
"content-encoding": "gzip",
},
timeout=case_timeout_seconds(),
)
assert response.status_code == 200, response.text

View file

@ -13,6 +13,7 @@ from pydantic import JsonValue
COUNTED_TOKENS: Final = 7
RELEVANCE: Final = 0.75
AZURE_DEPLOYMENT: Final = "gpt-url-join"
_USAGE: Final = {"input_tokens": 3, "output_tokens": 2}
@ -146,12 +147,20 @@ def _cohere_rerank(request: Request) -> Reply:
)
def _titan_embedding(_: Request) -> Reply:
return _json({"embedding": list(VECTOR), "inputTextTokenCount": 2})
_BY_PREFIX: Final[Mapping[str, Callable[[Request], Reply]]] = MappingProxyType(
{
"an": _anthropic,
"bi": lambda request: _json(anthropic_message(marker_of(request))),
"bc": _bedrock_converse,
"be": _titan_embedding,
"gm": _gemini,
"vx": _gemini,
"fd": _anthropic,
"az": _openai_like(AZURE_DEPLOYMENT),
"mi": _openai_like("mistral-small-latest"),
"db": _openai_like("databricks-meta-llama-3-3-70b-instruct"),
"co": _cohere_rerank,

View file

@ -1,6 +1,7 @@
import os
from collections.abc import Callable, Iterator
from pathlib import Path
from types import MappingProxyType
from typing import Final
import pytest
@ -8,6 +9,7 @@ import yaml
from integration._support.client import JSON_OBJECT, Gateway, gateway_from_environment
from integration._support.forward_proxy import ForwardProxy, refusing_forward_proxy
from integration._support.process import graceful_stop_seconds, owned_proxy
from integration._support.vertex import service_account_json
from integration.providers._api_base_trailing_slash_support import KEY, new_marker
from pydantic import JsonValue
@ -17,6 +19,12 @@ _RERANK: Final = "host-rerank"
_AZURE_CLAUDE: Final = "host-azure-claude"
_AZURE_COUNT: Final = "host-azure-count"
_EMPTY_BASE: Final = "host-empty-base"
_VERTEX_HOST: Final = "vertex-gateway"
_VERTEX_PROJECT: Final = "url-join-project"
_VERTEX_LOCATION: Final = "us-east5"
_VERTEX_CLAUDE_BASES: Final = MappingProxyType(
{"host-vertex-claude": f"http://{_VERTEX_HOST}", "host-vertex-claude-slash": f"http://{_VERTEX_HOST}/"}
)
_SCRUBBED_PREFIXES: Final = ("ANTHROPIC_", "AZURE_AI_", "INFINITY_")
_PROXY_VARIABLES: Final = frozenset({"HTTPS_PROXY", "HTTP_PROXY", "ALL_PROXY", "NO_PROXY"})
_load_yaml: Final[Callable[[str], object]] = yaml.safe_load
@ -28,7 +36,20 @@ def _deployment(name: str, model: str, api_base: str) -> dict[str, JsonValue]:
return {"model_name": name, "litellm_params": {"model": model, "api_base": api_base, "api_key": KEY}}
def _config(directory: Path) -> Path:
def _vertex_claude_deployment(name: str, api_base: str, token_url: str) -> dict[str, JsonValue]:
return {
"model_name": name,
"litellm_params": {
"model": "vertex_ai/claude-haiku-5-5",
"api_base": api_base,
"vertex_project": _VERTEX_PROJECT,
"vertex_location": _VERTEX_LOCATION,
"vertex_credentials": service_account_json(_VERTEX_PROJECT, token_url),
},
}
def _config(directory: Path, token_url: str) -> Path:
base: Final = JSON_OBJECT.validate_python(_load_yaml(Path("tests/integration/proxy_config.yaml").read_text()))
configuration: Final = {
**base,
@ -38,6 +59,7 @@ def _config(directory: Path) -> Path:
_deployment(_AZURE_CLAUDE, "azure_ai/claude-haiku-5-5", f"http://{_AZURE_HOST}/anthropic"),
_deployment(_AZURE_COUNT, "azure_ai/claude-haiku-5-5", f"http://{_AZURE_HOST}"),
_deployment(_EMPTY_BASE, "anthropic/claude-haiku-5-5", ""),
*(_vertex_claude_deployment(name, base, token_url) for name, base in _VERTEX_CLAUDE_BASES.items()),
],
"router_settings": {"disable_cooldowns": True, "num_retries": 0},
}
@ -72,7 +94,7 @@ def isolated(forward: ForwardProxy, tmp_path_factory: pytest.TempPathFactory) ->
"NO_PROXY": "127.0.0.1,localhost",
"PROXY_BATCH_POLLING_ENABLED": "false",
},
config=_config(directory),
config=_config(directory, shared.upstream_url.rstrip("/")),
remove_environment=_scrubbed(),
) as candidate:
yield candidate
@ -138,3 +160,14 @@ def test_azure_claude_count_tokens_on_a_bare_host_starting_with_anthropic_adds_t
def test_empty_api_base_falls_back_to_the_provider_default_host(isolated: Gateway, forward: ForwardProxy) -> None:
targets: Final = _targets_after(forward, lambda: _chat(isolated, _EMPTY_BASE))
assert targets == {"api.anthropic.com:443"}
@pytest.mark.parametrize("model", tuple(_VERTEX_CLAUDE_BASES), ids=("no-slash", "slash"))
def test_vertex_partner_claude_on_a_bare_host_reaches_one_single_slash_url(
isolated: Gateway, forward: ForwardProxy, model: str
) -> None:
targets: Final = _targets_after(forward, lambda: _chat(isolated, model))
assert targets == {
f"http://{_VERTEX_HOST}/v1/projects/{_VERTEX_PROJECT}/locations/{_VERTEX_LOCATION}"
"/publishers/anthropic/models/claude-haiku-5-5:rawPredict"
}

View file

@ -7,6 +7,7 @@ import openai
import pytest
from integration._support.client import Gateway, Scenario, eventually, list_value, object_value, string_value
from integration._support.database import read_rows
from integration._support.vertex import service_account_json
from integration._support.wire import Reply, Request, wire_server
from integration.providers._api_base_trailing_slash_support import (
KEY,
@ -22,7 +23,7 @@ from integration.providers._api_base_trailing_slash_support import (
reply_text,
request_json,
)
from integration.providers._api_base_url_join_support import COUNTED_TOKENS, RELEVANCE, join_peer
from integration.providers._api_base_url_join_support import AZURE_DEPLOYMENT, COUNTED_TOKENS, RELEVANCE, join_peer
from pydantic import JsonValue
_HAIKU: Final = "anthropic.claude-haiku-5-5"
@ -49,10 +50,20 @@ _RESPONSES_ROWS: Final = (_ANTHROPIC, _GEMINI)
_EMBEDDING_ROWS: Final = (
Row("mistral/mistral-embed", "/mi/v1", "/embeddings"),
Row("gemini/gemini-embedding-2", "/gm/v1beta", "/models/gemini-embedding-2:batchEmbedContents"),
Row("bedrock/amazon.titan-embed-text-v2:0", "/be", "/model/amazon.titan-embed-text-v2%3A0/invoke"),
)
_TRANSCRIPTION: Final = Row("mistral/voxtral-mini-latest", "/mi/v1", "/audio/transcriptions")
_FULL_ROUTE_SUFFIXES: Final = ("", "/", "?beta=true")
_FULL_ROUTE_IDS: Final = ("exact", "slash", "query")
_FULL_ROUTE_SUFFIXES: Final = (("", ""), ("/", ""), ("?beta=true", "?beta=true"), ("/?beta=true", "?beta=true"))
_FULL_ROUTE_IDS: Final = ("exact", "slash", "query", "slash-query")
_VERTEX_PROJECT: Final = "url-join-project"
_VERTEX_LOCATION: Final = "us-east5"
_VERTEX_GEMINI_PATH: Final = (
f"/vx/v1/projects/{_VERTEX_PROJECT}/locations/{_VERTEX_LOCATION}/publishers/google/models/gemini-3.8-flash"
)
_AZURE_V1_BASES: Final = ("/az", "/az/", "/az/openai/v1", "/az/openai/v1/")
_AZURE_V1_IDS: Final = ("host", "host-slash", "route", "route-slash")
_AZURE_ANTHROPIC_BASES: Final = ("/fd/anthropic", "/fd/anthropic/v1/messages")
_AZURE_ANTHROPIC_IDS: Final = ("anthropic-root", "messages-route")
def _ids(rows: tuple[Row, ...]) -> tuple[str, ...]:
@ -65,6 +76,17 @@ def _deploy(scenario: Scenario, row: Row, api_base: str) -> str:
return scenario.model(model=row.model, api_base=api_base, api_key=KEY)
def _deploy_vertex(gateway: Gateway, scenario: Scenario, model: str, api_base: str) -> str:
return scenario.model(
model=model,
api_base=api_base,
api_key=None,
vertex_project=_VERTEX_PROJECT,
vertex_location=_VERTEX_LOCATION,
vertex_credentials=service_account_json(_VERTEX_PROJECT, gateway.upstream_url.rstrip("/")),
)
def _openai(gateway: Gateway) -> openai.OpenAI:
return openai.OpenAI(base_url=str(gateway.client.base_url) + "/v1", api_key=gateway.key, max_retries=0)
@ -80,6 +102,13 @@ def _chat(gateway: Gateway, model: str, marker: str) -> str | None:
return completion.choices[0].message.content
def _streamed_chat(gateway: Gateway, model: str, marker: str) -> str:
stream: Final = _openai(gateway).chat.completions.create(
model=model, messages=[{"role": "user", "content": marker}], stream=True
)
return "".join(chunk.choices[0].delta.content or "" for chunk in stream if chunk.choices)
def _message_text(gateway: Gateway, model: str, marker: str) -> list[str]:
message: Final = _anthropic(gateway).messages.create(
model=model, max_tokens=16, messages=[{"role": "user", "content": marker}]
@ -270,22 +299,74 @@ def test_cache_hit_on_a_slash_base_skips_the_upstream_and_logs_its_own_row(gatew
assert "//" not in urlsplit(string_value(hit["api_base"])).path, hit
@pytest.mark.parametrize("suffix", _FULL_ROUTE_SUFFIXES, ids=_FULL_ROUTE_IDS)
def test_chat_base_that_already_names_the_route_keeps_its_path_and_query(gateway: Gateway, suffix: str) -> None:
@pytest.mark.parametrize(("suffix", "sent"), _FULL_ROUTE_SUFFIXES, ids=_FULL_ROUTE_IDS)
def test_chat_base_that_already_names_the_route_keeps_its_path_and_query(
gateway: Gateway, suffix: str, sent: str
) -> None:
marker: Final = new_marker()
with wire_server(join_peer) as wire, gateway.scenario() as scenario:
model: Final = _deploy(scenario, _ANTHROPIC, f"{wire.url}/an/v1/messages{suffix}")
assert _chat(gateway, model, marker) == reply_text(marker)
assert_one_request(wire.drain(), f"/an/v1/messages{suffix.rstrip('/')}", marker)
assert_one_request(wire.drain(), f"/an/v1/messages{sent}", marker)
@pytest.mark.parametrize("suffix", _FULL_ROUTE_SUFFIXES, ids=_FULL_ROUTE_IDS)
def test_messages_base_that_already_names_the_route_keeps_its_path_and_query(gateway: Gateway, suffix: str) -> None:
@pytest.mark.parametrize(("suffix", "sent"), _FULL_ROUTE_SUFFIXES, ids=_FULL_ROUTE_IDS)
def test_messages_base_that_already_names_the_route_keeps_its_path_and_query(
gateway: Gateway, suffix: str, sent: str
) -> None:
marker: Final = new_marker()
with wire_server(join_peer) as wire, gateway.scenario() as scenario:
model: Final = _deploy(scenario, _ANTHROPIC, f"{wire.url}/an/v1/messages{suffix}")
assert _message_text(gateway, model, marker) == [reply_text(marker)]
assert_one_request(wire.drain(), f"/an/v1/messages{suffix.rstrip('/')}", marker)
assert_one_request(wire.drain(), f"/an/v1/messages{sent}", marker)
@pytest.mark.parametrize("trailing", TRAILING, ids=TRAILING_IDS)
def test_vertex_model_url_base_calls_its_method_with_no_slash_before_the_colon(gateway: Gateway, trailing: str) -> None:
marker: Final = new_marker()
with wire_server(join_peer) as wire, gateway.scenario() as scenario:
model: Final = _deploy_vertex(
gateway, scenario, "vertex_ai/gemini-3.8-flash", f"{wire.url}{_VERTEX_GEMINI_PATH}{trailing}"
)
assert _chat(gateway, model, marker) == reply_text(marker)
assert_one_request(wire.drain(), f"{_VERTEX_GEMINI_PATH}:generateContent", marker, keyed=False)
@pytest.mark.parametrize("trailing", TRAILING, ids=TRAILING_IDS)
def test_vertex_model_url_base_streams_with_no_slash_before_the_colon(gateway: Gateway, trailing: str) -> None:
marker: Final = new_marker()
with wire_server(join_peer) as wire, gateway.scenario() as scenario:
model: Final = _deploy_vertex(
gateway, scenario, "vertex_ai/gemini-3.8-flash", f"{wire.url}{_VERTEX_GEMINI_PATH}{trailing}"
)
assert _streamed_chat(gateway, model, marker) == reply_text(marker)
assert_one_request(wire.drain(), f"{_VERTEX_GEMINI_PATH}:streamGenerateContent?alt=sse", marker, keyed=False)
@pytest.mark.parametrize("base", _AZURE_V1_BASES, ids=_AZURE_V1_IDS)
def test_azure_v1_chat_reaches_one_openai_v1_route(gateway: Gateway, base: str) -> None:
marker: Final = new_marker()
with wire_server(join_peer) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model=f"azure/{AZURE_DEPLOYMENT}", api_base=f"{wire.url}{base}", api_key=KEY, api_version="v1"
)
assert _chat(gateway, model, marker) == reply_text(marker)
assert_one_request(wire.drain(), "/az/openai/v1/chat/completions", marker)
@pytest.mark.parametrize("trailing", TRAILING, ids=TRAILING_IDS)
@pytest.mark.parametrize("base", _AZURE_ANTHROPIC_BASES, ids=_AZURE_ANTHROPIC_IDS)
def test_azure_ai_count_tokens_reaches_one_count_tokens_route(gateway: Gateway, base: str, trailing: str) -> None:
marker: Final = new_marker()
with wire_server(join_peer) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model="azure_ai/claude-haiku-5-5", api_base=f"{wire.url}{base}{trailing}", api_key=KEY
)
response: Final = gateway.request(
"POST", "/v1/messages/count_tokens", {"model": model, "messages": [{"role": "user", "content": marker}]}
)
assert body_of(response)["input_tokens"] == COUNTED_TOKENS
assert_one_request(wire.drain(), "/fd/anthropic/v1/messages/count_tokens", marker)
@pytest.mark.parametrize("suffix", ("", "/", "?tier=a"), ids=("exact", "slash", "query"))

View file

@ -1,5 +1,6 @@
import json
from collections.abc import Iterator
from collections.abc import Callable, Iterator
from types import MappingProxyType
from typing import Final
import anthropic
@ -8,6 +9,7 @@ import pytest
from integration._support.client import Gateway, gateway_from_environment
from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process
from integration._support.wire import Reply, Request, wire_server
from litellm.types.llms.anthropic import ANTHROPIC_OAUTH_BETA_HEADER
from pydantic import JsonValue, TypeAdapter
pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
@ -26,6 +28,14 @@ _PROVIDER_KEY_VARIABLES: Final = (
"AZURE_TENANT_ID",
"OPENAI_API_KEY",
)
_ENVIRONMENT_KEYS: Final = MappingProxyType(
{
"ANTHROPIC_API_KEY": "synthetic-anthropic-env-key",
"AZURE_API_KEY": "synthetic-azure-env-key",
"OPENAI_API_KEY": "synthetic-openai-env-key",
}
)
_SUBSCRIPTION_TOKEN: Final = "sk-ant-oat01-synthetic-subscription-token"
@pytest.fixture(scope="module")
@ -44,14 +54,14 @@ def keyless_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[OwnedPro
@pytest.fixture(scope="module")
def azure_environment_key_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[OwnedProxy]:
def environment_key_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[OwnedProxy]:
with (
gateway_from_environment() as gateway,
owned_proxy_process(
gateway,
tmp_path_factory.mktemp("count-tokens-azure-environment-key"),
{"AZURE_API_KEY": "synthetic-azure-env-key"},
remove_environment=tuple(name for name in _PROVIDER_KEY_VARIABLES if name != "AZURE_API_KEY"),
tmp_path_factory.mktemp("count-tokens-environment-keys"),
_ENVIRONMENT_KEYS,
remove_environment=tuple(name for name in _PROVIDER_KEY_VARIABLES if name not in _ENVIRONMENT_KEYS),
workers=1,
) as owned,
):
@ -147,9 +157,9 @@ def test_anthropic_count_without_a_provider_credential_is_an_anthropic_401(
def test_azure_ai_anthropic_count_sends_the_azure_environment_key_as_x_api_key(
azure_environment_key_proxy: OwnedProxy,
environment_key_proxy: OwnedProxy,
) -> None:
gateway: Final = azure_environment_key_proxy.gateway
gateway: Final = environment_key_proxy.gateway
def count(request: Request) -> Reply:
assert request.headers.get("x-api-key") == "synthetic-azure-env-key", request.headers
@ -257,3 +267,122 @@ def test_google_count_tokens_errors_use_the_google_envelope(keyless_proxy: Owned
assert _JSON_OBJECT.validate_json(malformed.content) == {
"error": {"code": 400, "message": "Invalid Google countTokens request body", "status": "INVALID_ARGUMENT"}
}
def _counted(tokens: int) -> Reply:
return Reply(body=json.dumps({"object": "response.input_tokens", "input_tokens": tokens}).encode())
def _anthropic_count(model: str) -> tuple[str, dict[str, JsonValue]]:
return "/v1/messages/count_tokens", {"model": model, "messages": _MESSAGES}
def _responses_count(model: str) -> tuple[str, dict[str, JsonValue]]:
return "/v1/responses/input_tokens", {"model": model, "input": "Count this synthetic request."}
@pytest.mark.parametrize(
("provider_model", "base_path", "count_request", "header", "credential"),
(
("anthropic/claude-sonnet-5-5", "", _anthropic_count, "x-api-key", "synthetic-anthropic-env-key"),
("openai/gpt-4o-mini", "/v1", _responses_count, "authorization", "Bearer synthetic-openai-env-key"),
),
ids=("anthropic", "openai"),
)
def test_count_uses_a_provider_key_set_only_in_the_proxy_environment(
environment_key_proxy: OwnedProxy,
provider_model: str,
base_path: str,
count_request: Callable[[str], tuple[str, dict[str, JsonValue]]],
header: str,
credential: str,
) -> None:
gateway: Final = environment_key_proxy.gateway
with wire_server(lambda _request: _counted(47)) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model=provider_model, api_base=f"{wire.url}{base_path}", api_key=None, num_retries=0
)
path, body = count_request(model)
counted: Final = gateway.request("POST", path, body)
assert counted.status_code == 200, counted.text
assert _JSON_OBJECT.validate_json(counted.content)["input_tokens"] == 47, counted.text
received: Final = wire.drain()
assert [request.headers.get(header) for request in received] == [credential], received
def _google_count(model: str) -> tuple[str, dict[str, JsonValue]]:
return f"/v1beta/models/{model}:countTokens", {
"contents": [{"role": "user", "parts": [{"text": "Count this synthetic request."}]}]
}
def _utils_count(model: str) -> tuple[str, dict[str, JsonValue]]:
return "/utils/token_counter?call_endpoint=true", {"model": model, "messages": _MESSAGES}
@pytest.mark.parametrize(
("count_request", "google_status"),
((_google_count, "UNAUTHENTICATED"), (_utils_count, None)),
ids=("google", "utils"),
)
def test_count_entrypoints_report_a_missing_provider_key_in_their_own_shape(
keyless_proxy: OwnedProxy,
count_request: Callable[[str], tuple[str, dict[str, JsonValue]]],
google_status: str | None,
) -> None:
gateway: Final = keyless_proxy.gateway
with wire_server(_never_called) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model="anthropic/claude-sonnet-5-5", api_base=wire.url, api_key=None, num_retries=0
)
path, body = count_request(model)
rejected: Final = gateway.request("POST", path, body)
assert rejected.status_code == 401, rejected.text
error: Final = _JSON_OBJECT.validate_json(rejected.content)["error"]
assert isinstance(error, dict), rejected.text
assert f"No Anthropic API key is set for model {model}" in str(error["message"]), rejected.text
if google_status is not None:
assert (error["code"], error["status"]) == (401, google_status), rejected.text
assert wire.drain() == ()
def test_anthropic_count_uses_the_callers_subscription_token_when_the_deployment_has_no_key(
keyless_proxy: OwnedProxy,
) -> None:
gateway: Final = keyless_proxy.gateway
with wire_server(lambda _request: _counted(41)) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model="anthropic/claude-sonnet-5-5", api_base=wire.url, api_key=None, num_retries=0
)
counted: Final = gateway.request(
"POST",
"/v1/messages/count_tokens",
{"model": model, "messages": _MESSAGES},
headers={"Authorization": f"Bearer {_SUBSCRIPTION_TOKEN}", "x-litellm-api-key": f"Bearer {gateway.key}"},
)
assert counted.status_code == 200, counted.text
assert _JSON_OBJECT.validate_json(counted.content)["input_tokens"] == 41, counted.text
(received,) = wire.drain()
assert received.headers.get("authorization") == f"Bearer {_SUBSCRIPTION_TOKEN}", received.headers
assert "x-api-key" not in received.headers, received.headers
assert ANTHROPIC_OAUTH_BETA_HEADER in received.headers.get("anthropic-beta", ""), received.headers
def test_anthropic_count_keeps_the_deployments_key_over_the_callers_subscription_token(
keyless_proxy: OwnedProxy,
) -> None:
gateway: Final = keyless_proxy.gateway
with wire_server(lambda _request: _counted(43)) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model="anthropic/claude-sonnet-5-5", api_base=wire.url, api_key="synthetic-deployment-key", num_retries=0
)
counted: Final = gateway.request(
"POST",
"/v1/messages/count_tokens",
{"model": model, "messages": _MESSAGES},
headers={"Authorization": f"Bearer {_SUBSCRIPTION_TOKEN}", "x-litellm-api-key": f"Bearer {gateway.key}"},
)
assert counted.status_code == 200, counted.text
(received,) = wire.drain()
assert received.headers.get("x-api-key") == "synthetic-deployment-key", received.headers
assert _SUBSCRIPTION_TOKEN not in json.dumps(dict(received.headers)), received.headers

View file

@ -48,6 +48,7 @@ VERTEX_ACCESS_TOKEN: Final = "ya29.scripted-delete-token"
AWS_ACCESS_KEY_ID: Final = "AKIAIOSFODNN7EXAMPLE"
AWS_SECRET_ACCESS_KEY: Final = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
AWS_REGION: Final = "us-east-1"
S3_OBJECT_BODY: Final = b'{"recordId":"r1","modelOutput":{"stop_reason":"end_turn"}}\n'
_ForbiddenSurface = Literal[
"upload_sdk_extra_body",
"upload_form",
@ -708,6 +709,86 @@ def test_non_admin_cannot_delete_a_raw_gcs_uri_before_vertex_auth_or_storage(gat
assert token_requests == (), "non-admin raw GCS delete reached the Vertex token endpoint"
def test_non_admin_still_reads_an_ordinary_provider_file_id(gateway: Gateway) -> None:
with wire_server(_openai_backend()) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model="openai/gpt-4o-mini", api_base=wire.url + "/v1", api_key="provider-key-" + uuid.uuid4().hex[:8]
)
_wait_until_every_worker_serves(gateway, model)
member: Final = _member(scenario, model)
file_id: Final = "file-" + uuid.uuid4().hex
read: Final = gateway.request("GET", f"/v1/files/{file_id}/content", key=member.key, params={"model": model})
assert read.status_code == 200, read.text
assert read.content == b"file-content\n", read.content
assert [
(request.method, urlsplit(request.target).path) for request in _drained_other_than_model_list_probes(wire)
] == [("GET", f"/v1/files/{file_id}/content")]
@pytest.mark.parametrize(
("method", "route_suffix", "verb"),
[("GET", "/content", "retrieved"), ("DELETE", "", "deleted")],
ids=["content", "delete"],
)
def test_non_admin_cannot_reach_vertex_with_a_twice_encoded_raw_gcs_uri(
gateway: Gateway, method: Literal["GET", "DELETE"], route_suffix: str, verb: str
) -> None:
with (
wire_server(_vertex_token_backend) as token_wire,
gateway.scenario() as scenario,
):
model: Final = scenario.model(
model=VERTEX_MODEL,
api_key=None,
vertex_project=VERTEX_PROJECT,
vertex_location=VERTEX_LOCATION,
vertex_credentials=_vertex_credentials(token_wire.url),
gcs_bucket_name=BUCKET,
)
_wait_until_every_worker_serves(gateway, model)
caller_user: Final = scenario.user(user_role="internal_user")
caller_key: Final = scenario.key(user_id=caller_user, models=[model])
object_uri: Final = f"gs://{BUCKET}/litellm-vertex-files/litellm-vertex-files-obj.jsonl"
twice_encoded: Final = quote(quote(object_uri, safe=""), safe="")
token_wire.drain()
denied: Final = gateway.request(
method, f"/v1/files/{twice_encoded}{route_suffix}", key=caller_key, params={"model": model}
)
token_requests: Final = token_wire.drain()
assert denied.status_code == 403, f"{denied.text}; token requests={token_requests}"
assert _error_message(denied, 403) == (
f"Raw cloud storage file ids can only be {verb} by a proxy admin key. "
"Use the LiteLLM managed file id returned when the file was created."
), denied.text
assert token_requests == (), f"non-admin twice-encoded raw GCS {method} reached the Vertex token endpoint"
def _assert_signed_empty_body_s3_request(request: Request, method: Literal["GET", "DELETE"], wire_url: str) -> None:
assert request.method == method, request.method
request_date: Final = request.headers["x-amz-date"]
content_hash: Final = hashlib.sha256(b"").hexdigest()
assert request.headers["x-amz-content-sha256"] == content_hash, request.headers
aws_request: Final = AWSRequest(
method=method,
url=wire_url + request.target,
headers={
"host": request.headers["host"],
"x-amz-date": request_date,
"x-amz-content-sha256": content_hash,
},
)
aws_request.context["timestamp"] = request_date
auth: Final = S3SigV4Auth(Credentials(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY), "s3", AWS_REGION)
signature: Final = auth.signature(auth.string_to_sign(aws_request, auth.canonical_request(aws_request)), aws_request)
expected_authorization: Final = (
f"AWS4-HMAC-SHA256 Credential={AWS_ACCESS_KEY_ID}/{request_date[:8]}/{AWS_REGION}/s3/aws4_request, "
f"SignedHeaders={auth.signed_headers(auth.headers_to_sign(aws_request))}, Signature={signature}"
)
assert request.headers["authorization"] == expected_authorization, request.headers["authorization"]
def test_proxy_admin_deletes_a_raw_s3_uri_with_one_signed_delete(gateway: Gateway) -> None:
with wire_server(_s3_backend()) as wire, gateway.scenario() as scenario:
model: Final = _bedrock_model(scenario, wire)
@ -721,29 +802,30 @@ def test_proxy_admin_deletes_a_raw_s3_uri_with_one_signed_delete(gateway: Gatewa
assert parsed.deleted is True and parsed.id == object_uri, deleted.text
(request,) = _drained_other_than_model_list_probes(wire)
assert request.target == f"/{BUCKET}/litellm-bedrock-files-obj.jsonl", request.target
request_date: Final = request.headers["x-amz-date"]
content_hash: Final = hashlib.sha256(b"").hexdigest()
assert request.headers["x-amz-content-sha256"] == content_hash, request.headers
aws_request: Final = AWSRequest(
method="DELETE",
url=wire.url + request.target,
headers={
"host": request.headers["host"],
"x-amz-date": request_date,
"x-amz-content-sha256": content_hash,
},
_assert_signed_empty_body_s3_request(request, "DELETE", wire.url)
def _s3_object_backend(request: Request) -> Reply:
if request.method == "GET":
return Reply(body=S3_OBJECT_BODY, content_type="application/octet-stream")
return _json_reply({"error": {"message": f"unscripted {request.method} {request.target}"}}, 404)
def test_proxy_admin_reads_a_raw_s3_uri_with_one_signed_get(gateway: Gateway) -> None:
with wire_server(_s3_object_backend) as wire, gateway.scenario() as scenario:
model: Final = _bedrock_model(scenario, wire)
_wait_until_every_worker_serves(gateway, model)
object_key: Final = f"litellm-batch-outputs/{uuid.uuid4().hex}.jsonl.out"
object_uri: Final = f"s3://{BUCKET}/{object_key}"
read: Final = gateway.request(
"GET", "/v1/files/" + quote(object_uri, safe="") + "/content", params={"model": model}
)
aws_request.context["timestamp"] = request_date
auth: Final = S3SigV4Auth(Credentials(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY), "s3", AWS_REGION)
signature: Final = auth.signature(
auth.string_to_sign(aws_request, auth.canonical_request(aws_request)), aws_request
)
expected_authorization: Final = (
f"AWS4-HMAC-SHA256 Credential={AWS_ACCESS_KEY_ID}/{request_date[:8]}/{AWS_REGION}/s3/aws4_request, "
f"SignedHeaders={auth.signed_headers(auth.headers_to_sign(aws_request))}, Signature={signature}"
)
assert request.headers["authorization"] == expected_authorization, request.headers["authorization"]
assert request.method == "DELETE", request.method
storage_requests: Final = _drained_other_than_model_list_probes(wire)
assert read.status_code == 200, f"{read.text}; storage requests={storage_requests}"
assert read.content == S3_OBJECT_BODY, read.content
assert [request.target for request in storage_requests] == [f"/{BUCKET}/{object_key}"], storage_requests
_assert_signed_empty_body_s3_request(storage_requests[0], "GET", wire.url)
def _granting_vertex_token_backend(request: Request) -> Reply:

View file

@ -642,6 +642,15 @@ def _key_budget(scenario: Scenario, budget_owner: str, model_budget: JsonValue)
return {}
def _assert_the_next_create_exceeds_the_model_budget(scenario: Scenario, key: str, model: str) -> None:
refused: Final = eventually(
lambda: _next_create(scenario, key, model),
lambda response: response.status_code != 200,
seconds=SPEND_LANDING_SECONDS,
)
assert f"exceeded budget for model={model}" in refused.text, refused.text
@pytest.mark.timeout(240)
@pytest.mark.parametrize("budget_owner", ["key", "team", "user", "budget_tier"])
def test_a_billed_background_response_counts_toward_its_creators_project_and_model_budget(
@ -663,12 +672,24 @@ def test_a_billed_background_response_counts_toward_its_creators_project_and_mod
lambda spends: spends[0] > 0 and costs(spends[0])(spends[1]),
seconds=SPEND_LANDING_SECONDS,
)
refused: Final = eventually(
lambda: _next_create(scenario, key, upstream.model),
lambda response: response.status_code != 200,
seconds=SPEND_LANDING_SECONDS,
_assert_the_next_create_exceeds_the_model_budget(scenario, key, upstream.model)
@pytest.mark.timeout(240)
def test_a_billed_background_response_on_a_team_model_counts_toward_the_budget_of_the_model_name_it_was_created_with(
gateway: Gateway,
) -> None:
with gateway.scenario() as scenario:
team_id: Final = scenario.team()
upstream: Final = scripted_upstream(scenario, background_routes("completed"), model_info={"team_id": team_id})
key: Final = scenario.key(
team_id=team_id, model_max_budget={upstream.model: {"max_budget": PRICE / 2, "budget_duration": "1d"}}
)
assert f"exceeded budget for model={upstream.model}" in refused.text, refused.text
response_id: Final = create_background_response(scenario, key, upstream.model)
read: Final = read_response(gateway, key, response_id)
assert (read,) in (settling_reads("completed", 1), free_reads("completed", 1))
eventually(lambda: key_spend(key), lambda spend: spend > 0, seconds=SPEND_LANDING_SECONDS)
_assert_the_next_create_exceeds_the_model_budget(scenario, key, upstream.model)
@pytest.mark.timeout(240)

View file

@ -167,3 +167,51 @@ def test_provider_500_failure_rows_count_deepseek_v4_prompt_tokens_with_deepseek
short: Final = _failure_row_prompt_tokens(gateway, model, key, SHORT_TEXT)
assert wire.drain()
assert_counted_with_deepseek(chinese - short, chinese_minus_short)
COMPACTION_INSTRUCTIONS: Final[JsonValue] = {"role": "system", "content": "answer briefly"}
COMPACTED_HISTORY: Final[tuple[JsonValue, ...]] = (
{"role": "user", "content": "a long turn " * 200},
{"role": "assistant", "content": "a long reply " * 200},
)
COMPACTION_ITEM: Final[JsonValue] = {
"type": "compaction",
"encrypted_content": json.dumps({"type": "compaction", "content": "the cat is named Biscuit"}),
}
TURN_AFTER_COMPACTION: Final[JsonValue] = {"role": "user", "content": "what is the cat's name?"}
def _failed_responses_prompt_tokens(gateway: Gateway, model: str, key: str, items: list[JsonValue]) -> int:
failed: Final = gateway.request("POST", "/v1/responses", {"model": model, "input": items}, key=key)
assert failed.status_code == 500 and "synthetic provider outage" in failed.text, failed.text
rows: Final = eventually(
lambda: read_rows(
'SELECT status, prompt_tokens FROM "LiteLLM_SpendLogs" WHERE request_id=%s',
(failed.headers["x-litellm-call-id"],),
),
lambda values: len(values) == 1,
seconds=70,
)
assert rows[0]["status"] == "failure", rows
prompt_tokens: Final = rows[0]["prompt_tokens"]
assert isinstance(prompt_tokens, int) and prompt_tokens > 0, rows
return prompt_tokens
def test_a_failed_responses_request_counts_its_input_from_the_newest_compaction_item(gateway: Gateway) -> None:
with wire_server(_provider_500) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(api_base=wire.url + "/v1", num_retries=0)
key: Final = scenario.key(models=[model])
resent: Final = _failed_responses_prompt_tokens(
gateway, model, key, [COMPACTION_INSTRUCTIONS, *COMPACTED_HISTORY, COMPACTION_ITEM, TURN_AFTER_COMPACTION]
)
compacted: Final = _failed_responses_prompt_tokens(
gateway, model, key, [COMPACTION_INSTRUCTIONS, COMPACTION_ITEM, TURN_AFTER_COMPACTION]
)
uncompacted: Final = _failed_responses_prompt_tokens(
gateway, model, key, [COMPACTION_INSTRUCTIONS, *COMPACTED_HISTORY, TURN_AFTER_COMPACTION]
)
assert len(wire.drain()) == 3
assert resent == compacted
assert resent < uncompacted

View file

@ -0,0 +1,73 @@
import json
import uuid
from hashlib import sha256
from typing import Final
from integration._support.client import Gateway, eventually
from integration._support.database import read_rows
from integration._support.deepseek_tokens import (
CHINESE_TEXT,
SHORT_TEXT,
assert_counted_with_deepseek,
chinese_minus_short,
)
from integration._support.wire import Reply, Request, wire_server
from pydantic import JsonValue
_DEEPSEEK_V4: Final = "DeepSeek-V4-Flash-0731"
def _data_frame(frame: dict[str, JsonValue]) -> bytes:
return b"data: " + json.dumps(frame).encode() + b"\n\n"
def _stream_without_usage(request: Request) -> Reply:
assert request.method == "POST"
chunk: Final[dict[str, JsonValue]] = {
"id": f"chatcmpl-{uuid.uuid4().hex}",
"object": "chat.completion.chunk",
"created": 1,
"model": _DEEPSEEK_V4,
}
return Reply(
content_type="text/event-stream",
chunks=(
_data_frame({**chunk, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "OK"}}]}),
_data_frame({**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}),
b"data: [DONE]\n\n",
),
)
def _streamed_prompt_tokens(gateway: Gateway, key: str, model: str, text: str) -> int:
response: Final = gateway.request(
"POST",
"/v1/chat/completions",
{"model": model, "messages": [{"role": "user", "content": text}], "stream": True},
key=key,
)
assert response.status_code == 200, response.text
assert response.text.rstrip().endswith("data: [DONE]"), response.text
rows: Final = eventually(
lambda: read_rows(
'SELECT status, prompt_tokens FROM "LiteLLM_SpendLogs" WHERE api_key=%s',
(sha256(key.encode()).hexdigest(),),
),
lambda values: len(values) == 1,
seconds=70,
)
assert rows[0]["status"] == "success", rows
prompt_tokens: Final = rows[0]["prompt_tokens"]
assert isinstance(prompt_tokens, int), rows
return prompt_tokens
def test_a_deepseek_v4_stream_without_usage_bills_prompt_tokens_counted_with_deepseeks_tokenizer(
gateway: Gateway,
) -> None:
with wire_server(_stream_without_usage) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"azure_ai/{_DEEPSEEK_V4}", api_base=wire.url, num_retries=0)
chinese: Final = _streamed_prompt_tokens(gateway, scenario.key(models=[model]), model, CHINESE_TEXT)
short: Final = _streamed_prompt_tokens(gateway, scenario.key(models=[model]), model, SHORT_TEXT)
assert len(wire.drain()) == 2
assert_counted_with_deepseek(chinese - short, chinese_minus_short)

View file

@ -221,6 +221,45 @@ class TestCountTokensUsesWorkloadIdentity:
assert result.total_tokens == 13
assert route.calls.last.request.headers["x-api-key"] == FAKE_REGULAR_KEY
@pytest.mark.asyncio
@pytest.mark.parametrize(
("deployment_key", "expected_credentials"),
[
(None, {"authorization": f"Bearer {FAKE_OAUTH_TOKEN}", "x-api-key": None}),
(FAKE_REGULAR_KEY, {"authorization": None, "x-api-key": FAKE_REGULAR_KEY}),
],
ids=["keyless-deployment-uses-the-callers-token", "deployment-key-wins-over-the-callers-token"],
)
async def test_a_callers_subscription_token_counts_only_when_no_key_is_configured(
self,
monkeypatch: pytest.MonkeyPatch,
httpx_transport_clients: None,
deployment_key: str | None,
expected_credentials: dict[str, str | None],
) -> None:
for name in ("ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_API_BASE", "ANTHROPIC_BASE_URL"):
monkeypatch.delenv(name, raising=False)
monkeypatch.setattr(litellm, "anthropic_key", None)
monkeypatch.setattr(litellm, "api_key", None)
with respx.mock(assert_all_called=True) as router:
route = router.post("https://api.anthropic.com/v1/messages/count_tokens").mock(
return_value=httpx.Response(200, json={"input_tokens": 17})
)
result = await AnthropicTokenCounter(client_oauth_token=FAKE_OAUTH_TOKEN).count_tokens(
model_to_use="claude-sonnet-4-5",
messages=[{"role": "user", "content": "hi"}],
contents=None,
deployment={"litellm_params": {"model": "anthropic/claude-sonnet-4-5", "api_key": deployment_key}},
request_model="claude-sonnet-4-5",
)
assert result is not None
assert result.total_tokens == 17
sent = route.calls.last.request.headers
assert {name: sent.get(name) for name in expected_credentials} == expected_credentials
assert (ANTHROPIC_OAUTH_BETA_HEADER in sent["anthropic-beta"].split(",")) is (deployment_key is None)
@pytest.mark.asyncio
async def test_a_failed_mint_degrades_like_an_anthropic_error(self, monkeypatch):
from litellm.llms.anthropic.count_tokens import token_counter as token_counter_module

View file

@ -121,3 +121,105 @@ def test_a_rejection_logs_the_retry_and_advises_opting_out_only_when_native_outp
assert _MODEL in caplog.text
assert ("supports_bedrock_native_structured_output: false" in caplog.text) is advises_opting_out
_STRICT_FALLBACK: Final = NativeOutputFallback(optional_params={"tools": "without_strict"}, litellm_params={})
# Bedrock's 400s for strict tools, seen live on us.anthropic.claude-sonnet-4-6 from 2026-10-20 to 2026-10-28 and quoted
# in issue #34388, each answered 200 once the request dropped strict
_STRICT_TOOLS_REJECTIONS: Final = tuple(
BedrockError(status_code=400, message=f'{{"message":"The model returned the following errors: {text}"}}')
for text in (
"Too many strict tools (21). The maximum number of strict tools supported is 20.",
"Schemas contains too many optional parameters (27), which would make grammar compilation inefficient.",
"Schemas contains too many parameters with union types (18 parameters with type arrays or anyOf). "
"This causes exponential compilation cost.",
"The compiled grammar is too large, which would cause performance issues.",
"tools.0.custom: Invalid schema: Schema type is missing for schema: {'description': 'tag'}",
"tools.3.custom: For 'integer' type, property 'minimum' is not supported",
"Compiled grammar size (410.4MB) exceeds maximum allowed size (300MB)",
"tools.1.custom.strict: Extra inputs are not permitted",
"Grammar compilation timed out.",
)
)
_TOO_MANY_STRICT_TOOLS: Final = _STRICT_TOOLS_REJECTIONS[0]
# Bedrock's 400s for requests with a strict tool that fail for other reasons, seen live on
# us.anthropic.claude-sonnet-4-6 on 2026-10-21, which echo tool names and tool call ids
_OTHER_400S_WITH_A_STRICT_TOOL: Final = tuple(
BedrockError(status_code=400, message=f'{{"message":"{text}"}}')
for text in (
"The tool check_grammar is already defined at toolConfig.tools.1.",
"Expected toolResult blocks at messages.0.content for the following Ids: call_grammar_9",
"tool_use ids were found without tool_result blocks immediately after: call_grammar_1",
)
)
def _call_with_strict_tools_fallback(
bedrock: ScriptedBedrock, fallback: NativeOutputFallback | None, strict_tools_fallback: NativeOutputFallback | None
) -> object:
return _resolve(
call_with_native_output_fallback(
bedrock,
None if fallback is None else lambda: fallback,
_MODEL,
strict_tools_fallback=None if strict_tools_fallback is None else lambda: strict_tools_fallback,
)
)
@_MODES
@pytest.mark.parametrize("rejection", _STRICT_TOOLS_REJECTIONS, ids=lambda error: error.message[60:100])
def test_a_strict_tools_rejection_is_sent_again_once_with_the_strict_tools_fallback(
asynchronous: bool, rejection: BedrockError
) -> None:
bedrock: Final = ScriptedBedrock(errors=(rejection, None), asynchronous=asynchronous)
response: Final = _call_with_strict_tools_fallback(bedrock, None, _STRICT_FALLBACK)
assert isinstance(response, ModelResponse) and response.id == "answer-2"
assert bedrock.sent == (None, _STRICT_FALLBACK)
@_MODES
@pytest.mark.parametrize(
("error", "strict_tools_fallback"),
[
(_OTHER_400, _STRICT_FALLBACK),
*((error, _STRICT_FALLBACK) for error in _OTHER_400S_WITH_A_STRICT_TOOL),
(BedrockError(status_code=500, message=_TOO_MANY_STRICT_TOOLS.message), _STRICT_FALLBACK),
(_TOO_MANY_STRICT_TOOLS, None),
],
ids=[
"other_400",
"duplicate_tool_name",
"missing_tool_result",
"orphan_tool_use",
"server_error",
"no_strict_tools_fallback",
],
)
def test_no_second_request_without_both_a_strict_tools_rejection_and_a_strict_tools_fallback(
asynchronous: bool, error: BedrockError, strict_tools_fallback: NativeOutputFallback | None
) -> None:
bedrock: Final = ScriptedBedrock(errors=(error,), asynchronous=asynchronous)
with pytest.raises(BedrockError) as raised:
_call_with_strict_tools_fallback(bedrock, None, strict_tools_fallback)
assert raised.value is error
assert bedrock.sent == (None,)
@pytest.mark.parametrize(
("error", "retried_with"),
[(_REJECTION, _FALLBACK), (_TOO_MANY_STRICT_TOOLS, _STRICT_FALLBACK)],
ids=["native_output_rejection", "strict_tools_rejection"],
)
def test_each_rejection_is_sent_again_with_its_own_fallback_when_both_are_set(
error: BedrockError, retried_with: NativeOutputFallback
) -> None:
bedrock: Final = ScriptedBedrock(errors=(error, None), asynchronous=False)
_call_with_strict_tools_fallback(bedrock, _FALLBACK, _STRICT_FALLBACK)
assert bedrock.sent == (None, retried_with)

View file

@ -6,7 +6,9 @@ from typing import Final
from litellm.llms.bedrock.common_utils import(
AmazonBedrockGlobalConfig,
BedrockModelInfo,
NativeOutputFallback,
bedrock_native_output_is_opt_in,
bedrock_strict_tools_fallback,
deployment_model_info,
extract_model_name_from_bedrock_arn,
get_bedrock_base_model,
@ -2150,3 +2152,80 @@ def test_bedrock_response_stream_shape_is_unavailable_when_botocore_describes_th
assert shape is None
(logged,) = warning.call_args_list
assert "only-in-the-description" not in logged.args[0] % logged.args[1:]
def test_bedrock_strict_tools_fallback_drops_only_strict_and_keeps_everything_else():
strict_tool: Final = {
"type": "function",
"cache_control": {"type": "ephemeral"},
"function": {"name": "lookup", "description": "Look up", "strict": True, "parameters": {"type": "object"}},
}
plain_tool: Final = {"type": "function", "function": {"name": "plain", "parameters": {"type": "object"}}}
anthropic_tool: Final = {"name": "native", "input_schema": {"type": "object"}}
bedrock_tool: Final = {"toolSpec": {"name": "spec", "strict": True, "inputSchema": {"json": {"type": "object"}}}}
litellm_params: Final = {"drop_params": True}
fallback: Final = bedrock_strict_tools_fallback(
"us.anthropic.claude-sonnet-4-6",
{"tools": [strict_tool, plain_tool, anthropic_tool, bedrock_tool], "maxTokens": 16, "stream": True},
litellm_params,
)
assert fallback == NativeOutputFallback(
optional_params={
"tools": [
{**strict_tool, "function": {"name": "lookup", "description": "Look up", "parameters": {"type": "object"}}},
plain_tool,
anthropic_tool,
{"toolSpec": {"name": "spec", "inputSchema": {"json": {"type": "object"}}}},
],
"maxTokens": 16,
"stream": True,
},
litellm_params=litellm_params,
)
assert strict_tool["function"]["strict"] is True
assert bedrock_tool["toolSpec"]["strict"] is True
@pytest.mark.parametrize(
"optional_params",
[
{"maxTokens": 16},
{"tools": [{"type": "function", "function": {"name": "plain", "parameters": {}}}]},
{"tools": [{"toolSpec": {"name": "spec", "inputSchema": {"json": {}}}}]},
],
ids=["no_tools", "no_strict_tool", "no_strict_tool_spec"],
)
def test_bedrock_strict_tools_fallback_is_none_without_a_strict_tool(optional_params: dict[str, object]) -> None:
assert bedrock_strict_tools_fallback("us.anthropic.claude-sonnet-4-6", optional_params, {}) is None
_CLAUDE_PROFILE_ARN: Final = "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/claude"
@pytest.mark.parametrize(
("model", "litellm_params"),
[
("us.anthropic.claude-sonnet-4-6", {}),
("us.amazon.nova-pro-v1:0", {}),
(_CLAUDE_PROFILE_ARN, {"base_model": "bedrock/us.anthropic.claude-sonnet-4-6"}),
(_CLAUDE_PROFILE_ARN, {}),
],
ids=["claude", "nova", "claude_profile_with_base_model", "profile_without_base_model"],
)
@pytest.mark.parametrize("strict", [True, False])
def test_bedrock_strict_tools_fallback_retries_only_when_the_request_sent_strict(
model: str, litellm_params: dict[str, object], strict: bool
) -> None:
from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig
tools: Final = [{"type": "function", "function": {"name": "lookup", "strict": strict, "parameters": {}}}]
request: Final = AmazonConverseConfig().transform_request(
model, [{"role": "user", "content": "hi"}], {"tools": tools}, {**litellm_params}, {}
)
sent_strict: Final = any("strict" in tool["toolSpec"] for tool in request["toolConfig"]["tools"])
fallback: Final = bedrock_strict_tools_fallback(model, {"tools": tools}, litellm_params)
assert (fallback is not None) == sent_strict

View file

@ -0,0 +1,145 @@
import json
from typing import Final
import httpx
import pytest
import respx
import litellm
from litellm.caching.llm_caching_handler import LLMClientCache
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
from litellm.utils import ProviderConfigManager
AIAND_API_BASE = "https://api.aiand.com/v1"
def test_aiand_chat_provider(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setenv("AIAND_API_KEY", "test-key")
provider_config = JSONProviderRegistry.get("aiand")
assert provider_config is not None
assert litellm.LlmProviders.AIAND.value == "aiand"
assert "aiand" in litellm.openai_compatible_providers
assert provider_config.base_url == AIAND_API_BASE
assert provider_config.api_key_env == "AIAND_API_KEY"
assert provider_config.api_base_env == "AIAND_API_BASE"
model, provider, api_key, api_base = get_llm_provider("aiand/openai/gpt-oss-120b")
assert model == "openai/gpt-oss-120b"
assert provider == "aiand"
assert api_key == "test-key"
assert api_base == AIAND_API_BASE
def test_aiand_completion_request(respx_mock: respx.MockRouter):
messages = [{"role": "user", "content": "Hello"}]
route = respx_mock.post(f"{AIAND_API_BASE}/chat/completions").mock(
return_value=httpx.Response(
200,
json={
"id": "chatcmpl-test",
"object": "chat.completion",
"created": 1,
"model": "openai/gpt-oss-120b",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "Hello"},
"finish_reason": "stop",
}
],
"usage": {
"prompt_tokens": 1,
"completion_tokens": 1,
"total_tokens": 2,
},
},
)
)
litellm.completion(
model="aiand/openai/gpt-oss-120b",
api_key="test-key",
messages=messages,
)
assert route.called
request = route.calls.last.request
assert str(request.url) == f"{AIAND_API_BASE}/chat/completions"
assert request.headers["Authorization"] == "Bearer test-key"
request_body = json.loads(request.read())
assert request_body["model"] == "openai/gpt-oss-120b"
assert request_body["messages"] == messages
def test_aiand_responses_provider():
config = ProviderConfigManager.get_provider_responses_api_config(
provider="aiand",
model="aiand/openai/gpt-oss-120b",
)
assert config is not None
assert config.custom_llm_provider == "aiand"
assert config.get_complete_url(api_base=None, litellm_params={}) == f"{AIAND_API_BASE}/responses"
def _aiand_catalog() -> frozenset[str]:
return frozenset(name for name, info in litellm.model_cost.items() if info.get("litellm_provider") == "aiand")
def test_aiand_wildcard_lists_its_priced_models():
catalog: Final = _aiand_catalog()
assert catalog
assert frozenset(get_known_models_from_wildcard("aiand/*")) == catalog
def test_aiand_calls_are_billed_from_their_cost_rows():
catalog: Final = _aiand_catalog()
assert catalog
for model in sorted(catalog):
row: Final = litellm.model_cost[model]
prompt_cost, completion_cost = litellm.cost_per_token(model=model, prompt_tokens=1000, completion_tokens=500)
assert prompt_cost == pytest.approx(1000 * row["input_cost_per_token"])
assert completion_cost == pytest.approx(500 * row["output_cost_per_token"])
assert prompt_cost > 0 and completion_cost > 0
@pytest.mark.asyncio
async def test_aiand_anthropic_messages_reach_ai_and_with_the_thinking_budget(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
thinking: Final = {"type": "enabled", "budget_tokens": 1024}
with respx.mock() as upstream:
route: Final = upstream.post(f"{AIAND_API_BASE}/messages").respond(
200,
json={
"id": "msg_aiand",
"type": "message",
"role": "assistant",
"model": "zai-org/glm-5.3",
"content": [{"type": "thinking", "thinking": "A greeting", "signature": ""}, {"type": "text", "text": "Hello"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 4, "output_tokens": 3},
},
)
response: Final = await litellm.anthropic.messages.acreate(
model="aiand/zai-org/glm-5.3",
messages=[{"role": "user", "content": "Say hello"}],
max_tokens=2048,
thinking=thinking,
api_key="aiand-test-key",
)
request: Final = route.calls.last.request
body: Final = json.loads(request.content)
assert route.call_count == 1
assert body["model"] == "zai-org/glm-5.3"
assert body["thinking"] == thinking
assert "reasoning_effort" not in body
assert [block["type"] for block in response["content"]] == ["thinking", "text"]

View file

@ -53,6 +53,7 @@ from litellm.proxy.litellm_pre_call_utils import (
add_provider_specific_headers_to_request,
check_if_token_is_service_account,
clean_headers,
forwarded_anthropic_oauth_token,
get_dynamic_logging_metadata,
get_metadata_variable_name,
move_guardrails_to_metadata,
@ -1393,6 +1394,8 @@ async def test_add_litellm_data_to_request_strips_string_encoded_admin_injection
("fallback_depth", True),
("_target_order", 2),
("attempted_targets", ["forged"]),
("_excluded_deployment_ids", ["pinned-deployment"]),
("_retry_skipped_deployment_ids", ["pinned-deployment"]),
],
)
async def test_add_litellm_data_to_request_strips_forged_fallback_hop_state(
@ -1477,6 +1480,7 @@ async def test_add_litellm_data_to_request_strips_user_control_fields():
"_managed_background_response": True,
"_guardrail_pipelines": [{"name": "spoofed"}],
"_pipeline_managed_guardrails": ["evaded"],
"_failover_excluded_ids": ["pinned-deployment"],
"safe_user_metadata": "kept",
}
data = {
@ -1537,6 +1541,7 @@ async def test_add_litellm_data_to_request_strips_user_control_fields():
"_managed_background_response",
"_guardrail_pipelines",
"_pipeline_managed_guardrails",
"_failover_excluded_ids",
}
assert "litellm_metadata" not in updated
for stripped_key in stripped_keys:
@ -5962,6 +5967,25 @@ def test_clean_headers_strips_x_api_key_when_byok_disabled():
assert "x-api-key" not in result
@pytest.mark.parametrize(
("headers", "expected_token"),
[
(
{"authorization": "Bearer sk-ant-oat01-caller", "x-litellm-api-key": "Bearer sk-proxy"},
"sk-ant-oat01-caller",
),
({"authorization": "Bearer sk-ant-oat01-caller"}, None),
({"authorization": "Bearer sk-ant-api03-caller", "x-litellm-api-key": "Bearer sk-proxy"}, None),
({"x-litellm-api-key": "Bearer sk-proxy"}, None),
],
ids=["oauth-beside-proxy-key", "oauth-used-as-proxy-key", "non-oauth-authorization", "no-authorization"],
)
def test_forwarded_anthropic_oauth_token_follows_chats_forwarding_rule(
headers: dict[str, str], expected_token: str | None
) -> None:
assert forwarded_anthropic_oauth_token(Headers(headers)) == expected_token
def test_clean_headers_strips_x_api_key_when_byok_enabled_but_x_api_key_was_auth_header():
"""
Anti-replay regression: even when forward_llm_provider_auth_headers=True,

View file

@ -364,6 +364,7 @@ async def test_anthropic_messages_count_tokens_endpoint():
output_format=None,
context_management=None,
request_litellm_params=None,
client_oauth_token=None,
):
assert call_endpoint == True, "Should be called with call_endpoint=True for Anthropic endpoint"
assert request.model == "claude-3-sonnet-20240229"
@ -444,6 +445,7 @@ async def test_anthropic_messages_count_tokens_with_non_anthropic_model():
output_format=None,
context_management=None,
request_litellm_params=None,
client_oauth_token=None,
):
assert call_endpoint == True, "Should be called with call_endpoint=True for Anthropic endpoint"
assert request.model == "gpt-4"
@ -1262,6 +1264,7 @@ async def test_anthropic_endpoint_returns_anthropic_error_format():
output_format=None,
context_management=None,
request_litellm_params=None,
client_oauth_token=None,
):
raise ProxyException(
message='{"detail":{"message":"Input is too long for requested model."}}',
@ -1319,6 +1322,7 @@ async def test_anthropic_endpoint_403_permission_error_format():
output_format=None,
context_management=None,
request_litellm_params=None,
client_oauth_token=None,
):
raise ProxyException(
message='{"Message":"Bearer Token has expired"}',
@ -1376,6 +1380,7 @@ async def test_anthropic_endpoint_429_rate_limit_error_format():
output_format=None,
context_management=None,
request_litellm_params=None,
client_oauth_token=None,
):
raise ProxyException(
message="Rate limit exceeded",

View file

@ -9,6 +9,7 @@ import pytest
import litellm
from litellm.caching.dual_cache import DualCache
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.router_utils.pre_call_checks.deployment_affinity_check import (
DeploymentAffinityCheck,
)
@ -1209,3 +1210,92 @@ async def test_virtual_key_hash_wins_over_user_id_for_affinity():
parent_otel_span=None,
)
assert other_key_same_user == healthy_deployments
class _ForbiddenError(Exception):
status_code: Final = 403
@pytest.mark.asyncio
@pytest.mark.parametrize(
("pre_call_check", "metadata"),
[
("session_affinity", {"session_id": "sess-retry"}),
("deployment_affinity", {"user_api_key_hash": "hashed-key-retry"}),
],
)
async def test_retry_reaches_healthy_sibling_when_affinity_pins_deployment_that_refused(
pre_call_check: str, metadata: dict[str, str]
):
router: Final = litellm.Router(
model_list=[
{
"model_name": "test-model",
"litellm_params": {
"model": "gpt-4o",
"api_key": "bad",
"mock_response": _ForbiddenError("A refuses"),
"weight": 1,
},
"model_info": {"id": "A"},
},
{
"model_name": "test-model",
"litellm_params": {
"model": "gpt-4o",
"api_key": "good",
"mock_response": "ok from B",
"weight": 0,
},
"model_info": {"id": "B"},
},
],
routing_strategy="simple-shuffle",
num_retries=1,
retry_after=0,
disable_cooldowns=True,
optional_pre_call_checks=[pre_call_check],
)
responses: Final = [
await router.acompletion(
model="test-model",
messages=[{"role": "user", "content": "hi"}],
metadata=dict(metadata),
)
for _ in range(2)
]
assert [response._hidden_params["model_id"] for response in responses] == ["B", "B"]
assert [response._hidden_params["additional_headers"]["x-litellm-attempted-retries"] for response in responses] == [
1,
1,
]
@pytest.mark.asyncio
async def test_previous_response_id_pin_holds_even_after_its_deployment_failed():
callback: Final = DeploymentAffinityCheck(
cache=DualCache(),
ttl_seconds=60,
enable_user_key_affinity=True,
enable_responses_api_affinity=True,
)
healthy_deployments: Final = [
{"model_name": "gpt-4o", "litellm_params": {"model": "gpt-4o"}, "model_info": {"id": "A"}},
{"model_name": "gpt-4o", "litellm_params": {"model": "gpt-4o"}, "model_info": {"id": "B"}},
]
assert await callback.async_filter_deployments(
model="gpt-4o",
healthy_deployments=healthy_deployments,
messages=None,
request_kwargs={
"previous_response_id": ResponsesAPIRequestUtils._build_responses_api_response_id(
custom_llm_provider="openai", model_id="A", response_id="resp_upstream"
),
"_excluded_deployment_ids": ["A"],
"metadata": {"user_api_key_hash": "hashed-key"},
},
parent_otel_span=None,
) == [healthy_deployments[0]]

View file

@ -16,6 +16,7 @@ The mechanism works without any cache and supports two encoding strategies:
"""
import time
from typing import Final
from unittest.mock import AsyncMock, patch
import pytest
@ -2606,3 +2607,111 @@ async def test_affinity_strips_for_team_and_pattern_routes():
assert result is sibling_pool
assert not request_kwargs["input"][1].get("encrypted_content")
def _region_deployment(region: str) -> dict[str, object]:
return {
"model_name": "gpt-5.4",
"litellm_params": {
"model": "openai/gpt-5.4",
"api_base": f"https://{region}.example.com/v1",
"api_key": f"key-{region}",
},
"model_info": {"id": region},
}
def _reasoning_input_minted_by(origin: str) -> list[dict[str, object]]:
return [
{
"type": "reasoning",
"id": ResponsesAPIRequestUtils._build_encrypted_item_id(origin, "rs_test"),
"encrypted_content": ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id("gAAAAA-blob", origin),
"summary": [{"type": "summary_text", "text": "scattering"}],
},
{"role": "user", "content": "and sunsets?"},
]
def _previous_response_id_from(origin: str) -> str:
return ResponsesAPIRequestUtils._build_responses_api_response_id("openai", origin, "resp_test")
_ORIGIN_FAILED_THIS_REQUEST: Final = pytest.mark.parametrize(
"failed_in_request",
[{"_excluded_deployment_ids": ["region-a"]}, {"_retry_skipped_deployment_ids": ("region-a",)}],
)
@pytest.mark.asyncio
@_ORIGIN_FAILED_THIS_REQUEST
@pytest.mark.parametrize("continuing", [{}, {"previous_response_id": _previous_response_id_from("region-b")}])
async def test_affinity_strips_and_dispatches_when_origin_already_failed_this_request(
failed_in_request: dict[str, object], continuing: dict[str, object]
):
from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import (
EncryptedContentAffinityCheck,
)
router: Final = litellm.Router(model_list=[_region_deployment("region-a"), _region_deployment("region-b")])
group: Final = router.model_list
reasoning_input: Final = _reasoning_input_minted_by("region-a")
result: Final = await EncryptedContentAffinityCheck(router=router).async_filter_deployments(
model="gpt-5.4",
healthy_deployments=group,
messages=None,
request_kwargs={"litellm_metadata": {}, "input": reasoning_input, **failed_in_request, **continuing},
)
assert result is group
assert reasoning_input[0] == {"type": "reasoning", "summary": [{"type": "summary_text", "text": "scattering"}]}
@pytest.mark.asyncio
@_ORIGIN_FAILED_THIS_REQUEST
async def test_affinity_keeps_origin_and_reasoning_when_previous_response_id_binds_the_failed_origin(
failed_in_request: dict[str, object],
):
from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import (
EncryptedContentAffinityCheck,
)
pool: Final = (_region_deployment("region-a"), _region_deployment("region-b"))
reasoning_input: Final = _reasoning_input_minted_by("region-a")
original_reasoning: Final = dict(reasoning_input[0])
assert await EncryptedContentAffinityCheck(router=litellm.Router(model_list=list(pool))).async_filter_deployments(
model="gpt-5.4",
healthy_deployments=list(pool),
messages=None,
request_kwargs={
"litellm_metadata": {},
"input": reasoning_input,
"previous_response_id": _previous_response_id_from("region-a"),
**failed_in_request,
},
) == [pool[0]]
assert reasoning_input[0] == original_reasoning
@pytest.mark.asyncio
@pytest.mark.parametrize("regions", [("region-a",), ("region-a", "region-b")])
async def test_affinity_keeps_reasoning_when_every_deployment_already_refused_the_request(regions: tuple[str, ...]):
from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import (
EncryptedContentAffinityCheck,
)
router: Final = litellm.Router(model_list=[_region_deployment(region) for region in regions])
reasoning_input: Final = _reasoning_input_minted_by("region-a")
original_reasoning: Final = dict(reasoning_input[0])
result: Final = await EncryptedContentAffinityCheck(router=router).async_filter_deployments(
model="gpt-5.4",
healthy_deployments=router.model_list,
messages=None,
request_kwargs={"litellm_metadata": {}, "input": reasoning_input, "_retry_skipped_deployment_ids": regions},
)
assert [deployment["model_info"]["id"] for deployment in result] == ["region-a"]
assert reasoning_input[0] == original_reasoning

View file

@ -313,6 +313,27 @@ async def test_async_filter_deployments_does_not_pin_when_target_order_is_set():
assert filtered == deployments
@pytest.mark.asyncio
@pytest.mark.parametrize(
"failed_in_request",
[{"_excluded_deployment_ids": ["dep-2"]}, {"_retry_skipped_deployment_ids": ("dep-2",)}],
)
async def test_async_filter_deployments_does_not_pin_to_a_deployment_that_failed_this_request(
failed_in_request: dict[str, object],
):
cache: Final = DualCache()
check: Final = PromptCachingDeploymentCheck(cache=cache)
messages: Final = _messages(word_count=5000)
await PromptCachingCache(cache=cache).async_add_model_id(model_id="dep-2", messages=messages, tools=None)
assert await check.async_filter_deployments(
model=MODEL_GROUP_ALIAS,
healthy_deployments=_deployments("anthropic/claude-opus-4-6", "anthropic/claude-opus-4-6"),
messages=messages,
request_kwargs=failed_in_request,
) == _deployments("anthropic/claude-opus-4-6", "anthropic/claude-opus-4-6")
@pytest.mark.asyncio
async def test_async_filter_deployments_narrows_for_group_whose_model_minimum_is_lower():
"""

View file

@ -1,10 +1,12 @@
import asyncio
import copy
import datetime
import importlib
import itertools
import random
import time
from collections.abc import Mapping
import uuid
from collections.abc import Mapping, Sequence
from typing import Final
from unittest.mock import MagicMock, patch
@ -682,6 +684,45 @@ class TestDeploymentLevelAllowedFails:
)
def _gpt_4_group_with_a_sibling() -> Sequence[Mapping[str, object]]:
return (
{"model_name": "gpt-4", "litellm_params": {"model": "openai/gpt-4"}, "model_info": {"id": "primary"}},
{"model_name": "gpt-4", "litellm_params": {"model": "openai/gpt-4"}, "model_info": {"id": "secondary"}},
)
def _primary_cooldown_decisions_on_repeated_503s(router: Router) -> tuple[bool, ...]:
exception: Final = litellm.ServiceUnavailableError("down", "openai", "gpt-4")
return tuple(
_should_cooldown_deployment(
litellm_router_instance=router,
deployment="primary",
exception_status=503,
original_exception=exception,
)
for _ in range(litellm.allowed_fails + 1)
)
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
class TestRouterLevelAllowedFailsEqualToDefault:
def test_allowed_fails_equal_to_the_default_cools_down_after_that_many_fails(self):
router: Final = _make_router(model_list=list(_gpt_4_group_with_a_sibling()), allowed_fails=litellm.allowed_fails)
assert _primary_cooldown_decisions_on_repeated_503s(router) == (False,) * litellm.allowed_fails + (True,)
def test_unset_allowed_fails_keeps_the_rate_based_default(self):
router: Final = _make_router(model_list=list(_gpt_4_group_with_a_sibling()))
assert _primary_cooldown_decisions_on_repeated_503s(router) == (False,) * (litellm.allowed_fails + 1)
def test_default_allowed_fails_saved_through_update_settings_keeps_the_rate_based_default(self):
router: Final = _make_router(model_list=list(_gpt_4_group_with_a_sibling()))
router.update_settings(allowed_fails=litellm.allowed_fails)
assert _primary_cooldown_decisions_on_repeated_503s(router) == (False,) * (litellm.allowed_fails + 1)
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
class TestDeploymentLevelAllowedFailsPolicyByExceptionType:
def test_rate_limit_error_triggers_cooldown_with_zero_threshold(self):
@ -2931,6 +2972,104 @@ async def test_bad_request_error_does_not_cool_down_a_deployment(
assert tuple(cooldown_deployments) == ()
async def _complete_like_the_proxy(router: Router, model: str) -> str:
logging_obj, request = litellm.utils.function_setup(
original_function="acompletion",
rules_obj=litellm.utils.Rules(),
start_time=datetime.datetime(2026, 1, 1),
model=model,
messages=[{"role": "user", "content": "hi"}],
litellm_call_id=str(uuid.uuid4()),
)
response: Final = await router.acompletion(**request, litellm_logging_obj=logging_obj)
model_id: Final = response._hidden_params["model_id"]
assert isinstance(model_id, str)
return model_id
_CHAT_COMPLETION_OK: Final = {
"id": "chatcmpl-1",
"object": "chat.completion",
"created": 1,
"model": "gpt-4o",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
}
def _openai_deployment(model_name: str, host: str, deployment_id: str) -> Mapping[str, object]:
return {
"model_name": model_name,
"litellm_params": {"model": "openai/gpt-4o", "api_key": "k", "api_base": f"https://{host}/v1"},
"model_info": {"id": deployment_id},
}
@pytest.mark.asyncio
async def test_fallback_deployment_stays_up_while_the_primary_is_in_cooldown(
respx_mock: respx.MockRouter,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
respx_mock.post("https://primary.test/v1/chat/completions").mock(
return_value=httpx.Response(503, json={"error": {"message": "down", "type": "server_error"}})
)
backup_route: Final = respx_mock.post("https://backup.test/v1/chat/completions").mock(
return_value=httpx.Response(200, json=_CHAT_COMPLETION_OK)
)
router: Final = litellm.Router(
model_list=[
_openai_deployment("primary", "primary.test", "primary-dep"),
_openai_deployment("backup", "backup.test", "backup-dep"),
],
fallbacks=[{"primary": ["backup"]}],
allowed_fails=litellm.allowed_fails,
num_retries=0,
)
served_by: Final = [await _complete_like_the_proxy(router, "primary") for _ in range(20)]
cooldown_deployments: Final = await async_get_cooldown_deployments(
litellm_router_instance=router,
parent_otel_span=None,
)
assert served_by == ["backup-dep"] * 20
assert len(backup_route.calls) == 20
assert tuple(cooldown_deployments) == ("primary-dep",)
@pytest.mark.asyncio
async def test_fallback_group_stays_up_after_a_bad_key_cools_the_primary_under_the_rate_based_default(
respx_mock: respx.MockRouter,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
respx_mock.post("https://primary.test/v1/chat/completions").mock(
return_value=httpx.Response(401, json={"error": {"message": "bad key", "type": "invalid_request_error"}})
)
respx_mock.post("https://backup.test/v1/chat/completions").mock(
return_value=httpx.Response(200, json=_CHAT_COMPLETION_OK)
)
router: Final = litellm.Router(
model_list=[
_openai_deployment("primary", "primary.test", "primary-dep"),
_openai_deployment("backup", "backup.test", "backup-1"),
_openai_deployment("backup", "backup.test", "backup-2"),
],
fallbacks=[{"primary": ["backup"]}],
num_retries=0,
)
served_by: Final = [await _complete_like_the_proxy(router, "primary") for _ in range(12)]
cooldown_deployments: Final = await async_get_cooldown_deployments(
litellm_router_instance=router,
parent_otel_span=None,
)
assert set(served_by) <= {"backup-1", "backup-2"}
assert tuple(cooldown_deployments) == ("primary-dep",)
TIMED_OUT_DEPLOYMENT: Final = {
"model_name": "test-model",
"litellm_params": {"model": "openai/test-model", "api_key": "test-key"},

View file

@ -14,7 +14,7 @@ from dataclasses import dataclass
from datetime import datetime
from importlib import import_module
from pathlib import Path
from typing import Final
from typing import Final, Literal
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
@ -44,12 +44,13 @@ from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kw
from litellm.litellm_core_utils.get_litellm_params import stored_control_options
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt
from litellm.llms.custom_httpx.http_handler import HTTPHandler
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.types.litellm_params import ControlOptions
from litellm.types.llms.openai import AllMessageValues, HttpxBinaryResponseContent
from litellm.types.prompts.init_prompts import PromptSpec
from litellm.types.utils import Delta, ModelResponseStream, StandardCallbackDynamicParams, StreamingChoices, Usage
from tests.unit.llms.bedrock.slow_upstream import (
CONVERSE_ANSWER,
STREAM_TIMEOUT_SECONDS,
UPSTREAM_FIRST_BYTE_SECONDS,
slow_upstream_sync_client,
@ -9950,3 +9951,126 @@ async def test_converse_fallback_prices_the_current_attempt(
else:
client.close()
configuration.reset_rust_configuration()
_LOOKUP_FUNCTION: Final = {
"name": "lookup_record",
"description": "Look up a record",
"parameters": {
"type": "object",
"properties": {"query": {"type": "string"}},
"required": ["query"],
"additionalProperties": False,
},
}
_STRICT_LOOKUP_TOOL: Final = {"type": "function", "function": {**_LOOKUP_FUNCTION, "strict": True}}
# Bedrock's 400 for a request past Claude's strict tool limits, seen live on us.anthropic.claude-sonnet-4-6 on 2026-10-20
_TOO_MANY_STRICT_TOOLS: Final = (
"The model returned the following errors: Too many strict tools (21). The maximum number of strict tools "
"supported is 20. Try reducing the number of tools marked as strict."
)
@dataclass(slots=True)
class _StrictToolsLimitRejectionFirst:
stream: Literal["off", "fake", "real"]
sent: tuple[tuple[str, dict[str, object]], ...] = ()
def __call__(self, request: httpx.Request) -> httpx.Response:
self.sent = (*self.sent, (request.url.path.rsplit("/", 1)[-1], json.loads(request.content)))
if len(self.sent) == 1:
return httpx.Response(400, json={"message": _TOO_MANY_STRICT_TOOLS})
return httpx.Response(200, content=b"" if self.stream == "real" else CONVERSE_ANSWER)
async def _converse_with_a_strict_tools_limit_rejection_first(
asynchronous: bool,
stream: Literal["off", "fake", "real"],
drop_params: bool | None,
tool: Mapping[str, object] = _STRICT_LOOKUP_TOOL,
) -> tuple[tuple[tuple[str, dict[str, object]], ...], litellm.BadRequestError | None]:
respond: Final = _StrictToolsLimitRejectionFirst(stream)
kwargs: Final = {
"model": "bedrock/us.anthropic.claude-sonnet-4-6",
"messages": [{"role": "user", "content": "Find invoice 42"}],
"tools": [tool],
"aws_access_key_id": "AKIASTRICTTOOLS",
"aws_secret_access_key": "strict-tools-secret",
"aws_region_name": "us-east-1",
"num_retries": 0,
"drop_params": drop_params,
"stream": stream != "off",
"fake_stream": stream == "fake",
}
try:
chunks: Final = (
await _acompletion_chunks(AsyncHTTPHandler(transport=httpx.MockTransport(respond)), kwargs, stream)
if asynchronous
else _completion_chunks(
HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))), kwargs, stream
)
)
except litellm.BadRequestError as error:
return respond.sent, error
assert chunks
return respond.sent, None
async def _acompletion_chunks(
client: AsyncHTTPHandler, kwargs: Mapping[str, object], stream: Literal["off", "fake", "real"]
) -> tuple[object, ...]:
response: Final = await litellm.acompletion(client=client, **kwargs)
return tuple([chunk async for chunk in response]) if stream == "fake" else (response,)
def _completion_chunks(
client: HTTPHandler, kwargs: Mapping[str, object], stream: Literal["off", "fake", "real"]
) -> tuple[object, ...]:
response: Final = litellm.completion(client=client, **kwargs)
return tuple(response) if stream == "fake" else (response,)
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
@pytest.mark.parametrize("stream", ["off", "fake", "real"], ids=["completion", "fake_stream", "stream"])
async def test_bedrock_strict_tools_limit_rejection_is_sent_again_without_strict_under_drop_params(
asynchronous: bool, stream: Literal["off", "fake", "real"], local_model_cost_map: None
) -> None:
sent, error = await _converse_with_a_strict_tools_limit_rejection_first(asynchronous, stream, drop_params=True)
assert error is None
(first_path, first_body), (second_path, second_body) = sent
first_tool_spec: Final = first_body["toolConfig"]["tools"][0]["toolSpec"]
assert first_tool_spec["strict"] is True
assert first_path == second_path == ("converse-stream" if stream == "real" else "converse")
assert second_body == {
**first_body,
"toolConfig": {
**first_body["toolConfig"],
"tools": [{"toolSpec": {key: value for key, value in first_tool_spec.items() if key != "strict"}}],
},
}
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
async def test_bedrock_strict_tools_limit_rejection_reaches_the_caller_without_drop_params(
asynchronous: bool, local_model_cost_map: None
) -> None:
sent, error = await _converse_with_a_strict_tools_limit_rejection_first(asynchronous, stream="off", drop_params=None)
assert error is not None and "Too many strict tools" in str(error)
assert len(sent) == 1
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
async def test_bedrock_strict_tools_limit_rejection_is_not_sent_again_when_no_tool_sent_strict(
asynchronous: bool, local_model_cost_map: None
) -> None:
sent, error = await _converse_with_a_strict_tools_limit_rejection_first(
asynchronous,
stream="off",
drop_params=True,
tool={"type": "function", "function": {**_LOOKUP_FUNCTION, "strict": False}},
)
assert error is not None and "Too many strict tools" in str(error)
assert len(sent) == 1

View file

@ -2,7 +2,7 @@ from __future__ import annotations
import json
import re
from collections.abc import Mapping
from collections.abc import Iterator, Mapping
from pathlib import Path
from types import MappingProxyType
from typing import Final
@ -224,6 +224,50 @@ def test_dated_variants_carry_base_alias_service_tier_pricing(prices: dict):
)
PRIORITY_SUFFIX: Final = "_priority"
MIN_PRIORITY_MARKUP: Final = 1.0
MAX_PRIORITY_MARKUP: Final = 5.0
def _rate(value: JsonValue | None) -> float | None:
return float(value) if isinstance(value, (int, float)) and not isinstance(value, bool) else None
def _priority_markup(entry: Mapping[str, JsonValue], key: str) -> float | None:
if not key.endswith(PRIORITY_SUFFIX):
return None
priority: Final = _rate(entry[key])
standard: Final = _rate(entry.get(key.removesuffix(PRIORITY_SUFFIX)))
if priority is None or standard is None or standard <= 0:
return None
return priority / standard
def priority_markups(prices: Mapping[str, JsonValue]) -> Iterator[tuple[str, str, float]]:
for name, entry in prices.items():
if not isinstance(entry, dict):
continue
for key in entry:
if (markup := _priority_markup(entry, key)) is not None:
yield name, key, markup
@pytest.mark.parametrize("path", (PRICES_PATH, BACKUP_PRICES_PATH), ids=("main", "backup"))
def test_priority_rates_stay_within_a_premium_of_their_standard_rate(path: Path) -> None:
prices: Final = TypeAdapter(dict[str, JsonValue]).validate_json(path.read_bytes())
markups: Final = tuple(priority_markups(prices))
assert markups
overpriced: Final = tuple(
f"{name}.{key} is {markup:.1f}x its standard rate"
for name, key, markup in markups
if not MIN_PRIORITY_MARKUP <= markup <= MAX_PRIORITY_MARKUP
)
assert overpriced == (), (
"a priority rate is a premium on the standard rate, so one below it or far above it was likely copied from "
"another model; drop the key when the provider sells no priority tier for the model:\n" + "\n".join(overpriced)
)
# Azure Foundry GPT-6.1 Sol announcement, 2026-09-29: https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-6-1-sol-in-microsoft-foundry-advanced-intelligence-optimized-for/4560811
AZURE_GPT_6_1_SOL_REGIONAL_PREMIUM: Final = MappingProxyType({"us": 1.1, "eu": 1.2, "apac": 1.2})

View file

@ -21239,9 +21239,9 @@ def test_router_retry_skip_stamp_feeds_deployment_ids_to_skip_on_retry(
],
)
def test_router_as_retry_skipped_deployment_ids_keeps_only_a_tuple_of_strings(value, expected):
from litellm.router import _as_retry_skipped_deployment_ids
from litellm.router_utils.common_utils import as_retry_skipped_deployment_ids
assert _as_retry_skipped_deployment_ids(value) == expected
assert as_retry_skipped_deployment_ids(value) == expected
@pytest.mark.parametrize(

View file

@ -8,13 +8,14 @@ cross-group fallback runs.
"""
from collections import Counter
from typing import Optional
from typing import Final, Optional
from unittest.mock import AsyncMock, patch
import pytest
import litellm
from litellm import Router
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
from litellm.utils import get_excluded_filtered_deployments
@ -944,3 +945,77 @@ async def test_failover_falls_through_to_external_fallback_when_remaining_in_coo
)
assert response._hidden_params["model_id"] == "fallback"
@pytest.mark.asyncio
@pytest.mark.parametrize(
("pre_call_check", "metadata"),
[
("session_affinity", {"session_id": "sess-1"}),
("deployment_affinity", {"user_api_key_hash": "hashed-key-1"}),
],
)
async def test_failover_reaches_healthy_sibling_when_affinity_pins_failed_deployment(
pre_call_check: str, metadata: dict[str, str]
):
router: Final = Router(
model_list=[
{
"model_name": "test-model",
"litellm_params": {
"model": "gpt-4o",
"api_key": "bad",
"mock_response": Exception("A down"),
"weight": 1,
},
"model_info": {"id": "A"},
},
{
"model_name": "test-model",
"litellm_params": {
"model": "gpt-4o",
"api_key": "good",
"mock_response": "ok from B",
"weight": 0,
},
"model_info": {"id": "B"},
},
{
"model_name": "fallback-model",
"litellm_params": {
"model": "gpt-4o",
"api_key": "good",
"mock_response": "ok from fallback",
},
"model_info": {"id": "fallback"},
},
],
routing_strategy="simple-shuffle",
num_retries=0,
disable_cooldowns=True,
enable_weighted_failover=True,
optional_pre_call_checks=[pre_call_check],
fallbacks=[{"test-model": ["fallback-model"]}],
)
while_a_is_down: Final = await router.acompletion(
model="test-model",
messages=[{"role": "user", "content": "hi"}],
metadata=dict(metadata),
)
router.upsert_deployment(
Deployment(
model_name="test-model",
litellm_params=LiteLLM_Params(model="gpt-4o", api_key="bad", mock_response="ok from A", weight=1),
model_info=ModelInfo(id="A"),
)
)
after_a_recovers: Final = await router.acompletion(
model="test-model",
messages=[{"role": "user", "content": "hi"}],
metadata=dict(metadata),
)
assert while_a_is_down._hidden_params["model_id"] == "B"
assert while_a_is_down._hidden_params["additional_headers"]["x-litellm-attempted-fallbacks"] == 1
assert after_a_recovers._hidden_params["model_id"] == "A"