mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge remote-tracking branch 'origin/main' into litellm_responses_comparison_response_id_decode
This commit is contained in:
commit
ee414f9b05
45 changed files with 2659 additions and 104 deletions
|
|
@ -710,6 +710,7 @@ transcribe_models: Set = set()
|
|||
gigachat_models: Set = set()
|
||||
llamagate_models: Set = set()
|
||||
coralbricks_models: Set[str] = set()
|
||||
aiand_models: Set[str] = set()
|
||||
reducto_models: Set = set()
|
||||
bedrock_mantle_models: Set = set()
|
||||
|
||||
|
|
@ -1022,6 +1023,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
|
|||
llamagate_models.add(key)
|
||||
elif value.get("litellm_provider") == "coralbricks":
|
||||
coralbricks_models.add(key)
|
||||
elif value.get("litellm_provider") == "aiand":
|
||||
aiand_models.add(key)
|
||||
elif value.get("litellm_provider") == "reducto":
|
||||
reducto_models.add(key)
|
||||
elif value.get("litellm_provider") == "bedrock_mantle":
|
||||
|
|
@ -1159,6 +1162,7 @@ model_list = list(
|
|||
| reducto_models
|
||||
| bedrock_mantle_models
|
||||
| coralbricks_models
|
||||
| aiand_models
|
||||
| set(clarifai_models)
|
||||
)
|
||||
|
||||
|
|
@ -1283,6 +1287,7 @@ def _build_models_by_provider() -> dict:
|
|||
"gigachat": gigachat_models,
|
||||
"llamagate": llamagate_models,
|
||||
"coralbricks": coralbricks_models,
|
||||
"aiand": aiand_models,
|
||||
"reducto": reducto_models,
|
||||
"bedrock_mantle": bedrock_mantle_models,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1035,6 +1035,7 @@ openai_compatible_endpoints: Final[list[str]] = [
|
|||
"https://api-inference.modelscope.cn/v1",
|
||||
"https://api.moonshot.ai/v1",
|
||||
"https://api.publicai.co/v1",
|
||||
"https://api.aiand.com/v1",
|
||||
"https://api.synthetic.new/openai/v1",
|
||||
"https://serverless.tensormesh.ai/v1",
|
||||
"https://api.stima.tech/v1",
|
||||
|
|
@ -1100,6 +1101,7 @@ openai_compatible_providers: Final[list[str]] = [
|
|||
"novita",
|
||||
"meta_llama",
|
||||
"publicai", # PublicAI - JSON-configured provider
|
||||
"aiand", # ai& - JSON-configured provider
|
||||
"synthetic", # Synthetic - JSON-configured provider
|
||||
"tensormesh", # Tensormesh - JSON-configured provider
|
||||
"apertis", # Apertis - JSON-configured provider
|
||||
|
|
|
|||
|
|
@ -27,6 +27,9 @@ anthropic_count_tokens_handler: Final = AnthropicCountTokensHandler()
|
|||
class AnthropicTokenCounter(BaseTokenCounter):
|
||||
"""Token counter implementation for Anthropic provider using the CountTokens API."""
|
||||
|
||||
def __init__(self, client_oauth_token: str | None = None) -> None:
|
||||
self.client_oauth_token: Final = client_oauth_token
|
||||
|
||||
def should_use_token_counting_api(
|
||||
self,
|
||||
custom_llm_provider: str | None = None,
|
||||
|
|
@ -95,12 +98,17 @@ class AnthropicTokenCounter(BaseTokenCounter):
|
|||
api_base: Final = litellm_params.get("api_base")
|
||||
|
||||
try:
|
||||
auth_header: Final = await AnthropicModelInfo.aget_auth_header(
|
||||
configured_auth_header: Final = await AnthropicModelInfo.aget_auth_header(
|
||||
api_key=litellm_params.get("api_key") or litellm.anthropic_key or litellm.api_key,
|
||||
api_base=api_base,
|
||||
litellm_params=litellm_params,
|
||||
allow_workload_identity=True,
|
||||
)
|
||||
auth_header: Final = (
|
||||
configured_auth_header
|
||||
if configured_auth_header is not None or self.client_oauth_token is None
|
||||
else AnthropicModelInfo.get_auth_header(api_key=self.client_oauth_token, api_base=api_base)
|
||||
)
|
||||
if auth_header is None:
|
||||
return TokenCountResponse(
|
||||
total_tokens=0,
|
||||
|
|
|
|||
|
|
@ -1,11 +1,14 @@
|
|||
from collections.abc import Callable, Coroutine
|
||||
from functools import partial
|
||||
from typing import Final, TypeAlias
|
||||
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.bedrock.common_utils import (
|
||||
NativeOutputFallback,
|
||||
is_bedrock_native_output_rejection,
|
||||
is_bedrock_strict_tools_rejection,
|
||||
warn_bedrock_native_output_rejected,
|
||||
warn_bedrock_strict_tools_dropped,
|
||||
)
|
||||
from litellm.types.utils import ModelResponse
|
||||
from litellm.utils import CustomStreamWrapper
|
||||
|
|
@ -14,12 +17,13 @@ NativeOutputResponse: TypeAlias = ModelResponse | CustomStreamWrapper
|
|||
NativeOutputAttemptResult: TypeAlias = NativeOutputResponse | Coroutine[object, object, NativeOutputResponse]
|
||||
NativeOutputAttempt: TypeAlias = Callable[[NativeOutputFallback | None], NativeOutputAttemptResult]
|
||||
NativeOutputFallbackFactory: TypeAlias = Callable[[], NativeOutputFallback | None]
|
||||
_RetryFor: TypeAlias = Callable[[BaseLLMException], NativeOutputFallback | None]
|
||||
|
||||
|
||||
def _fallback_for(
|
||||
error: BaseLLMException, fallback: NativeOutputFallbackFactory, model: str
|
||||
def _native_output_retry(
|
||||
error: BaseLLMException, fallback: NativeOutputFallbackFactory | None, model: str
|
||||
) -> NativeOutputFallback | None:
|
||||
if not is_bedrock_native_output_rejection(error.status_code, error.message):
|
||||
if fallback is None or not is_bedrock_native_output_rejection(error.status_code, error.message):
|
||||
return None
|
||||
retry: Final = fallback()
|
||||
if retry is not None:
|
||||
|
|
@ -27,16 +31,38 @@ def _fallback_for(
|
|||
return retry
|
||||
|
||||
|
||||
def _strict_tools_retry(
|
||||
error: BaseLLMException, fallback: NativeOutputFallbackFactory | None, model: str
|
||||
) -> NativeOutputFallback | None:
|
||||
if fallback is None or not is_bedrock_strict_tools_rejection(error.status_code, error.message):
|
||||
return None
|
||||
retry: Final = fallback()
|
||||
if retry is not None:
|
||||
warn_bedrock_strict_tools_dropped(model)
|
||||
return retry
|
||||
|
||||
|
||||
def _fallback_for(
|
||||
error: BaseLLMException,
|
||||
fallback: NativeOutputFallbackFactory | None,
|
||||
strict_tools_fallback: NativeOutputFallbackFactory | None,
|
||||
model: str,
|
||||
) -> NativeOutputFallback | None:
|
||||
native_output_retry: Final = _native_output_retry(error, fallback, model)
|
||||
if native_output_retry is not None:
|
||||
return native_output_retry
|
||||
return _strict_tools_retry(error, strict_tools_fallback, model)
|
||||
|
||||
|
||||
async def _await_with_native_output_fallback(
|
||||
first_attempt: Coroutine[object, object, NativeOutputResponse],
|
||||
attempt: NativeOutputAttempt,
|
||||
fallback: NativeOutputFallbackFactory,
|
||||
model: str,
|
||||
retry_for: _RetryFor,
|
||||
) -> NativeOutputResponse:
|
||||
try:
|
||||
return await first_attempt
|
||||
except BaseLLMException as error:
|
||||
retry: Final = _fallback_for(error, fallback, model)
|
||||
retry: Final = retry_for(error)
|
||||
if retry is None:
|
||||
raise
|
||||
second_attempt: Final = attempt(retry)
|
||||
|
|
@ -47,16 +73,20 @@ def call_with_native_output_fallback(
|
|||
attempt: NativeOutputAttempt,
|
||||
fallback: NativeOutputFallbackFactory | None,
|
||||
model: str,
|
||||
strict_tools_fallback: NativeOutputFallbackFactory | None = None,
|
||||
) -> NativeOutputAttemptResult:
|
||||
if fallback is None:
|
||||
if fallback is None and strict_tools_fallback is None:
|
||||
return attempt(None)
|
||||
retry_for: Final = partial(
|
||||
_fallback_for, fallback=fallback, strict_tools_fallback=strict_tools_fallback, model=model
|
||||
)
|
||||
try:
|
||||
first_attempt: Final = attempt(None)
|
||||
except BaseLLMException as error:
|
||||
retry: Final = _fallback_for(error, fallback, model)
|
||||
retry: Final = retry_for(error)
|
||||
if retry is None:
|
||||
raise
|
||||
return attempt(retry)
|
||||
if isinstance(first_attempt, Coroutine):
|
||||
return _await_with_native_output_fallback(first_attempt, attempt, fallback, model)
|
||||
return _await_with_native_output_fallback(first_attempt, attempt, retry_for)
|
||||
return first_attempt
|
||||
|
|
|
|||
|
|
@ -362,6 +362,60 @@ class NativeOutputFallback:
|
|||
litellm_params: Mapping[str, object]
|
||||
|
||||
|
||||
_BEDROCK_STRICT_TOOLS_REJECTION: Final = re.compile(
|
||||
r"Too many strict tools \(\d+\)"
|
||||
r"|Schemas contains too many (?:optional parameters|parameters with union types)"
|
||||
r"|[Cc]ompiled grammar (?:is too large|size)"
|
||||
r"|tools\.\d+\.custom(?:\.strict)?: "
|
||||
r"|Grammar compilation timed out"
|
||||
)
|
||||
_TOOLS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...])
|
||||
_TOOL_SPEC_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
def is_bedrock_strict_tools_rejection(status_code: int, error_text: str) -> bool:
|
||||
return status_code == 400 and _BEDROCK_STRICT_TOOLS_REJECTION.search(error_text) is not None
|
||||
|
||||
|
||||
def warn_bedrock_strict_tools_dropped(model: str) -> None:
|
||||
verbose_logger.warning(
|
||||
"Bedrock rejected the strict tool schemas for model=%s, so litellm sent the request again without `strict` "
|
||||
"because drop_params is set.",
|
||||
model,
|
||||
)
|
||||
|
||||
|
||||
def _without_strict(tool: Mapping[str, object]) -> Mapping[str, object]:
|
||||
spec_key: Final = next((key for key in ("function", "toolSpec") if key in tool), None)
|
||||
if spec_key is None:
|
||||
return tool
|
||||
spec: Final = _TOOL_SPEC_ADAPTER.validate_python(tool[spec_key])
|
||||
return {**tool, spec_key: {key: value for key, value in spec.items() if key != "strict"}}
|
||||
|
||||
|
||||
def _sends_strict(tool: Mapping[str, object], supports_strict_tools: bool) -> bool:
|
||||
if "toolSpec" in tool:
|
||||
return "strict" in _TOOL_SPEC_ADAPTER.validate_python(tool["toolSpec"])
|
||||
if "function" not in tool:
|
||||
return False
|
||||
return supports_strict_tools and bool(_TOOL_SPEC_ADAPTER.validate_python(tool["function"]).get("strict"))
|
||||
|
||||
|
||||
def bedrock_strict_tools_fallback(
|
||||
model: str, optional_params: Mapping[str, object], litellm_params: Mapping[str, object]
|
||||
) -> NativeOutputFallback | None:
|
||||
tools: Final = _TOOLS_ADAPTER.validate_python(optional_params.get("tools", ()))
|
||||
supports_strict_tools: Final = bedrock_converse_supports_strict_tools(
|
||||
bedrock_capability_model(model, litellm_params.get("base_model"))
|
||||
)
|
||||
if not any(_sends_strict(tool, supports_strict_tools) for tool in tools):
|
||||
return None
|
||||
return NativeOutputFallback(
|
||||
optional_params={**optional_params, "tools": [_without_strict(tool) for tool in tools]},
|
||||
litellm_params=litellm_params,
|
||||
)
|
||||
|
||||
|
||||
def requests_output_config_format(optional_params: Mapping[str, object]) -> bool:
|
||||
match optional_params:
|
||||
case {"output_config": {"format": object()}}:
|
||||
|
|
|
|||
|
|
@ -107,6 +107,12 @@
|
|||
"api_key_env": "AIHUBMIX_API_KEY",
|
||||
"api_base_env": "AIHUBMIX_API_BASE"
|
||||
},
|
||||
"aiand": {
|
||||
"base_url": "https://api.aiand.com/v1",
|
||||
"api_key_env": "AIAND_API_KEY",
|
||||
"api_base_env": "AIAND_API_BASE",
|
||||
"supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/messages"]
|
||||
},
|
||||
"crusoe": {
|
||||
"base_url": "https://api.inference.crusoecloud.com/v1",
|
||||
"api_key_env": "CRUSOE_API_KEY",
|
||||
|
|
|
|||
|
|
@ -123,6 +123,7 @@ from litellm.llms.bedrock.common_utils import (
|
|||
BedrockModelInfo,
|
||||
NativeOutputFallback,
|
||||
bedrock_route_for_request,
|
||||
bedrock_strict_tools_fallback,
|
||||
deployment_model_info,
|
||||
without_bedrock_native_output_opt_in,
|
||||
without_bedrock_route_prefix,
|
||||
|
|
@ -206,7 +207,12 @@ from litellm.utils import (
|
|||
|
||||
from ._logging import verbose_logger
|
||||
from .caching.caching import disable_cache, enable_cache, update_cache
|
||||
from .litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params, safe_deep_copy
|
||||
from .litellm_core_utils.core_helpers import (
|
||||
as_str_mapping,
|
||||
drops_unsupported_params,
|
||||
normalize_drop_params,
|
||||
safe_deep_copy,
|
||||
)
|
||||
from .litellm_core_utils.fallback_utils import (
|
||||
async_completion_with_fallbacks,
|
||||
completion_with_fallbacks,
|
||||
|
|
@ -4378,6 +4384,11 @@ def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResu
|
|||
else None
|
||||
),
|
||||
converse_model,
|
||||
strict_tools_fallback=(
|
||||
partial(bedrock_strict_tools_fallback, converse_model, converse_params, dict(litellm_params))
|
||||
if drops_unsupported_params(litellm_params.get("drop_params"))
|
||||
else None
|
||||
),
|
||||
)
|
||||
elif bedrock_route == "converse_like":
|
||||
model = model.replace("converse_like/", "")
|
||||
|
|
|
|||
|
|
@ -37504,7 +37504,6 @@
|
|||
"input_cost_per_token": 5e-08,
|
||||
"input_cost_per_token_batches": 2.5e-08,
|
||||
"input_cost_per_token_flex": 2.5e-08,
|
||||
"input_cost_per_token_priority": 2.5e-06,
|
||||
"litellm_provider": "openai",
|
||||
"max_input_tokens": 272000,
|
||||
"max_output_tokens": 128000,
|
||||
|
|
@ -37552,7 +37551,6 @@
|
|||
"deprecation_date": "2026-12-11",
|
||||
"input_cost_per_token": 5e-08,
|
||||
"input_cost_per_token_batches": 2.5e-08,
|
||||
"input_cost_per_token_priority": 2.5e-06,
|
||||
"input_cost_per_token_flex": 2.5e-08,
|
||||
"litellm_provider": "openai",
|
||||
"max_input_tokens": 272000,
|
||||
|
|
@ -69827,6 +69825,201 @@
|
|||
"supports_reasoning": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"aiand/deepseek-ai/deepseek-v4-flash": {
|
||||
"cache_read_input_token_cost": 8e-08,
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"litellm_provider": "aiand",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 384000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.5e-07,
|
||||
"source": "https://api.aiand.com/v1/models",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
"aiand/deepseek-ai/deepseek-v4-pro": {
|
||||
"cache_read_input_token_cost": 2.5e-07,
|
||||
"input_cost_per_token": 1e-06,
|
||||
"litellm_provider": "aiand",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 384000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.5e-06,
|
||||
"source": "https://api.aiand.com/v1/models",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
"aiand/deepseek-ai/deepseek-v4.1-flash": {
|
||||
"cache_read_input_token_cost": 2e-08,
|
||||
"input_cost_per_token": 3e-07,
|
||||
"litellm_provider": "aiand",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 384000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 6e-07,
|
||||
"source": "https://api.aiand.com/v1/models",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"aiand/google/gemma-4-31b-it": {
|
||||
"cache_read_input_token_cost": 5e-08,
|
||||
"input_cost_per_token": 2e-07,
|
||||
"litellm_provider": "aiand",
|
||||
"max_input_tokens": 262144,
|
||||
"max_output_tokens": 32768,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 5e-07,
|
||||
"source": "https://api.aiand.com/v1/models",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"aiand/moonshotai/kimi-k2.7-code": {
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
"input_cost_per_token": 7.5e-07,
|
||||
"litellm_provider": "aiand",
|
||||
"max_input_tokens": 262144,
|
||||
"max_output_tokens": 262144,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.5e-06,
|
||||
"source": "https://api.aiand.com/v1/models",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"aiand/moonshotai/kimi-k3": {
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
"input_cost_per_token": 3e-06,
|
||||
"litellm_provider": "aiand",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 131072,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.25e-05,
|
||||
"source": "https://api.aiand.com/v1/models",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"aiand/motif-technologies/motif-3": {
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
"input_cost_per_token": 5e-07,
|
||||
"litellm_provider": "aiand",
|
||||
"max_input_tokens": 262144,
|
||||
"max_output_tokens": 262144,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2e-06,
|
||||
"source": "https://api.aiand.com/v1/models",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
"aiand/openai/gpt-oss-120b": {
|
||||
"cache_read_input_token_cost": 8e-08,
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"litellm_provider": "aiand",
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 32768,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 6e-07,
|
||||
"source": "https://api.aiand.com/v1/models",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
"aiand/qwen/qwen3.6-27b": {
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
"input_cost_per_token": 3.2e-07,
|
||||
"litellm_provider": "aiand",
|
||||
"max_input_tokens": 262144,
|
||||
"max_output_tokens": 65536,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.2e-06,
|
||||
"source": "https://api.aiand.com/v1/models",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"aiand/qwen/qwen3.8-27b": {
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
"input_cost_per_token": 4e-07,
|
||||
"litellm_provider": "aiand",
|
||||
"max_input_tokens": 262144,
|
||||
"max_output_tokens": 32768,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3e-06,
|
||||
"source": "https://api.aiand.com/v1/models",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"aiand/zai-org/glm-5.2": {
|
||||
"cache_read_input_token_cost": 3e-07,
|
||||
"input_cost_per_token": 1e-06,
|
||||
"litellm_provider": "aiand",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 131072,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4e-06,
|
||||
"source": "https://api.aiand.com/v1/models",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
"aiand/zai-org/glm-5.3": {
|
||||
"cache_read_input_token_cost": 3e-07,
|
||||
"input_cost_per_token": 1e-06,
|
||||
"litellm_provider": "aiand",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 131072,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4e-06,
|
||||
"source": "https://api.aiand.com/v1/models",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
"aiand/zai-org/glm-5.3-flash": {
|
||||
"cache_read_input_token_cost": 3e-08,
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"litellm_provider": "aiand",
|
||||
"max_input_tokens": 1048550,
|
||||
"max_output_tokens": 131072,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 5e-07,
|
||||
"source": "https://api.aiand.com/v1/models",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"bespoke/nimble-latest": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "bespoke",
|
||||
|
|
|
|||
|
|
@ -189,6 +189,32 @@
|
|||
"generateContent": false
|
||||
}
|
||||
},
|
||||
"aiand": {
|
||||
"display_name": "ai& (`aiand`)",
|
||||
"url": "https://docs.aiand.com/api/chat-completions/",
|
||||
"endpoints": {
|
||||
"chat_completions": true,
|
||||
"responses": true,
|
||||
"text_completion": true,
|
||||
"embeddings": false,
|
||||
"image_generations": false,
|
||||
"image_edits": false,
|
||||
"audio_transcriptions": false,
|
||||
"audio_speech": false,
|
||||
"rerank": false,
|
||||
"ocr": false,
|
||||
"search": false,
|
||||
"moderations": false,
|
||||
"realtime": false,
|
||||
"video_generations": false,
|
||||
"batches": false,
|
||||
"messages": true,
|
||||
"interactions": true,
|
||||
"systemone": false,
|
||||
"a2a": true,
|
||||
"generateContent": true
|
||||
}
|
||||
},
|
||||
"amazon_nova": {
|
||||
"display_name": "Amazon Nova (`amazon_nova`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/amazon_nova",
|
||||
|
|
|
|||
|
|
@ -50,7 +50,10 @@ from litellm.proxy.common_utils.openai_error_payload import (
|
|||
with_litellm_call_id,
|
||||
)
|
||||
from litellm.proxy.common_utils.token_count_settings import count_source_fields
|
||||
from litellm.proxy.litellm_pre_call_utils import should_auto_drop_params_for_agentic_cli
|
||||
from litellm.proxy.litellm_pre_call_utils import (
|
||||
forwarded_anthropic_oauth_token,
|
||||
should_auto_drop_params_for_agentic_cli,
|
||||
)
|
||||
from litellm.router import Router
|
||||
|
||||
router: Final = APIRouter()
|
||||
|
|
@ -412,6 +415,7 @@ async def count_tokens(
|
|||
request=token_request,
|
||||
call_endpoint=True,
|
||||
messages_format="anthropic",
|
||||
client_oauth_token=forwarded_anthropic_oauth_token(request.headers),
|
||||
**_count_tokens_body_options(data, auto_drop_params),
|
||||
)
|
||||
except UnsupportedContentTypeError as e:
|
||||
|
|
|
|||
|
|
@ -363,6 +363,8 @@ _UNTRUSTED_ROOT_CONTROL_FIELDS: Final = (
|
|||
"fallback_depth",
|
||||
"_target_order",
|
||||
"attempted_targets",
|
||||
"_excluded_deployment_ids",
|
||||
"_retry_skipped_deployment_ids",
|
||||
"proxy_server_request",
|
||||
"standard_logging_object",
|
||||
"secret_fields",
|
||||
|
|
@ -444,6 +446,7 @@ _UNTRUSTED_METADATA_CONTROL_FIELDS: Final = (
|
|||
"_pipeline_managed_guardrails",
|
||||
"client_disconnected",
|
||||
"error_information",
|
||||
"_failover_excluded_ids",
|
||||
PRE_CALL_EXECUTED_GUARDRAILS_KEY,
|
||||
)
|
||||
|
||||
|
|
@ -1412,12 +1415,34 @@ def _tenant_service_name(user_api_key_dict: UserAPIKeyAuth) -> str | None:
|
|||
)
|
||||
|
||||
|
||||
def litellm_auth_header_name(headers: Headers) -> str:
|
||||
if "x-litellm-api-key" in headers:
|
||||
return "x-litellm-api-key"
|
||||
if "authorization" in headers:
|
||||
return "authorization"
|
||||
return "x-api-key"
|
||||
|
||||
|
||||
def forwarded_anthropic_oauth_token(headers: Headers) -> str | None:
|
||||
from litellm.llms.anthropic.common_utils import is_anthropic_oauth_key
|
||||
|
||||
forwarded: Final = clean_headers(headers, authenticated_with_header=litellm_auth_header_name(headers))
|
||||
return next(
|
||||
(
|
||||
value.removeprefix("Bearer ")
|
||||
for header, value in forwarded.items()
|
||||
if header.lower() == "authorization" and is_anthropic_oauth_key(value)
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def clean_headers(
|
||||
headers: Headers,
|
||||
litellm_key_header_name: str | None = None,
|
||||
forward_llm_provider_auth_headers: bool = False,
|
||||
authenticated_with_header: str | None = None,
|
||||
) -> dict:
|
||||
) -> dict[str, str]:
|
||||
"""
|
||||
Removes litellm api key from headers
|
||||
|
||||
|
|
@ -2510,26 +2535,13 @@ async def add_litellm_data_to_request(
|
|||
forward_llm_auth = general_settings.get("forward_llm_provider_auth_headers", False)
|
||||
if not forward_llm_auth:
|
||||
forward_llm_auth = getattr(litellm, "forward_llm_provider_auth_headers", False)
|
||||
# Determine which header was used for authentication
|
||||
# This enables forwarding provider keys (e.g., x-api-key) when they weren't used for LiteLLM auth
|
||||
authenticated_with_header = None
|
||||
if "x-litellm-api-key" in request.headers:
|
||||
# If x-litellm-api-key is present, it was used for auth
|
||||
authenticated_with_header = "x-litellm-api-key"
|
||||
elif "authorization" in request.headers:
|
||||
# Authorization header was used for auth
|
||||
authenticated_with_header = "authorization"
|
||||
else:
|
||||
# x-api-key or another header was used for auth
|
||||
authenticated_with_header = "x-api-key"
|
||||
|
||||
_headers: Final[dict[str, str]] = clean_headers(
|
||||
request.headers,
|
||||
litellm_key_header_name=(
|
||||
general_settings.get("litellm_key_header_name") if general_settings is not None else None
|
||||
),
|
||||
forward_llm_provider_auth_headers=forward_llm_auth,
|
||||
authenticated_with_header=authenticated_with_header,
|
||||
authenticated_with_header=litellm_auth_header_name(request.headers),
|
||||
)
|
||||
_strip_untrusted_request_header_controls(
|
||||
_headers,
|
||||
|
|
|
|||
|
|
@ -14628,6 +14628,7 @@ async def run_thread(
|
|||
# )
|
||||
# async def get_available_routes(user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth)):
|
||||
from litellm.litellm_core_utils.tool_fields import DropPolicy, drop_policy
|
||||
from litellm.llms.anthropic.count_tokens.token_counter import AnthropicTokenCounter
|
||||
from litellm.llms.base_llm.base_utils import BaseTokenCounter
|
||||
from litellm.llms.openai.responses.count_tokens.token_counter import OpenAITokenCounter
|
||||
from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient
|
||||
|
|
@ -14649,6 +14650,14 @@ from litellm.responses.litellm_completion_transformation.count_request import (
|
|||
)
|
||||
|
||||
|
||||
def _with_client_oauth_token(
|
||||
counter: BaseTokenCounter | None, client_oauth_token: str | None
|
||||
) -> BaseTokenCounter | None:
|
||||
if client_oauth_token is None or not isinstance(counter, AnthropicTokenCounter):
|
||||
return counter
|
||||
return AnthropicTokenCounter(client_oauth_token=client_oauth_token)
|
||||
|
||||
|
||||
def _get_provider_token_counter(
|
||||
deployment: dict, model_to_use: str
|
||||
) -> tuple[BaseTokenCounter | None, str | None, str | None]:
|
||||
|
|
@ -15264,6 +15273,7 @@ async def count_request_tokens(
|
|||
context_management: JsonValue = None,
|
||||
request_litellm_params: Mapping[str, JsonValue] = MappingProxyType({}),
|
||||
responses_request: Mapping[str, object] | None = None,
|
||||
client_oauth_token: str | None = None,
|
||||
) -> TokenCountResponse:
|
||||
prompt: Final = request.prompt
|
||||
messages: Final = request.messages
|
||||
|
|
@ -15321,7 +15331,8 @@ async def count_request_tokens(
|
|||
custom_llm_provider: str | None = None
|
||||
if call_endpoint is True and deployment is not None:
|
||||
# Auto-route to the correct provider based on model
|
||||
provider_counter, _model, custom_llm_provider = _get_provider_token_counter(deployment, model_to_use)
|
||||
routed_counter, _model, custom_llm_provider = _get_provider_token_counter(deployment, model_to_use)
|
||||
provider_counter = _with_client_oauth_token(routed_counter, client_oauth_token)
|
||||
if _model is not None:
|
||||
model_to_use = _model
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,32 @@
|
|||
[
|
||||
{
|
||||
"provider": "AIAND",
|
||||
"provider_display_name": "ai&",
|
||||
"litellm_provider": "aiand",
|
||||
"credential_fields": [
|
||||
{
|
||||
"key": "api_base",
|
||||
"label": "API Base",
|
||||
"placeholder": "https://api.aiand.com/v1",
|
||||
"tooltip": null,
|
||||
"required": false,
|
||||
"field_type": "text",
|
||||
"options": null,
|
||||
"default_value": null
|
||||
},
|
||||
{
|
||||
"key": "api_key",
|
||||
"label": "API Key",
|
||||
"placeholder": null,
|
||||
"tooltip": null,
|
||||
"required": true,
|
||||
"field_type": "password",
|
||||
"options": null,
|
||||
"default_value": null
|
||||
}
|
||||
],
|
||||
"default_model_placeholder": "aiand/openai/gpt-oss-120b"
|
||||
},
|
||||
{
|
||||
"provider": "SCALEDOWN",
|
||||
"provider_display_name": "ScaleDown",
|
||||
|
|
|
|||
|
|
@ -212,6 +212,7 @@ from litellm.router_utils.clientside_credential_handler import (
|
|||
)
|
||||
from litellm.router_utils.common_utils import (
|
||||
MID_STREAM_CONTINUATION_KWARG,
|
||||
as_retry_skipped_deployment_ids,
|
||||
deployments_the_caller_may_use,
|
||||
filter_team_based_models,
|
||||
filter_web_search_deployments,
|
||||
|
|
@ -238,11 +239,12 @@ from litellm.router_utils.cooldown_handlers import (
|
|||
async_get_cooldown_deployments_with_debug_info,
|
||||
get_cooldown_deployment_info,
|
||||
get_cooldown_deployments,
|
||||
is_advisor_orchestration_failure,
|
||||
is_background_response_poll_not_found,
|
||||
is_caller_scoped_auth_failure,
|
||||
is_caller_timeout_408,
|
||||
is_failure_outside_selected_deployment,
|
||||
is_video_lookup_not_found,
|
||||
mark_deployment_selection_failure,
|
||||
set_cooldown_deployments,
|
||||
)
|
||||
from litellm.router_utils.deployment_pricing import deployment_pricing_models
|
||||
|
|
@ -684,10 +686,6 @@ def _configured_model_info(deployment: Mapping[str, object]) -> Mapping[str, obj
|
|||
return _MODEL_INFO_ADAPTER.validate_python(deployment.get("model_info") or {})
|
||||
|
||||
|
||||
def _as_retry_skipped_deployment_ids(value: object) -> tuple[str, ...]:
|
||||
return tuple(item for item in value if isinstance(item, str)) if isinstance(value, tuple) else ()
|
||||
|
||||
|
||||
def _silent_experiment_targets(silent_model: object) -> tuple[str, ...]:
|
||||
if silent_model is None:
|
||||
return ()
|
||||
|
|
@ -1519,6 +1517,7 @@ class Router:
|
|||
self.allowed_fails = allowed_fails
|
||||
else:
|
||||
self.allowed_fails = litellm.allowed_fails
|
||||
self.explicit_allowed_fails = allowed_fails
|
||||
self.cooldown_time = cooldown_time or DEFAULT_COOLDOWN_TIME_SECONDS
|
||||
self.cooldown_cache = CooldownCache(cache=self.cache, default_cooldown_time=self.cooldown_time)
|
||||
self.disable_cooldowns = disable_cooldowns
|
||||
|
|
@ -8940,7 +8939,7 @@ class Router:
|
|||
return ()
|
||||
if litellm.should_retry(status_code):
|
||||
return ()
|
||||
already_skipped_ids: Final = _as_retry_skipped_deployment_ids(already_skipped)
|
||||
already_skipped_ids: Final = as_retry_skipped_deployment_ids(already_skipped)
|
||||
skipped: Final = tuple(sorted(frozenset((*already_skipped_ids, failed_deployment_id))))
|
||||
verbose_router_logger.debug(
|
||||
"Retry skips deployments that already answered %s to this request: %s", status_code, skipped
|
||||
|
|
@ -9579,10 +9578,10 @@ class Router:
|
|||
try:
|
||||
exception: Final = kwargs.get("exception", None)
|
||||
|
||||
if is_advisor_orchestration_failure(exception):
|
||||
if is_failure_outside_selected_deployment(exception):
|
||||
verbose_router_logger.debug(
|
||||
"Router: Exiting 'deployment_callback_on_failure' without cooldown. "
|
||||
"Failure originated from advisor orchestration, not the selected deployment."
|
||||
"Failure came from advisor orchestration or deployment selection, not the selected deployment."
|
||||
)
|
||||
return False
|
||||
|
||||
|
|
@ -15184,7 +15183,7 @@ class Router:
|
|||
## RETRY SKIP ## -> drop deployments that already refused this request with a
|
||||
## non-retryable status, unless that leaves nothing, so the caller still gets
|
||||
## the provider's own error instead of a no-deployments error.
|
||||
_retry_skipped_deployment_ids: Final = _as_retry_skipped_deployment_ids(
|
||||
_retry_skipped_deployment_ids: Final = as_retry_skipped_deployment_ids(
|
||||
request_kwargs.pop("_retry_skipped_deployment_ids", None) if request_kwargs else None
|
||||
)
|
||||
healthy_deployments = (
|
||||
|
|
@ -15473,6 +15472,7 @@ class Router:
|
|||
phase_event(_DEPLOYMENT_SELECTED_EVENT, pick_attributes)
|
||||
return deployment
|
||||
except Exception as e:
|
||||
mark_deployment_selection_failure(e)
|
||||
traceback_exception: Final = traceback.format_exc()
|
||||
# if router rejects call -> log to langfuse/otel/etc.
|
||||
if request_kwargs is not None:
|
||||
|
|
@ -15641,6 +15641,7 @@ class Router:
|
|||
|
||||
return deployment
|
||||
except Exception as e:
|
||||
mark_deployment_selection_failure(e)
|
||||
traceback_exception: Final = traceback.format_exc()
|
||||
if request_kwargs is not None:
|
||||
logging_obj: Final = request_kwargs.get("litellm_logging_obj", None)
|
||||
|
|
@ -16414,7 +16415,7 @@ class Router:
|
|||
)
|
||||
|
||||
## RETRY SKIP ## -> see async counterpart in async_get_healthy_deployments.
|
||||
_retry_skipped_deployment_ids: Final = _as_retry_skipped_deployment_ids(
|
||||
_retry_skipped_deployment_ids: Final = as_retry_skipped_deployment_ids(
|
||||
request_kwargs.pop("_retry_skipped_deployment_ids", None) if request_kwargs else None
|
||||
)
|
||||
healthy_deployments = (
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ from collections.abc import Mapping, Sequence
|
|||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, TypeVar
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.types.llms.openai import OpenAIFileObject
|
||||
|
|
@ -38,6 +38,7 @@ _OBJECT_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[object, object]]] = TypeAdapt
|
|||
_DEPLOYMENT_ROWS_ADAPTER: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter(
|
||||
tuple[Mapping[str, object], ...]
|
||||
)
|
||||
_DEPLOYMENT_IDS_ADAPTER: Final = TypeAdapter(tuple[str, ...])
|
||||
|
||||
|
||||
def with_admin_api_key_secret_reference(deployment: Deployment) -> Deployment:
|
||||
|
|
@ -57,6 +58,31 @@ def without_router_only_kwargs(kwargs: Mapping[str, _V]) -> dict[str, _V]:
|
|||
return {key: value for key, value in kwargs.items() if key not in ROUTER_ONLY_CALL_KWARGS}
|
||||
|
||||
|
||||
def as_retry_skipped_deployment_ids(value: object) -> tuple[str, ...]:
|
||||
if not isinstance(value, tuple):
|
||||
return ()
|
||||
items: Final[tuple[object, ...]] = value
|
||||
return tuple(item for item in items if isinstance(item, str))
|
||||
|
||||
|
||||
def as_excluded_deployment_ids(value: object) -> tuple[str, ...]:
|
||||
if value is None:
|
||||
return ()
|
||||
try:
|
||||
return _DEPLOYMENT_IDS_ADAPTER.validate_python(value)
|
||||
except ValidationError:
|
||||
return ()
|
||||
|
||||
|
||||
def deployment_ids_failed_in_request(request_kwargs: Mapping[str, object]) -> frozenset[str]:
|
||||
return frozenset(
|
||||
(
|
||||
*as_excluded_deployment_ids(request_kwargs.get("_excluded_deployment_ids")),
|
||||
*as_retry_skipped_deployment_ids(request_kwargs.get("_retry_skipped_deployment_ids")),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def request_user_api_key_auth(request_kwargs: Mapping[str, object]) -> object:
|
||||
"""The caller's auth object, from whichever metadata bucket this surface writes to."""
|
||||
metadata_value: Final = request_kwargs.get("metadata")
|
||||
|
|
|
|||
|
|
@ -57,6 +57,7 @@ else:
|
|||
Span = Any
|
||||
|
||||
_ADVISOR_ORCHESTRATION_FAILURE_ATTR: Final = "_litellm_advisor_orchestration_failure"
|
||||
_DEPLOYMENT_SELECTION_FAILURE_ATTR: Final = "_litellm_deployment_selection_failure"
|
||||
_CREDENTIAL_VALUES_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
|
|
@ -79,6 +80,16 @@ def is_advisor_orchestration_failure(exception: BaseException | None) -> bool:
|
|||
return bool(getattr(exception, _ADVISOR_ORCHESTRATION_FAILURE_ATTR, False))
|
||||
|
||||
|
||||
def mark_deployment_selection_failure(exception: BaseException) -> None:
|
||||
setattr(exception, _DEPLOYMENT_SELECTION_FAILURE_ATTR, True)
|
||||
|
||||
|
||||
def is_failure_outside_selected_deployment(exception: BaseException | None) -> bool:
|
||||
return is_advisor_orchestration_failure(exception) or bool(
|
||||
getattr(exception, _DEPLOYMENT_SELECTION_FAILURE_ATTR, False)
|
||||
)
|
||||
|
||||
|
||||
_BACKGROUND_RESPONSE_POLL_CALL_ORIGINS: Final[frozenset[InternalCallOrigin]] = frozenset(
|
||||
{BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, BACKGROUND_RESPONSE_STATUS_POLL_CALL_ORIGIN}
|
||||
)
|
||||
|
|
@ -781,14 +792,17 @@ def _is_allowed_fails_set_on_router(
|
|||
litellm_router_instance: LitellmRouter,
|
||||
) -> bool:
|
||||
"""
|
||||
Check if Router.allowed_fails is set or is Non-default Value
|
||||
Check if Router.allowed_fails was passed to the Router or is a Non-default Value
|
||||
|
||||
Returns:
|
||||
- True if Router.allowed_fails is set or is Non-default Value
|
||||
- False if Router.allowed_fails is None or is Default Value
|
||||
- True if allowed_fails was passed to the Router, even when it equals the default
|
||||
- True if Router.allowed_fails is a Non-default Value
|
||||
- False if Router.allowed_fails is None, or is the Default Value and was not passed
|
||||
"""
|
||||
if litellm_router_instance.allowed_fails is None:
|
||||
return False
|
||||
if litellm_router_instance.explicit_allowed_fails is not None:
|
||||
return True
|
||||
if litellm_router_instance.allowed_fails != litellm.allowed_fails:
|
||||
return True
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ from litellm.caching.dual_cache import DualCache
|
|||
from litellm.constants import SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY
|
||||
from litellm.integrations.custom_logger import CustomLogger, Span
|
||||
from litellm.responses.utils import ResponsesAPIRequestUtils
|
||||
from litellm.router_utils.common_utils import deployment_ids_failed_in_request
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.utils import CallTypes
|
||||
|
||||
|
|
@ -429,6 +430,9 @@ class DeploymentAffinityCheck(CustomLogger):
|
|||
session_affinity_active: Final = (
|
||||
enable_session_id or self._get_marker_session_affinity_ttl(request_kwargs=request_kwargs) is not None
|
||||
)
|
||||
failed_deployment_ids: Final = deployment_ids_failed_in_request(
|
||||
cast(Mapping[str, object], request_kwargs) # cast-ok: request kwargs keys are param names
|
||||
)
|
||||
user_key: Final = (
|
||||
self.get_user_key_from_request_kwargs(request_kwargs=request_kwargs)
|
||||
if (session_affinity_active or enable_user_key)
|
||||
|
|
@ -458,7 +462,7 @@ class DeploymentAffinityCheck(CustomLogger):
|
|||
elif isinstance(session_cache_result, str):
|
||||
session_model_id = session_cache_result
|
||||
|
||||
if session_model_id:
|
||||
if session_model_id and session_model_id not in failed_deployment_ids:
|
||||
session_deployment: Final = self._find_deployment_by_model_id(
|
||||
healthy_deployments=typed_healthy_deployments,
|
||||
model_id=session_model_id,
|
||||
|
|
@ -493,7 +497,7 @@ class DeploymentAffinityCheck(CustomLogger):
|
|||
# Backwards / safety: allow raw string values.
|
||||
model_id = cache_result
|
||||
|
||||
if not model_id:
|
||||
if not model_id or model_id in failed_deployment_ids:
|
||||
return typed_healthy_deployments
|
||||
|
||||
deployment = self._find_deployment_by_model_id(
|
||||
|
|
|
|||
|
|
@ -51,6 +51,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
strip_encrypted_reasoning_from_messages,
|
||||
)
|
||||
from litellm.responses.utils import ResponsesAPIRequestUtils
|
||||
from litellm.router_utils.common_utils import as_excluded_deployment_ids, as_retry_skipped_deployment_ids
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.router import Deployment
|
||||
from litellm.utils import get_order_filtered_deployments
|
||||
|
|
@ -182,6 +183,14 @@ class EncryptedContentAffinityCheck(CustomLogger):
|
|||
None,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _deployment_model_id(deployment: Mapping[str, object]) -> str | None:
|
||||
model_info: Final = deployment.get("model_info")
|
||||
if not isinstance(model_info, Mapping):
|
||||
return None
|
||||
deployment_id: Final = cast(Mapping[str, object], model_info).get("id") # cast-ok: narrowed by isinstance
|
||||
return None if deployment_id is None else str(deployment_id)
|
||||
|
||||
@staticmethod
|
||||
def _find_deployment_by_model_id(healthy_deployments: list[dict], model_id: str) -> dict | None:
|
||||
for deployment in healthy_deployments:
|
||||
|
|
@ -345,7 +354,8 @@ class EncryptedContentAffinityCheck(CustomLogger):
|
|||
typed_healthy_deployments: Final = cast(list[dict[str, object]], healthy_deployments)
|
||||
if not self._is_enabled_for_model_group(model):
|
||||
return typed_healthy_deployments
|
||||
target_order: Final = _REQUEST_KWARGS_ADAPTER.validate_python(request_kwargs).get("_target_order")
|
||||
typed_request_kwargs: Final = _REQUEST_KWARGS_ADAPTER.validate_python(request_kwargs)
|
||||
target_order: Final = typed_request_kwargs.get("_target_order")
|
||||
candidates: Final = (
|
||||
get_order_filtered_deployments(typed_healthy_deployments, target_order=target_order)
|
||||
if isinstance(target_order, int)
|
||||
|
|
@ -375,8 +385,31 @@ class EncryptedContentAffinityCheck(CustomLogger):
|
|||
model_id,
|
||||
)
|
||||
|
||||
previous_response_id: Final = typed_request_kwargs.get("previous_response_id")
|
||||
previous_response_origin: Final = ResponsesAPIRequestUtils.get_model_id_from_response_id(
|
||||
None if previous_response_id is None else str(previous_response_id)
|
||||
)
|
||||
previous_response_binds_origin: Final = previous_response_origin == model_id
|
||||
excluded_ids: Final[frozenset[str]] = (
|
||||
frozenset()
|
||||
if previous_response_binds_origin
|
||||
else frozenset(as_excluded_deployment_ids(typed_request_kwargs.get("_excluded_deployment_ids")))
|
||||
)
|
||||
retry_skipped_ids: Final[frozenset[str]] = (
|
||||
frozenset()
|
||||
if previous_response_binds_origin
|
||||
else frozenset(as_retry_skipped_deployment_ids(typed_request_kwargs.get("_retry_skipped_deployment_ids")))
|
||||
)
|
||||
not_excluded_candidates: Final = [
|
||||
candidate for candidate in candidates if self._deployment_model_id(candidate) not in excluded_ids
|
||||
]
|
||||
untried_candidates: Final = [
|
||||
candidate
|
||||
for candidate in not_excluded_candidates
|
||||
if self._deployment_model_id(candidate) not in retry_skipped_ids
|
||||
] or not_excluded_candidates
|
||||
deployment: Final = self._find_deployment_by_model_id(
|
||||
healthy_deployments=candidates,
|
||||
healthy_deployments=untried_candidates,
|
||||
model_id=model_id,
|
||||
)
|
||||
if deployment is not None:
|
||||
|
|
@ -392,7 +425,7 @@ class EncryptedContentAffinityCheck(CustomLogger):
|
|||
|
||||
# Follow-up switched model_name (LIT-2531): pin by Azure resource instead.
|
||||
boundary_matches, _originating = self._find_deployments_on_same_encryption_boundary(
|
||||
healthy_deployments=candidates,
|
||||
healthy_deployments=untried_candidates,
|
||||
model_id=model_id,
|
||||
)
|
||||
if boundary_matches:
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ from litellm.integrations.anthropic_cache_control_hook import (
|
|||
from litellm.integrations.custom_logger import CustomLogger, Span
|
||||
from litellm.litellm_core_utils.token_counter import offload_token_count
|
||||
from litellm.llms.bedrock.common_utils import bedrock_cache_point_ttl_supported, bedrock_capability_model
|
||||
from litellm.router_utils.common_utils import deployment_ids_failed_in_request
|
||||
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam
|
||||
from litellm.types.utils import CallTypes, StandardLoggingPayload
|
||||
from litellm.utils import (
|
||||
|
|
@ -152,7 +153,9 @@ class PromptCachingDeploymentCheck(CustomLogger):
|
|||
request_tools,
|
||||
),
|
||||
)
|
||||
if model_id_dict is not None:
|
||||
if model_id_dict is not None and model_id_dict["model_id"] not in deployment_ids_failed_in_request(
|
||||
request_data or {}
|
||||
):
|
||||
model_id: Final = model_id_dict["model_id"]
|
||||
for deployment in healthy_deployments:
|
||||
if deployment["model_info"]["id"] == model_id:
|
||||
|
|
|
|||
|
|
@ -4574,6 +4574,7 @@ class LlmProviders(str, Enum):
|
|||
POE = "poe"
|
||||
CHUTES = "chutes"
|
||||
NEOSANTARA = "neosantara"
|
||||
AIAND = "aiand"
|
||||
PARASAIL = "parasail"
|
||||
XIAOMI_MIMO = "xiaomi_mimo"
|
||||
TENSORMESH = "tensormesh"
|
||||
|
|
|
|||
|
|
@ -37504,7 +37504,6 @@
|
|||
"input_cost_per_token": 5e-08,
|
||||
"input_cost_per_token_batches": 2.5e-08,
|
||||
"input_cost_per_token_flex": 2.5e-08,
|
||||
"input_cost_per_token_priority": 2.5e-06,
|
||||
"litellm_provider": "openai",
|
||||
"max_input_tokens": 272000,
|
||||
"max_output_tokens": 128000,
|
||||
|
|
@ -37552,7 +37551,6 @@
|
|||
"deprecation_date": "2026-12-11",
|
||||
"input_cost_per_token": 5e-08,
|
||||
"input_cost_per_token_batches": 2.5e-08,
|
||||
"input_cost_per_token_priority": 2.5e-06,
|
||||
"input_cost_per_token_flex": 2.5e-08,
|
||||
"litellm_provider": "openai",
|
||||
"max_input_tokens": 272000,
|
||||
|
|
@ -69827,6 +69825,201 @@
|
|||
"supports_reasoning": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"aiand/deepseek-ai/deepseek-v4-flash": {
|
||||
"cache_read_input_token_cost": 8e-08,
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"litellm_provider": "aiand",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 384000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.5e-07,
|
||||
"source": "https://api.aiand.com/v1/models",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
"aiand/deepseek-ai/deepseek-v4-pro": {
|
||||
"cache_read_input_token_cost": 2.5e-07,
|
||||
"input_cost_per_token": 1e-06,
|
||||
"litellm_provider": "aiand",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 384000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2.5e-06,
|
||||
"source": "https://api.aiand.com/v1/models",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
"aiand/deepseek-ai/deepseek-v4.1-flash": {
|
||||
"cache_read_input_token_cost": 2e-08,
|
||||
"input_cost_per_token": 3e-07,
|
||||
"litellm_provider": "aiand",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 384000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 6e-07,
|
||||
"source": "https://api.aiand.com/v1/models",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"aiand/google/gemma-4-31b-it": {
|
||||
"cache_read_input_token_cost": 5e-08,
|
||||
"input_cost_per_token": 2e-07,
|
||||
"litellm_provider": "aiand",
|
||||
"max_input_tokens": 262144,
|
||||
"max_output_tokens": 32768,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 5e-07,
|
||||
"source": "https://api.aiand.com/v1/models",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"aiand/moonshotai/kimi-k2.7-code": {
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
"input_cost_per_token": 7.5e-07,
|
||||
"litellm_provider": "aiand",
|
||||
"max_input_tokens": 262144,
|
||||
"max_output_tokens": 262144,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.5e-06,
|
||||
"source": "https://api.aiand.com/v1/models",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"aiand/moonshotai/kimi-k3": {
|
||||
"cache_read_input_token_cost": 5e-07,
|
||||
"input_cost_per_token": 3e-06,
|
||||
"litellm_provider": "aiand",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 131072,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.25e-05,
|
||||
"source": "https://api.aiand.com/v1/models",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"aiand/motif-technologies/motif-3": {
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
"input_cost_per_token": 5e-07,
|
||||
"litellm_provider": "aiand",
|
||||
"max_input_tokens": 262144,
|
||||
"max_output_tokens": 262144,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 2e-06,
|
||||
"source": "https://api.aiand.com/v1/models",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
"aiand/openai/gpt-oss-120b": {
|
||||
"cache_read_input_token_cost": 8e-08,
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"litellm_provider": "aiand",
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 32768,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 6e-07,
|
||||
"source": "https://api.aiand.com/v1/models",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
"aiand/qwen/qwen3.6-27b": {
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
"input_cost_per_token": 3.2e-07,
|
||||
"litellm_provider": "aiand",
|
||||
"max_input_tokens": 262144,
|
||||
"max_output_tokens": 65536,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.2e-06,
|
||||
"source": "https://api.aiand.com/v1/models",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"aiand/qwen/qwen3.8-27b": {
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
"input_cost_per_token": 4e-07,
|
||||
"litellm_provider": "aiand",
|
||||
"max_input_tokens": 262144,
|
||||
"max_output_tokens": 32768,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3e-06,
|
||||
"source": "https://api.aiand.com/v1/models",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"aiand/zai-org/glm-5.2": {
|
||||
"cache_read_input_token_cost": 3e-07,
|
||||
"input_cost_per_token": 1e-06,
|
||||
"litellm_provider": "aiand",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 131072,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4e-06,
|
||||
"source": "https://api.aiand.com/v1/models",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
"aiand/zai-org/glm-5.3": {
|
||||
"cache_read_input_token_cost": 3e-07,
|
||||
"input_cost_per_token": 1e-06,
|
||||
"litellm_provider": "aiand",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 131072,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4e-06,
|
||||
"source": "https://api.aiand.com/v1/models",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
"aiand/zai-org/glm-5.3-flash": {
|
||||
"cache_read_input_token_cost": 3e-08,
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"litellm_provider": "aiand",
|
||||
"max_input_tokens": 1048550,
|
||||
"max_output_tokens": 131072,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 5e-07,
|
||||
"source": "https://api.aiand.com/v1/models",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"bespoke/nimble-latest": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "bespoke",
|
||||
|
|
|
|||
|
|
@ -189,6 +189,32 @@
|
|||
"generateContent": false
|
||||
}
|
||||
},
|
||||
"aiand": {
|
||||
"display_name": "ai& (`aiand`)",
|
||||
"url": "https://docs.aiand.com/api/chat-completions/",
|
||||
"endpoints": {
|
||||
"chat_completions": true,
|
||||
"responses": true,
|
||||
"text_completion": true,
|
||||
"embeddings": false,
|
||||
"image_generations": false,
|
||||
"image_edits": false,
|
||||
"audio_transcriptions": false,
|
||||
"audio_speech": false,
|
||||
"rerank": false,
|
||||
"ocr": false,
|
||||
"search": false,
|
||||
"moderations": false,
|
||||
"realtime": false,
|
||||
"video_generations": false,
|
||||
"batches": false,
|
||||
"messages": true,
|
||||
"interactions": true,
|
||||
"systemone": false,
|
||||
"a2a": true,
|
||||
"generateContent": true
|
||||
}
|
||||
},
|
||||
"amazon_nova": {
|
||||
"display_name": "Amazon Nova (`amazon_nova`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/amazon_nova",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,392 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import gzip
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import threading
|
||||
import uuid
|
||||
from collections.abc import Callable, Iterator, Mapping
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import yaml
|
||||
from integration._support.budget import case_timeout_seconds
|
||||
from integration._support.client import (
|
||||
JSON_OBJECT,
|
||||
Gateway,
|
||||
eventually,
|
||||
gateway_from_environment,
|
||||
list_value,
|
||||
object_value,
|
||||
string_value,
|
||||
)
|
||||
from integration._support.process import graceful_stop_seconds, owned_proxy_process
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
from pydantic import JsonValue
|
||||
|
||||
MIB: Final = 1024 * 1024
|
||||
CHAT: Final = "/v1/chat/completions"
|
||||
CHAT_MODEL: Final = "compressed-chat"
|
||||
OTHER_MODEL: Final = "compressed-chat-other"
|
||||
ANTHROPIC_MODEL: Final = "claude-haiku-5-5"
|
||||
GATED_PREFIX: Final = "gated-"
|
||||
ORIGIN: Final = "http://browser.example"
|
||||
MIDDLEWARE_HEADERS: Final = (
|
||||
"access-control-allow-origin",
|
||||
"x-frame-options",
|
||||
"content-security-policy",
|
||||
"x-content-type-options",
|
||||
)
|
||||
ZSTD_MAGIC: Final = b"\x28\xb5\x2f\xfd"
|
||||
ZSTD_SINGLE_SEGMENT_FOUR_BYTE_SIZE: Final = b"\xa0"
|
||||
ZSTD_MAX_BLOCK: Final = 128 * 1024
|
||||
NO_HEADERS: Final[Mapping[str, str]] = MappingProxyType({})
|
||||
|
||||
pytestmark: Final = [
|
||||
pytest.mark.timeout(2 * graceful_stop_seconds() + 120),
|
||||
pytest.mark.xdist_group("compressed-request-bodies"),
|
||||
]
|
||||
|
||||
|
||||
def zstd_frame(data: bytes) -> bytes:
|
||||
blocks: Final = tuple(data[start : start + ZSTD_MAX_BLOCK] for start in range(0, len(data), ZSTD_MAX_BLOCK))
|
||||
framed: Final = blocks or (b"",)
|
||||
return (
|
||||
ZSTD_MAGIC
|
||||
+ ZSTD_SINGLE_SEGMENT_FOUR_BYTE_SIZE
|
||||
+ len(data).to_bytes(4, "little")
|
||||
+ b"".join(
|
||||
(len(block) << 3 | (index == len(framed) - 1)).to_bytes(3, "little") + block
|
||||
for index, block in enumerate(framed)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def gzip_body(data: bytes) -> bytes:
|
||||
return gzip.compress(data, mtime=0)
|
||||
|
||||
|
||||
def as_sent(data: bytes) -> bytes:
|
||||
return data
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Rig:
|
||||
gateway: Gateway
|
||||
upstream: Wire
|
||||
gate: threading.Event
|
||||
limited_key: str
|
||||
|
||||
def calls(self) -> tuple[Request, ...]:
|
||||
return tuple(request for request in self.upstream.drain() if request.method == "POST")
|
||||
|
||||
|
||||
def _chat_completion() -> bytes:
|
||||
return json.dumps(
|
||||
{
|
||||
"id": f"chatcmpl-{uuid.uuid4().hex}",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "gpt-4o-mini",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "scripted"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5},
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
def _stream_frame(identity: str, delta: Mapping[str, JsonValue], finish_reason: str | None) -> bytes:
|
||||
chunk: Final = {
|
||||
"id": identity,
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "gpt-4o-mini",
|
||||
"choices": [{"index": 0, "delta": dict(delta), "finish_reason": finish_reason}],
|
||||
}
|
||||
return b"data: " + json.dumps(chunk).encode() + b"\n\n"
|
||||
|
||||
|
||||
def _chat_stream(gate: threading.Event | None) -> Reply:
|
||||
identity: Final = f"chatcmpl-{uuid.uuid4().hex}"
|
||||
return Reply(
|
||||
content_type="text/event-stream",
|
||||
chunks=(
|
||||
_stream_frame(identity, {"role": "assistant", "content": "streamed "}, None),
|
||||
_stream_frame(identity, {"content": "answer"}, "stop"),
|
||||
b"data: [DONE]\n\n",
|
||||
),
|
||||
gate_after_first=gate,
|
||||
)
|
||||
|
||||
|
||||
def _anthropic_message() -> bytes:
|
||||
return json.dumps(
|
||||
{
|
||||
"id": f"msg_{uuid.uuid4().hex}",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": ANTHROPIC_MODEL,
|
||||
"content": [{"type": "text", "text": "scripted"}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 3, "output_tokens": 2},
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
def _respond(gate: threading.Event, request: Request) -> Reply:
|
||||
if request.method == "GET":
|
||||
return Reply(body=b'{"object":"list","data":[]}')
|
||||
if request.target == "/v1/messages":
|
||||
return Reply(body=_anthropic_message())
|
||||
body: Final = JSON_OBJECT.validate_json(request.body)
|
||||
if body.get("stream") is not True:
|
||||
return Reply(body=_chat_completion())
|
||||
text: Final = string_value(object_value(list_value(body["messages"])[0])["content"])
|
||||
return _chat_stream(gate if text.startswith(GATED_PREFIX) else None)
|
||||
|
||||
|
||||
def _deployment(name: str, upstream: Wire) -> Mapping[str, JsonValue]:
|
||||
return {
|
||||
"model_name": name,
|
||||
"litellm_params": {"model": "openai/gpt-4o-mini", "api_base": f"{upstream.url}/v1", "api_key": "sk-fixture"},
|
||||
}
|
||||
|
||||
|
||||
def _config(directory: Path, upstream: Wire) -> Path:
|
||||
base: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
|
||||
configuration: Final = {
|
||||
**base,
|
||||
"model_list": [_deployment(CHAT_MODEL, upstream), _deployment(OTHER_MODEL, upstream)],
|
||||
"general_settings": {
|
||||
**object_value(base.get("general_settings", {})),
|
||||
"max_request_size_mb": 1,
|
||||
"enable_claude_code_gateway": True,
|
||||
},
|
||||
"router_settings": {**object_value(base.get("router_settings", {})), "num_retries": 0},
|
||||
}
|
||||
path: Final = directory / "compressed-request-bodies.yaml"
|
||||
path.write_text(yaml.safe_dump(configuration))
|
||||
return path
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]:
|
||||
assert os.environ.get("LITELLM_LICENSE"), (
|
||||
"LITELLM_LICENSE must be set: max_request_size_mb is an enterprise feature"
|
||||
)
|
||||
gate: Final = threading.Event()
|
||||
directory: Final = tmp_path_factory.mktemp("compressed-request-bodies")
|
||||
with gateway_from_environment() as root, wire_server(partial(_respond, gate)) as upstream:
|
||||
overrides: Final = {
|
||||
"ANTHROPIC_API_BASE": upstream.url,
|
||||
"ANTHROPIC_API_KEY": "sk-ant-synthetic",
|
||||
"MAX_DECOMPRESSED_REQUEST_BODIES_IN_FLIGHT_MB": "1",
|
||||
}
|
||||
with (
|
||||
owned_proxy_process(root, directory, overrides, config=_config(directory, upstream), workers=1) as owned,
|
||||
owned.gateway.scenario() as scenario,
|
||||
):
|
||||
yield Rig(owned.gateway, upstream, gate, scenario.key(models=[CHAT_MODEL]))
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def empty_upstream_log(rig: Rig) -> None:
|
||||
rig.upstream.drain()
|
||||
|
||||
|
||||
def _headers(key: str, coding: str | None, extra: Mapping[str, str] = NO_HEADERS) -> Mapping[str, str]:
|
||||
return {
|
||||
"Authorization": f"Bearer {key}",
|
||||
"content-type": "application/json",
|
||||
**({"content-encoding": coding} if coding is not None else {}),
|
||||
**extra,
|
||||
}
|
||||
|
||||
|
||||
def _send(
|
||||
rig: Rig,
|
||||
data: bytes,
|
||||
coding: str | None,
|
||||
*,
|
||||
key: str | None = None,
|
||||
extra: Mapping[str, str] = NO_HEADERS,
|
||||
) -> httpx.Response:
|
||||
return rig.gateway.client.post(
|
||||
CHAT,
|
||||
content=data,
|
||||
headers=_headers(rig.gateway.key if key is None else key, coding, extra),
|
||||
timeout=case_timeout_seconds(),
|
||||
)
|
||||
|
||||
|
||||
def _chat_body(model: str, text: str, *, stream: bool = False) -> bytes:
|
||||
return json.dumps(
|
||||
{
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": text}],
|
||||
"stream": stream,
|
||||
"cache": {"no-cache": True},
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
def _error_type(response: httpx.Response) -> JsonValue:
|
||||
return object_value(JSON_OBJECT.validate_json(response.content)["error"])["type"]
|
||||
|
||||
|
||||
def _answer(response: httpx.Response) -> JsonValue:
|
||||
choice: Final = object_value(list_value(JSON_OBJECT.validate_json(response.content)["choices"])[0])
|
||||
return object_value(choice["message"])["content"]
|
||||
|
||||
|
||||
def _stream(gateway: Gateway, data: bytes, first_text: threading.Event) -> tuple[int, str]:
|
||||
with (
|
||||
httpx.Client(base_url=gateway.client.base_url, timeout=case_timeout_seconds(), trust_env=False) as client,
|
||||
client.stream("POST", CHAT, content=data, headers=_headers(gateway.key, "gzip")) as response,
|
||||
):
|
||||
pieces: Final = response.iter_text()
|
||||
first: Final = next(pieces, "")
|
||||
first_text.set()
|
||||
return response.status_code, first + "".join(pieces)
|
||||
|
||||
|
||||
CODINGS: Final = (
|
||||
pytest.param("zstd", zstd_frame, id="zstd"),
|
||||
pytest.param("gzip", gzip_body, id="gzip"),
|
||||
pytest.param("gzip", as_sent, id="plain-labeled-gzip"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("coding", "encode"), CODINGS)
|
||||
def test_a_compressed_chat_body_reaches_the_provider_as_its_plain_twin(
|
||||
rig: Rig, coding: str, encode: Callable[[bytes], bytes]
|
||||
) -> None:
|
||||
body: Final = _chat_body(CHAT_MODEL, f"twin {uuid.uuid4().hex}")
|
||||
plain: Final = _send(rig, body, None)
|
||||
compressed: Final = _send(rig, encode(body), coding)
|
||||
assert plain.status_code == 200, plain.text
|
||||
assert compressed.status_code == 200, compressed.text
|
||||
assert _answer(compressed) == _answer(plain)
|
||||
assert compressed.headers["x-litellm-response-cost"] == plain.headers["x-litellm-response-cost"]
|
||||
calls: Final = rig.calls()
|
||||
assert len(calls) == 2, [call.target for call in calls]
|
||||
assert JSON_OBJECT.validate_json(calls[1].body) == JSON_OBJECT.validate_json(calls[0].body)
|
||||
assert "content-encoding" not in calls[1].headers, calls[1].headers
|
||||
|
||||
|
||||
def test_auth_reads_the_model_from_a_compressed_body(rig: Rig) -> None:
|
||||
refused: Final = _send(rig, zstd_frame(_chat_body(OTHER_MODEL, "outside the key")), "zstd", key=rig.limited_key)
|
||||
assert refused.status_code == 403, refused.text
|
||||
assert _error_type(refused) == "key_model_access_denied", refused.text
|
||||
assert rig.calls() == ()
|
||||
served: Final = _send(rig, zstd_frame(_chat_body(CHAT_MODEL, "inside the key")), "zstd", key=rig.limited_key)
|
||||
assert served.status_code == 200, served.text
|
||||
assert len(rig.calls()) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("coding", "encode"),
|
||||
(pytest.param("zstd", zstd_frame, id="zstd"), pytest.param("gzip", as_sent, id="plain-labeled-gzip")),
|
||||
)
|
||||
def test_a_compressed_pass_through_body_reaches_the_vendor_decoded(
|
||||
rig: Rig, coding: str, encode: Callable[[bytes], bytes]
|
||||
) -> None:
|
||||
body: Final = json.dumps(
|
||||
{
|
||||
"model": ANTHROPIC_MODEL,
|
||||
"max_tokens": 8,
|
||||
"messages": [{"role": "user", "content": f"pass through {uuid.uuid4().hex}"}],
|
||||
}
|
||||
).encode()
|
||||
response: Final = rig.gateway.client.post(
|
||||
"/anthropic/v1/messages",
|
||||
content=encode(body),
|
||||
headers={
|
||||
"x-litellm-api-key": rig.gateway.key,
|
||||
"anthropic-version": "2023-06-01",
|
||||
"content-type": "application/json",
|
||||
"content-encoding": coding,
|
||||
},
|
||||
timeout=case_timeout_seconds(),
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
calls: Final = rig.calls()
|
||||
assert [call.target for call in calls] == ["/v1/messages"]
|
||||
assert JSON_OBJECT.validate_json(calls[0].body) == JSON_OBJECT.validate_json(body)
|
||||
assert "content-encoding" not in calls[0].headers, calls[0].headers
|
||||
|
||||
|
||||
def test_a_gzip_bomb_gets_a_413_with_the_headers_other_errors_carry(rig: Rig) -> None:
|
||||
origin: Final = {"origin": ORIGIN}
|
||||
rejected: Final = _send(rig, gzip_body(b"\0" * 800_000), "gzip", extra=origin)
|
||||
unauthorized: Final = _send(
|
||||
rig, _chat_body(CHAT_MODEL, "unknown key"), None, key=f"sk-unknown-{uuid.uuid4().hex}", extra=origin
|
||||
)
|
||||
assert rejected.status_code == 413, rejected.text
|
||||
assert unauthorized.status_code == 401, unauthorized.text
|
||||
assert unauthorized.headers.get("access-control-allow-origin") is not None, dict(unauthorized.headers)
|
||||
assert {name: rejected.headers.get(name) for name in MIDDLEWARE_HEADERS} == {
|
||||
name: unauthorized.headers.get(name) for name in MIDDLEWARE_HEADERS
|
||||
}
|
||||
assert rig.calls() == ()
|
||||
|
||||
|
||||
def test_the_operator_size_limit_holds_a_compressed_body_to_its_plain_twins_limit(rig: Rig) -> None:
|
||||
over: Final = _chat_body(CHAT_MODEL, secrets.token_hex(600_000))
|
||||
compressed_over: Final = gzip_body(over)
|
||||
assert len(compressed_over) < MIB < len(over)
|
||||
plain: Final = _send(rig, over, None)
|
||||
rejected: Final = _send(rig, compressed_over, "gzip")
|
||||
served: Final = _send(rig, gzip_body(_chat_body(CHAT_MODEL, secrets.token_hex(300_000))), "gzip")
|
||||
assert plain.status_code == 413, plain.text
|
||||
assert rejected.status_code == plain.status_code, rejected.text
|
||||
assert served.status_code == 200, served.text
|
||||
assert len(rig.calls()) == 1
|
||||
|
||||
|
||||
def test_the_decoded_body_budget_turns_away_a_second_body_until_the_stream_ends(rig: Rig) -> None:
|
||||
held: Final = gzip_body(_chat_body(CHAT_MODEL, GATED_PREFIX + secrets.token_hex(300_000), stream=True))
|
||||
second: Final = gzip_body(_chat_body(CHAT_MODEL, secrets.token_hex(300_000)))
|
||||
keyless: Final = _send(rig, second, "gzip", key=f"sk-unknown-{uuid.uuid4().hex}")
|
||||
assert keyless.status_code == 401, keyless.text
|
||||
first_text: Final = threading.Event()
|
||||
with ThreadPoolExecutor(1) as pool:
|
||||
streaming: Final = pool.submit(_stream, rig.gateway, held, first_text)
|
||||
try:
|
||||
eventually(lambda: first_text.is_set() or streaming.done(), bool, seconds=case_timeout_seconds())
|
||||
reached: Final = rig.calls()
|
||||
turned_away: Final = _send(rig, second, "gzip")
|
||||
finally:
|
||||
rig.gate.set()
|
||||
status, streamed = streaming.result(timeout=case_timeout_seconds())
|
||||
assert len(reached) == 1, streamed
|
||||
assert turned_away.status_code == 503, turned_away.text
|
||||
assert _error_type(turned_away) == "overloaded_error", turned_away.text
|
||||
assert turned_away.headers.get("retry-after") == "1", dict(turned_away.headers)
|
||||
assert status == 200, streamed
|
||||
assert "streamed " in streamed and "answer" in streamed and "data: [DONE]" in streamed, streamed
|
||||
assert rig.calls() == ()
|
||||
after_the_stream: Final = _send(rig, second, "gzip")
|
||||
assert after_the_stream.status_code == 200, after_the_stream.text
|
||||
assert len(rig.calls()) == 1
|
||||
|
||||
|
||||
def test_a_claude_code_otlp_export_keeps_a_two_member_gzip_body(rig: Rig) -> None:
|
||||
response: Final = rig.gateway.client.post(
|
||||
"/claude_code_gateway/v1/metrics",
|
||||
content=gzip_body(b"first member") + gzip_body(b"second member"),
|
||||
headers={
|
||||
"Authorization": f"Bearer {rig.gateway.key}",
|
||||
"content-type": "application/x-protobuf",
|
||||
"content-encoding": "gzip",
|
||||
},
|
||||
timeout=case_timeout_seconds(),
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
|
|
@ -13,6 +13,7 @@ from pydantic import JsonValue
|
|||
|
||||
COUNTED_TOKENS: Final = 7
|
||||
RELEVANCE: Final = 0.75
|
||||
AZURE_DEPLOYMENT: Final = "gpt-url-join"
|
||||
_USAGE: Final = {"input_tokens": 3, "output_tokens": 2}
|
||||
|
||||
|
||||
|
|
@ -146,12 +147,20 @@ def _cohere_rerank(request: Request) -> Reply:
|
|||
)
|
||||
|
||||
|
||||
def _titan_embedding(_: Request) -> Reply:
|
||||
return _json({"embedding": list(VECTOR), "inputTextTokenCount": 2})
|
||||
|
||||
|
||||
_BY_PREFIX: Final[Mapping[str, Callable[[Request], Reply]]] = MappingProxyType(
|
||||
{
|
||||
"an": _anthropic,
|
||||
"bi": lambda request: _json(anthropic_message(marker_of(request))),
|
||||
"bc": _bedrock_converse,
|
||||
"be": _titan_embedding,
|
||||
"gm": _gemini,
|
||||
"vx": _gemini,
|
||||
"fd": _anthropic,
|
||||
"az": _openai_like(AZURE_DEPLOYMENT),
|
||||
"mi": _openai_like("mistral-small-latest"),
|
||||
"db": _openai_like("databricks-meta-llama-3-3-70b-instruct"),
|
||||
"co": _cohere_rerank,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import os
|
||||
from collections.abc import Callable, Iterator
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
|
@ -8,6 +9,7 @@ import yaml
|
|||
from integration._support.client import JSON_OBJECT, Gateway, gateway_from_environment
|
||||
from integration._support.forward_proxy import ForwardProxy, refusing_forward_proxy
|
||||
from integration._support.process import graceful_stop_seconds, owned_proxy
|
||||
from integration._support.vertex import service_account_json
|
||||
from integration.providers._api_base_trailing_slash_support import KEY, new_marker
|
||||
from pydantic import JsonValue
|
||||
|
||||
|
|
@ -17,6 +19,12 @@ _RERANK: Final = "host-rerank"
|
|||
_AZURE_CLAUDE: Final = "host-azure-claude"
|
||||
_AZURE_COUNT: Final = "host-azure-count"
|
||||
_EMPTY_BASE: Final = "host-empty-base"
|
||||
_VERTEX_HOST: Final = "vertex-gateway"
|
||||
_VERTEX_PROJECT: Final = "url-join-project"
|
||||
_VERTEX_LOCATION: Final = "us-east5"
|
||||
_VERTEX_CLAUDE_BASES: Final = MappingProxyType(
|
||||
{"host-vertex-claude": f"http://{_VERTEX_HOST}", "host-vertex-claude-slash": f"http://{_VERTEX_HOST}/"}
|
||||
)
|
||||
_SCRUBBED_PREFIXES: Final = ("ANTHROPIC_", "AZURE_AI_", "INFINITY_")
|
||||
_PROXY_VARIABLES: Final = frozenset({"HTTPS_PROXY", "HTTP_PROXY", "ALL_PROXY", "NO_PROXY"})
|
||||
_load_yaml: Final[Callable[[str], object]] = yaml.safe_load
|
||||
|
|
@ -28,7 +36,20 @@ def _deployment(name: str, model: str, api_base: str) -> dict[str, JsonValue]:
|
|||
return {"model_name": name, "litellm_params": {"model": model, "api_base": api_base, "api_key": KEY}}
|
||||
|
||||
|
||||
def _config(directory: Path) -> Path:
|
||||
def _vertex_claude_deployment(name: str, api_base: str, token_url: str) -> dict[str, JsonValue]:
|
||||
return {
|
||||
"model_name": name,
|
||||
"litellm_params": {
|
||||
"model": "vertex_ai/claude-haiku-5-5",
|
||||
"api_base": api_base,
|
||||
"vertex_project": _VERTEX_PROJECT,
|
||||
"vertex_location": _VERTEX_LOCATION,
|
||||
"vertex_credentials": service_account_json(_VERTEX_PROJECT, token_url),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _config(directory: Path, token_url: str) -> Path:
|
||||
base: Final = JSON_OBJECT.validate_python(_load_yaml(Path("tests/integration/proxy_config.yaml").read_text()))
|
||||
configuration: Final = {
|
||||
**base,
|
||||
|
|
@ -38,6 +59,7 @@ def _config(directory: Path) -> Path:
|
|||
_deployment(_AZURE_CLAUDE, "azure_ai/claude-haiku-5-5", f"http://{_AZURE_HOST}/anthropic"),
|
||||
_deployment(_AZURE_COUNT, "azure_ai/claude-haiku-5-5", f"http://{_AZURE_HOST}"),
|
||||
_deployment(_EMPTY_BASE, "anthropic/claude-haiku-5-5", ""),
|
||||
*(_vertex_claude_deployment(name, base, token_url) for name, base in _VERTEX_CLAUDE_BASES.items()),
|
||||
],
|
||||
"router_settings": {"disable_cooldowns": True, "num_retries": 0},
|
||||
}
|
||||
|
|
@ -72,7 +94,7 @@ def isolated(forward: ForwardProxy, tmp_path_factory: pytest.TempPathFactory) ->
|
|||
"NO_PROXY": "127.0.0.1,localhost",
|
||||
"PROXY_BATCH_POLLING_ENABLED": "false",
|
||||
},
|
||||
config=_config(directory),
|
||||
config=_config(directory, shared.upstream_url.rstrip("/")),
|
||||
remove_environment=_scrubbed(),
|
||||
) as candidate:
|
||||
yield candidate
|
||||
|
|
@ -138,3 +160,14 @@ def test_azure_claude_count_tokens_on_a_bare_host_starting_with_anthropic_adds_t
|
|||
def test_empty_api_base_falls_back_to_the_provider_default_host(isolated: Gateway, forward: ForwardProxy) -> None:
|
||||
targets: Final = _targets_after(forward, lambda: _chat(isolated, _EMPTY_BASE))
|
||||
assert targets == {"api.anthropic.com:443"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", tuple(_VERTEX_CLAUDE_BASES), ids=("no-slash", "slash"))
|
||||
def test_vertex_partner_claude_on_a_bare_host_reaches_one_single_slash_url(
|
||||
isolated: Gateway, forward: ForwardProxy, model: str
|
||||
) -> None:
|
||||
targets: Final = _targets_after(forward, lambda: _chat(isolated, model))
|
||||
assert targets == {
|
||||
f"http://{_VERTEX_HOST}/v1/projects/{_VERTEX_PROJECT}/locations/{_VERTEX_LOCATION}"
|
||||
"/publishers/anthropic/models/claude-haiku-5-5:rawPredict"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import openai
|
|||
import pytest
|
||||
from integration._support.client import Gateway, Scenario, eventually, list_value, object_value, string_value
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.vertex import service_account_json
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from integration.providers._api_base_trailing_slash_support import (
|
||||
KEY,
|
||||
|
|
@ -22,7 +23,7 @@ from integration.providers._api_base_trailing_slash_support import (
|
|||
reply_text,
|
||||
request_json,
|
||||
)
|
||||
from integration.providers._api_base_url_join_support import COUNTED_TOKENS, RELEVANCE, join_peer
|
||||
from integration.providers._api_base_url_join_support import AZURE_DEPLOYMENT, COUNTED_TOKENS, RELEVANCE, join_peer
|
||||
from pydantic import JsonValue
|
||||
|
||||
_HAIKU: Final = "anthropic.claude-haiku-5-5"
|
||||
|
|
@ -49,10 +50,20 @@ _RESPONSES_ROWS: Final = (_ANTHROPIC, _GEMINI)
|
|||
_EMBEDDING_ROWS: Final = (
|
||||
Row("mistral/mistral-embed", "/mi/v1", "/embeddings"),
|
||||
Row("gemini/gemini-embedding-2", "/gm/v1beta", "/models/gemini-embedding-2:batchEmbedContents"),
|
||||
Row("bedrock/amazon.titan-embed-text-v2:0", "/be", "/model/amazon.titan-embed-text-v2%3A0/invoke"),
|
||||
)
|
||||
_TRANSCRIPTION: Final = Row("mistral/voxtral-mini-latest", "/mi/v1", "/audio/transcriptions")
|
||||
_FULL_ROUTE_SUFFIXES: Final = ("", "/", "?beta=true")
|
||||
_FULL_ROUTE_IDS: Final = ("exact", "slash", "query")
|
||||
_FULL_ROUTE_SUFFIXES: Final = (("", ""), ("/", ""), ("?beta=true", "?beta=true"), ("/?beta=true", "?beta=true"))
|
||||
_FULL_ROUTE_IDS: Final = ("exact", "slash", "query", "slash-query")
|
||||
_VERTEX_PROJECT: Final = "url-join-project"
|
||||
_VERTEX_LOCATION: Final = "us-east5"
|
||||
_VERTEX_GEMINI_PATH: Final = (
|
||||
f"/vx/v1/projects/{_VERTEX_PROJECT}/locations/{_VERTEX_LOCATION}/publishers/google/models/gemini-3.8-flash"
|
||||
)
|
||||
_AZURE_V1_BASES: Final = ("/az", "/az/", "/az/openai/v1", "/az/openai/v1/")
|
||||
_AZURE_V1_IDS: Final = ("host", "host-slash", "route", "route-slash")
|
||||
_AZURE_ANTHROPIC_BASES: Final = ("/fd/anthropic", "/fd/anthropic/v1/messages")
|
||||
_AZURE_ANTHROPIC_IDS: Final = ("anthropic-root", "messages-route")
|
||||
|
||||
|
||||
def _ids(rows: tuple[Row, ...]) -> tuple[str, ...]:
|
||||
|
|
@ -65,6 +76,17 @@ def _deploy(scenario: Scenario, row: Row, api_base: str) -> str:
|
|||
return scenario.model(model=row.model, api_base=api_base, api_key=KEY)
|
||||
|
||||
|
||||
def _deploy_vertex(gateway: Gateway, scenario: Scenario, model: str, api_base: str) -> str:
|
||||
return scenario.model(
|
||||
model=model,
|
||||
api_base=api_base,
|
||||
api_key=None,
|
||||
vertex_project=_VERTEX_PROJECT,
|
||||
vertex_location=_VERTEX_LOCATION,
|
||||
vertex_credentials=service_account_json(_VERTEX_PROJECT, gateway.upstream_url.rstrip("/")),
|
||||
)
|
||||
|
||||
|
||||
def _openai(gateway: Gateway) -> openai.OpenAI:
|
||||
return openai.OpenAI(base_url=str(gateway.client.base_url) + "/v1", api_key=gateway.key, max_retries=0)
|
||||
|
||||
|
|
@ -80,6 +102,13 @@ def _chat(gateway: Gateway, model: str, marker: str) -> str | None:
|
|||
return completion.choices[0].message.content
|
||||
|
||||
|
||||
def _streamed_chat(gateway: Gateway, model: str, marker: str) -> str:
|
||||
stream: Final = _openai(gateway).chat.completions.create(
|
||||
model=model, messages=[{"role": "user", "content": marker}], stream=True
|
||||
)
|
||||
return "".join(chunk.choices[0].delta.content or "" for chunk in stream if chunk.choices)
|
||||
|
||||
|
||||
def _message_text(gateway: Gateway, model: str, marker: str) -> list[str]:
|
||||
message: Final = _anthropic(gateway).messages.create(
|
||||
model=model, max_tokens=16, messages=[{"role": "user", "content": marker}]
|
||||
|
|
@ -270,22 +299,74 @@ def test_cache_hit_on_a_slash_base_skips_the_upstream_and_logs_its_own_row(gatew
|
|||
assert "//" not in urlsplit(string_value(hit["api_base"])).path, hit
|
||||
|
||||
|
||||
@pytest.mark.parametrize("suffix", _FULL_ROUTE_SUFFIXES, ids=_FULL_ROUTE_IDS)
|
||||
def test_chat_base_that_already_names_the_route_keeps_its_path_and_query(gateway: Gateway, suffix: str) -> None:
|
||||
@pytest.mark.parametrize(("suffix", "sent"), _FULL_ROUTE_SUFFIXES, ids=_FULL_ROUTE_IDS)
|
||||
def test_chat_base_that_already_names_the_route_keeps_its_path_and_query(
|
||||
gateway: Gateway, suffix: str, sent: str
|
||||
) -> None:
|
||||
marker: Final = new_marker()
|
||||
with wire_server(join_peer) as wire, gateway.scenario() as scenario:
|
||||
model: Final = _deploy(scenario, _ANTHROPIC, f"{wire.url}/an/v1/messages{suffix}")
|
||||
assert _chat(gateway, model, marker) == reply_text(marker)
|
||||
assert_one_request(wire.drain(), f"/an/v1/messages{suffix.rstrip('/')}", marker)
|
||||
assert_one_request(wire.drain(), f"/an/v1/messages{sent}", marker)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("suffix", _FULL_ROUTE_SUFFIXES, ids=_FULL_ROUTE_IDS)
|
||||
def test_messages_base_that_already_names_the_route_keeps_its_path_and_query(gateway: Gateway, suffix: str) -> None:
|
||||
@pytest.mark.parametrize(("suffix", "sent"), _FULL_ROUTE_SUFFIXES, ids=_FULL_ROUTE_IDS)
|
||||
def test_messages_base_that_already_names_the_route_keeps_its_path_and_query(
|
||||
gateway: Gateway, suffix: str, sent: str
|
||||
) -> None:
|
||||
marker: Final = new_marker()
|
||||
with wire_server(join_peer) as wire, gateway.scenario() as scenario:
|
||||
model: Final = _deploy(scenario, _ANTHROPIC, f"{wire.url}/an/v1/messages{suffix}")
|
||||
assert _message_text(gateway, model, marker) == [reply_text(marker)]
|
||||
assert_one_request(wire.drain(), f"/an/v1/messages{suffix.rstrip('/')}", marker)
|
||||
assert_one_request(wire.drain(), f"/an/v1/messages{sent}", marker)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("trailing", TRAILING, ids=TRAILING_IDS)
|
||||
def test_vertex_model_url_base_calls_its_method_with_no_slash_before_the_colon(gateway: Gateway, trailing: str) -> None:
|
||||
marker: Final = new_marker()
|
||||
with wire_server(join_peer) as wire, gateway.scenario() as scenario:
|
||||
model: Final = _deploy_vertex(
|
||||
gateway, scenario, "vertex_ai/gemini-3.8-flash", f"{wire.url}{_VERTEX_GEMINI_PATH}{trailing}"
|
||||
)
|
||||
assert _chat(gateway, model, marker) == reply_text(marker)
|
||||
assert_one_request(wire.drain(), f"{_VERTEX_GEMINI_PATH}:generateContent", marker, keyed=False)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("trailing", TRAILING, ids=TRAILING_IDS)
|
||||
def test_vertex_model_url_base_streams_with_no_slash_before_the_colon(gateway: Gateway, trailing: str) -> None:
|
||||
marker: Final = new_marker()
|
||||
with wire_server(join_peer) as wire, gateway.scenario() as scenario:
|
||||
model: Final = _deploy_vertex(
|
||||
gateway, scenario, "vertex_ai/gemini-3.8-flash", f"{wire.url}{_VERTEX_GEMINI_PATH}{trailing}"
|
||||
)
|
||||
assert _streamed_chat(gateway, model, marker) == reply_text(marker)
|
||||
assert_one_request(wire.drain(), f"{_VERTEX_GEMINI_PATH}:streamGenerateContent?alt=sse", marker, keyed=False)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("base", _AZURE_V1_BASES, ids=_AZURE_V1_IDS)
|
||||
def test_azure_v1_chat_reaches_one_openai_v1_route(gateway: Gateway, base: str) -> None:
|
||||
marker: Final = new_marker()
|
||||
with wire_server(join_peer) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model=f"azure/{AZURE_DEPLOYMENT}", api_base=f"{wire.url}{base}", api_key=KEY, api_version="v1"
|
||||
)
|
||||
assert _chat(gateway, model, marker) == reply_text(marker)
|
||||
assert_one_request(wire.drain(), "/az/openai/v1/chat/completions", marker)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("trailing", TRAILING, ids=TRAILING_IDS)
|
||||
@pytest.mark.parametrize("base", _AZURE_ANTHROPIC_BASES, ids=_AZURE_ANTHROPIC_IDS)
|
||||
def test_azure_ai_count_tokens_reaches_one_count_tokens_route(gateway: Gateway, base: str, trailing: str) -> None:
|
||||
marker: Final = new_marker()
|
||||
with wire_server(join_peer) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model="azure_ai/claude-haiku-5-5", api_base=f"{wire.url}{base}{trailing}", api_key=KEY
|
||||
)
|
||||
response: Final = gateway.request(
|
||||
"POST", "/v1/messages/count_tokens", {"model": model, "messages": [{"role": "user", "content": marker}]}
|
||||
)
|
||||
assert body_of(response)["input_tokens"] == COUNTED_TOKENS
|
||||
assert_one_request(wire.drain(), "/fd/anthropic/v1/messages/count_tokens", marker)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("suffix", ("", "/", "?tier=a"), ids=("exact", "slash", "query"))
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import json
|
||||
from collections.abc import Iterator
|
||||
from collections.abc import Callable, Iterator
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import anthropic
|
||||
|
|
@ -8,6 +9,7 @@ import pytest
|
|||
from integration._support.client import Gateway, gateway_from_environment
|
||||
from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_OAUTH_BETA_HEADER
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
|
||||
|
|
@ -26,6 +28,14 @@ _PROVIDER_KEY_VARIABLES: Final = (
|
|||
"AZURE_TENANT_ID",
|
||||
"OPENAI_API_KEY",
|
||||
)
|
||||
_ENVIRONMENT_KEYS: Final = MappingProxyType(
|
||||
{
|
||||
"ANTHROPIC_API_KEY": "synthetic-anthropic-env-key",
|
||||
"AZURE_API_KEY": "synthetic-azure-env-key",
|
||||
"OPENAI_API_KEY": "synthetic-openai-env-key",
|
||||
}
|
||||
)
|
||||
_SUBSCRIPTION_TOKEN: Final = "sk-ant-oat01-synthetic-subscription-token"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
|
|
@ -44,14 +54,14 @@ def keyless_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[OwnedPro
|
|||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def azure_environment_key_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[OwnedProxy]:
|
||||
def environment_key_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[OwnedProxy]:
|
||||
with (
|
||||
gateway_from_environment() as gateway,
|
||||
owned_proxy_process(
|
||||
gateway,
|
||||
tmp_path_factory.mktemp("count-tokens-azure-environment-key"),
|
||||
{"AZURE_API_KEY": "synthetic-azure-env-key"},
|
||||
remove_environment=tuple(name for name in _PROVIDER_KEY_VARIABLES if name != "AZURE_API_KEY"),
|
||||
tmp_path_factory.mktemp("count-tokens-environment-keys"),
|
||||
_ENVIRONMENT_KEYS,
|
||||
remove_environment=tuple(name for name in _PROVIDER_KEY_VARIABLES if name not in _ENVIRONMENT_KEYS),
|
||||
workers=1,
|
||||
) as owned,
|
||||
):
|
||||
|
|
@ -147,9 +157,9 @@ def test_anthropic_count_without_a_provider_credential_is_an_anthropic_401(
|
|||
|
||||
|
||||
def test_azure_ai_anthropic_count_sends_the_azure_environment_key_as_x_api_key(
|
||||
azure_environment_key_proxy: OwnedProxy,
|
||||
environment_key_proxy: OwnedProxy,
|
||||
) -> None:
|
||||
gateway: Final = azure_environment_key_proxy.gateway
|
||||
gateway: Final = environment_key_proxy.gateway
|
||||
|
||||
def count(request: Request) -> Reply:
|
||||
assert request.headers.get("x-api-key") == "synthetic-azure-env-key", request.headers
|
||||
|
|
@ -257,3 +267,122 @@ def test_google_count_tokens_errors_use_the_google_envelope(keyless_proxy: Owned
|
|||
assert _JSON_OBJECT.validate_json(malformed.content) == {
|
||||
"error": {"code": 400, "message": "Invalid Google countTokens request body", "status": "INVALID_ARGUMENT"}
|
||||
}
|
||||
|
||||
|
||||
def _counted(tokens: int) -> Reply:
|
||||
return Reply(body=json.dumps({"object": "response.input_tokens", "input_tokens": tokens}).encode())
|
||||
|
||||
|
||||
def _anthropic_count(model: str) -> tuple[str, dict[str, JsonValue]]:
|
||||
return "/v1/messages/count_tokens", {"model": model, "messages": _MESSAGES}
|
||||
|
||||
|
||||
def _responses_count(model: str) -> tuple[str, dict[str, JsonValue]]:
|
||||
return "/v1/responses/input_tokens", {"model": model, "input": "Count this synthetic request."}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("provider_model", "base_path", "count_request", "header", "credential"),
|
||||
(
|
||||
("anthropic/claude-sonnet-5-5", "", _anthropic_count, "x-api-key", "synthetic-anthropic-env-key"),
|
||||
("openai/gpt-4o-mini", "/v1", _responses_count, "authorization", "Bearer synthetic-openai-env-key"),
|
||||
),
|
||||
ids=("anthropic", "openai"),
|
||||
)
|
||||
def test_count_uses_a_provider_key_set_only_in_the_proxy_environment(
|
||||
environment_key_proxy: OwnedProxy,
|
||||
provider_model: str,
|
||||
base_path: str,
|
||||
count_request: Callable[[str], tuple[str, dict[str, JsonValue]]],
|
||||
header: str,
|
||||
credential: str,
|
||||
) -> None:
|
||||
gateway: Final = environment_key_proxy.gateway
|
||||
with wire_server(lambda _request: _counted(47)) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model=provider_model, api_base=f"{wire.url}{base_path}", api_key=None, num_retries=0
|
||||
)
|
||||
path, body = count_request(model)
|
||||
counted: Final = gateway.request("POST", path, body)
|
||||
assert counted.status_code == 200, counted.text
|
||||
assert _JSON_OBJECT.validate_json(counted.content)["input_tokens"] == 47, counted.text
|
||||
received: Final = wire.drain()
|
||||
assert [request.headers.get(header) for request in received] == [credential], received
|
||||
|
||||
|
||||
def _google_count(model: str) -> tuple[str, dict[str, JsonValue]]:
|
||||
return f"/v1beta/models/{model}:countTokens", {
|
||||
"contents": [{"role": "user", "parts": [{"text": "Count this synthetic request."}]}]
|
||||
}
|
||||
|
||||
|
||||
def _utils_count(model: str) -> tuple[str, dict[str, JsonValue]]:
|
||||
return "/utils/token_counter?call_endpoint=true", {"model": model, "messages": _MESSAGES}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("count_request", "google_status"),
|
||||
((_google_count, "UNAUTHENTICATED"), (_utils_count, None)),
|
||||
ids=("google", "utils"),
|
||||
)
|
||||
def test_count_entrypoints_report_a_missing_provider_key_in_their_own_shape(
|
||||
keyless_proxy: OwnedProxy,
|
||||
count_request: Callable[[str], tuple[str, dict[str, JsonValue]]],
|
||||
google_status: str | None,
|
||||
) -> None:
|
||||
gateway: Final = keyless_proxy.gateway
|
||||
with wire_server(_never_called) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model="anthropic/claude-sonnet-5-5", api_base=wire.url, api_key=None, num_retries=0
|
||||
)
|
||||
path, body = count_request(model)
|
||||
rejected: Final = gateway.request("POST", path, body)
|
||||
assert rejected.status_code == 401, rejected.text
|
||||
error: Final = _JSON_OBJECT.validate_json(rejected.content)["error"]
|
||||
assert isinstance(error, dict), rejected.text
|
||||
assert f"No Anthropic API key is set for model {model}" in str(error["message"]), rejected.text
|
||||
if google_status is not None:
|
||||
assert (error["code"], error["status"]) == (401, google_status), rejected.text
|
||||
assert wire.drain() == ()
|
||||
|
||||
|
||||
def test_anthropic_count_uses_the_callers_subscription_token_when_the_deployment_has_no_key(
|
||||
keyless_proxy: OwnedProxy,
|
||||
) -> None:
|
||||
gateway: Final = keyless_proxy.gateway
|
||||
with wire_server(lambda _request: _counted(41)) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model="anthropic/claude-sonnet-5-5", api_base=wire.url, api_key=None, num_retries=0
|
||||
)
|
||||
counted: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/messages/count_tokens",
|
||||
{"model": model, "messages": _MESSAGES},
|
||||
headers={"Authorization": f"Bearer {_SUBSCRIPTION_TOKEN}", "x-litellm-api-key": f"Bearer {gateway.key}"},
|
||||
)
|
||||
assert counted.status_code == 200, counted.text
|
||||
assert _JSON_OBJECT.validate_json(counted.content)["input_tokens"] == 41, counted.text
|
||||
(received,) = wire.drain()
|
||||
assert received.headers.get("authorization") == f"Bearer {_SUBSCRIPTION_TOKEN}", received.headers
|
||||
assert "x-api-key" not in received.headers, received.headers
|
||||
assert ANTHROPIC_OAUTH_BETA_HEADER in received.headers.get("anthropic-beta", ""), received.headers
|
||||
|
||||
|
||||
def test_anthropic_count_keeps_the_deployments_key_over_the_callers_subscription_token(
|
||||
keyless_proxy: OwnedProxy,
|
||||
) -> None:
|
||||
gateway: Final = keyless_proxy.gateway
|
||||
with wire_server(lambda _request: _counted(43)) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model="anthropic/claude-sonnet-5-5", api_base=wire.url, api_key="synthetic-deployment-key", num_retries=0
|
||||
)
|
||||
counted: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/messages/count_tokens",
|
||||
{"model": model, "messages": _MESSAGES},
|
||||
headers={"Authorization": f"Bearer {_SUBSCRIPTION_TOKEN}", "x-litellm-api-key": f"Bearer {gateway.key}"},
|
||||
)
|
||||
assert counted.status_code == 200, counted.text
|
||||
(received,) = wire.drain()
|
||||
assert received.headers.get("x-api-key") == "synthetic-deployment-key", received.headers
|
||||
assert _SUBSCRIPTION_TOKEN not in json.dumps(dict(received.headers)), received.headers
|
||||
|
|
|
|||
|
|
@ -48,6 +48,7 @@ VERTEX_ACCESS_TOKEN: Final = "ya29.scripted-delete-token"
|
|||
AWS_ACCESS_KEY_ID: Final = "AKIAIOSFODNN7EXAMPLE"
|
||||
AWS_SECRET_ACCESS_KEY: Final = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
|
||||
AWS_REGION: Final = "us-east-1"
|
||||
S3_OBJECT_BODY: Final = b'{"recordId":"r1","modelOutput":{"stop_reason":"end_turn"}}\n'
|
||||
_ForbiddenSurface = Literal[
|
||||
"upload_sdk_extra_body",
|
||||
"upload_form",
|
||||
|
|
@ -708,6 +709,86 @@ def test_non_admin_cannot_delete_a_raw_gcs_uri_before_vertex_auth_or_storage(gat
|
|||
assert token_requests == (), "non-admin raw GCS delete reached the Vertex token endpoint"
|
||||
|
||||
|
||||
def test_non_admin_still_reads_an_ordinary_provider_file_id(gateway: Gateway) -> None:
|
||||
with wire_server(_openai_backend()) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model="openai/gpt-4o-mini", api_base=wire.url + "/v1", api_key="provider-key-" + uuid.uuid4().hex[:8]
|
||||
)
|
||||
_wait_until_every_worker_serves(gateway, model)
|
||||
member: Final = _member(scenario, model)
|
||||
file_id: Final = "file-" + uuid.uuid4().hex
|
||||
|
||||
read: Final = gateway.request("GET", f"/v1/files/{file_id}/content", key=member.key, params={"model": model})
|
||||
assert read.status_code == 200, read.text
|
||||
assert read.content == b"file-content\n", read.content
|
||||
assert [
|
||||
(request.method, urlsplit(request.target).path) for request in _drained_other_than_model_list_probes(wire)
|
||||
] == [("GET", f"/v1/files/{file_id}/content")]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("method", "route_suffix", "verb"),
|
||||
[("GET", "/content", "retrieved"), ("DELETE", "", "deleted")],
|
||||
ids=["content", "delete"],
|
||||
)
|
||||
def test_non_admin_cannot_reach_vertex_with_a_twice_encoded_raw_gcs_uri(
|
||||
gateway: Gateway, method: Literal["GET", "DELETE"], route_suffix: str, verb: str
|
||||
) -> None:
|
||||
with (
|
||||
wire_server(_vertex_token_backend) as token_wire,
|
||||
gateway.scenario() as scenario,
|
||||
):
|
||||
model: Final = scenario.model(
|
||||
model=VERTEX_MODEL,
|
||||
api_key=None,
|
||||
vertex_project=VERTEX_PROJECT,
|
||||
vertex_location=VERTEX_LOCATION,
|
||||
vertex_credentials=_vertex_credentials(token_wire.url),
|
||||
gcs_bucket_name=BUCKET,
|
||||
)
|
||||
_wait_until_every_worker_serves(gateway, model)
|
||||
caller_user: Final = scenario.user(user_role="internal_user")
|
||||
caller_key: Final = scenario.key(user_id=caller_user, models=[model])
|
||||
object_uri: Final = f"gs://{BUCKET}/litellm-vertex-files/litellm-vertex-files-obj.jsonl"
|
||||
twice_encoded: Final = quote(quote(object_uri, safe=""), safe="")
|
||||
token_wire.drain()
|
||||
|
||||
denied: Final = gateway.request(
|
||||
method, f"/v1/files/{twice_encoded}{route_suffix}", key=caller_key, params={"model": model}
|
||||
)
|
||||
token_requests: Final = token_wire.drain()
|
||||
assert denied.status_code == 403, f"{denied.text}; token requests={token_requests}"
|
||||
assert _error_message(denied, 403) == (
|
||||
f"Raw cloud storage file ids can only be {verb} by a proxy admin key. "
|
||||
"Use the LiteLLM managed file id returned when the file was created."
|
||||
), denied.text
|
||||
assert token_requests == (), f"non-admin twice-encoded raw GCS {method} reached the Vertex token endpoint"
|
||||
|
||||
|
||||
def _assert_signed_empty_body_s3_request(request: Request, method: Literal["GET", "DELETE"], wire_url: str) -> None:
|
||||
assert request.method == method, request.method
|
||||
request_date: Final = request.headers["x-amz-date"]
|
||||
content_hash: Final = hashlib.sha256(b"").hexdigest()
|
||||
assert request.headers["x-amz-content-sha256"] == content_hash, request.headers
|
||||
aws_request: Final = AWSRequest(
|
||||
method=method,
|
||||
url=wire_url + request.target,
|
||||
headers={
|
||||
"host": request.headers["host"],
|
||||
"x-amz-date": request_date,
|
||||
"x-amz-content-sha256": content_hash,
|
||||
},
|
||||
)
|
||||
aws_request.context["timestamp"] = request_date
|
||||
auth: Final = S3SigV4Auth(Credentials(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY), "s3", AWS_REGION)
|
||||
signature: Final = auth.signature(auth.string_to_sign(aws_request, auth.canonical_request(aws_request)), aws_request)
|
||||
expected_authorization: Final = (
|
||||
f"AWS4-HMAC-SHA256 Credential={AWS_ACCESS_KEY_ID}/{request_date[:8]}/{AWS_REGION}/s3/aws4_request, "
|
||||
f"SignedHeaders={auth.signed_headers(auth.headers_to_sign(aws_request))}, Signature={signature}"
|
||||
)
|
||||
assert request.headers["authorization"] == expected_authorization, request.headers["authorization"]
|
||||
|
||||
|
||||
def test_proxy_admin_deletes_a_raw_s3_uri_with_one_signed_delete(gateway: Gateway) -> None:
|
||||
with wire_server(_s3_backend()) as wire, gateway.scenario() as scenario:
|
||||
model: Final = _bedrock_model(scenario, wire)
|
||||
|
|
@ -721,29 +802,30 @@ def test_proxy_admin_deletes_a_raw_s3_uri_with_one_signed_delete(gateway: Gatewa
|
|||
assert parsed.deleted is True and parsed.id == object_uri, deleted.text
|
||||
(request,) = _drained_other_than_model_list_probes(wire)
|
||||
assert request.target == f"/{BUCKET}/litellm-bedrock-files-obj.jsonl", request.target
|
||||
request_date: Final = request.headers["x-amz-date"]
|
||||
content_hash: Final = hashlib.sha256(b"").hexdigest()
|
||||
assert request.headers["x-amz-content-sha256"] == content_hash, request.headers
|
||||
aws_request: Final = AWSRequest(
|
||||
method="DELETE",
|
||||
url=wire.url + request.target,
|
||||
headers={
|
||||
"host": request.headers["host"],
|
||||
"x-amz-date": request_date,
|
||||
"x-amz-content-sha256": content_hash,
|
||||
},
|
||||
_assert_signed_empty_body_s3_request(request, "DELETE", wire.url)
|
||||
|
||||
|
||||
def _s3_object_backend(request: Request) -> Reply:
|
||||
if request.method == "GET":
|
||||
return Reply(body=S3_OBJECT_BODY, content_type="application/octet-stream")
|
||||
return _json_reply({"error": {"message": f"unscripted {request.method} {request.target}"}}, 404)
|
||||
|
||||
|
||||
def test_proxy_admin_reads_a_raw_s3_uri_with_one_signed_get(gateway: Gateway) -> None:
|
||||
with wire_server(_s3_object_backend) as wire, gateway.scenario() as scenario:
|
||||
model: Final = _bedrock_model(scenario, wire)
|
||||
_wait_until_every_worker_serves(gateway, model)
|
||||
object_key: Final = f"litellm-batch-outputs/{uuid.uuid4().hex}.jsonl.out"
|
||||
object_uri: Final = f"s3://{BUCKET}/{object_key}"
|
||||
|
||||
read: Final = gateway.request(
|
||||
"GET", "/v1/files/" + quote(object_uri, safe="") + "/content", params={"model": model}
|
||||
)
|
||||
aws_request.context["timestamp"] = request_date
|
||||
auth: Final = S3SigV4Auth(Credentials(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY), "s3", AWS_REGION)
|
||||
signature: Final = auth.signature(
|
||||
auth.string_to_sign(aws_request, auth.canonical_request(aws_request)), aws_request
|
||||
)
|
||||
expected_authorization: Final = (
|
||||
f"AWS4-HMAC-SHA256 Credential={AWS_ACCESS_KEY_ID}/{request_date[:8]}/{AWS_REGION}/s3/aws4_request, "
|
||||
f"SignedHeaders={auth.signed_headers(auth.headers_to_sign(aws_request))}, Signature={signature}"
|
||||
)
|
||||
assert request.headers["authorization"] == expected_authorization, request.headers["authorization"]
|
||||
assert request.method == "DELETE", request.method
|
||||
storage_requests: Final = _drained_other_than_model_list_probes(wire)
|
||||
assert read.status_code == 200, f"{read.text}; storage requests={storage_requests}"
|
||||
assert read.content == S3_OBJECT_BODY, read.content
|
||||
assert [request.target for request in storage_requests] == [f"/{BUCKET}/{object_key}"], storage_requests
|
||||
_assert_signed_empty_body_s3_request(storage_requests[0], "GET", wire.url)
|
||||
|
||||
|
||||
def _granting_vertex_token_backend(request: Request) -> Reply:
|
||||
|
|
|
|||
|
|
@ -642,6 +642,15 @@ def _key_budget(scenario: Scenario, budget_owner: str, model_budget: JsonValue)
|
|||
return {}
|
||||
|
||||
|
||||
def _assert_the_next_create_exceeds_the_model_budget(scenario: Scenario, key: str, model: str) -> None:
|
||||
refused: Final = eventually(
|
||||
lambda: _next_create(scenario, key, model),
|
||||
lambda response: response.status_code != 200,
|
||||
seconds=SPEND_LANDING_SECONDS,
|
||||
)
|
||||
assert f"exceeded budget for model={model}" in refused.text, refused.text
|
||||
|
||||
|
||||
@pytest.mark.timeout(240)
|
||||
@pytest.mark.parametrize("budget_owner", ["key", "team", "user", "budget_tier"])
|
||||
def test_a_billed_background_response_counts_toward_its_creators_project_and_model_budget(
|
||||
|
|
@ -663,12 +672,24 @@ def test_a_billed_background_response_counts_toward_its_creators_project_and_mod
|
|||
lambda spends: spends[0] > 0 and costs(spends[0])(spends[1]),
|
||||
seconds=SPEND_LANDING_SECONDS,
|
||||
)
|
||||
refused: Final = eventually(
|
||||
lambda: _next_create(scenario, key, upstream.model),
|
||||
lambda response: response.status_code != 200,
|
||||
seconds=SPEND_LANDING_SECONDS,
|
||||
_assert_the_next_create_exceeds_the_model_budget(scenario, key, upstream.model)
|
||||
|
||||
|
||||
@pytest.mark.timeout(240)
|
||||
def test_a_billed_background_response_on_a_team_model_counts_toward_the_budget_of_the_model_name_it_was_created_with(
|
||||
gateway: Gateway,
|
||||
) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
team_id: Final = scenario.team()
|
||||
upstream: Final = scripted_upstream(scenario, background_routes("completed"), model_info={"team_id": team_id})
|
||||
key: Final = scenario.key(
|
||||
team_id=team_id, model_max_budget={upstream.model: {"max_budget": PRICE / 2, "budget_duration": "1d"}}
|
||||
)
|
||||
assert f"exceeded budget for model={upstream.model}" in refused.text, refused.text
|
||||
response_id: Final = create_background_response(scenario, key, upstream.model)
|
||||
read: Final = read_response(gateway, key, response_id)
|
||||
assert (read,) in (settling_reads("completed", 1), free_reads("completed", 1))
|
||||
eventually(lambda: key_spend(key), lambda spend: spend > 0, seconds=SPEND_LANDING_SECONDS)
|
||||
_assert_the_next_create_exceeds_the_model_budget(scenario, key, upstream.model)
|
||||
|
||||
|
||||
@pytest.mark.timeout(240)
|
||||
|
|
|
|||
|
|
@ -167,3 +167,51 @@ def test_provider_500_failure_rows_count_deepseek_v4_prompt_tokens_with_deepseek
|
|||
short: Final = _failure_row_prompt_tokens(gateway, model, key, SHORT_TEXT)
|
||||
assert wire.drain()
|
||||
assert_counted_with_deepseek(chinese - short, chinese_minus_short)
|
||||
|
||||
|
||||
COMPACTION_INSTRUCTIONS: Final[JsonValue] = {"role": "system", "content": "answer briefly"}
|
||||
COMPACTED_HISTORY: Final[tuple[JsonValue, ...]] = (
|
||||
{"role": "user", "content": "a long turn " * 200},
|
||||
{"role": "assistant", "content": "a long reply " * 200},
|
||||
)
|
||||
COMPACTION_ITEM: Final[JsonValue] = {
|
||||
"type": "compaction",
|
||||
"encrypted_content": json.dumps({"type": "compaction", "content": "the cat is named Biscuit"}),
|
||||
}
|
||||
TURN_AFTER_COMPACTION: Final[JsonValue] = {"role": "user", "content": "what is the cat's name?"}
|
||||
|
||||
|
||||
def _failed_responses_prompt_tokens(gateway: Gateway, model: str, key: str, items: list[JsonValue]) -> int:
|
||||
failed: Final = gateway.request("POST", "/v1/responses", {"model": model, "input": items}, key=key)
|
||||
assert failed.status_code == 500 and "synthetic provider outage" in failed.text, failed.text
|
||||
rows: Final = eventually(
|
||||
lambda: read_rows(
|
||||
'SELECT status, prompt_tokens FROM "LiteLLM_SpendLogs" WHERE request_id=%s',
|
||||
(failed.headers["x-litellm-call-id"],),
|
||||
),
|
||||
lambda values: len(values) == 1,
|
||||
seconds=70,
|
||||
)
|
||||
assert rows[0]["status"] == "failure", rows
|
||||
prompt_tokens: Final = rows[0]["prompt_tokens"]
|
||||
assert isinstance(prompt_tokens, int) and prompt_tokens > 0, rows
|
||||
return prompt_tokens
|
||||
|
||||
|
||||
def test_a_failed_responses_request_counts_its_input_from_the_newest_compaction_item(gateway: Gateway) -> None:
|
||||
with wire_server(_provider_500) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(api_base=wire.url + "/v1", num_retries=0)
|
||||
key: Final = scenario.key(models=[model])
|
||||
resent: Final = _failed_responses_prompt_tokens(
|
||||
gateway, model, key, [COMPACTION_INSTRUCTIONS, *COMPACTED_HISTORY, COMPACTION_ITEM, TURN_AFTER_COMPACTION]
|
||||
)
|
||||
compacted: Final = _failed_responses_prompt_tokens(
|
||||
gateway, model, key, [COMPACTION_INSTRUCTIONS, COMPACTION_ITEM, TURN_AFTER_COMPACTION]
|
||||
)
|
||||
uncompacted: Final = _failed_responses_prompt_tokens(
|
||||
gateway, model, key, [COMPACTION_INSTRUCTIONS, *COMPACTED_HISTORY, TURN_AFTER_COMPACTION]
|
||||
)
|
||||
assert len(wire.drain()) == 3
|
||||
|
||||
assert resent == compacted
|
||||
assert resent < uncompacted
|
||||
|
|
|
|||
73
tests/integration/spend/test_streamed_usage_estimate.py
Normal file
73
tests/integration/spend/test_streamed_usage_estimate.py
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
import json
|
||||
import uuid
|
||||
from hashlib import sha256
|
||||
from typing import Final
|
||||
|
||||
from integration._support.client import Gateway, eventually
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.deepseek_tokens import (
|
||||
CHINESE_TEXT,
|
||||
SHORT_TEXT,
|
||||
assert_counted_with_deepseek,
|
||||
chinese_minus_short,
|
||||
)
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from pydantic import JsonValue
|
||||
|
||||
_DEEPSEEK_V4: Final = "DeepSeek-V4-Flash-0731"
|
||||
|
||||
|
||||
def _data_frame(frame: dict[str, JsonValue]) -> bytes:
|
||||
return b"data: " + json.dumps(frame).encode() + b"\n\n"
|
||||
|
||||
|
||||
def _stream_without_usage(request: Request) -> Reply:
|
||||
assert request.method == "POST"
|
||||
chunk: Final[dict[str, JsonValue]] = {
|
||||
"id": f"chatcmpl-{uuid.uuid4().hex}",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": _DEEPSEEK_V4,
|
||||
}
|
||||
return Reply(
|
||||
content_type="text/event-stream",
|
||||
chunks=(
|
||||
_data_frame({**chunk, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "OK"}}]}),
|
||||
_data_frame({**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}),
|
||||
b"data: [DONE]\n\n",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _streamed_prompt_tokens(gateway: Gateway, key: str, model: str, text: str) -> int:
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": model, "messages": [{"role": "user", "content": text}], "stream": True},
|
||||
key=key,
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.text.rstrip().endswith("data: [DONE]"), response.text
|
||||
rows: Final = eventually(
|
||||
lambda: read_rows(
|
||||
'SELECT status, prompt_tokens FROM "LiteLLM_SpendLogs" WHERE api_key=%s',
|
||||
(sha256(key.encode()).hexdigest(),),
|
||||
),
|
||||
lambda values: len(values) == 1,
|
||||
seconds=70,
|
||||
)
|
||||
assert rows[0]["status"] == "success", rows
|
||||
prompt_tokens: Final = rows[0]["prompt_tokens"]
|
||||
assert isinstance(prompt_tokens, int), rows
|
||||
return prompt_tokens
|
||||
|
||||
|
||||
def test_a_deepseek_v4_stream_without_usage_bills_prompt_tokens_counted_with_deepseeks_tokenizer(
|
||||
gateway: Gateway,
|
||||
) -> None:
|
||||
with wire_server(_stream_without_usage) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"azure_ai/{_DEEPSEEK_V4}", api_base=wire.url, num_retries=0)
|
||||
chinese: Final = _streamed_prompt_tokens(gateway, scenario.key(models=[model]), model, CHINESE_TEXT)
|
||||
short: Final = _streamed_prompt_tokens(gateway, scenario.key(models=[model]), model, SHORT_TEXT)
|
||||
assert len(wire.drain()) == 2
|
||||
assert_counted_with_deepseek(chinese - short, chinese_minus_short)
|
||||
|
|
@ -221,6 +221,45 @@ class TestCountTokensUsesWorkloadIdentity:
|
|||
assert result.total_tokens == 13
|
||||
assert route.calls.last.request.headers["x-api-key"] == FAKE_REGULAR_KEY
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("deployment_key", "expected_credentials"),
|
||||
[
|
||||
(None, {"authorization": f"Bearer {FAKE_OAUTH_TOKEN}", "x-api-key": None}),
|
||||
(FAKE_REGULAR_KEY, {"authorization": None, "x-api-key": FAKE_REGULAR_KEY}),
|
||||
],
|
||||
ids=["keyless-deployment-uses-the-callers-token", "deployment-key-wins-over-the-callers-token"],
|
||||
)
|
||||
async def test_a_callers_subscription_token_counts_only_when_no_key_is_configured(
|
||||
self,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
httpx_transport_clients: None,
|
||||
deployment_key: str | None,
|
||||
expected_credentials: dict[str, str | None],
|
||||
) -> None:
|
||||
for name in ("ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_API_BASE", "ANTHROPIC_BASE_URL"):
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
monkeypatch.setattr(litellm, "anthropic_key", None)
|
||||
monkeypatch.setattr(litellm, "api_key", None)
|
||||
|
||||
with respx.mock(assert_all_called=True) as router:
|
||||
route = router.post("https://api.anthropic.com/v1/messages/count_tokens").mock(
|
||||
return_value=httpx.Response(200, json={"input_tokens": 17})
|
||||
)
|
||||
result = await AnthropicTokenCounter(client_oauth_token=FAKE_OAUTH_TOKEN).count_tokens(
|
||||
model_to_use="claude-sonnet-4-5",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
contents=None,
|
||||
deployment={"litellm_params": {"model": "anthropic/claude-sonnet-4-5", "api_key": deployment_key}},
|
||||
request_model="claude-sonnet-4-5",
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert result.total_tokens == 17
|
||||
sent = route.calls.last.request.headers
|
||||
assert {name: sent.get(name) for name in expected_credentials} == expected_credentials
|
||||
assert (ANTHROPIC_OAUTH_BETA_HEADER in sent["anthropic-beta"].split(",")) is (deployment_key is None)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_failed_mint_degrades_like_an_anthropic_error(self, monkeypatch):
|
||||
from litellm.llms.anthropic.count_tokens import token_counter as token_counter_module
|
||||
|
|
|
|||
|
|
@ -121,3 +121,105 @@ def test_a_rejection_logs_the_retry_and_advises_opting_out_only_when_native_outp
|
|||
|
||||
assert _MODEL in caplog.text
|
||||
assert ("supports_bedrock_native_structured_output: false" in caplog.text) is advises_opting_out
|
||||
|
||||
|
||||
_STRICT_FALLBACK: Final = NativeOutputFallback(optional_params={"tools": "without_strict"}, litellm_params={})
|
||||
# Bedrock's 400s for strict tools, seen live on us.anthropic.claude-sonnet-4-6 from 2026-10-20 to 2026-10-28 and quoted
|
||||
# in issue #34388, each answered 200 once the request dropped strict
|
||||
_STRICT_TOOLS_REJECTIONS: Final = tuple(
|
||||
BedrockError(status_code=400, message=f'{{"message":"The model returned the following errors: {text}"}}')
|
||||
for text in (
|
||||
"Too many strict tools (21). The maximum number of strict tools supported is 20.",
|
||||
"Schemas contains too many optional parameters (27), which would make grammar compilation inefficient.",
|
||||
"Schemas contains too many parameters with union types (18 parameters with type arrays or anyOf). "
|
||||
"This causes exponential compilation cost.",
|
||||
"The compiled grammar is too large, which would cause performance issues.",
|
||||
"tools.0.custom: Invalid schema: Schema type is missing for schema: {'description': 'tag'}",
|
||||
"tools.3.custom: For 'integer' type, property 'minimum' is not supported",
|
||||
"Compiled grammar size (410.4MB) exceeds maximum allowed size (300MB)",
|
||||
"tools.1.custom.strict: Extra inputs are not permitted",
|
||||
"Grammar compilation timed out.",
|
||||
)
|
||||
)
|
||||
_TOO_MANY_STRICT_TOOLS: Final = _STRICT_TOOLS_REJECTIONS[0]
|
||||
# Bedrock's 400s for requests with a strict tool that fail for other reasons, seen live on
|
||||
# us.anthropic.claude-sonnet-4-6 on 2026-10-21, which echo tool names and tool call ids
|
||||
_OTHER_400S_WITH_A_STRICT_TOOL: Final = tuple(
|
||||
BedrockError(status_code=400, message=f'{{"message":"{text}"}}')
|
||||
for text in (
|
||||
"The tool check_grammar is already defined at toolConfig.tools.1.",
|
||||
"Expected toolResult blocks at messages.0.content for the following Ids: call_grammar_9",
|
||||
"tool_use ids were found without tool_result blocks immediately after: call_grammar_1",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _call_with_strict_tools_fallback(
|
||||
bedrock: ScriptedBedrock, fallback: NativeOutputFallback | None, strict_tools_fallback: NativeOutputFallback | None
|
||||
) -> object:
|
||||
return _resolve(
|
||||
call_with_native_output_fallback(
|
||||
bedrock,
|
||||
None if fallback is None else lambda: fallback,
|
||||
_MODEL,
|
||||
strict_tools_fallback=None if strict_tools_fallback is None else lambda: strict_tools_fallback,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@_MODES
|
||||
@pytest.mark.parametrize("rejection", _STRICT_TOOLS_REJECTIONS, ids=lambda error: error.message[60:100])
|
||||
def test_a_strict_tools_rejection_is_sent_again_once_with_the_strict_tools_fallback(
|
||||
asynchronous: bool, rejection: BedrockError
|
||||
) -> None:
|
||||
bedrock: Final = ScriptedBedrock(errors=(rejection, None), asynchronous=asynchronous)
|
||||
|
||||
response: Final = _call_with_strict_tools_fallback(bedrock, None, _STRICT_FALLBACK)
|
||||
|
||||
assert isinstance(response, ModelResponse) and response.id == "answer-2"
|
||||
assert bedrock.sent == (None, _STRICT_FALLBACK)
|
||||
|
||||
|
||||
@_MODES
|
||||
@pytest.mark.parametrize(
|
||||
("error", "strict_tools_fallback"),
|
||||
[
|
||||
(_OTHER_400, _STRICT_FALLBACK),
|
||||
*((error, _STRICT_FALLBACK) for error in _OTHER_400S_WITH_A_STRICT_TOOL),
|
||||
(BedrockError(status_code=500, message=_TOO_MANY_STRICT_TOOLS.message), _STRICT_FALLBACK),
|
||||
(_TOO_MANY_STRICT_TOOLS, None),
|
||||
],
|
||||
ids=[
|
||||
"other_400",
|
||||
"duplicate_tool_name",
|
||||
"missing_tool_result",
|
||||
"orphan_tool_use",
|
||||
"server_error",
|
||||
"no_strict_tools_fallback",
|
||||
],
|
||||
)
|
||||
def test_no_second_request_without_both_a_strict_tools_rejection_and_a_strict_tools_fallback(
|
||||
asynchronous: bool, error: BedrockError, strict_tools_fallback: NativeOutputFallback | None
|
||||
) -> None:
|
||||
bedrock: Final = ScriptedBedrock(errors=(error,), asynchronous=asynchronous)
|
||||
|
||||
with pytest.raises(BedrockError) as raised:
|
||||
_call_with_strict_tools_fallback(bedrock, None, strict_tools_fallback)
|
||||
|
||||
assert raised.value is error
|
||||
assert bedrock.sent == (None,)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("error", "retried_with"),
|
||||
[(_REJECTION, _FALLBACK), (_TOO_MANY_STRICT_TOOLS, _STRICT_FALLBACK)],
|
||||
ids=["native_output_rejection", "strict_tools_rejection"],
|
||||
)
|
||||
def test_each_rejection_is_sent_again_with_its_own_fallback_when_both_are_set(
|
||||
error: BedrockError, retried_with: NativeOutputFallback
|
||||
) -> None:
|
||||
bedrock: Final = ScriptedBedrock(errors=(error, None), asynchronous=False)
|
||||
|
||||
_call_with_strict_tools_fallback(bedrock, _FALLBACK, _STRICT_FALLBACK)
|
||||
|
||||
assert bedrock.sent == (None, retried_with)
|
||||
|
|
|
|||
|
|
@ -6,7 +6,9 @@ from typing import Final
|
|||
from litellm.llms.bedrock.common_utils import(
|
||||
AmazonBedrockGlobalConfig,
|
||||
BedrockModelInfo,
|
||||
NativeOutputFallback,
|
||||
bedrock_native_output_is_opt_in,
|
||||
bedrock_strict_tools_fallback,
|
||||
deployment_model_info,
|
||||
extract_model_name_from_bedrock_arn,
|
||||
get_bedrock_base_model,
|
||||
|
|
@ -2150,3 +2152,80 @@ def test_bedrock_response_stream_shape_is_unavailable_when_botocore_describes_th
|
|||
assert shape is None
|
||||
(logged,) = warning.call_args_list
|
||||
assert "only-in-the-description" not in logged.args[0] % logged.args[1:]
|
||||
|
||||
|
||||
def test_bedrock_strict_tools_fallback_drops_only_strict_and_keeps_everything_else():
|
||||
strict_tool: Final = {
|
||||
"type": "function",
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
"function": {"name": "lookup", "description": "Look up", "strict": True, "parameters": {"type": "object"}},
|
||||
}
|
||||
plain_tool: Final = {"type": "function", "function": {"name": "plain", "parameters": {"type": "object"}}}
|
||||
anthropic_tool: Final = {"name": "native", "input_schema": {"type": "object"}}
|
||||
bedrock_tool: Final = {"toolSpec": {"name": "spec", "strict": True, "inputSchema": {"json": {"type": "object"}}}}
|
||||
litellm_params: Final = {"drop_params": True}
|
||||
|
||||
fallback: Final = bedrock_strict_tools_fallback(
|
||||
"us.anthropic.claude-sonnet-4-6",
|
||||
{"tools": [strict_tool, plain_tool, anthropic_tool, bedrock_tool], "maxTokens": 16, "stream": True},
|
||||
litellm_params,
|
||||
)
|
||||
|
||||
assert fallback == NativeOutputFallback(
|
||||
optional_params={
|
||||
"tools": [
|
||||
{**strict_tool, "function": {"name": "lookup", "description": "Look up", "parameters": {"type": "object"}}},
|
||||
plain_tool,
|
||||
anthropic_tool,
|
||||
{"toolSpec": {"name": "spec", "inputSchema": {"json": {"type": "object"}}}},
|
||||
],
|
||||
"maxTokens": 16,
|
||||
"stream": True,
|
||||
},
|
||||
litellm_params=litellm_params,
|
||||
)
|
||||
assert strict_tool["function"]["strict"] is True
|
||||
assert bedrock_tool["toolSpec"]["strict"] is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"optional_params",
|
||||
[
|
||||
{"maxTokens": 16},
|
||||
{"tools": [{"type": "function", "function": {"name": "plain", "parameters": {}}}]},
|
||||
{"tools": [{"toolSpec": {"name": "spec", "inputSchema": {"json": {}}}}]},
|
||||
],
|
||||
ids=["no_tools", "no_strict_tool", "no_strict_tool_spec"],
|
||||
)
|
||||
def test_bedrock_strict_tools_fallback_is_none_without_a_strict_tool(optional_params: dict[str, object]) -> None:
|
||||
assert bedrock_strict_tools_fallback("us.anthropic.claude-sonnet-4-6", optional_params, {}) is None
|
||||
|
||||
|
||||
_CLAUDE_PROFILE_ARN: Final = "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/claude"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "litellm_params"),
|
||||
[
|
||||
("us.anthropic.claude-sonnet-4-6", {}),
|
||||
("us.amazon.nova-pro-v1:0", {}),
|
||||
(_CLAUDE_PROFILE_ARN, {"base_model": "bedrock/us.anthropic.claude-sonnet-4-6"}),
|
||||
(_CLAUDE_PROFILE_ARN, {}),
|
||||
],
|
||||
ids=["claude", "nova", "claude_profile_with_base_model", "profile_without_base_model"],
|
||||
)
|
||||
@pytest.mark.parametrize("strict", [True, False])
|
||||
def test_bedrock_strict_tools_fallback_retries_only_when_the_request_sent_strict(
|
||||
model: str, litellm_params: dict[str, object], strict: bool
|
||||
) -> None:
|
||||
from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig
|
||||
|
||||
tools: Final = [{"type": "function", "function": {"name": "lookup", "strict": strict, "parameters": {}}}]
|
||||
request: Final = AmazonConverseConfig().transform_request(
|
||||
model, [{"role": "user", "content": "hi"}], {"tools": tools}, {**litellm_params}, {}
|
||||
)
|
||||
sent_strict: Final = any("strict" in tool["toolSpec"] for tool in request["toolConfig"]["tools"])
|
||||
|
||||
fallback: Final = bedrock_strict_tools_fallback(model, {"tools": tools}, litellm_params)
|
||||
|
||||
assert (fallback is not None) == sent_strict
|
||||
|
|
|
|||
145
tests/unit/llms/openai_like/test_aiand_provider.py
Normal file
145
tests/unit/llms/openai_like/test_aiand_provider.py
Normal file
|
|
@ -0,0 +1,145 @@
|
|||
import json
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
import litellm
|
||||
from litellm.caching.llm_caching_handler import LLMClientCache
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
|
||||
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
||||
AIAND_API_BASE = "https://api.aiand.com/v1"
|
||||
|
||||
|
||||
def test_aiand_chat_provider(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setenv("AIAND_API_KEY", "test-key")
|
||||
|
||||
provider_config = JSONProviderRegistry.get("aiand")
|
||||
assert provider_config is not None
|
||||
assert litellm.LlmProviders.AIAND.value == "aiand"
|
||||
assert "aiand" in litellm.openai_compatible_providers
|
||||
assert provider_config.base_url == AIAND_API_BASE
|
||||
assert provider_config.api_key_env == "AIAND_API_KEY"
|
||||
assert provider_config.api_base_env == "AIAND_API_BASE"
|
||||
|
||||
model, provider, api_key, api_base = get_llm_provider("aiand/openai/gpt-oss-120b")
|
||||
assert model == "openai/gpt-oss-120b"
|
||||
assert provider == "aiand"
|
||||
assert api_key == "test-key"
|
||||
assert api_base == AIAND_API_BASE
|
||||
|
||||
|
||||
def test_aiand_completion_request(respx_mock: respx.MockRouter):
|
||||
messages = [{"role": "user", "content": "Hello"}]
|
||||
route = respx_mock.post(f"{AIAND_API_BASE}/chat/completions").mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "chatcmpl-test",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "openai/gpt-oss-120b",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "Hello"},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"prompt_tokens": 1,
|
||||
"completion_tokens": 1,
|
||||
"total_tokens": 2,
|
||||
},
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
litellm.completion(
|
||||
model="aiand/openai/gpt-oss-120b",
|
||||
api_key="test-key",
|
||||
messages=messages,
|
||||
)
|
||||
|
||||
assert route.called
|
||||
request = route.calls.last.request
|
||||
assert str(request.url) == f"{AIAND_API_BASE}/chat/completions"
|
||||
assert request.headers["Authorization"] == "Bearer test-key"
|
||||
request_body = json.loads(request.read())
|
||||
assert request_body["model"] == "openai/gpt-oss-120b"
|
||||
assert request_body["messages"] == messages
|
||||
|
||||
|
||||
def test_aiand_responses_provider():
|
||||
config = ProviderConfigManager.get_provider_responses_api_config(
|
||||
provider="aiand",
|
||||
model="aiand/openai/gpt-oss-120b",
|
||||
)
|
||||
|
||||
assert config is not None
|
||||
assert config.custom_llm_provider == "aiand"
|
||||
assert config.get_complete_url(api_base=None, litellm_params={}) == f"{AIAND_API_BASE}/responses"
|
||||
|
||||
|
||||
def _aiand_catalog() -> frozenset[str]:
|
||||
return frozenset(name for name, info in litellm.model_cost.items() if info.get("litellm_provider") == "aiand")
|
||||
|
||||
|
||||
def test_aiand_wildcard_lists_its_priced_models():
|
||||
catalog: Final = _aiand_catalog()
|
||||
|
||||
assert catalog
|
||||
assert frozenset(get_known_models_from_wildcard("aiand/*")) == catalog
|
||||
|
||||
|
||||
def test_aiand_calls_are_billed_from_their_cost_rows():
|
||||
catalog: Final = _aiand_catalog()
|
||||
assert catalog
|
||||
|
||||
for model in sorted(catalog):
|
||||
row: Final = litellm.model_cost[model]
|
||||
prompt_cost, completion_cost = litellm.cost_per_token(model=model, prompt_tokens=1000, completion_tokens=500)
|
||||
|
||||
assert prompt_cost == pytest.approx(1000 * row["input_cost_per_token"])
|
||||
assert completion_cost == pytest.approx(500 * row["output_cost_per_token"])
|
||||
assert prompt_cost > 0 and completion_cost > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aiand_anthropic_messages_reach_ai_and_with_the_thinking_budget(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
||||
thinking: Final = {"type": "enabled", "budget_tokens": 1024}
|
||||
with respx.mock() as upstream:
|
||||
route: Final = upstream.post(f"{AIAND_API_BASE}/messages").respond(
|
||||
200,
|
||||
json={
|
||||
"id": "msg_aiand",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "zai-org/glm-5.3",
|
||||
"content": [{"type": "thinking", "thinking": "A greeting", "signature": ""}, {"type": "text", "text": "Hello"}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 4, "output_tokens": 3},
|
||||
},
|
||||
)
|
||||
response: Final = await litellm.anthropic.messages.acreate(
|
||||
model="aiand/zai-org/glm-5.3",
|
||||
messages=[{"role": "user", "content": "Say hello"}],
|
||||
max_tokens=2048,
|
||||
thinking=thinking,
|
||||
api_key="aiand-test-key",
|
||||
)
|
||||
|
||||
request: Final = route.calls.last.request
|
||||
body: Final = json.loads(request.content)
|
||||
assert route.call_count == 1
|
||||
assert body["model"] == "zai-org/glm-5.3"
|
||||
assert body["thinking"] == thinking
|
||||
assert "reasoning_effort" not in body
|
||||
assert [block["type"] for block in response["content"]] == ["thinking", "text"]
|
||||
|
|
@ -53,6 +53,7 @@ from litellm.proxy.litellm_pre_call_utils import (
|
|||
add_provider_specific_headers_to_request,
|
||||
check_if_token_is_service_account,
|
||||
clean_headers,
|
||||
forwarded_anthropic_oauth_token,
|
||||
get_dynamic_logging_metadata,
|
||||
get_metadata_variable_name,
|
||||
move_guardrails_to_metadata,
|
||||
|
|
@ -1393,6 +1394,8 @@ async def test_add_litellm_data_to_request_strips_string_encoded_admin_injection
|
|||
("fallback_depth", True),
|
||||
("_target_order", 2),
|
||||
("attempted_targets", ["forged"]),
|
||||
("_excluded_deployment_ids", ["pinned-deployment"]),
|
||||
("_retry_skipped_deployment_ids", ["pinned-deployment"]),
|
||||
],
|
||||
)
|
||||
async def test_add_litellm_data_to_request_strips_forged_fallback_hop_state(
|
||||
|
|
@ -1477,6 +1480,7 @@ async def test_add_litellm_data_to_request_strips_user_control_fields():
|
|||
"_managed_background_response": True,
|
||||
"_guardrail_pipelines": [{"name": "spoofed"}],
|
||||
"_pipeline_managed_guardrails": ["evaded"],
|
||||
"_failover_excluded_ids": ["pinned-deployment"],
|
||||
"safe_user_metadata": "kept",
|
||||
}
|
||||
data = {
|
||||
|
|
@ -1537,6 +1541,7 @@ async def test_add_litellm_data_to_request_strips_user_control_fields():
|
|||
"_managed_background_response",
|
||||
"_guardrail_pipelines",
|
||||
"_pipeline_managed_guardrails",
|
||||
"_failover_excluded_ids",
|
||||
}
|
||||
assert "litellm_metadata" not in updated
|
||||
for stripped_key in stripped_keys:
|
||||
|
|
@ -5962,6 +5967,25 @@ def test_clean_headers_strips_x_api_key_when_byok_disabled():
|
|||
assert "x-api-key" not in result
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("headers", "expected_token"),
|
||||
[
|
||||
(
|
||||
{"authorization": "Bearer sk-ant-oat01-caller", "x-litellm-api-key": "Bearer sk-proxy"},
|
||||
"sk-ant-oat01-caller",
|
||||
),
|
||||
({"authorization": "Bearer sk-ant-oat01-caller"}, None),
|
||||
({"authorization": "Bearer sk-ant-api03-caller", "x-litellm-api-key": "Bearer sk-proxy"}, None),
|
||||
({"x-litellm-api-key": "Bearer sk-proxy"}, None),
|
||||
],
|
||||
ids=["oauth-beside-proxy-key", "oauth-used-as-proxy-key", "non-oauth-authorization", "no-authorization"],
|
||||
)
|
||||
def test_forwarded_anthropic_oauth_token_follows_chats_forwarding_rule(
|
||||
headers: dict[str, str], expected_token: str | None
|
||||
) -> None:
|
||||
assert forwarded_anthropic_oauth_token(Headers(headers)) == expected_token
|
||||
|
||||
|
||||
def test_clean_headers_strips_x_api_key_when_byok_enabled_but_x_api_key_was_auth_header():
|
||||
"""
|
||||
Anti-replay regression: even when forward_llm_provider_auth_headers=True,
|
||||
|
|
|
|||
|
|
@ -364,6 +364,7 @@ async def test_anthropic_messages_count_tokens_endpoint():
|
|||
output_format=None,
|
||||
context_management=None,
|
||||
request_litellm_params=None,
|
||||
client_oauth_token=None,
|
||||
):
|
||||
assert call_endpoint == True, "Should be called with call_endpoint=True for Anthropic endpoint"
|
||||
assert request.model == "claude-3-sonnet-20240229"
|
||||
|
|
@ -444,6 +445,7 @@ async def test_anthropic_messages_count_tokens_with_non_anthropic_model():
|
|||
output_format=None,
|
||||
context_management=None,
|
||||
request_litellm_params=None,
|
||||
client_oauth_token=None,
|
||||
):
|
||||
assert call_endpoint == True, "Should be called with call_endpoint=True for Anthropic endpoint"
|
||||
assert request.model == "gpt-4"
|
||||
|
|
@ -1262,6 +1264,7 @@ async def test_anthropic_endpoint_returns_anthropic_error_format():
|
|||
output_format=None,
|
||||
context_management=None,
|
||||
request_litellm_params=None,
|
||||
client_oauth_token=None,
|
||||
):
|
||||
raise ProxyException(
|
||||
message='{"detail":{"message":"Input is too long for requested model."}}',
|
||||
|
|
@ -1319,6 +1322,7 @@ async def test_anthropic_endpoint_403_permission_error_format():
|
|||
output_format=None,
|
||||
context_management=None,
|
||||
request_litellm_params=None,
|
||||
client_oauth_token=None,
|
||||
):
|
||||
raise ProxyException(
|
||||
message='{"Message":"Bearer Token has expired"}',
|
||||
|
|
@ -1376,6 +1380,7 @@ async def test_anthropic_endpoint_429_rate_limit_error_format():
|
|||
output_format=None,
|
||||
context_management=None,
|
||||
request_litellm_params=None,
|
||||
client_oauth_token=None,
|
||||
):
|
||||
raise ProxyException(
|
||||
message="Rate limit exceeded",
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import pytest
|
|||
|
||||
import litellm
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.responses.utils import ResponsesAPIRequestUtils
|
||||
from litellm.router_utils.pre_call_checks.deployment_affinity_check import (
|
||||
DeploymentAffinityCheck,
|
||||
)
|
||||
|
|
@ -1209,3 +1210,92 @@ async def test_virtual_key_hash_wins_over_user_id_for_affinity():
|
|||
parent_otel_span=None,
|
||||
)
|
||||
assert other_key_same_user == healthy_deployments
|
||||
|
||||
|
||||
class _ForbiddenError(Exception):
|
||||
status_code: Final = 403
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("pre_call_check", "metadata"),
|
||||
[
|
||||
("session_affinity", {"session_id": "sess-retry"}),
|
||||
("deployment_affinity", {"user_api_key_hash": "hashed-key-retry"}),
|
||||
],
|
||||
)
|
||||
async def test_retry_reaches_healthy_sibling_when_affinity_pins_deployment_that_refused(
|
||||
pre_call_check: str, metadata: dict[str, str]
|
||||
):
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "test-model",
|
||||
"litellm_params": {
|
||||
"model": "gpt-4o",
|
||||
"api_key": "bad",
|
||||
"mock_response": _ForbiddenError("A refuses"),
|
||||
"weight": 1,
|
||||
},
|
||||
"model_info": {"id": "A"},
|
||||
},
|
||||
{
|
||||
"model_name": "test-model",
|
||||
"litellm_params": {
|
||||
"model": "gpt-4o",
|
||||
"api_key": "good",
|
||||
"mock_response": "ok from B",
|
||||
"weight": 0,
|
||||
},
|
||||
"model_info": {"id": "B"},
|
||||
},
|
||||
],
|
||||
routing_strategy="simple-shuffle",
|
||||
num_retries=1,
|
||||
retry_after=0,
|
||||
disable_cooldowns=True,
|
||||
optional_pre_call_checks=[pre_call_check],
|
||||
)
|
||||
|
||||
responses: Final = [
|
||||
await router.acompletion(
|
||||
model="test-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
metadata=dict(metadata),
|
||||
)
|
||||
for _ in range(2)
|
||||
]
|
||||
|
||||
assert [response._hidden_params["model_id"] for response in responses] == ["B", "B"]
|
||||
assert [response._hidden_params["additional_headers"]["x-litellm-attempted-retries"] for response in responses] == [
|
||||
1,
|
||||
1,
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_previous_response_id_pin_holds_even_after_its_deployment_failed():
|
||||
callback: Final = DeploymentAffinityCheck(
|
||||
cache=DualCache(),
|
||||
ttl_seconds=60,
|
||||
enable_user_key_affinity=True,
|
||||
enable_responses_api_affinity=True,
|
||||
)
|
||||
healthy_deployments: Final = [
|
||||
{"model_name": "gpt-4o", "litellm_params": {"model": "gpt-4o"}, "model_info": {"id": "A"}},
|
||||
{"model_name": "gpt-4o", "litellm_params": {"model": "gpt-4o"}, "model_info": {"id": "B"}},
|
||||
]
|
||||
|
||||
assert await callback.async_filter_deployments(
|
||||
model="gpt-4o",
|
||||
healthy_deployments=healthy_deployments,
|
||||
messages=None,
|
||||
request_kwargs={
|
||||
"previous_response_id": ResponsesAPIRequestUtils._build_responses_api_response_id(
|
||||
custom_llm_provider="openai", model_id="A", response_id="resp_upstream"
|
||||
),
|
||||
"_excluded_deployment_ids": ["A"],
|
||||
"metadata": {"user_api_key_hash": "hashed-key"},
|
||||
},
|
||||
parent_otel_span=None,
|
||||
) == [healthy_deployments[0]]
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ The mechanism works without any cache and supports two encoding strategies:
|
|||
"""
|
||||
|
||||
import time
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -2606,3 +2607,111 @@ async def test_affinity_strips_for_team_and_pattern_routes():
|
|||
|
||||
assert result is sibling_pool
|
||||
assert not request_kwargs["input"][1].get("encrypted_content")
|
||||
|
||||
|
||||
def _region_deployment(region: str) -> dict[str, object]:
|
||||
return {
|
||||
"model_name": "gpt-5.4",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-5.4",
|
||||
"api_base": f"https://{region}.example.com/v1",
|
||||
"api_key": f"key-{region}",
|
||||
},
|
||||
"model_info": {"id": region},
|
||||
}
|
||||
|
||||
|
||||
def _reasoning_input_minted_by(origin: str) -> list[dict[str, object]]:
|
||||
return [
|
||||
{
|
||||
"type": "reasoning",
|
||||
"id": ResponsesAPIRequestUtils._build_encrypted_item_id(origin, "rs_test"),
|
||||
"encrypted_content": ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id("gAAAAA-blob", origin),
|
||||
"summary": [{"type": "summary_text", "text": "scattering"}],
|
||||
},
|
||||
{"role": "user", "content": "and sunsets?"},
|
||||
]
|
||||
|
||||
|
||||
def _previous_response_id_from(origin: str) -> str:
|
||||
return ResponsesAPIRequestUtils._build_responses_api_response_id("openai", origin, "resp_test")
|
||||
|
||||
|
||||
_ORIGIN_FAILED_THIS_REQUEST: Final = pytest.mark.parametrize(
|
||||
"failed_in_request",
|
||||
[{"_excluded_deployment_ids": ["region-a"]}, {"_retry_skipped_deployment_ids": ("region-a",)}],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@_ORIGIN_FAILED_THIS_REQUEST
|
||||
@pytest.mark.parametrize("continuing", [{}, {"previous_response_id": _previous_response_id_from("region-b")}])
|
||||
async def test_affinity_strips_and_dispatches_when_origin_already_failed_this_request(
|
||||
failed_in_request: dict[str, object], continuing: dict[str, object]
|
||||
):
|
||||
from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import (
|
||||
EncryptedContentAffinityCheck,
|
||||
)
|
||||
|
||||
router: Final = litellm.Router(model_list=[_region_deployment("region-a"), _region_deployment("region-b")])
|
||||
group: Final = router.model_list
|
||||
reasoning_input: Final = _reasoning_input_minted_by("region-a")
|
||||
|
||||
result: Final = await EncryptedContentAffinityCheck(router=router).async_filter_deployments(
|
||||
model="gpt-5.4",
|
||||
healthy_deployments=group,
|
||||
messages=None,
|
||||
request_kwargs={"litellm_metadata": {}, "input": reasoning_input, **failed_in_request, **continuing},
|
||||
)
|
||||
|
||||
assert result is group
|
||||
assert reasoning_input[0] == {"type": "reasoning", "summary": [{"type": "summary_text", "text": "scattering"}]}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@_ORIGIN_FAILED_THIS_REQUEST
|
||||
async def test_affinity_keeps_origin_and_reasoning_when_previous_response_id_binds_the_failed_origin(
|
||||
failed_in_request: dict[str, object],
|
||||
):
|
||||
from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import (
|
||||
EncryptedContentAffinityCheck,
|
||||
)
|
||||
|
||||
pool: Final = (_region_deployment("region-a"), _region_deployment("region-b"))
|
||||
reasoning_input: Final = _reasoning_input_minted_by("region-a")
|
||||
original_reasoning: Final = dict(reasoning_input[0])
|
||||
|
||||
assert await EncryptedContentAffinityCheck(router=litellm.Router(model_list=list(pool))).async_filter_deployments(
|
||||
model="gpt-5.4",
|
||||
healthy_deployments=list(pool),
|
||||
messages=None,
|
||||
request_kwargs={
|
||||
"litellm_metadata": {},
|
||||
"input": reasoning_input,
|
||||
"previous_response_id": _previous_response_id_from("region-a"),
|
||||
**failed_in_request,
|
||||
},
|
||||
) == [pool[0]]
|
||||
assert reasoning_input[0] == original_reasoning
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("regions", [("region-a",), ("region-a", "region-b")])
|
||||
async def test_affinity_keeps_reasoning_when_every_deployment_already_refused_the_request(regions: tuple[str, ...]):
|
||||
from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import (
|
||||
EncryptedContentAffinityCheck,
|
||||
)
|
||||
|
||||
router: Final = litellm.Router(model_list=[_region_deployment(region) for region in regions])
|
||||
reasoning_input: Final = _reasoning_input_minted_by("region-a")
|
||||
original_reasoning: Final = dict(reasoning_input[0])
|
||||
|
||||
result: Final = await EncryptedContentAffinityCheck(router=router).async_filter_deployments(
|
||||
model="gpt-5.4",
|
||||
healthy_deployments=router.model_list,
|
||||
messages=None,
|
||||
request_kwargs={"litellm_metadata": {}, "input": reasoning_input, "_retry_skipped_deployment_ids": regions},
|
||||
)
|
||||
|
||||
assert [deployment["model_info"]["id"] for deployment in result] == ["region-a"]
|
||||
assert reasoning_input[0] == original_reasoning
|
||||
|
|
|
|||
|
|
@ -313,6 +313,27 @@ async def test_async_filter_deployments_does_not_pin_when_target_order_is_set():
|
|||
assert filtered == deployments
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"failed_in_request",
|
||||
[{"_excluded_deployment_ids": ["dep-2"]}, {"_retry_skipped_deployment_ids": ("dep-2",)}],
|
||||
)
|
||||
async def test_async_filter_deployments_does_not_pin_to_a_deployment_that_failed_this_request(
|
||||
failed_in_request: dict[str, object],
|
||||
):
|
||||
cache: Final = DualCache()
|
||||
check: Final = PromptCachingDeploymentCheck(cache=cache)
|
||||
messages: Final = _messages(word_count=5000)
|
||||
await PromptCachingCache(cache=cache).async_add_model_id(model_id="dep-2", messages=messages, tools=None)
|
||||
|
||||
assert await check.async_filter_deployments(
|
||||
model=MODEL_GROUP_ALIAS,
|
||||
healthy_deployments=_deployments("anthropic/claude-opus-4-6", "anthropic/claude-opus-4-6"),
|
||||
messages=messages,
|
||||
request_kwargs=failed_in_request,
|
||||
) == _deployments("anthropic/claude-opus-4-6", "anthropic/claude-opus-4-6")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_filter_deployments_narrows_for_group_whose_model_minimum_is_lower():
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -1,10 +1,12 @@
|
|||
import asyncio
|
||||
import copy
|
||||
import datetime
|
||||
import importlib
|
||||
import itertools
|
||||
import random
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
import uuid
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
|
|
@ -682,6 +684,45 @@ class TestDeploymentLevelAllowedFails:
|
|||
)
|
||||
|
||||
|
||||
def _gpt_4_group_with_a_sibling() -> Sequence[Mapping[str, object]]:
|
||||
return (
|
||||
{"model_name": "gpt-4", "litellm_params": {"model": "openai/gpt-4"}, "model_info": {"id": "primary"}},
|
||||
{"model_name": "gpt-4", "litellm_params": {"model": "openai/gpt-4"}, "model_info": {"id": "secondary"}},
|
||||
)
|
||||
|
||||
|
||||
def _primary_cooldown_decisions_on_repeated_503s(router: Router) -> tuple[bool, ...]:
|
||||
exception: Final = litellm.ServiceUnavailableError("down", "openai", "gpt-4")
|
||||
return tuple(
|
||||
_should_cooldown_deployment(
|
||||
litellm_router_instance=router,
|
||||
deployment="primary",
|
||||
exception_status=503,
|
||||
original_exception=exception,
|
||||
)
|
||||
for _ in range(litellm.allowed_fails + 1)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
class TestRouterLevelAllowedFailsEqualToDefault:
|
||||
def test_allowed_fails_equal_to_the_default_cools_down_after_that_many_fails(self):
|
||||
router: Final = _make_router(model_list=list(_gpt_4_group_with_a_sibling()), allowed_fails=litellm.allowed_fails)
|
||||
|
||||
assert _primary_cooldown_decisions_on_repeated_503s(router) == (False,) * litellm.allowed_fails + (True,)
|
||||
|
||||
def test_unset_allowed_fails_keeps_the_rate_based_default(self):
|
||||
router: Final = _make_router(model_list=list(_gpt_4_group_with_a_sibling()))
|
||||
|
||||
assert _primary_cooldown_decisions_on_repeated_503s(router) == (False,) * (litellm.allowed_fails + 1)
|
||||
|
||||
def test_default_allowed_fails_saved_through_update_settings_keeps_the_rate_based_default(self):
|
||||
router: Final = _make_router(model_list=list(_gpt_4_group_with_a_sibling()))
|
||||
router.update_settings(allowed_fails=litellm.allowed_fails)
|
||||
|
||||
assert _primary_cooldown_decisions_on_repeated_503s(router) == (False,) * (litellm.allowed_fails + 1)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
|
||||
class TestDeploymentLevelAllowedFailsPolicyByExceptionType:
|
||||
def test_rate_limit_error_triggers_cooldown_with_zero_threshold(self):
|
||||
|
|
@ -2931,6 +2972,104 @@ async def test_bad_request_error_does_not_cool_down_a_deployment(
|
|||
assert tuple(cooldown_deployments) == ()
|
||||
|
||||
|
||||
async def _complete_like_the_proxy(router: Router, model: str) -> str:
|
||||
logging_obj, request = litellm.utils.function_setup(
|
||||
original_function="acompletion",
|
||||
rules_obj=litellm.utils.Rules(),
|
||||
start_time=datetime.datetime(2026, 1, 1),
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
litellm_call_id=str(uuid.uuid4()),
|
||||
)
|
||||
response: Final = await router.acompletion(**request, litellm_logging_obj=logging_obj)
|
||||
model_id: Final = response._hidden_params["model_id"]
|
||||
assert isinstance(model_id, str)
|
||||
return model_id
|
||||
|
||||
|
||||
_CHAT_COMPLETION_OK: Final = {
|
||||
"id": "chatcmpl-1",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "gpt-4o",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
||||
}
|
||||
|
||||
|
||||
def _openai_deployment(model_name: str, host: str, deployment_id: str) -> Mapping[str, object]:
|
||||
return {
|
||||
"model_name": model_name,
|
||||
"litellm_params": {"model": "openai/gpt-4o", "api_key": "k", "api_base": f"https://{host}/v1"},
|
||||
"model_info": {"id": deployment_id},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fallback_deployment_stays_up_while_the_primary_is_in_cooldown(
|
||||
respx_mock: respx.MockRouter,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
respx_mock.post("https://primary.test/v1/chat/completions").mock(
|
||||
return_value=httpx.Response(503, json={"error": {"message": "down", "type": "server_error"}})
|
||||
)
|
||||
backup_route: Final = respx_mock.post("https://backup.test/v1/chat/completions").mock(
|
||||
return_value=httpx.Response(200, json=_CHAT_COMPLETION_OK)
|
||||
)
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
_openai_deployment("primary", "primary.test", "primary-dep"),
|
||||
_openai_deployment("backup", "backup.test", "backup-dep"),
|
||||
],
|
||||
fallbacks=[{"primary": ["backup"]}],
|
||||
allowed_fails=litellm.allowed_fails,
|
||||
num_retries=0,
|
||||
)
|
||||
|
||||
served_by: Final = [await _complete_like_the_proxy(router, "primary") for _ in range(20)]
|
||||
cooldown_deployments: Final = await async_get_cooldown_deployments(
|
||||
litellm_router_instance=router,
|
||||
parent_otel_span=None,
|
||||
)
|
||||
|
||||
assert served_by == ["backup-dep"] * 20
|
||||
assert len(backup_route.calls) == 20
|
||||
assert tuple(cooldown_deployments) == ("primary-dep",)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fallback_group_stays_up_after_a_bad_key_cools_the_primary_under_the_rate_based_default(
|
||||
respx_mock: respx.MockRouter,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
respx_mock.post("https://primary.test/v1/chat/completions").mock(
|
||||
return_value=httpx.Response(401, json={"error": {"message": "bad key", "type": "invalid_request_error"}})
|
||||
)
|
||||
respx_mock.post("https://backup.test/v1/chat/completions").mock(
|
||||
return_value=httpx.Response(200, json=_CHAT_COMPLETION_OK)
|
||||
)
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
_openai_deployment("primary", "primary.test", "primary-dep"),
|
||||
_openai_deployment("backup", "backup.test", "backup-1"),
|
||||
_openai_deployment("backup", "backup.test", "backup-2"),
|
||||
],
|
||||
fallbacks=[{"primary": ["backup"]}],
|
||||
num_retries=0,
|
||||
)
|
||||
|
||||
served_by: Final = [await _complete_like_the_proxy(router, "primary") for _ in range(12)]
|
||||
cooldown_deployments: Final = await async_get_cooldown_deployments(
|
||||
litellm_router_instance=router,
|
||||
parent_otel_span=None,
|
||||
)
|
||||
|
||||
assert set(served_by) <= {"backup-1", "backup-2"}
|
||||
assert tuple(cooldown_deployments) == ("primary-dep",)
|
||||
|
||||
|
||||
TIMED_OUT_DEPLOYMENT: Final = {
|
||||
"model_name": "test-model",
|
||||
"litellm_params": {"model": "openai/test-model", "api_key": "test-key"},
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ from dataclasses import dataclass
|
|||
from datetime import datetime
|
||||
from importlib import import_module
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
from typing import Final, Literal
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
|
|
@ -44,12 +44,13 @@ from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kw
|
|||
from litellm.litellm_core_utils.get_litellm_params import stored_control_options
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.types.litellm_params import ControlOptions
|
||||
from litellm.types.llms.openai import AllMessageValues, HttpxBinaryResponseContent
|
||||
from litellm.types.prompts.init_prompts import PromptSpec
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StandardCallbackDynamicParams, StreamingChoices, Usage
|
||||
from tests.unit.llms.bedrock.slow_upstream import (
|
||||
CONVERSE_ANSWER,
|
||||
STREAM_TIMEOUT_SECONDS,
|
||||
UPSTREAM_FIRST_BYTE_SECONDS,
|
||||
slow_upstream_sync_client,
|
||||
|
|
@ -9950,3 +9951,126 @@ async def test_converse_fallback_prices_the_current_attempt(
|
|||
else:
|
||||
client.close()
|
||||
configuration.reset_rust_configuration()
|
||||
|
||||
|
||||
_LOOKUP_FUNCTION: Final = {
|
||||
"name": "lookup_record",
|
||||
"description": "Look up a record",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"query": {"type": "string"}},
|
||||
"required": ["query"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
}
|
||||
_STRICT_LOOKUP_TOOL: Final = {"type": "function", "function": {**_LOOKUP_FUNCTION, "strict": True}}
|
||||
# Bedrock's 400 for a request past Claude's strict tool limits, seen live on us.anthropic.claude-sonnet-4-6 on 2026-10-20
|
||||
_TOO_MANY_STRICT_TOOLS: Final = (
|
||||
"The model returned the following errors: Too many strict tools (21). The maximum number of strict tools "
|
||||
"supported is 20. Try reducing the number of tools marked as strict."
|
||||
)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _StrictToolsLimitRejectionFirst:
|
||||
stream: Literal["off", "fake", "real"]
|
||||
sent: tuple[tuple[str, dict[str, object]], ...] = ()
|
||||
|
||||
def __call__(self, request: httpx.Request) -> httpx.Response:
|
||||
self.sent = (*self.sent, (request.url.path.rsplit("/", 1)[-1], json.loads(request.content)))
|
||||
if len(self.sent) == 1:
|
||||
return httpx.Response(400, json={"message": _TOO_MANY_STRICT_TOOLS})
|
||||
return httpx.Response(200, content=b"" if self.stream == "real" else CONVERSE_ANSWER)
|
||||
|
||||
|
||||
async def _converse_with_a_strict_tools_limit_rejection_first(
|
||||
asynchronous: bool,
|
||||
stream: Literal["off", "fake", "real"],
|
||||
drop_params: bool | None,
|
||||
tool: Mapping[str, object] = _STRICT_LOOKUP_TOOL,
|
||||
) -> tuple[tuple[tuple[str, dict[str, object]], ...], litellm.BadRequestError | None]:
|
||||
respond: Final = _StrictToolsLimitRejectionFirst(stream)
|
||||
kwargs: Final = {
|
||||
"model": "bedrock/us.anthropic.claude-sonnet-4-6",
|
||||
"messages": [{"role": "user", "content": "Find invoice 42"}],
|
||||
"tools": [tool],
|
||||
"aws_access_key_id": "AKIASTRICTTOOLS",
|
||||
"aws_secret_access_key": "strict-tools-secret",
|
||||
"aws_region_name": "us-east-1",
|
||||
"num_retries": 0,
|
||||
"drop_params": drop_params,
|
||||
"stream": stream != "off",
|
||||
"fake_stream": stream == "fake",
|
||||
}
|
||||
try:
|
||||
chunks: Final = (
|
||||
await _acompletion_chunks(AsyncHTTPHandler(transport=httpx.MockTransport(respond)), kwargs, stream)
|
||||
if asynchronous
|
||||
else _completion_chunks(
|
||||
HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))), kwargs, stream
|
||||
)
|
||||
)
|
||||
except litellm.BadRequestError as error:
|
||||
return respond.sent, error
|
||||
assert chunks
|
||||
return respond.sent, None
|
||||
|
||||
|
||||
async def _acompletion_chunks(
|
||||
client: AsyncHTTPHandler, kwargs: Mapping[str, object], stream: Literal["off", "fake", "real"]
|
||||
) -> tuple[object, ...]:
|
||||
response: Final = await litellm.acompletion(client=client, **kwargs)
|
||||
return tuple([chunk async for chunk in response]) if stream == "fake" else (response,)
|
||||
|
||||
|
||||
def _completion_chunks(
|
||||
client: HTTPHandler, kwargs: Mapping[str, object], stream: Literal["off", "fake", "real"]
|
||||
) -> tuple[object, ...]:
|
||||
response: Final = litellm.completion(client=client, **kwargs)
|
||||
return tuple(response) if stream == "fake" else (response,)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
|
||||
@pytest.mark.parametrize("stream", ["off", "fake", "real"], ids=["completion", "fake_stream", "stream"])
|
||||
async def test_bedrock_strict_tools_limit_rejection_is_sent_again_without_strict_under_drop_params(
|
||||
asynchronous: bool, stream: Literal["off", "fake", "real"], local_model_cost_map: None
|
||||
) -> None:
|
||||
sent, error = await _converse_with_a_strict_tools_limit_rejection_first(asynchronous, stream, drop_params=True)
|
||||
|
||||
assert error is None
|
||||
(first_path, first_body), (second_path, second_body) = sent
|
||||
first_tool_spec: Final = first_body["toolConfig"]["tools"][0]["toolSpec"]
|
||||
assert first_tool_spec["strict"] is True
|
||||
assert first_path == second_path == ("converse-stream" if stream == "real" else "converse")
|
||||
assert second_body == {
|
||||
**first_body,
|
||||
"toolConfig": {
|
||||
**first_body["toolConfig"],
|
||||
"tools": [{"toolSpec": {key: value for key, value in first_tool_spec.items() if key != "strict"}}],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
|
||||
async def test_bedrock_strict_tools_limit_rejection_reaches_the_caller_without_drop_params(
|
||||
asynchronous: bool, local_model_cost_map: None
|
||||
) -> None:
|
||||
sent, error = await _converse_with_a_strict_tools_limit_rejection_first(asynchronous, stream="off", drop_params=None)
|
||||
|
||||
assert error is not None and "Too many strict tools" in str(error)
|
||||
assert len(sent) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
|
||||
async def test_bedrock_strict_tools_limit_rejection_is_not_sent_again_when_no_tool_sent_strict(
|
||||
asynchronous: bool, local_model_cost_map: None
|
||||
) -> None:
|
||||
sent, error = await _converse_with_a_strict_tools_limit_rejection_first(
|
||||
asynchronous,
|
||||
stream="off",
|
||||
drop_params=True,
|
||||
tool={"type": "function", "function": {**_LOOKUP_FUNCTION, "strict": False}},
|
||||
)
|
||||
|
||||
assert error is not None and "Too many strict tools" in str(error)
|
||||
assert len(sent) == 1
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ from __future__ import annotations
|
|||
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Iterator, Mapping
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
|
@ -224,6 +224,50 @@ def test_dated_variants_carry_base_alias_service_tier_pricing(prices: dict):
|
|||
)
|
||||
|
||||
|
||||
PRIORITY_SUFFIX: Final = "_priority"
|
||||
MIN_PRIORITY_MARKUP: Final = 1.0
|
||||
MAX_PRIORITY_MARKUP: Final = 5.0
|
||||
|
||||
|
||||
def _rate(value: JsonValue | None) -> float | None:
|
||||
return float(value) if isinstance(value, (int, float)) and not isinstance(value, bool) else None
|
||||
|
||||
|
||||
def _priority_markup(entry: Mapping[str, JsonValue], key: str) -> float | None:
|
||||
if not key.endswith(PRIORITY_SUFFIX):
|
||||
return None
|
||||
priority: Final = _rate(entry[key])
|
||||
standard: Final = _rate(entry.get(key.removesuffix(PRIORITY_SUFFIX)))
|
||||
if priority is None or standard is None or standard <= 0:
|
||||
return None
|
||||
return priority / standard
|
||||
|
||||
|
||||
def priority_markups(prices: Mapping[str, JsonValue]) -> Iterator[tuple[str, str, float]]:
|
||||
for name, entry in prices.items():
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
for key in entry:
|
||||
if (markup := _priority_markup(entry, key)) is not None:
|
||||
yield name, key, markup
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", (PRICES_PATH, BACKUP_PRICES_PATH), ids=("main", "backup"))
|
||||
def test_priority_rates_stay_within_a_premium_of_their_standard_rate(path: Path) -> None:
|
||||
prices: Final = TypeAdapter(dict[str, JsonValue]).validate_json(path.read_bytes())
|
||||
markups: Final = tuple(priority_markups(prices))
|
||||
assert markups
|
||||
overpriced: Final = tuple(
|
||||
f"{name}.{key} is {markup:.1f}x its standard rate"
|
||||
for name, key, markup in markups
|
||||
if not MIN_PRIORITY_MARKUP <= markup <= MAX_PRIORITY_MARKUP
|
||||
)
|
||||
assert overpriced == (), (
|
||||
"a priority rate is a premium on the standard rate, so one below it or far above it was likely copied from "
|
||||
"another model; drop the key when the provider sells no priority tier for the model:\n" + "\n".join(overpriced)
|
||||
)
|
||||
|
||||
|
||||
# Azure Foundry GPT-6.1 Sol announcement, 2026-09-29: https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-6-1-sol-in-microsoft-foundry-advanced-intelligence-optimized-for/4560811
|
||||
AZURE_GPT_6_1_SOL_REGIONAL_PREMIUM: Final = MappingProxyType({"us": 1.1, "eu": 1.2, "apac": 1.2})
|
||||
|
||||
|
|
|
|||
|
|
@ -21239,9 +21239,9 @@ def test_router_retry_skip_stamp_feeds_deployment_ids_to_skip_on_retry(
|
|||
],
|
||||
)
|
||||
def test_router_as_retry_skipped_deployment_ids_keeps_only_a_tuple_of_strings(value, expected):
|
||||
from litellm.router import _as_retry_skipped_deployment_ids
|
||||
from litellm.router_utils.common_utils import as_retry_skipped_deployment_ids
|
||||
|
||||
assert _as_retry_skipped_deployment_ids(value) == expected
|
||||
assert as_retry_skipped_deployment_ids(value) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
|
|||
|
|
@ -8,13 +8,14 @@ cross-group fallback runs.
|
|||
"""
|
||||
|
||||
from collections import Counter
|
||||
from typing import Optional
|
||||
from typing import Final, Optional
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
|
||||
from litellm.utils import get_excluded_filtered_deployments
|
||||
|
||||
|
||||
|
|
@ -944,3 +945,77 @@ async def test_failover_falls_through_to_external_fallback_when_remaining_in_coo
|
|||
)
|
||||
|
||||
assert response._hidden_params["model_id"] == "fallback"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("pre_call_check", "metadata"),
|
||||
[
|
||||
("session_affinity", {"session_id": "sess-1"}),
|
||||
("deployment_affinity", {"user_api_key_hash": "hashed-key-1"}),
|
||||
],
|
||||
)
|
||||
async def test_failover_reaches_healthy_sibling_when_affinity_pins_failed_deployment(
|
||||
pre_call_check: str, metadata: dict[str, str]
|
||||
):
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "test-model",
|
||||
"litellm_params": {
|
||||
"model": "gpt-4o",
|
||||
"api_key": "bad",
|
||||
"mock_response": Exception("A down"),
|
||||
"weight": 1,
|
||||
},
|
||||
"model_info": {"id": "A"},
|
||||
},
|
||||
{
|
||||
"model_name": "test-model",
|
||||
"litellm_params": {
|
||||
"model": "gpt-4o",
|
||||
"api_key": "good",
|
||||
"mock_response": "ok from B",
|
||||
"weight": 0,
|
||||
},
|
||||
"model_info": {"id": "B"},
|
||||
},
|
||||
{
|
||||
"model_name": "fallback-model",
|
||||
"litellm_params": {
|
||||
"model": "gpt-4o",
|
||||
"api_key": "good",
|
||||
"mock_response": "ok from fallback",
|
||||
},
|
||||
"model_info": {"id": "fallback"},
|
||||
},
|
||||
],
|
||||
routing_strategy="simple-shuffle",
|
||||
num_retries=0,
|
||||
disable_cooldowns=True,
|
||||
enable_weighted_failover=True,
|
||||
optional_pre_call_checks=[pre_call_check],
|
||||
fallbacks=[{"test-model": ["fallback-model"]}],
|
||||
)
|
||||
|
||||
while_a_is_down: Final = await router.acompletion(
|
||||
model="test-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
metadata=dict(metadata),
|
||||
)
|
||||
router.upsert_deployment(
|
||||
Deployment(
|
||||
model_name="test-model",
|
||||
litellm_params=LiteLLM_Params(model="gpt-4o", api_key="bad", mock_response="ok from A", weight=1),
|
||||
model_info=ModelInfo(id="A"),
|
||||
)
|
||||
)
|
||||
after_a_recovers: Final = await router.acompletion(
|
||||
model="test-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
metadata=dict(metadata),
|
||||
)
|
||||
|
||||
assert while_a_is_down._hidden_params["model_id"] == "B"
|
||||
assert while_a_is_down._hidden_params["additional_headers"]["x-litellm-attempted-fallbacks"] == 1
|
||||
assert after_a_recovers._hidden_params["model_id"] == "A"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue