mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge remote-tracking branch 'origin/main' into litellm_responses_comparison_response_id_decode
This commit is contained in:
commit
19681de431
36 changed files with 2317 additions and 295 deletions
|
|
@ -198,6 +198,11 @@ def _genai_message_part(part: object) -> _GenAIMessagePart | None:
|
|||
return ChatCompletionTextObject(type="text", text=text) if isinstance(text, str) else _genai_media_part(fields)
|
||||
|
||||
|
||||
def _genai_content_parts(value: object) -> Sequence[object]:
|
||||
single_part: Final = _validated(_CONFIG_FIELDS, value)
|
||||
return (single_part,) if single_part is not None else _validated(_GENAI_CONTENT_PARTS, value) or ()
|
||||
|
||||
|
||||
def _genai_message_parts(parts: object) -> tuple[_GenAIMessagePart, ...]:
|
||||
items: Final = _validated(_GENAI_CONTENT_PARTS, parts) or ()
|
||||
return tuple(converted for part in items if (converted := _genai_message_part(part)) is not None)
|
||||
|
|
@ -248,7 +253,7 @@ def _genai_content_messages(content: object) -> tuple[AllMessageValues, ...]:
|
|||
fields: Final = _validated(_CONFIG_FIELDS, content)
|
||||
if fields is None:
|
||||
return ()
|
||||
parts: Final = _validated(_GENAI_CONTENT_PARTS, fields.get("parts", ())) or ()
|
||||
parts: Final = _genai_content_parts(fields.get("parts", ()))
|
||||
converted: Final = _genai_message_parts(parts)
|
||||
role: Final = fields.get("role", "user")
|
||||
if role == "user":
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ from ..constants import (
|
|||
)
|
||||
from ..errors import AnthropicContextManagementError
|
||||
from ..placeholders import build_cleared_tool_result_content
|
||||
from .compact import count_system_tokens, history_after_client_compaction
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.types.llms.openai import ChatCompletionToolParam
|
||||
|
|
@ -203,9 +204,10 @@ def _trigger_met(
|
|||
model: str,
|
||||
messages: list[dict[str, object]],
|
||||
tools: list[dict[str, object]] | None,
|
||||
system: str | Sequence[Mapping[str, object]] | None,
|
||||
counted_tool_uses: int,
|
||||
) -> tuple[bool, int | None]:
|
||||
"""Return (trigger_met, input_tokens if counted for reuse)."""
|
||||
"""Return (trigger_met, message and tool tokens for reuse); like the API, input_tokens adds the system prompt."""
|
||||
trigger_type: Final = trigger.get("type", "input_tokens")
|
||||
threshold = trigger.get("value")
|
||||
|
||||
|
|
@ -216,14 +218,17 @@ def _trigger_met(
|
|||
|
||||
if not isinstance(threshold, int):
|
||||
threshold = DEFAULT_INPUT_TOKENS_TRIGGER
|
||||
current_tokens: Final = litellm.token_counter(
|
||||
message_tokens: Final = litellm.token_counter(
|
||||
model=model,
|
||||
messages=messages,
|
||||
tools=cast("list[ChatCompletionToolParam] | None", tools),
|
||||
)
|
||||
if message_tokens > threshold:
|
||||
return True, message_tokens
|
||||
current_tokens: Final = message_tokens + count_system_tokens(model, system)
|
||||
verbose_logger.debug("context_management polyfill: current_tokens: %s", current_tokens)
|
||||
verbose_logger.debug("context_management polyfill: threshold: %s", threshold)
|
||||
return current_tokens > threshold, current_tokens
|
||||
return current_tokens > threshold, message_tokens
|
||||
|
||||
|
||||
def _resolve_keep_count(keep: dict[str, object]) -> int:
|
||||
|
|
@ -237,7 +242,7 @@ def _resolve_keep_count(keep: dict[str, object]) -> int:
|
|||
|
||||
|
||||
def _last_completed_tool_use_id(
|
||||
messages: list[dict[str, object]],
|
||||
messages: Sequence[Mapping[str, object]],
|
||||
) -> str | None:
|
||||
"""Latest completed tool_result id; never cleared."""
|
||||
last_id: str | None = None
|
||||
|
|
@ -253,7 +258,7 @@ def _last_completed_tool_use_id(
|
|||
|
||||
|
||||
def _clear_tool_results(
|
||||
messages: list[dict[str, object]], ids_to_clear: set[str]
|
||||
messages: list[dict[str, object]], ids_to_clear: Set[str]
|
||||
) -> tuple[list[dict[str, object]], int]:
|
||||
"""Clear matching tool_result content; return (messages, cleared_count)."""
|
||||
cleared = 0
|
||||
|
|
@ -311,7 +316,11 @@ def apply_clear_tool_uses_20250919(
|
|||
system: str | list[dict[str, object]] | None,
|
||||
edit_spec: ClearToolUsesEditSpec,
|
||||
) -> tuple[list[dict[str, object]], AppliedEdit | None]:
|
||||
"""Apply clear_tool_uses to an edit ``check_clear_tool_uses_spec`` returned; return (messages, AppliedEdit or None)."""
|
||||
"""Apply clear_tool_uses to an edit ``check_clear_tool_uses_spec`` returned; return (messages, AppliedEdit or None).
|
||||
|
||||
Like the API, it reads only the history after a client compaction block and counts the block's summary as system.
|
||||
"""
|
||||
kept, counted_system = history_after_client_compaction(messages, system)
|
||||
fields: Final = _ClearToolUsesFields.model_validate(edit_spec)
|
||||
clear_at_least: Final = 0 if fields.clear_at_least is None else fields.clear_at_least.value
|
||||
exclude_tools: Final = frozenset(fields.exclude_tools or ())
|
||||
|
|
@ -326,7 +335,7 @@ def apply_clear_tool_uses_20250919(
|
|||
"value": DEFAULT_KEEP_TOOL_USES,
|
||||
}
|
||||
|
||||
blocks: Final = _all_blocks(messages)
|
||||
blocks: Final = _all_blocks(kept)
|
||||
results: Final = _blocks_by_id(blocks, "tool_result", "tool_use_id")
|
||||
uses: Final = _blocks_by_id(blocks, "tool_use", "id")
|
||||
tiny: Final = _tiny_result_ids(results)
|
||||
|
|
@ -335,15 +344,15 @@ def apply_clear_tool_uses_20250919(
|
|||
for tool_use_id, name in _tool_uses_in_order(blocks)
|
||||
if name not in exclude_tools and (tool_use_id not in tiny or _clears_input(name, input_tools))
|
||||
)
|
||||
met, counted_tokens = _trigger_met(trigger, model, messages, tools, len(clearable))
|
||||
if not met:
|
||||
return messages, None
|
||||
|
||||
keep_count: Final = _resolve_keep_count(keep)
|
||||
if len(clearable) <= keep_count:
|
||||
return messages, None
|
||||
|
||||
last_completed_id: Final = _last_completed_tool_use_id(messages)
|
||||
met, counted_tokens = _trigger_met(trigger, model, kept, tools, counted_system, len(clearable))
|
||||
if not met:
|
||||
return messages, None
|
||||
|
||||
last_completed_id: Final = _last_completed_tool_use_id(kept)
|
||||
candidates: Final = tuple(
|
||||
(tool_use_id, name)
|
||||
for tool_use_id, name in clearable[: len(clearable) - keep_count]
|
||||
|
|
@ -358,7 +367,7 @@ def apply_clear_tool_uses_20250919(
|
|||
_select_in_batches(candidate_ids, tokens, clear_at_least) if clear_at_least > 0 else candidate_ids
|
||||
)
|
||||
|
||||
edited, cleared_count = _clear_tool_results(messages, set(ids_to_clear))
|
||||
edited, cleared_count = _clear_tool_results(messages, ids_to_clear)
|
||||
verbose_logger.debug("context_management polyfill: edited: %s", edited)
|
||||
if cleared_count == 0:
|
||||
return messages, None
|
||||
|
|
@ -370,11 +379,12 @@ def apply_clear_tool_uses_20250919(
|
|||
counted_tokens
|
||||
if counted_tokens is not None
|
||||
else litellm.token_counter(
|
||||
model=model, messages=messages, tools=cast("list[ChatCompletionToolParam] | None", tools)
|
||||
model=model, messages=kept, tools=cast("list[ChatCompletionToolParam] | None", tools)
|
||||
)
|
||||
)
|
||||
kept_after, _ = history_after_client_compaction(final_messages, system)
|
||||
tokens_after: Final = litellm.token_counter(
|
||||
model=model, messages=final_messages, tools=cast("list[ChatCompletionToolParam] | None", tools)
|
||||
model=model, messages=kept_after, tools=cast("list[ChatCompletionToolParam] | None", tools)
|
||||
)
|
||||
cleared_input_tokens: Final = max(tokens_before - tokens_after, 0)
|
||||
|
||||
|
|
|
|||
|
|
@ -189,6 +189,21 @@ def _strip_compaction_blocks(
|
|||
return cleaned
|
||||
|
||||
|
||||
def history_after_client_compaction(
|
||||
messages: Sequence[dict[str, object]],
|
||||
system: str | Sequence[Mapping[str, object]] | None,
|
||||
) -> tuple[list[dict[str, object]], str | Sequence[Mapping[str, object]] | None]:
|
||||
"""Return the history the API keeps after the latest compaction block, without it, and the system with its summary."""
|
||||
sliced, block = _slice_around_compaction_block(messages)
|
||||
if block is None:
|
||||
return list(messages), system
|
||||
tail: Final = _strip_compaction_blocks(sliced)
|
||||
summary: Final = block.get("content")
|
||||
if not isinstance(summary, str) or not summary:
|
||||
return tail, system
|
||||
return tail, _augment_system_with_summary(_system_to_text(system) or None, summary)
|
||||
|
||||
|
||||
def _augment_system_with_summary(
|
||||
system: str | list[dict[str, object]] | None,
|
||||
summary_text: str,
|
||||
|
|
@ -320,14 +335,16 @@ def _count_effective_tokens(
|
|||
content: Final = compaction_block.get("content") or ""
|
||||
if content:
|
||||
total += litellm.token_counter(model=model, text=content)
|
||||
return total + count_system_tokens(model, system)
|
||||
|
||||
|
||||
def count_system_tokens(model: str, system: str | Sequence[Mapping[str, object]] | None) -> int:
|
||||
system_text: Final = _system_to_text(system)
|
||||
if system_text:
|
||||
total += litellm.token_counter(model=model, text=system_text)
|
||||
return total
|
||||
return litellm.token_counter(model=model, text=system_text) if system_text else 0
|
||||
|
||||
|
||||
def _system_to_text(
|
||||
system: str | list[dict[str, object]] | None,
|
||||
system: str | Sequence[Mapping[str, object]] | None,
|
||||
) -> str:
|
||||
"""Flatten an Anthropic-style ``system`` value into a single string for
|
||||
token counting. Returns ``""`` when ``system`` carries no text."""
|
||||
|
|
@ -338,7 +355,7 @@ def _system_to_text(
|
|||
return "\n".join(
|
||||
text
|
||||
for block in system
|
||||
if isinstance(block, dict)
|
||||
if isinstance(block, Mapping)
|
||||
and block.get("type") == "text"
|
||||
and isinstance(text := block.get("text"), str)
|
||||
and text
|
||||
|
|
|
|||
|
|
@ -3816,6 +3816,16 @@ class ManagementEndpointLoggingPayload(LiteLLMPydanticObjectBase):
|
|||
end_time: datetime | None = None
|
||||
|
||||
|
||||
def proxy_exception_status_code(message: str, code: str) -> str:
|
||||
# Litellm router.py returns "No healthy deployment available" when there are no deployments available
|
||||
# Should map to 429 errors https://github.com/BerriAI/litellm/issues/2487
|
||||
if "No healthy deployment available" in message or "No deployments available" in message:
|
||||
return "429"
|
||||
if RouterErrors.no_deployments_with_tag_routing.value in message:
|
||||
return "401"
|
||||
return code
|
||||
|
||||
|
||||
class ProxyException(Exception):
|
||||
# NOTE: DO NOT MODIFY THIS
|
||||
# This is used to map exactly to OPENAI Exceptions
|
||||
|
|
@ -3844,13 +3854,7 @@ class ProxyException(Exception):
|
|||
headers[k] = str(v)
|
||||
self.headers = headers or {}
|
||||
self.provider_specific_fields = provider_specific_fields
|
||||
# rules for proxyExceptions
|
||||
# Litellm router.py returns "No healthy deployment available" when there are no deployments available
|
||||
# Should map to 429 errors https://github.com/BerriAI/litellm/issues/2487
|
||||
if "No healthy deployment available" in self.message or "No deployments available" in self.message:
|
||||
self.code = "429"
|
||||
elif RouterErrors.no_deployments_with_tag_routing.value in self.message:
|
||||
self.code = "401"
|
||||
self.code = proxy_exception_status_code(self.message, self.code)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""Converts the ProxyException instance to a dictionary."""
|
||||
|
|
|
|||
|
|
@ -1063,6 +1063,15 @@ def is_mcp_discovery_request(route: str, request_body: Mapping[str, object]) ->
|
|||
return isinstance(method, str) and method in MCP_ZERO_SPEND_JSONRPC_METHODS
|
||||
|
||||
|
||||
class ModelMaxBudgetHolder(Protocol):
|
||||
@property
|
||||
def model_max_budget(self) -> Mapping[str, object] | None: ...
|
||||
|
||||
|
||||
def model_max_budget_of(holder: ModelMaxBudgetHolder) -> Mapping[str, object] | None:
|
||||
return holder.model_max_budget
|
||||
|
||||
|
||||
def route_skips_budget_checks(route: str) -> bool:
|
||||
return route not in BUDGET_ENFORCED_SIDE_EFFECT_ROUTES and (
|
||||
route in MODEL_DISCOVERY_ROUTES or not RouteChecks.is_llm_api_route(route=route)
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ from litellm.integrations.otel.model.config import is_otel_v2_enabled
|
|||
from litellm.integrations.otel.runtime import phase_event, phase_span, seed_request_identity
|
||||
from litellm.litellm_core_utils.dd_tracing import tracer
|
||||
from litellm.litellm_core_utils.dot_notation_indexing import get_nested_value
|
||||
from litellm.models.end_user import LiteLLM_EndUserTable
|
||||
from litellm.proxy._types import *
|
||||
from litellm.proxy.agent_endpoints.auth.agent_caller import agent_caller_from_headers
|
||||
from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module exports
|
||||
|
|
@ -85,6 +86,7 @@ from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module expo
|
|||
jwt_key_mapping_cache_key,
|
||||
key_access_group_grants_model,
|
||||
key_model_aliases_for_auth_check,
|
||||
model_max_budget_of,
|
||||
resolve_and_validate_end_user_id,
|
||||
resolve_default_end_user_budget,
|
||||
route_skips_budget_checks,
|
||||
|
|
@ -384,6 +386,16 @@ class _TeamModelBudgetLimiter(Protocol):
|
|||
) -> bool: ...
|
||||
|
||||
|
||||
class _EndUserModelBudgetLimiter(Protocol):
|
||||
async def is_end_user_within_model_budget(
|
||||
self,
|
||||
end_user_id: str,
|
||||
end_user_model_max_budget: Mapping[str, object],
|
||||
model: str,
|
||||
team_id: str | None = None,
|
||||
) -> bool: ...
|
||||
|
||||
|
||||
class _EveryModelBudgetLimiter(Protocol):
|
||||
async def is_key_within_model_budget(self, user_api_key_dict: UserAPIKeyAuth, model: str) -> bool: ...
|
||||
|
||||
|
|
@ -530,6 +542,49 @@ async def _check_team_model_budget(
|
|||
)
|
||||
|
||||
|
||||
async def _end_user_model_max_budget(
|
||||
end_user_id: str | None,
|
||||
end_user_object: LiteLLM_EndUserTable | None,
|
||||
key_end_user_budget_id: str | None,
|
||||
prisma_client: PrismaClient | None,
|
||||
user_api_key_cache: UserApiKeyCache,
|
||||
parent_otel_span: Span | None,
|
||||
) -> Mapping[str, object] | None:
|
||||
budget_table: Final = (
|
||||
end_user_object.litellm_budget_table
|
||||
if end_user_object is not None
|
||||
else (
|
||||
await resolve_default_end_user_budget(
|
||||
prisma_client=prisma_client,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
key_end_user_budget_id=key_end_user_budget_id,
|
||||
parent_otel_span=parent_otel_span,
|
||||
)
|
||||
if end_user_id and prisma_client is not None
|
||||
else None
|
||||
)
|
||||
)
|
||||
return model_max_budget_of(budget_table) if budget_table is not None else None
|
||||
|
||||
|
||||
async def _check_end_user_model_budget(
|
||||
end_user_id: str | None,
|
||||
end_user_model_max_budget: Mapping[str, object] | None,
|
||||
model_max_budget_limiter: _EndUserModelBudgetLimiter,
|
||||
models: Sequence[str],
|
||||
team_id: str | None,
|
||||
) -> None:
|
||||
if not end_user_id or not end_user_model_max_budget:
|
||||
return
|
||||
for model_name in models:
|
||||
await model_max_budget_limiter.is_end_user_within_model_budget(
|
||||
end_user_id=end_user_id,
|
||||
end_user_model_max_budget=end_user_model_max_budget,
|
||||
model=model_name,
|
||||
team_id=team_id,
|
||||
)
|
||||
|
||||
|
||||
async def _is_within_every_model_budget(
|
||||
valid_token: UserAPIKeyAuth,
|
||||
model_max_budget_limiter: _EveryModelBudgetLimiter,
|
||||
|
|
@ -2094,6 +2149,10 @@ async def _resolve_object_permission_for_unresolvable_team(
|
|||
)
|
||||
|
||||
|
||||
def _custom_auth_token_for_this_request(response: object) -> UserAPIKeyAuth:
|
||||
return UserAPIKeyAuth.model_validate(response).model_copy()
|
||||
|
||||
|
||||
async def user_api_key_auth_builder(
|
||||
request: Request,
|
||||
api_key: str,
|
||||
|
|
@ -2172,7 +2231,7 @@ async def user_api_key_auth_builder(
|
|||
request=request, api_key=api_key, user_custom_auth=user_custom_auth
|
||||
)
|
||||
if response is not None and isinstance(response, UserAPIKeyAuth):
|
||||
validated = UserAPIKeyAuth.model_validate(response)
|
||||
validated = _custom_auth_token_for_this_request(response)
|
||||
if getattr(litellm, "enable_post_custom_auth_checks", False):
|
||||
validated = await _run_post_custom_auth_checks(
|
||||
valid_token=validated,
|
||||
|
|
@ -2188,7 +2247,7 @@ async def user_api_key_auth_builder(
|
|||
custom_auth_api_key = True
|
||||
elif user_custom_auth is not None:
|
||||
response = await user_custom_auth(request=request, api_key=api_key)
|
||||
validated = UserAPIKeyAuth.model_validate(response)
|
||||
validated = _custom_auth_token_for_this_request(response)
|
||||
if getattr(litellm, "enable_post_custom_auth_checks", False):
|
||||
validated = await _run_post_custom_auth_checks(
|
||||
valid_token=validated,
|
||||
|
|
@ -3129,7 +3188,7 @@ async def validate_resolved_virtual_key( # noqa: C901 # Preserve ordering of e
|
|||
)
|
||||
|
||||
# Recompute after a potential budget-fallback rewrite so
|
||||
# the end-user check below validates the final model
|
||||
# the user check below validates the final model
|
||||
current_model = _get_model_from_request_context(
|
||||
request_data=request_data,
|
||||
route=route,
|
||||
|
|
@ -3147,23 +3206,6 @@ async def validate_resolved_virtual_key( # noqa: C901 # Preserve ordering of e
|
|||
models=current_models,
|
||||
)
|
||||
|
||||
# Check 5b. End-user model max budget
|
||||
end_user_mmb: Final = valid_token.end_user_model_max_budget
|
||||
if (
|
||||
end_user_mmb is not None
|
||||
and isinstance(end_user_mmb, dict)
|
||||
and len(end_user_mmb) > 0
|
||||
and current_models
|
||||
and valid_token.end_user_id is not None
|
||||
):
|
||||
for model_name in current_models:
|
||||
await model_max_budget_limiter.is_end_user_within_model_budget(
|
||||
end_user_id=valid_token.end_user_id,
|
||||
end_user_model_max_budget=end_user_mmb,
|
||||
model=model_name,
|
||||
team_id=valid_token.team_id,
|
||||
)
|
||||
|
||||
# Check 6: Additional Common Checks across jwt + key auth
|
||||
if valid_token.team_id is not None:
|
||||
try:
|
||||
|
|
@ -3733,9 +3775,7 @@ async def run_centralized_common_checks(
|
|||
project_object: Final[LiteLLM_ProjectTableCachedObj | None] = (
|
||||
None if isinstance(project_result, BaseException) else project_result
|
||||
)
|
||||
end_user_object: Final[LiteLLM_EndUserTable | None] = (
|
||||
None if isinstance(end_user_result, BaseException) else end_user_result
|
||||
)
|
||||
end_user_object: Final = end_user_result if isinstance(end_user_result, LiteLLM_EndUserTable) else None
|
||||
global_proxy_spend: float | None = None if isinstance(global_spend_result, BaseException) else global_spend_result
|
||||
carry_team_and_user_budget_state(
|
||||
valid_token=user_api_key_auth_obj,
|
||||
|
|
@ -3795,6 +3835,21 @@ async def run_centralized_common_checks(
|
|||
parent_otel_span=parent_otel_span,
|
||||
keep_token_limits=user_custom_auth is not None,
|
||||
)
|
||||
end_user_model_max_budget: Final = await _end_user_model_max_budget(
|
||||
end_user_id=end_user_id,
|
||||
end_user_object=end_user_object,
|
||||
key_end_user_budget_id=key_end_user_budget_id,
|
||||
prisma_client=prisma_client,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
parent_otel_span=parent_otel_span,
|
||||
)
|
||||
if end_user_id and (
|
||||
not user_api_key_auth_obj.authenticated_by_custom_auth
|
||||
or user_api_key_auth_obj.end_user_model_max_budget is None
|
||||
):
|
||||
user_api_key_auth_obj.end_user_model_max_budget = ( # rebind-ok: the spend hook charges from this token
|
||||
dict(end_user_model_max_budget) if end_user_model_max_budget else None
|
||||
)
|
||||
|
||||
skip_budget_checks: Final = _should_skip_budget_checks(
|
||||
request_data=request_data,
|
||||
|
|
@ -3887,6 +3942,21 @@ async def run_centralized_common_checks(
|
|||
user_api_key_cache=user_api_key_cache,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
await _check_end_user_model_budget(
|
||||
end_user_id=end_user_id,
|
||||
end_user_model_max_budget=user_api_key_auth_obj.end_user_model_max_budget,
|
||||
model_max_budget_limiter=model_max_budget_limiter,
|
||||
team_id=user_api_key_auth_obj.team_id,
|
||||
models=_get_model_names_for_budget_checks(
|
||||
model=_get_model_from_request_context(
|
||||
request_data=request_data,
|
||||
route=route,
|
||||
request=request,
|
||||
llm_router=llm_router,
|
||||
team_id=user_api_key_auth_obj.team_id,
|
||||
)
|
||||
),
|
||||
)
|
||||
if sub_call and RouteChecks.is_llm_api_route(route=route):
|
||||
await virtual_key_max_budget_check(
|
||||
valid_token=user_api_key_auth_obj,
|
||||
|
|
|
|||
|
|
@ -5,16 +5,18 @@ import shutil
|
|||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from collections import Counter
|
||||
from collections.abc import Callable, Hashable, Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from itertools import chain
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final, TypeAlias, TypeVar
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import click
|
||||
import requests
|
||||
from pydantic import JsonValue, TypeAdapter, ValidationError
|
||||
from pydantic import ConfigDict, JsonValue, TypeAdapter, ValidationError
|
||||
|
||||
from litellm.proxy.common_utils.codex_model_catalog import (
|
||||
CODEX_BASE_INSTRUCTIONS_PATH,
|
||||
|
|
@ -497,6 +499,113 @@ def _fetch_model_listing(
|
|||
return listing.data
|
||||
|
||||
|
||||
class _DeploymentParams(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
model: str
|
||||
custom_llm_provider: str | None = None
|
||||
api_base: str | None = None
|
||||
litellm_credential_name: str | None = None
|
||||
reasoning_effort: JsonValue = None
|
||||
|
||||
|
||||
class _DeploymentInfo(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
team_public_model_name: str | None = None
|
||||
blocked: bool | None = None
|
||||
|
||||
|
||||
class _Deployment(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
model_name: str
|
||||
litellm_params: _DeploymentParams
|
||||
model_info: _DeploymentInfo | None = None
|
||||
|
||||
|
||||
class _DeploymentListing(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
data: tuple[_Deployment, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ModelUpstream:
|
||||
"""The OpenAI model every deployment behind a public name routes to, and the reasoning effort they all pin."""
|
||||
|
||||
model: str
|
||||
reasoning_effort: str | None = None
|
||||
|
||||
|
||||
_NO_UPSTREAMS: Final[Mapping[str, ModelUpstream]] = MappingProxyType({})
|
||||
_Pinned: Final = TypeVar("_Pinned", bound=Hashable)
|
||||
|
||||
|
||||
def _public_name(deployment: _Deployment) -> str:
|
||||
team_name: Final = deployment.model_info.team_public_model_name if deployment.model_info is not None else None
|
||||
return team_name or deployment.model_name
|
||||
|
||||
|
||||
def _is_paused(deployment: _Deployment) -> bool:
|
||||
return deployment.model_info is not None and deployment.model_info.blocked is True
|
||||
|
||||
|
||||
def _is_openai_host(params: _DeploymentParams) -> bool:
|
||||
if params.api_base is None:
|
||||
return not params.litellm_credential_name
|
||||
host: Final = urlsplit(params.api_base).hostname or ""
|
||||
return host == "api.openai.com" or host.endswith(".api.openai.com")
|
||||
|
||||
|
||||
def _openai_model(params: _DeploymentParams) -> str | None:
|
||||
if params.custom_llm_provider not in (None, "openai") or not _is_openai_host(params):
|
||||
return None
|
||||
provider, slash, name = params.model.partition("/")
|
||||
if not slash:
|
||||
return params.model
|
||||
return name if provider == "openai" else None
|
||||
|
||||
|
||||
def _pinned_effort(params: _DeploymentParams) -> str | None:
|
||||
return params.reasoning_effort if isinstance(params.reasoning_effort, str) else None
|
||||
|
||||
|
||||
def _shared_by_every_deployment(values: Iterable[tuple[str, _Pinned]]) -> Mapping[str, _Pinned]:
|
||||
distinct: Final = frozenset(values)
|
||||
values_per_name: Final = Counter(name for name, _ in distinct)
|
||||
return MappingProxyType({name: value for name, value in distinct if values_per_name[name] == 1})
|
||||
|
||||
|
||||
def _upstreams_by_public_name(deployments: Sequence[_Deployment]) -> Mapping[str, ModelUpstream]:
|
||||
live: Final = tuple(d for d in deployments if not _is_paused(d))
|
||||
models: Final = _shared_by_every_deployment((_public_name(d), _openai_model(d.litellm_params)) for d in live)
|
||||
efforts: Final = _shared_by_every_deployment((_public_name(d), _pinned_effort(d.litellm_params)) for d in live)
|
||||
return MappingProxyType(
|
||||
{name: ModelUpstream(model, efforts.get(name)) for name, model in models.items() if model is not None}
|
||||
)
|
||||
|
||||
|
||||
def _fetch_model_upstreams(
|
||||
base_url: str,
|
||||
api_key: str,
|
||||
*,
|
||||
get: Callable[..., requests.Response],
|
||||
) -> Mapping[str, ModelUpstream]:
|
||||
url: Final = base_url.rstrip("/") + "/v1/model/info"
|
||||
try:
|
||||
resp: Final = get(url, headers=MappingProxyType({"Authorization": f"Bearer {api_key}"}), timeout=10)
|
||||
except requests.RequestException:
|
||||
return _NO_UPSTREAMS
|
||||
if resp.status_code != 200:
|
||||
return _NO_UPSTREAMS
|
||||
try:
|
||||
listing: Final = _DeploymentListing.model_validate_json(resp.content)
|
||||
except ValidationError:
|
||||
return _NO_UPSTREAMS
|
||||
return _upstreams_by_public_name(listing.data)
|
||||
|
||||
|
||||
class _OpenCodeLimit(LiteLLMBaseModel):
|
||||
context: int
|
||||
output: int
|
||||
|
|
@ -576,8 +685,21 @@ def opencode_model_sync_env(
|
|||
return MappingProxyType({OPENCODE_CONFIG_CONTENT_ENV: opencode_provider_config(base_url, listing)})
|
||||
|
||||
|
||||
def _codex_row(model: ListedModel, upstream: ModelUpstream | None) -> CodexCatalogRow:
|
||||
return CodexCatalogRow(
|
||||
id=model.id,
|
||||
mode=model.mode,
|
||||
max_input_tokens=model.max_input_tokens,
|
||||
upstream_model=upstream.model if upstream is not None else None,
|
||||
reasoning_effort=upstream.reasoning_effort if upstream is not None else None,
|
||||
)
|
||||
|
||||
|
||||
def codex_model_catalog(
|
||||
models: Sequence[ListedModel], stock: Sequence[CodexStockModel], instructions: str
|
||||
models: Sequence[ListedModel],
|
||||
stock: Sequence[CodexStockModel],
|
||||
instructions: str,
|
||||
upstreams: Mapping[str, ModelUpstream],
|
||||
) -> str | None:
|
||||
"""The `model_catalog_json` body listing the proxy's chat models, or None if there are none.
|
||||
|
||||
|
|
@ -586,14 +708,23 @@ def codex_model_catalog(
|
|||
installed Codex knows keeps that Codex's own entry and the proxy only
|
||||
decides its place in the picker: the listing orders it, lists it even when
|
||||
Codex hides it or keeps it off the API, and keeps Codex's upgrade nudge only
|
||||
when the model it points at is served too. A model Codex does not know gets the fallback
|
||||
when the model it points at is served too. An alias of such a model, per
|
||||
`upstreams`, takes that entry too, minus the defaults Codex picks for the
|
||||
model: no service tier or upgrade nudge, the reasoning effort its
|
||||
deployments pin, and a window no larger than its listed max_input_tokens.
|
||||
A model Codex does not know gets the fallback
|
||||
entry, with the same base instructions Codex itself uses so the agent never
|
||||
runs without a system prompt. The catalog is the one `/v1/models?client_version=...`
|
||||
serves, built from the installed Codex's own entries instead of the proxy's vendored ones.
|
||||
runs without a system prompt. Apart from aliases, the catalog is the one
|
||||
`/v1/models?client_version=...` serves, built from the installed Codex's own
|
||||
entries instead of the proxy's vendored ones.
|
||||
"""
|
||||
rows: Final = tuple(CodexCatalogRow(id=m.id, mode=m.mode, max_input_tokens=m.max_input_tokens) for m in models)
|
||||
rows: Final = tuple(_codex_row(m, upstreams.get(m.id)) for m in models)
|
||||
body: Final = codex_models_response_json(
|
||||
rows, stock=MappingProxyType({m.slug: m for m in stock}), instructions=instructions, byte_limit=None
|
||||
rows,
|
||||
stock=MappingProxyType({m.slug: m for m in stock}),
|
||||
instructions=instructions,
|
||||
byte_limit=None,
|
||||
alias_defaults="deployment",
|
||||
)
|
||||
return body.json if body.listed else None
|
||||
|
||||
|
|
@ -678,12 +809,14 @@ def codex_model_sync_args(
|
|||
Codex has no env or inline equivalent of OPENCODE_CONFIG_CONTENT: the catalog
|
||||
must be a file, so it is written under $CODEX_HOME (default ~/.codex) and
|
||||
atomically replaced on every launch. The Codex at `binary` first lists its
|
||||
own models, so the ones the proxy serves keep that Codex's entries, and then
|
||||
own models, so the ones the proxy serves keep that Codex's entries, an alias
|
||||
included when every unpaused deployment /v1/model/info lists for it routes
|
||||
to the same OpenAI model and Codex knows that model, and then
|
||||
reads the file back once before it is handed over. The key never lands in
|
||||
the file. A failed fetch, read, listing, write or read-back is reported
|
||||
rather than raised: Codex still launches with its built-in catalog and takes
|
||||
a proxy model by name via -m, and a rejected file stays on disk to be looked
|
||||
at.
|
||||
at. An unavailable /v1/model/info only leaves every alias on the fallback entry.
|
||||
"""
|
||||
listing: Final = _fetch_model_listing(base_url, api_key, get=get)
|
||||
if isinstance(listing, ModelSyncSkipped):
|
||||
|
|
@ -700,7 +833,8 @@ def codex_model_sync_args(
|
|||
stock: Final = _stock_codex_models(binary, base_env, run=run)
|
||||
if isinstance(stock, ModelSyncSkipped):
|
||||
return stock
|
||||
catalog: Final = codex_model_catalog(listing, stock, instructions)
|
||||
upstreams: Final = _fetch_model_upstreams(base_url, api_key, get=get)
|
||||
catalog: Final = codex_model_catalog(listing, stock, instructions, upstreams)
|
||||
if catalog is None:
|
||||
return ModelSyncSkipped(f"{base_url.rstrip('/')}/v1/models lists no chat models")
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ from functools import cache
|
|||
from itertools import accumulate, pairwise
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType, NoneType
|
||||
from typing import TYPE_CHECKING, Annotated, Final, Literal
|
||||
from typing import TYPE_CHECKING, Annotated, Final, Literal, TypeAlias
|
||||
|
||||
from pydantic import ConfigDict, StringConstraints, TypeAdapter, ValidationError
|
||||
|
||||
|
|
@ -110,6 +110,12 @@ class CodexStockUpgrade(LiteLLMBaseModel):
|
|||
model: str
|
||||
|
||||
|
||||
class CodexReasoningLevel(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="allow")
|
||||
|
||||
effort: str
|
||||
|
||||
|
||||
class CodexStockModel(LiteLLMBaseModel):
|
||||
"""One `ModelInfo` entry as Codex ships it or prints it from `codex debug models`.
|
||||
|
||||
|
|
@ -128,6 +134,8 @@ class CodexStockModel(LiteLLMBaseModel):
|
|||
upgrade: CodexStockUpgrade | None = None
|
||||
service_tiers: tuple[CodexServiceTier, ...] = ()
|
||||
default_service_tier: str | None = None
|
||||
supported_reasoning_levels: tuple[CodexReasoningLevel, ...] = ()
|
||||
context_window: int | None = None
|
||||
|
||||
|
||||
class CodexStockCatalog(LiteLLMBaseModel):
|
||||
|
|
@ -135,19 +143,21 @@ class CodexStockCatalog(LiteLLMBaseModel):
|
|||
|
||||
|
||||
CodexCatalogEntry = CodexFallbackModel | CodexStockModel
|
||||
CodexAliasDefaults: TypeAlias = Literal["codex", "deployment"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CodexCatalogRow:
|
||||
"""What the catalog needs to know about one listed model: its public id, the listing's mode and
|
||||
input limit, the upstream model behind it, and the `model_info` values the operator set;
|
||||
`service_tiers` holds one raw `model_info.service_tiers` value per deployment behind the id, None
|
||||
where a deployment sets none."""
|
||||
input limit, the upstream model behind it, the reasoning effort its deployments pin, and the
|
||||
`model_info` values the operator set; `service_tiers` holds one raw `model_info.service_tiers`
|
||||
value per deployment behind the id, None where a deployment sets none."""
|
||||
|
||||
id: str
|
||||
mode: str | None = None
|
||||
max_input_tokens: int | None = None
|
||||
upstream_model: str | None = None
|
||||
reasoning_effort: str | None = None
|
||||
display_name: str | None = None
|
||||
service_tiers: tuple[object, ...] = ()
|
||||
|
||||
|
|
@ -254,12 +264,42 @@ def _stock_for(row: CodexCatalogRow, stock: Mapping[str, CodexStockModel]) -> Co
|
|||
return upstream if upstream is not None else stock.get(row.upstream_model.partition("/")[2])
|
||||
|
||||
|
||||
def _alias_context_window(row: CodexCatalogRow, stock_model: CodexStockModel) -> Mapping[str, int]:
|
||||
if row.max_input_tokens is None or stock_model.context_window is None:
|
||||
return MappingProxyType({})
|
||||
return MappingProxyType({"context_window": min(stock_model.context_window, row.max_input_tokens)})
|
||||
|
||||
|
||||
def _alias_reasoning_level(row: CodexCatalogRow, stock_model: CodexStockModel) -> str | None:
|
||||
efforts: Final = frozenset(level.effort for level in stock_model.supported_reasoning_levels)
|
||||
return row.reasoning_effort if row.reasoning_effort in efforts else None
|
||||
|
||||
|
||||
def _deployment_defaults(row: CodexCatalogRow, stock_model: CodexStockModel) -> Mapping[str, object]:
|
||||
"""What an alias's deployments decide in place of the defaults Codex picks for the model they route to:
|
||||
no service tier, verbosity, upgrade nudge or startup promo of Codex's own, so what the deployment
|
||||
pins (or OpenAI's API default) stays in force, the pinned reasoning effort when Codex offers it and
|
||||
none otherwise, and a window no larger than the alias's own limit."""
|
||||
return MappingProxyType(
|
||||
{
|
||||
"service_tiers": (),
|
||||
"default_service_tier": None,
|
||||
"default_verbosity": None,
|
||||
"upgrade": None,
|
||||
"availability_nux": None,
|
||||
"default_reasoning_level": _alias_reasoning_level(row, stock_model),
|
||||
**_alias_context_window(row, stock_model),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _entry(
|
||||
index: int,
|
||||
row: CodexCatalogRow,
|
||||
stock: Mapping[str, CodexStockModel],
|
||||
served: frozenset[str],
|
||||
instructions: str,
|
||||
alias_defaults: CodexAliasDefaults,
|
||||
) -> CodexCatalogEntry:
|
||||
stock_model: Final = _stock_for(row, stock)
|
||||
stock_tiers: Final = MappingProxyType(
|
||||
|
|
@ -278,7 +318,8 @@ def _entry(
|
|||
upgrade: Final = (
|
||||
stock_model.upgrade if stock_model.upgrade is not None and stock_model.upgrade.model in served else None
|
||||
)
|
||||
stock_name: Final = stock_model.display_name if row.id == stock_model.slug else row.id
|
||||
is_alias: Final = row.id != stock_model.slug
|
||||
stock_name: Final = row.id if is_alias else stock_model.display_name
|
||||
listing: Final = {
|
||||
"slug": row.id,
|
||||
"display_name": row.display_name if row.display_name is not None else stock_name,
|
||||
|
|
@ -286,6 +327,7 @@ def _entry(
|
|||
"visibility": "list",
|
||||
"supported_in_api": True,
|
||||
"upgrade": upgrade,
|
||||
**(_deployment_defaults(row, stock_model) if is_alias and alias_defaults == "deployment" else {}),
|
||||
}
|
||||
if configured is None:
|
||||
return stock_model.model_copy(update=listing)
|
||||
|
|
@ -351,6 +393,7 @@ def codex_models_response_json(
|
|||
stock: Mapping[str, CodexStockModel] | None = None,
|
||||
instructions: str | None = None,
|
||||
byte_limit: int | None = CODEX_CATALOG_BYTE_LIMIT,
|
||||
alias_defaults: CodexAliasDefaults = "codex",
|
||||
) -> CodexCatalogBody:
|
||||
"""Codex's `ModelsResponse` body for `rows`, in listing order, cut to Codex's byte limit.
|
||||
|
||||
|
|
@ -359,13 +402,18 @@ def codex_models_response_json(
|
|||
byte limit silently and keeps its bundled one, so the body holds only the entries that fit,
|
||||
the ones offering a service tier taken first and the rest in listing order, an entry too large
|
||||
for the bytes left passed over, every kept entry at its listing position; `byte_limit=None`
|
||||
keeps every entry.
|
||||
keeps every entry. A row listed under another name than the Codex model its `upstream_model`
|
||||
names (an alias) takes that model's entry; `alias_defaults="deployment"` drops the defaults
|
||||
Codex picks for that model in favor of what the alias's deployments pin, while `"codex"` keeps
|
||||
them, as Codex would use them for the model's own name.
|
||||
"""
|
||||
known: Final = bundled_codex_models() if stock is None else stock
|
||||
prompt: Final = codex_base_instructions() if instructions is None else instructions
|
||||
drivable: Final = tuple(row for row in rows if _codex_can_drive(row))
|
||||
served: Final = frozenset(row.id for row in drivable)
|
||||
entries: Final = tuple(_entry(index, row, known, served, prompt) for index, row in enumerate(drivable))
|
||||
entries: Final = tuple(
|
||||
_entry(index, row, known, served, prompt, alias_defaults) for index, row in enumerate(drivable)
|
||||
)
|
||||
serialized: Final = tuple(_serialized(entry) for entry in entries)
|
||||
kept: Final = _kept_positions(entries, serialized, byte_limit)
|
||||
return CodexCatalogBody(
|
||||
|
|
|
|||
|
|
@ -2,14 +2,17 @@
|
|||
``type`` is a required string and ``param`` is nullable, neither of which the literal
|
||||
string ``"None"`` satisfies."""
|
||||
|
||||
import sys
|
||||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from fastapi import status
|
||||
from fastapi import HTTPException, status
|
||||
|
||||
from litellm.constants import STRINGIFIED_NONE
|
||||
from litellm.proxy._types import ProxyException
|
||||
from litellm.proxy._types import ProxyErrorTypes, ProxyException, proxy_exception_status_code
|
||||
from litellm.proxy.db.db_lookup_gate import DBLookupDeadlineExceeded
|
||||
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler, exception_chain
|
||||
|
||||
LITELLM_CALL_ID_HEADER: Final = "x-litellm-call-id"
|
||||
|
||||
|
|
@ -21,6 +24,19 @@ _OPENAI_ERROR_TYPE_BY_STATUS: Final[Mapping[int, str]] = MappingProxyType(
|
|||
}
|
||||
)
|
||||
|
||||
_AUTH_FAILURE_STATUS_CODES: Final = frozenset({status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN})
|
||||
|
||||
_FAILOVER_SQLSTATES: Final = frozenset({"25006", "57P01", "57P02", "57P03"})
|
||||
|
||||
_FAILOVER_REFUSALS: Final = (
|
||||
"is not currently accepting connections",
|
||||
"the database system is starting up",
|
||||
"the database system is shutting down",
|
||||
"the database system is in recovery mode",
|
||||
"the database system is not yet accepting connections",
|
||||
"terminating connection due to administrator command",
|
||||
)
|
||||
|
||||
|
||||
def attribute_of(value: object, name: str, default: object = None) -> object:
|
||||
return getattr(value, name, default)
|
||||
|
|
@ -61,6 +77,82 @@ def openai_error_param(exc: object) -> str | None:
|
|||
return carried if isinstance(carried, str) and carried != STRINGIFIED_NONE else None
|
||||
|
||||
|
||||
def proxy_exception_for(exc: Exception, default_status_code: int) -> ProxyException:
|
||||
match exc:
|
||||
case ProxyException():
|
||||
return exc
|
||||
case _ if _is_database_outage(exc):
|
||||
return ProxyException(
|
||||
message=PrismaDBExceptionHandler.database_unavailable_message(exc),
|
||||
type=ProxyErrorTypes.no_db_connection.value,
|
||||
param=None,
|
||||
code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
)
|
||||
case HTTPException(status_code=status_code, detail=detail):
|
||||
return _openai_shaped_proxy_exception(exc, str(detail), status_code)
|
||||
case _:
|
||||
return _openai_shaped_proxy_exception(
|
||||
exc, str(exc) or type(exc).__name__, error_status_code(exc, default_status_code)
|
||||
)
|
||||
|
||||
|
||||
def _is_database_outage(exc: Exception) -> bool:
|
||||
return any(map(_is_database_outage_link, exception_chain(exc)))
|
||||
|
||||
|
||||
def _is_database_outage_link(exc: BaseException) -> bool:
|
||||
match exc:
|
||||
case DBLookupDeadlineExceeded():
|
||||
return True
|
||||
case Exception() if _prisma_is_loaded():
|
||||
return _is_prisma_outage(exc)
|
||||
case _:
|
||||
return False
|
||||
|
||||
|
||||
def _prisma_is_loaded() -> bool:
|
||||
return sys.modules.get("prisma") is not None
|
||||
|
||||
|
||||
def _is_prisma_outage(exc: Exception) -> bool:
|
||||
if PrismaDBExceptionHandler.is_prisma_engine_internal_error(exc):
|
||||
return True
|
||||
if type(exc).__module__.partition(".")[0] != "prisma":
|
||||
return False
|
||||
return PrismaDBExceptionHandler.is_database_service_unavailable_error(exc) or _is_failover_refusal(exc)
|
||||
|
||||
|
||||
def _is_failover_refusal(exc: Exception) -> bool:
|
||||
if PrismaDBExceptionHandler.is_read_only_transaction_error(exc):
|
||||
return True
|
||||
if PrismaDBExceptionHandler.postgres_sqlstate(exc) in _FAILOVER_SQLSTATES:
|
||||
return True
|
||||
message: Final = str(exc).lower()
|
||||
return any(refusal in message for refusal in _FAILOVER_REFUSALS)
|
||||
|
||||
|
||||
def _openai_shaped_proxy_exception(exc: Exception, message: str, status_code: int) -> ProxyException:
|
||||
answered_status: Final = int(proxy_exception_status_code(message, str(status_code)))
|
||||
return ProxyException(
|
||||
message=message,
|
||||
type=_catch_all_error_type(exc, answered_status),
|
||||
param=openai_error_param(exc),
|
||||
code=answered_status,
|
||||
)
|
||||
|
||||
|
||||
def _catch_all_error_type(exc: Exception, status_code: int) -> str:
|
||||
if _rejects_the_caller(exc, status_code):
|
||||
return ProxyErrorTypes.auth_error.value
|
||||
return openai_error_type(exc, status_code)
|
||||
|
||||
|
||||
def _rejects_the_caller(exc: Exception, status_code: int) -> bool:
|
||||
raised_as_http_answer: Final = isinstance(exc, HTTPException)
|
||||
status_from_upstream: Final = error_status_code(exc, 0) == status_code
|
||||
return status_code in _AUTH_FAILURE_STATUS_CODES and (raised_as_http_answer or not status_from_upstream)
|
||||
|
||||
|
||||
def litellm_call_id_headers(litellm_call_id: str | None) -> dict[str, str] | None: # mutable-ok: ProxyException.headers
|
||||
if litellm_call_id is None:
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ _LOCK_TIMEOUT_SQLSTATE: Final = "55P03"
|
|||
_LOCK_TIMEOUT_PHRASE: Final = "canceling statement due to lock timeout"
|
||||
|
||||
|
||||
def _exception_chain(e: BaseException) -> Iterator[BaseException]:
|
||||
def exception_chain(e: BaseException) -> Iterator[BaseException]:
|
||||
current = e # rebind-ok: advances one link per iteration of the bounded walk
|
||||
for _ in range(_MAX_EXCEPTION_CHAIN_DEPTH):
|
||||
yield current
|
||||
|
|
@ -49,7 +49,7 @@ def _exception_chain(e: BaseException) -> Iterator[BaseException]:
|
|||
def _database_service_unavailable_errors(e: BaseException) -> tuple[Exception, ...]:
|
||||
return tuple(
|
||||
link
|
||||
for link in _exception_chain(e)
|
||||
for link in exception_chain(e)
|
||||
if isinstance(link, Exception) and PrismaDBExceptionHandler.is_database_service_unavailable_error(link)
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
from collections.abc import Mapping
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from typing import Annotated, Final
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, Response
|
||||
from fastapi.responses import JSONResponse, ORJSONResponse
|
||||
from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError
|
||||
from pydantic import BaseModel, BeforeValidator, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError
|
||||
|
||||
from litellm.litellm_core_utils.json_validation_rule import normalize_json_schema_types
|
||||
from litellm.proxy._types import *
|
||||
|
|
@ -27,14 +27,67 @@ from litellm.proxy.common_utils.token_count_settings import count_source_fields
|
|||
from litellm.types.llms.vertex_ai import TokenCountDetailsResponse
|
||||
|
||||
|
||||
def _google_repeated_field(value: JsonValue) -> JsonValue:
|
||||
match value:
|
||||
case None:
|
||||
return []
|
||||
case dict():
|
||||
return [value]
|
||||
case list():
|
||||
return [item for item in value if item is not None]
|
||||
case _:
|
||||
return value
|
||||
|
||||
|
||||
def _google_content(value: JsonValue) -> JsonValue:
|
||||
match value:
|
||||
case {"parts": parts}:
|
||||
return {**value, "parts": _google_repeated_field(parts)}
|
||||
case _:
|
||||
return value
|
||||
|
||||
|
||||
def _google_contents(value: JsonValue) -> JsonValue:
|
||||
match _google_repeated_field(value):
|
||||
case list() as contents:
|
||||
return [_google_content(content) for content in contents]
|
||||
case other:
|
||||
return other
|
||||
|
||||
|
||||
_GOOGLE_PART_TEXT_AND_METADATA_FIELDS: Final = frozenset(
|
||||
{"text", "thought", "thoughtSignature", "thought_signature", "partMetadata", "part_metadata"}
|
||||
)
|
||||
|
||||
|
||||
def _google_part_has_non_text_content(part: Mapping[str, JsonValue]) -> bool:
|
||||
return any(field is not None for key, field in part.items() if key not in _GOOGLE_PART_TEXT_AND_METADATA_FIELDS)
|
||||
|
||||
|
||||
def _google_system_part(value: JsonValue) -> JsonValue:
|
||||
match value:
|
||||
case dict() if value.get("text") is None and _google_part_has_non_text_content(value):
|
||||
return {key: field for key, field in value.items() if key != "text"}
|
||||
case dict():
|
||||
return {"text": None, **value}
|
||||
case _:
|
||||
return value
|
||||
|
||||
|
||||
_GOOGLE_REPEATED_FIELD: Final = BeforeValidator(_google_repeated_field)
|
||||
|
||||
|
||||
class _GoogleCountTokensTextPart(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
text: str
|
||||
text: str | None
|
||||
|
||||
|
||||
class _GoogleCountTokensSystemInstruction(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
parts: tuple[_GoogleCountTokensTextPart, ...]
|
||||
parts: Annotated[
|
||||
tuple[Annotated[_GoogleCountTokensTextPart, BeforeValidator(_google_system_part)], ...],
|
||||
_GOOGLE_REPEATED_FIELD,
|
||||
] = ()
|
||||
|
||||
|
||||
class _GoogleCountTokensFunctionDeclaration(BaseModel):
|
||||
|
|
@ -47,16 +100,16 @@ class _GoogleCountTokensFunctionDeclaration(BaseModel):
|
|||
|
||||
class _GoogleCountTokensTool(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, populate_by_name=True)
|
||||
function_declarations: tuple[_GoogleCountTokensFunctionDeclaration, ...] = Field(
|
||||
function_declarations: Annotated[tuple[_GoogleCountTokensFunctionDeclaration, ...], _GOOGLE_REPEATED_FIELD] = Field(
|
||||
default=(), alias="functionDeclarations"
|
||||
)
|
||||
|
||||
|
||||
class _GoogleCountTokensRequest(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="ignore", populate_by_name=True)
|
||||
contents: tuple[Mapping[str, JsonValue], ...] = ()
|
||||
contents: Annotated[tuple[Mapping[str, JsonValue], ...], BeforeValidator(_google_contents)] = ()
|
||||
system_instruction: _GoogleCountTokensSystemInstruction | None = Field(default=None, alias="systemInstruction")
|
||||
tools: tuple[_GoogleCountTokensTool, ...] = ()
|
||||
tools: Annotated[tuple[_GoogleCountTokensTool, ...], _GOOGLE_REPEATED_FIELD] = ()
|
||||
|
||||
|
||||
router: Final = APIRouter(
|
||||
|
|
@ -112,7 +165,11 @@ def _google_count_tokens_request(model_name: str, data: object) -> TokenCountReq
|
|||
GoogleGenAIAdapter().translate_generate_content_to_completion(model_name, contents).get("messages", [])
|
||||
)
|
||||
system: Final = (
|
||||
[{"type": "text", "text": part.text} for part in validated_data.system_instruction.parts]
|
||||
[
|
||||
{"type": "text", "text": part.text}
|
||||
for part in validated_data.system_instruction.parts
|
||||
if part.text is not None
|
||||
]
|
||||
if validated_data.system_instruction is not None
|
||||
else None
|
||||
)
|
||||
|
|
|
|||
|
|
@ -32,7 +32,6 @@ from litellm.proxy._types import (
|
|||
EnterpriseLicenseData,
|
||||
Litellm_EntityType,
|
||||
LitellmUserRoles,
|
||||
ProxyErrorTypes,
|
||||
ProxyException,
|
||||
SpecialModelNames,
|
||||
UserAPIKeyAuth,
|
||||
|
|
@ -47,6 +46,7 @@ from litellm.proxy.auth.auth_utils import (
|
|||
from litellm.proxy.auth.model_checks import get_key_models
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.common_utils import http_parsing_utils
|
||||
from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for
|
||||
from litellm.proxy.db.db_lookup_gate import db_lookup_stall_tracker
|
||||
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
|
||||
from litellm.proxy.db.health_check_latest import (
|
||||
|
|
@ -683,21 +683,9 @@ async def health_services_endpoint(
|
|||
except Exception as e:
|
||||
verbose_proxy_logger.error("litellm.proxy.proxy_server.health_services_endpoint(): Exception occured - %s", e)
|
||||
verbose_proxy_logger.debug(traceback.format_exc())
|
||||
if isinstance(e, HTTPException):
|
||||
raise ProxyException(
|
||||
message=getattr(e, "detail", f"Authentication Error({e})"),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR),
|
||||
)
|
||||
elif isinstance(e, ProxyException):
|
||||
raise e
|
||||
raise ProxyException(
|
||||
message="Authentication Error, " + str(e),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
if isinstance(e, ProxyException):
|
||||
raise
|
||||
raise proxy_exception_for(e, default_status_code=status.HTTP_500_INTERNAL_SERVER_ERROR) from e
|
||||
|
||||
|
||||
def convert_health_check_to_dict(check) -> dict:
|
||||
|
|
|
|||
|
|
@ -46,6 +46,7 @@ from litellm.proxy.auth.password_policy import (
|
|||
)
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast
|
||||
from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for
|
||||
from litellm.proxy.common_utils.user_api_key_cache import (
|
||||
AUTH_OBJECTS_TARGET,
|
||||
object_permission_cache_key,
|
||||
|
|
@ -1729,21 +1730,9 @@ async def user_update(
|
|||
except Exception as e:
|
||||
verbose_proxy_logger.exception("litellm.proxy.proxy_server.user_update(): Exception occured - %s", e)
|
||||
verbose_proxy_logger.debug(traceback.format_exc())
|
||||
if isinstance(e, HTTPException):
|
||||
raise ProxyException(
|
||||
message=getattr(e, "detail", f"Authentication Error({e})"),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
|
||||
)
|
||||
elif isinstance(e, ProxyException):
|
||||
raise e
|
||||
raise ProxyException(
|
||||
message="Authentication Error, " + str(e),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
if isinstance(e, ProxyException):
|
||||
raise
|
||||
raise proxy_exception_for(e, default_status_code=status.HTTP_400_BAD_REQUEST) from e
|
||||
|
||||
|
||||
async def bulk_update_processed_users(
|
||||
|
|
|
|||
|
|
@ -75,6 +75,7 @@ from litellm.proxy.common_utils.config_sync_pubsub import (
|
|||
coordination_redis_cache,
|
||||
publish_config_change,
|
||||
)
|
||||
from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for
|
||||
from litellm.proxy.common_utils.rbac_utils import check_org_admin_can_generate_keys
|
||||
from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time
|
||||
from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET, UserApiKeyCache
|
||||
|
|
@ -4087,21 +4088,9 @@ async def update_key_fn(
|
|||
# update based on remaining passed in values
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.exception("litellm.proxy.proxy_server.update_key_fn(): Exception occured - %s", e)
|
||||
if isinstance(e, HTTPException):
|
||||
raise ProxyException(
|
||||
message=getattr(e, "detail", f"Authentication Error({e})"),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
|
||||
)
|
||||
elif isinstance(e, ProxyException):
|
||||
raise e
|
||||
raise ProxyException(
|
||||
message="Authentication Error, " + str(e),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
if isinstance(e, ProxyException):
|
||||
raise
|
||||
raise proxy_exception_for(e, default_status_code=status.HTTP_400_BAD_REQUEST) from e
|
||||
|
||||
|
||||
@router.post(
|
||||
|
|
@ -7244,21 +7233,9 @@ async def list_keys(
|
|||
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.exception("Error in list_keys: %s", e)
|
||||
if isinstance(e, HTTPException):
|
||||
raise ProxyException(
|
||||
message=getattr(e, "detail", f"error({e})"),
|
||||
type=ProxyErrorTypes.internal_server_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=getattr(e, "status_code", fastapi.status.HTTP_500_INTERNAL_SERVER_ERROR),
|
||||
)
|
||||
elif isinstance(e, ProxyException):
|
||||
raise e
|
||||
raise ProxyException(
|
||||
message="Authentication Error, " + str(e),
|
||||
type=ProxyErrorTypes.internal_server_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=fastapi.status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
if isinstance(e, ProxyException):
|
||||
raise
|
||||
raise proxy_exception_for(e, default_status_code=fastapi.status.HTTP_500_INTERNAL_SERVER_ERROR) from e
|
||||
|
||||
|
||||
async def _apply_non_admin_alias_scope(
|
||||
|
|
@ -7402,21 +7379,9 @@ async def key_aliases(
|
|||
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.exception("Error in key_aliases: %s", e)
|
||||
if isinstance(e, HTTPException):
|
||||
raise ProxyException(
|
||||
message=getattr(e, "detail", f"error({e})"),
|
||||
type=ProxyErrorTypes.internal_server_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR),
|
||||
)
|
||||
elif isinstance(e, ProxyException):
|
||||
raise e
|
||||
raise ProxyException(
|
||||
message="Authentication Error, " + str(e),
|
||||
type=ProxyErrorTypes.internal_server_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
if isinstance(e, ProxyException):
|
||||
raise
|
||||
raise proxy_exception_for(e, default_status_code=status.HTTP_500_INTERNAL_SERVER_ERROR) from e
|
||||
|
||||
|
||||
def _validate_sort_params(
|
||||
|
|
|
|||
|
|
@ -77,6 +77,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import (
|
|||
decrypt_value_helper,
|
||||
encrypt_value_helper,
|
||||
)
|
||||
from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for
|
||||
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
|
||||
from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient
|
||||
from litellm.proxy.management.teams.authz import TEAM_ADMIN_ONLY, is_team_admin
|
||||
|
|
@ -2845,21 +2846,9 @@ async def delete_model(
|
|||
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.exception("Failed to delete model. Due to error - %s", e)
|
||||
if isinstance(e, HTTPException):
|
||||
raise ProxyException(
|
||||
message=getattr(e, "detail", f"Authentication Error({e})"),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
|
||||
)
|
||||
elif isinstance(e, ProxyException):
|
||||
raise e
|
||||
raise ProxyException(
|
||||
message="Authentication Error, " + str(e),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
if isinstance(e, ProxyException):
|
||||
raise
|
||||
raise proxy_exception_for(e, default_status_code=status.HTTP_400_BAD_REQUEST) from e
|
||||
|
||||
|
||||
async def delete_team_model_alias(
|
||||
|
|
@ -2899,6 +2888,13 @@ async def delete_team_model_alias(
|
|||
return removed_model_aliases
|
||||
|
||||
|
||||
def _model_not_added_to_db() -> HTTPException:
|
||||
return HTTPException(
|
||||
status_code=500,
|
||||
detail={"error": "Failed to add model to db. Check your server logs for more details."},
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/model/new",
|
||||
description="Allows adding new models to the model list in the config.yaml",
|
||||
|
|
@ -3080,6 +3076,8 @@ async def add_new_model(
|
|||
if isinstance(e, HTTPException):
|
||||
raise
|
||||
verbose_proxy_logger.exception("Exception in add_new_model: %s", e)
|
||||
if model_response is None:
|
||||
raise _model_not_added_to_db() from e
|
||||
|
||||
else:
|
||||
raise HTTPException(
|
||||
|
|
@ -3088,10 +3086,7 @@ async def add_new_model(
|
|||
)
|
||||
|
||||
if model_response is None:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail={"error": "Failed to add model to db. Check your server logs for more details."},
|
||||
)
|
||||
raise _model_not_added_to_db()
|
||||
|
||||
## CREATE AUDIT LOG ##
|
||||
asyncio.create_task(
|
||||
|
|
@ -3121,21 +3116,9 @@ async def add_new_model(
|
|||
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.exception("litellm.proxy.proxy_server.add_new_model(): Exception occured - %s", e)
|
||||
if isinstance(e, HTTPException):
|
||||
raise ProxyException(
|
||||
message=getattr(e, "detail", f"Authentication Error({e})"),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
|
||||
)
|
||||
elif isinstance(e, ProxyException):
|
||||
raise e
|
||||
raise ProxyException(
|
||||
message="Authentication Error, " + str(e),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
if isinstance(e, ProxyException):
|
||||
raise
|
||||
raise proxy_exception_for(e, default_status_code=status.HTTP_400_BAD_REQUEST) from e
|
||||
|
||||
|
||||
#### MODEL MANAGEMENT ####
|
||||
|
|
@ -3364,21 +3347,9 @@ async def update_model(
|
|||
return None if model_response is None else _model_write_response(model_response, member_write)
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.exception("litellm.proxy.proxy_server.update_model(): Exception occured - %s", e)
|
||||
if isinstance(e, HTTPException):
|
||||
raise ProxyException(
|
||||
message=getattr(e, "detail", f"Authentication Error({e})"),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
|
||||
)
|
||||
elif isinstance(e, ProxyException):
|
||||
raise e
|
||||
raise ProxyException(
|
||||
message="Authentication Error, " + str(e),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
if isinstance(e, ProxyException):
|
||||
raise
|
||||
raise proxy_exception_for(e, default_status_code=status.HTTP_400_BAD_REQUEST) from e
|
||||
|
||||
|
||||
@router.post(
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@ from litellm.proxy.auth.auth_checks import (
|
|||
)
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast
|
||||
from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for
|
||||
from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time
|
||||
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
|
||||
from litellm.proxy.management_endpoints.budget_management_endpoints import (
|
||||
|
|
@ -1396,21 +1397,9 @@ async def organization_member_add(
|
|||
)
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.exception("Error adding member to organization: %s", e)
|
||||
if isinstance(e, HTTPException):
|
||||
raise ProxyException(
|
||||
message=getattr(e, "detail", f"Authentication Error({e})"),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR),
|
||||
)
|
||||
elif isinstance(e, ProxyException):
|
||||
raise e
|
||||
raise ProxyException(
|
||||
message="Authentication Error, " + str(e),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
if isinstance(e, ProxyException):
|
||||
raise
|
||||
raise proxy_exception_for(e, default_status_code=status.HTTP_500_INTERNAL_SERVER_ERROR) from e
|
||||
|
||||
|
||||
async def find_member_if_email(user_email: str, prisma_client: PrismaClient) -> LiteLLM_UserTable:
|
||||
|
|
|
|||
|
|
@ -132,6 +132,7 @@ from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import (
|
|||
)
|
||||
from litellm.proxy.common_utils.callback_utils import encrypt_callback_vars
|
||||
from litellm.proxy.common_utils.json_merge_patch import apply_json_merge_patch
|
||||
from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for
|
||||
from litellm.proxy.common_utils.user_api_key_cache import (
|
||||
UserApiKeyCache,
|
||||
team_membership_auth_cache_key,
|
||||
|
|
@ -5526,21 +5527,9 @@ async def team_info(
|
|||
e,
|
||||
traceback.format_exc(),
|
||||
)
|
||||
if isinstance(e, HTTPException):
|
||||
raise ProxyException(
|
||||
message=getattr(e, "detail", f"Authentication Error({e})"),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
|
||||
)
|
||||
elif isinstance(e, ProxyException):
|
||||
raise e
|
||||
raise ProxyException(
|
||||
message="Authentication Error, " + str(e),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
if isinstance(e, ProxyException):
|
||||
raise
|
||||
raise proxy_exception_for(e, default_status_code=status.HTTP_400_BAD_REQUEST) from e
|
||||
|
||||
|
||||
@router.get(
|
||||
|
|
|
|||
|
|
@ -524,6 +524,7 @@ from litellm.proxy.common_utils.openai_error_payload import (
|
|||
LITELLM_CALL_ID_HEADER,
|
||||
headers_with_litellm_call_id,
|
||||
litellm_call_id_headers,
|
||||
proxy_exception_for,
|
||||
with_litellm_call_id,
|
||||
)
|
||||
from litellm.proxy.common_utils.periodic_reload_schedule import (
|
||||
|
|
@ -18536,25 +18537,13 @@ async def async_queue_request(
|
|||
await proxy_logging_obj.post_call_failure_hook(
|
||||
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
|
||||
)
|
||||
if isinstance(e, HTTPException):
|
||||
raise ProxyException(
|
||||
message=getattr(e, "detail", f"Authentication Error({e})"),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
|
||||
)
|
||||
elif isinstance(e, ProxyException):
|
||||
raise e
|
||||
elif isinstance(e, litellm.BadRequestError):
|
||||
if isinstance(e, litellm.BadRequestError):
|
||||
raise ProxyException(
|
||||
message=e.message, type="invalid_request_error", param=getattr(e, "param", None), code=e.status_code
|
||||
)
|
||||
raise ProxyException(
|
||||
message="Authentication Error, " + str(e),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
) from e
|
||||
if isinstance(e, ProxyException):
|
||||
raise
|
||||
raise proxy_exception_for(e, default_status_code=status.HTTP_400_BAD_REQUEST) from e
|
||||
|
||||
|
||||
@app.get("/fallback/login", tags=["experimental"], include_in_schema=False)
|
||||
|
|
@ -19833,21 +19822,9 @@ async def update_config(
|
|||
except Exception as e:
|
||||
verbose_proxy_logger.error("litellm.proxy.proxy_server.update_config(): Exception occured - %s", e)
|
||||
verbose_proxy_logger.debug(traceback.format_exc())
|
||||
if isinstance(e, HTTPException):
|
||||
raise ProxyException(
|
||||
message=getattr(e, "detail", f"Authentication Error({e})"),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
|
||||
)
|
||||
elif isinstance(e, ProxyException):
|
||||
raise e
|
||||
raise ProxyException(
|
||||
message="Authentication Error, " + str(e),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
if isinstance(e, ProxyException):
|
||||
raise
|
||||
raise proxy_exception_for(e, default_status_code=status.HTTP_400_BAD_REQUEST) from e
|
||||
|
||||
|
||||
### CONFIG GENERAL SETTINGS
|
||||
|
|
@ -21030,21 +21007,9 @@ async def get_config(
|
|||
}
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.exception("litellm.proxy.proxy_server.get_config(): Exception occured - %s", e)
|
||||
if isinstance(e, HTTPException):
|
||||
raise ProxyException(
|
||||
message=getattr(e, "detail", f"Authentication Error({e})"),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
|
||||
)
|
||||
elif isinstance(e, ProxyException):
|
||||
raise e
|
||||
raise ProxyException(
|
||||
message="Authentication Error, " + str(e),
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=getattr(e, "param", "None"),
|
||||
code=status.HTTP_400_BAD_REQUEST,
|
||||
)
|
||||
if isinstance(e, ProxyException):
|
||||
raise
|
||||
raise proxy_exception_for(e, default_status_code=status.HTTP_500_INTERNAL_SERVER_ERROR) from e
|
||||
|
||||
|
||||
@router.get(
|
||||
|
|
|
|||
|
|
@ -746,7 +746,7 @@ def test_internal_user_model_creation_prohibition_refuses_team_admin_but_not_pro
|
|||
"error": {
|
||||
"message": "Model creation is disabled for internal users by disable_model_add_for_internal_users.",
|
||||
"type": "auth_error",
|
||||
"param": "None",
|
||||
"param": None,
|
||||
"code": "403",
|
||||
}
|
||||
},
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ def _escalation_error(field: str) -> dict[str, JsonValue]:
|
|||
"error": {
|
||||
"message": f"{{'error': \"Non-admin users cannot modify '{field}' on their own record. Contact your proxy admin.\"}}",
|
||||
"type": "auth_error",
|
||||
"param": "None",
|
||||
"param": None,
|
||||
"code": "403",
|
||||
}
|
||||
}
|
||||
|
|
@ -49,7 +49,7 @@ def _role_change_error() -> dict[str, JsonValue]:
|
|||
"error": {
|
||||
"message": "Only proxy admins can modify user roles.",
|
||||
"type": "auth_error",
|
||||
"param": "None",
|
||||
"param": None,
|
||||
"code": "403",
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -255,8 +255,8 @@ def _team_not_found_body(team_id: str) -> dict[str, JsonValue]:
|
|||
return {
|
||||
"error": {
|
||||
"message": f"{{'message': 'Team not found, passed team id: {team_id}.'}}",
|
||||
"type": "auth_error",
|
||||
"param": "None",
|
||||
"type": "invalid_request_error",
|
||||
"param": None,
|
||||
"code": "404",
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -158,6 +158,18 @@ def test_native_function_response_preserves_json_when_media_is_malformed(invalid
|
|||
assert messages == [{"role": "tool", "tool_call_id": "call_lookup", "content": '{"ok": true}'}]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("role", ["user", "model"])
|
||||
def test_one_object_content_parts_translate_like_their_list_form(role: str) -> None:
|
||||
part: Final = {"text": "Reply with OK"}
|
||||
|
||||
def messages(parts: object) -> object:
|
||||
return GoogleGenAIAdapter().translate_generate_content_to_completion(
|
||||
model="anthropic/claude-haiku-5-5", contents=[{"role": role, "parts": parts}]
|
||||
)["messages"]
|
||||
|
||||
assert messages(part) == messages([part]) != []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("args", "expected_arguments"),
|
||||
[(None, "null"), ('{"query":"x"}', '"{\\"query\\":\\"x\\"}"')],
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from unittest.mock import AsyncMock, MagicMock
|
|||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.llms.anthropic.messages.adapters.handler import _run_polyfill_if_enabled
|
||||
from litellm.llms.anthropic.messages.adapters.handler import _run_polyfill_if_enabled, prepare_context_managed_request
|
||||
from litellm.llms.anthropic.messages.context_management import (
|
||||
AnthropicContextManagementError,
|
||||
PolyfillResult,
|
||||
|
|
@ -690,6 +690,115 @@ def test_tool_uses_trigger_does_not_count_excluded_tools(trigger_value: int, tri
|
|||
assert (applied is not None) == triggers
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"system",
|
||||
["Reference notes. " * 400, [{"type": "text", "text": "Reference notes. " * 400}]],
|
||||
ids=["string", "blocks"],
|
||||
)
|
||||
def test_input_tokens_trigger_counts_the_system_prompt(system: str | list[dict[str, object]]) -> None:
|
||||
"""Like the API, the trigger compares the whole prompt: a system prompt can push it over."""
|
||||
messages = _make_history(n_pairs=3)
|
||||
history_tokens: Final = litellm.token_counter(model=MODEL, messages=messages)
|
||||
edit_spec: Final = {
|
||||
"type": "clear_tool_uses_20250919",
|
||||
"trigger": {"type": "input_tokens", "value": history_tokens + 100},
|
||||
"keep": {"type": "tool_uses", "value": 1},
|
||||
}
|
||||
_, without_system = apply_clear_tool_uses_20250919(
|
||||
model=MODEL, messages=deepcopy(messages), tools=None, system=None, edit_spec=edit_spec
|
||||
)
|
||||
_, with_system = apply_clear_tool_uses_20250919(
|
||||
model=MODEL, messages=deepcopy(messages), tools=None, system=system, edit_spec=edit_spec
|
||||
)
|
||||
_, met_without_system = apply_clear_tool_uses_20250919(
|
||||
model=MODEL,
|
||||
messages=deepcopy(messages),
|
||||
tools=None,
|
||||
system=None,
|
||||
edit_spec={**edit_spec, "trigger": {"type": "input_tokens", "value": 0}},
|
||||
)
|
||||
assert without_system is None
|
||||
assert with_system is not None
|
||||
assert met_without_system is not None
|
||||
assert with_system["cleared_tool_uses"] == 2
|
||||
assert with_system["cleared_input_tokens"] == met_without_system["cleared_input_tokens"] > 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("compact_position", [None, "after", "before"], ids=["alone", "before_compact", "after_compact"])
|
||||
async def test_input_tokens_trigger_counts_a_client_compaction_summary(compact_position: str | None) -> None:
|
||||
"""A client compaction summary counts toward the trigger and calls beside it stay clearable, in any edit order."""
|
||||
tail: Final = _make_history(n_pairs=3)
|
||||
edit_spec: Final = {
|
||||
"type": "clear_tool_uses_20250919",
|
||||
"trigger": {"type": "input_tokens", "value": litellm.token_counter(model=MODEL, messages=tail) + 100},
|
||||
"keep": {"type": "tool_uses", "value": 1},
|
||||
}
|
||||
compaction: Final = {"type": "compaction", "content": "Earlier work. " * 400}
|
||||
compacted_history: Final = [tail[0], {**tail[1], "content": [compaction, *tail[1]["content"]]}, *tail[2:]]
|
||||
compact_spec: Final = {"type": "compact_20260112", "trigger": {"type": "input_tokens", "value": 50_000}}
|
||||
edits: Final = {None: [edit_spec], "after": [edit_spec, compact_spec], "before": [compact_spec, edit_spec]}[
|
||||
compact_position
|
||||
]
|
||||
|
||||
async def prepare(messages: list[dict[str, object]]) -> PolyfillResult | None:
|
||||
return await prepare_context_managed_request(
|
||||
model=MODEL,
|
||||
messages=messages,
|
||||
tools=None,
|
||||
system=None,
|
||||
context_management_spec={"edits": edits},
|
||||
litellm_metadata=None,
|
||||
additional_drop_params=None,
|
||||
llm_router=None,
|
||||
)
|
||||
|
||||
def cleared(result: PolyfillResult | None) -> list[object]:
|
||||
applied: Final = result.applied_edits if result is not None else []
|
||||
return [edit.get("cleared_tool_uses") for edit in applied if edit.get("type") == "clear_tool_uses_20250919"]
|
||||
|
||||
without_summary: Final = await prepare(deepcopy(tail))
|
||||
with_summary: Final = await prepare(deepcopy(compacted_history))
|
||||
|
||||
assert cleared(without_summary) == []
|
||||
assert cleared(with_summary) == [2]
|
||||
|
||||
|
||||
async def test_compact_after_clearing_replaces_the_client_compaction_summary(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from litellm.proxy.proxy_server import general_settings
|
||||
|
||||
monkeypatch.setitem(general_settings, COMPACT_SUMMARY_MODEL_SETTING_KEY, "summary-model")
|
||||
summary: Final = MagicMock()
|
||||
summary.choices = [MagicMock(message=MagicMock(content="<summary>Fresh summary</summary>"))]
|
||||
summary.usage = MagicMock(prompt_tokens=50, completion_tokens=100)
|
||||
router: Final = MagicMock()
|
||||
router.acompletion = AsyncMock(return_value=summary)
|
||||
tail: Final = _make_history(n_pairs=3)
|
||||
result: Final = await prepare_context_managed_request(
|
||||
model=MODEL,
|
||||
messages=[{"role": "assistant", "content": [{"type": "compaction", "content": "Old summary. " * 20_000}]}, *tail],
|
||||
tools=None,
|
||||
system="You are a helpful assistant.",
|
||||
context_management_spec={
|
||||
"edits": [
|
||||
{
|
||||
"type": "clear_tool_uses_20250919",
|
||||
"trigger": {"type": "input_tokens", "value": 1_000},
|
||||
"keep": {"type": "tool_uses", "value": 1},
|
||||
},
|
||||
{"type": "compact_20260112", "trigger": {"type": "input_tokens", "value": 50_000}},
|
||||
]
|
||||
},
|
||||
litellm_metadata=None,
|
||||
additional_drop_params=None,
|
||||
llm_router=router,
|
||||
)
|
||||
assert result is not None
|
||||
assert isinstance(result.system, str)
|
||||
assert "Fresh summary" in result.system
|
||||
assert "Old summary" not in result.system
|
||||
assert result.system.endswith("You are a helpful assistant.")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("clear_tool_inputs", "triggers"), [(False, False), (True, True)])
|
||||
def test_tool_uses_trigger_counts_tiny_results_only_when_their_inputs_are_cleared(
|
||||
clear_tool_inputs: bool, triggers: bool
|
||||
|
|
|
|||
|
|
@ -151,6 +151,7 @@ async def test_sync_editor_counts_tokens_off_the_event_loop():
|
|||
{
|
||||
"type": "clear_tool_uses_20250919",
|
||||
"trigger": {"type": "input_tokens", "value": 10_000_000},
|
||||
"keep": {"type": "tool_uses", "value": 0},
|
||||
}
|
||||
]
|
||||
},
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import os
|
|||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from functools import partial
|
||||
|
|
@ -37,6 +37,7 @@ from litellm.proxy.hooks.model_max_budget_limiter import (
|
|||
team_member_budget_entity_id,
|
||||
)
|
||||
from litellm.proxy._types import (
|
||||
JWTAuthBuilderResult,
|
||||
LiteLLMRoutes,
|
||||
LiteLLM_JWTAuth,
|
||||
LiteLLM_BudgetTable,
|
||||
|
|
@ -55,7 +56,7 @@ from litellm.proxy._types import (
|
|||
UserAPIKeyAuth,
|
||||
JWTRoutingOverride,
|
||||
)
|
||||
from litellm.proxy.auth.handle_jwt import JWTHandler
|
||||
from litellm.proxy.auth.handle_jwt import JWTAuthManager, JWTHandler
|
||||
from litellm.proxy.auth.auth_checks import (
|
||||
OrganizationNotFoundError,
|
||||
TeamNotFoundError,
|
||||
|
|
@ -724,6 +725,44 @@ async def test_user_custom_auth_runs_post_custom_auth_checks_when_opt_in():
|
|||
litellm.enable_post_custom_auth_checks = original_flag
|
||||
|
||||
|
||||
async def _authenticate_through_custom_auth() -> UserAPIKeyAuth:
|
||||
from starlette.datastructures import URL
|
||||
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth_builder
|
||||
|
||||
request: Final = Request(scope={"type": "http"})
|
||||
request._url = URL(url="/chat/completions")
|
||||
return await user_api_key_auth_builder(
|
||||
request=request,
|
||||
api_key="Bearer sk-custom-auth-shared",
|
||||
azure_api_key_header="",
|
||||
anthropic_api_key_header=None,
|
||||
google_ai_studio_api_key_header=None,
|
||||
azure_apim_header=None,
|
||||
request_data={},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_user_custom_auth_returning_one_shared_token_keeps_each_requests_end_user_apart(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
import litellm.proxy.proxy_server as _proxy_server_mod
|
||||
|
||||
shared_token: Final = UserAPIKeyAuth(api_key="sk-custom-auth-shared", user_id="custom-user-123")
|
||||
proxy_attrs: Final = _proxy_server_attrs_for_custom_auth(user_custom_auth=AsyncMock(return_value=shared_token))
|
||||
for attr, val in proxy_attrs.items():
|
||||
monkeypatch.setattr(_proxy_server_mod, attr, val, raising=False)
|
||||
monkeypatch.setattr(litellm, "enable_post_custom_auth_checks", False, raising=False)
|
||||
|
||||
capped_end_users_request: Final = await _authenticate_through_custom_auth()
|
||||
capped_end_users_request.end_user_id = "eu-capped"
|
||||
capped_end_users_request.end_user_model_max_budget = {"gpt-4o": {"max_budget": 0.00001}}
|
||||
next_end_users_request: Final = await _authenticate_through_custom_auth()
|
||||
|
||||
assert (next_end_users_request.end_user_id, next_end_users_request.end_user_model_max_budget) == (None, None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_enterprise_custom_auth_skips_post_custom_auth_checks_by_default():
|
||||
"""
|
||||
|
|
@ -5293,6 +5332,559 @@ async def test_centralized_common_checks_enforces_team_model_max_budget_from_the
|
|||
assert limiter.calls == [("t1", team_caps, key_caps, "gpt-4o")]
|
||||
|
||||
|
||||
_END_USER_MODEL_CAPS: Final = {"gpt-4o": {"budget_limit": 0.00004, "time_period": "1d"}}
|
||||
|
||||
|
||||
def _virtual_key_token_for_end_user() -> UserAPIKeyAuth:
|
||||
return UserAPIKeyAuth(api_key="sk-test", token="hashed", end_user_id="eu-1")
|
||||
|
||||
|
||||
def _master_key_token_cached_with_another_end_users_cap() -> UserAPIKeyAuth:
|
||||
return UserAPIKeyAuth(
|
||||
api_key=LITELLM_PROXY_MASTER_KEY_ALIAS,
|
||||
user_id="admin",
|
||||
user_role=LitellmUserRoles.PROXY_ADMIN,
|
||||
end_user_id="eu-1",
|
||||
end_user_model_max_budget={"gpt-4o": {"budget_limit": 1000.0, "time_period": "1d"}},
|
||||
)
|
||||
|
||||
|
||||
def _end_user_linked_to_the_capped_budget() -> LiteLLM_EndUserTable:
|
||||
return LiteLLM_EndUserTable(
|
||||
user_id="eu-1",
|
||||
blocked=False,
|
||||
budget_id="eu-budget",
|
||||
litellm_budget_table=LiteLLM_BudgetTable(budget_id="eu-budget", model_max_budget=_END_USER_MODEL_CAPS),
|
||||
)
|
||||
|
||||
|
||||
def _end_user_on_the_proxy_default_budget() -> LiteLLM_EndUserTable:
|
||||
return LiteLLM_EndUserTable(user_id="eu-1", blocked=False)
|
||||
|
||||
|
||||
def _no_end_user_row_yet() -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _jwt_token(end_user_id: str | None, end_user_object: LiteLLM_EndUserTable | None) -> UserAPIKeyAuth:
|
||||
return JWTAuthManager.user_api_key_auth_from_result(
|
||||
JWTAuthBuilderResult(
|
||||
is_proxy_admin=False,
|
||||
team_object=None,
|
||||
user_object=None,
|
||||
end_user_object=end_user_object,
|
||||
org_object=None,
|
||||
token="jwt",
|
||||
team_id=None,
|
||||
user_id=None,
|
||||
user_email=None,
|
||||
end_user_id=end_user_id,
|
||||
org_id=None,
|
||||
team_membership=None,
|
||||
jwt_claims={},
|
||||
agent_id=None,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _jwt_token_with_an_end_user_claim() -> UserAPIKeyAuth:
|
||||
return _jwt_token(end_user_id="eu-1", end_user_object=_end_user_linked_to_the_capped_budget())
|
||||
|
||||
|
||||
def _jwt_token_naming_the_end_user_only_in_the_body() -> UserAPIKeyAuth:
|
||||
return _jwt_token(end_user_id=None, end_user_object=None)
|
||||
|
||||
|
||||
async def _run_centralized_checks_for_end_user(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
token: UserAPIKeyAuth,
|
||||
end_user: LiteLLM_EndUserTable | None,
|
||||
gpt_4o_spend: float,
|
||||
model: str,
|
||||
route: str,
|
||||
default_budget_id: str | None,
|
||||
user_custom_auth: Callable[[Request, str], Awaitable[UserAPIKeyAuth]] | None,
|
||||
) -> None:
|
||||
from starlette.datastructures import URL
|
||||
|
||||
import litellm.proxy.proxy_server as _proxy_server_mod
|
||||
from litellm.proxy.common_utils.user_api_key_cache import end_user_cache_key
|
||||
from litellm.proxy.utils import ProxyLogging
|
||||
|
||||
monkeypatch.setattr(litellm, "max_end_user_budget_id", default_budget_id)
|
||||
user_api_key_cache: Final = UserApiKeyCache()
|
||||
if end_user is not None:
|
||||
await user_api_key_cache.async_set_cache(
|
||||
key=end_user_cache_key("eu-1"),
|
||||
value=end_user,
|
||||
model_type=LiteLLM_EndUserTable,
|
||||
)
|
||||
for budget_id in ("eu-default", "eu-key-budget"):
|
||||
await user_api_key_cache.async_set_cache(
|
||||
key=f"default_end_user_budget:{budget_id}",
|
||||
value=LiteLLM_BudgetTable(budget_id=budget_id, model_max_budget=_END_USER_MODEL_CAPS),
|
||||
model_type=LiteLLM_BudgetTable,
|
||||
)
|
||||
await user_api_key_cache.async_set_cache(
|
||||
key="admin",
|
||||
value=LiteLLM_UserTable(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN),
|
||||
model_type=LiteLLM_UserTable,
|
||||
)
|
||||
limiter: Final = PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=DualCache())
|
||||
await limiter.dual_cache.async_set_cache(
|
||||
key=model_budget_spend_cache_key(
|
||||
entity_type=Litellm_EntityType.END_USER,
|
||||
entity_id="eu-1",
|
||||
budget_model="gpt-4o",
|
||||
budget_duration="1d",
|
||||
),
|
||||
value=gpt_4o_spend,
|
||||
)
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{"model_name": "gpt-4o", "litellm_params": {"model": "openai/gpt-4o"}},
|
||||
{"model_name": "gpt-4o-mini", "litellm_params": {"model": "openai/gpt-4o-mini"}},
|
||||
],
|
||||
)
|
||||
attrs: Final = {
|
||||
**_proxy_attrs_for_centralized_checks(user_custom_auth=user_custom_auth),
|
||||
"general_settings": {
|
||||
"disable_budget_reservation": True,
|
||||
"custom_auth_run_common_checks": user_custom_auth is not None,
|
||||
},
|
||||
"llm_model_list": router.model_list,
|
||||
"llm_router": router,
|
||||
"prisma_client": MagicMock(),
|
||||
"user_api_key_cache": user_api_key_cache,
|
||||
"spend_counter_cache": DualCache(),
|
||||
"proxy_logging_obj": ProxyLogging(user_api_key_cache=user_api_key_cache),
|
||||
"model_max_budget_limiter": limiter,
|
||||
}
|
||||
originals: Final = {attribute: getattr(_proxy_server_mod, attribute, None) for attribute in attrs}
|
||||
request: Final = Request(scope={"type": "http"})
|
||||
request._url = URL(url=route)
|
||||
try:
|
||||
for attribute, value in attrs.items():
|
||||
setattr(_proxy_server_mod, attribute, value)
|
||||
await run_centralized_common_checks(
|
||||
user_api_key_auth_obj=token,
|
||||
request=request,
|
||||
request_data={"model": model, "user": "eu-1", "messages": [{"role": "user", "content": "hi"}]},
|
||||
route=route,
|
||||
)
|
||||
finally:
|
||||
for attribute, value in originals.items():
|
||||
setattr(_proxy_server_mod, attribute, value)
|
||||
|
||||
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("make_token", "make_end_user", "gpt_4o_spend", "model", "route", "must_raise"),
|
||||
[
|
||||
pytest.param(
|
||||
_virtual_key_token_for_end_user,
|
||||
_end_user_linked_to_the_capped_budget,
|
||||
0.0001,
|
||||
"gpt-4o",
|
||||
"/v1/chat/completions",
|
||||
True,
|
||||
id="virtual-key-over-cap",
|
||||
),
|
||||
pytest.param(
|
||||
_master_key_token_cached_with_another_end_users_cap,
|
||||
_end_user_linked_to_the_capped_budget,
|
||||
0.0001,
|
||||
"gpt-4o",
|
||||
"/v1/chat/completions",
|
||||
True,
|
||||
id="master-key-over-cap",
|
||||
),
|
||||
pytest.param(
|
||||
_jwt_token_with_an_end_user_claim,
|
||||
_end_user_linked_to_the_capped_budget,
|
||||
0.0001,
|
||||
"gpt-4o",
|
||||
"/v1/chat/completions",
|
||||
True,
|
||||
id="jwt-claim-over-cap",
|
||||
),
|
||||
pytest.param(
|
||||
_jwt_token_naming_the_end_user_only_in_the_body,
|
||||
_end_user_linked_to_the_capped_budget,
|
||||
0.0001,
|
||||
"gpt-4o",
|
||||
"/v1/chat/completions",
|
||||
True,
|
||||
id="jwt-body-user-over-cap",
|
||||
),
|
||||
pytest.param(
|
||||
_virtual_key_token_for_end_user,
|
||||
_no_end_user_row_yet,
|
||||
0.0001,
|
||||
"gpt-4o",
|
||||
"/v1/chat/completions",
|
||||
True,
|
||||
id="new-end-user-on-the-proxy-default-budget-over-cap",
|
||||
),
|
||||
pytest.param(
|
||||
_virtual_key_token_for_end_user,
|
||||
_end_user_on_the_proxy_default_budget,
|
||||
0.0001,
|
||||
"gpt-4o",
|
||||
"/v1/chat/completions",
|
||||
True,
|
||||
id="proxy-default-budget-over-cap",
|
||||
),
|
||||
pytest.param(
|
||||
_virtual_key_token_for_end_user,
|
||||
_end_user_linked_to_the_capped_budget,
|
||||
0.00001,
|
||||
"gpt-4o",
|
||||
"/v1/chat/completions",
|
||||
False,
|
||||
id="under-cap",
|
||||
),
|
||||
pytest.param(
|
||||
_virtual_key_token_for_end_user,
|
||||
_end_user_linked_to_the_capped_budget,
|
||||
0.0001,
|
||||
"gpt-4o-mini",
|
||||
"/v1/chat/completions",
|
||||
False,
|
||||
id="model-without-a-cap",
|
||||
),
|
||||
pytest.param(
|
||||
_master_key_token_cached_with_another_end_users_cap,
|
||||
_end_user_linked_to_the_capped_budget,
|
||||
0.0001,
|
||||
"gpt-4o",
|
||||
"/spend/calculate",
|
||||
False,
|
||||
id="budget-exempt-route",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_centralized_common_checks_enforces_the_end_users_model_max_budget(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
make_token: Callable[[], UserAPIKeyAuth],
|
||||
make_end_user: Callable[[], LiteLLM_EndUserTable | None],
|
||||
gpt_4o_spend: float,
|
||||
model: str,
|
||||
route: str,
|
||||
must_raise: bool,
|
||||
) -> None:
|
||||
token: Final = make_token()
|
||||
with pytest.raises(litellm.BudgetExceededError) if must_raise else nullcontext():
|
||||
await _run_centralized_checks_for_end_user(
|
||||
monkeypatch=monkeypatch,
|
||||
token=token,
|
||||
end_user=make_end_user(),
|
||||
gpt_4o_spend=gpt_4o_spend,
|
||||
model=model,
|
||||
route=route,
|
||||
default_budget_id="eu-default",
|
||||
user_custom_auth=None,
|
||||
)
|
||||
assert token.end_user_model_max_budget == _END_USER_MODEL_CAPS
|
||||
|
||||
|
||||
def _virtual_key_token_assigning_end_users_a_budget() -> UserAPIKeyAuth:
|
||||
return UserAPIKeyAuth(
|
||||
api_key="sk-test", token="hashed", end_user_id="eu-1", metadata={"end_user_budget_id": "eu-key-budget"}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("make_token", "make_end_user"),
|
||||
[
|
||||
pytest.param(
|
||||
_virtual_key_token_for_end_user,
|
||||
_end_user_linked_to_the_capped_budget,
|
||||
id="end-users-own-budget",
|
||||
),
|
||||
pytest.param(
|
||||
_virtual_key_token_assigning_end_users_a_budget,
|
||||
_end_user_on_the_proxy_default_budget,
|
||||
id="keys-end-user-budget",
|
||||
),
|
||||
pytest.param(
|
||||
_virtual_key_token_assigning_end_users_a_budget,
|
||||
_no_end_user_row_yet,
|
||||
id="keys-end-user-budget-for-a-new-end-user",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_centralized_common_checks_enforces_the_end_users_model_max_budget_without_a_proxy_default(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
make_token: Callable[[], UserAPIKeyAuth],
|
||||
make_end_user: Callable[[], LiteLLM_EndUserTable | None],
|
||||
) -> None:
|
||||
token: Final = make_token()
|
||||
with pytest.raises(litellm.BudgetExceededError):
|
||||
await _run_centralized_checks_for_end_user(
|
||||
monkeypatch=monkeypatch,
|
||||
token=token,
|
||||
end_user=make_end_user(),
|
||||
gpt_4o_spend=0.0001,
|
||||
model="gpt-4o",
|
||||
route="/v1/chat/completions",
|
||||
default_budget_id=None,
|
||||
user_custom_auth=None,
|
||||
)
|
||||
assert token.end_user_model_max_budget == _END_USER_MODEL_CAPS
|
||||
|
||||
|
||||
async def _custom_auth_that_is_never_called(request: Request, api_key: str) -> UserAPIKeyAuth:
|
||||
raise AssertionError("the shared checks never call custom auth")
|
||||
|
||||
|
||||
def _token_built_by_custom_auth_with_its_own_cap() -> UserAPIKeyAuth:
|
||||
return UserAPIKeyAuth(
|
||||
api_key="sk-custom", end_user_id="eu-1", end_user_model_max_budget=_END_USER_MODEL_CAPS
|
||||
).model_copy(update={"authenticated_by_custom_auth": True})
|
||||
|
||||
|
||||
def _master_key_token_custom_auth_handed_back_with_another_end_users_cap() -> UserAPIKeyAuth:
|
||||
return UserAPIKeyAuth(
|
||||
api_key=LITELLM_PROXY_MASTER_KEY_ALIAS,
|
||||
user_id="admin",
|
||||
user_role=LitellmUserRoles.PROXY_ADMIN,
|
||||
end_user_id="eu-1",
|
||||
end_user_model_max_budget=_END_USER_MODEL_CAPS,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("make_token", "must_raise", "expected_cap"),
|
||||
[
|
||||
pytest.param(
|
||||
_token_built_by_custom_auth_with_its_own_cap,
|
||||
True,
|
||||
_END_USER_MODEL_CAPS,
|
||||
id="custom-auth-token-keeps-its-own-cap",
|
||||
),
|
||||
pytest.param(
|
||||
_master_key_token_custom_auth_handed_back_with_another_end_users_cap,
|
||||
False,
|
||||
None,
|
||||
id="key-auth-token-behind-custom-auth-drops-another-end-users-cap",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_centralized_common_checks_keeps_only_a_cap_custom_auth_set(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
make_token: Callable[[], UserAPIKeyAuth],
|
||||
must_raise: bool,
|
||||
expected_cap: Mapping[str, object] | None,
|
||||
) -> None:
|
||||
token: Final = make_token()
|
||||
with pytest.raises(litellm.BudgetExceededError) if must_raise else nullcontext():
|
||||
await _run_centralized_checks_for_end_user(
|
||||
monkeypatch=monkeypatch,
|
||||
token=token,
|
||||
end_user=_end_user_on_the_proxy_default_budget(),
|
||||
gpt_4o_spend=0.0001,
|
||||
model="gpt-4o",
|
||||
route="/v1/chat/completions",
|
||||
default_budget_id=None,
|
||||
user_custom_auth=_custom_auth_that_is_never_called,
|
||||
)
|
||||
assert token.end_user_model_max_budget == expected_cap
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_centralized_common_checks_holds_the_end_user_to_the_model_a_team_member_fallback_picked(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from starlette.datastructures import URL
|
||||
|
||||
import litellm.proxy.proxy_server as _proxy_server_mod
|
||||
from litellm.proxy.common_utils.user_api_key_cache import end_user_cache_key
|
||||
from litellm.proxy.utils import ProxyLogging
|
||||
|
||||
budget_id: Final = "member-budget-1"
|
||||
user_api_key_cache: Final = UserApiKeyCache()
|
||||
await user_api_key_cache.async_set_cache(
|
||||
key=team_membership_reservation_cache_key(user_id="u1", team_id="t1"),
|
||||
value=LiteLLM_TeamMembership(user_id="u1", team_id="t1", budget_id=budget_id),
|
||||
model_type=LiteLLM_TeamMembership,
|
||||
)
|
||||
await user_api_key_cache.async_set_cache(
|
||||
key="team_id:t1",
|
||||
value=LiteLLM_TeamTableCachedObj(
|
||||
team_id="t1",
|
||||
models=["gpt-4o", "gpt-4o-mini"],
|
||||
metadata={"team_member_budget_id": budget_id},
|
||||
),
|
||||
model_type=LiteLLM_TeamTableCachedObj,
|
||||
)
|
||||
await user_api_key_cache.async_set_cache(
|
||||
key=f"team_member_default_budget:{budget_id}",
|
||||
value=LiteLLM_BudgetTable(
|
||||
budget_id=budget_id,
|
||||
model_max_budget={
|
||||
"gpt-4o": {"max_budget": 0.00004, "budget_duration": "1d"},
|
||||
"gpt-4o-mini": {"max_budget": 100.0, "budget_duration": "1d"},
|
||||
},
|
||||
),
|
||||
model_type=LiteLLM_BudgetTable,
|
||||
)
|
||||
await user_api_key_cache.async_set_cache(
|
||||
key="u1",
|
||||
value=LiteLLM_UserTable(user_id="u1", user_role=LitellmUserRoles.INTERNAL_USER),
|
||||
model_type=LiteLLM_UserTable,
|
||||
)
|
||||
await user_api_key_cache.async_set_cache(
|
||||
key=end_user_cache_key("eu-1"),
|
||||
value=_end_user_linked_to_the_capped_budget(),
|
||||
model_type=LiteLLM_EndUserTable,
|
||||
)
|
||||
limiter: Final = PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=DualCache())
|
||||
for entity_type, entity_id in (
|
||||
(Litellm_EntityType.TEAM_MEMBER, team_member_budget_entity_id(user_id="u1", team_id="t1")),
|
||||
(Litellm_EntityType.END_USER, "eu-1"),
|
||||
):
|
||||
await limiter.dual_cache.async_set_cache(
|
||||
key=model_budget_spend_cache_key(
|
||||
entity_type=entity_type,
|
||||
entity_id=entity_id,
|
||||
budget_model="gpt-4o",
|
||||
budget_duration="1d",
|
||||
),
|
||||
value=0.0001,
|
||||
)
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{"model_name": "gpt-4o", "litellm_params": {"model": "openai/gpt-4o"}},
|
||||
{"model_name": "gpt-4o-mini", "litellm_params": {"model": "openai/gpt-4o-mini"}},
|
||||
],
|
||||
fallbacks=[{"gpt-4o": ["gpt-4o-mini"]}],
|
||||
)
|
||||
attrs: Final = {
|
||||
**_proxy_attrs_for_centralized_checks(),
|
||||
"general_settings": {"disable_budget_reservation": True},
|
||||
"llm_model_list": router.model_list,
|
||||
"llm_router": router,
|
||||
"prisma_client": MagicMock(),
|
||||
"user_api_key_cache": user_api_key_cache,
|
||||
"spend_counter_cache": DualCache(),
|
||||
"proxy_logging_obj": ProxyLogging(user_api_key_cache=user_api_key_cache),
|
||||
"model_max_budget_limiter": limiter,
|
||||
}
|
||||
originals: Final = {attribute: getattr(_proxy_server_mod, attribute, None) for attribute in attrs}
|
||||
request: Final = Request(scope={"type": "http", "path_params": {}, "headers": []})
|
||||
request._url = URL(url="/v1/chat/completions")
|
||||
request_data: Final = {"model": "gpt-4o", "user": "eu-1", "messages": [{"role": "user", "content": "hi"}]}
|
||||
try:
|
||||
for attribute, value in attrs.items():
|
||||
setattr(_proxy_server_mod, attribute, value)
|
||||
await run_centralized_common_checks(
|
||||
user_api_key_auth_obj=UserAPIKeyAuth(
|
||||
api_key="sk-test",
|
||||
token="hashed",
|
||||
user_id="u1",
|
||||
team_id="t1",
|
||||
end_user_id="eu-1",
|
||||
models=["gpt-4o", "gpt-4o-mini"],
|
||||
user_role=LitellmUserRoles.INTERNAL_USER,
|
||||
),
|
||||
request=request,
|
||||
request_data=request_data,
|
||||
route="/v1/chat/completions",
|
||||
)
|
||||
finally:
|
||||
for attribute, value in originals.items():
|
||||
setattr(_proxy_server_mod, attribute, value)
|
||||
assert request_data["model"] == "gpt-4o-mini"
|
||||
|
||||
|
||||
def _access_groups_only_team_t1_puts_gpt_4o_in(model: str, team_id: str | None) -> frozenset[str]:
|
||||
return frozenset({"t1-group"}) if (model, team_id) == ("gpt-4o", "t1") else frozenset()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("team_id", "must_raise"),
|
||||
[
|
||||
pytest.param("t1", True, id="team-whose-access-group-the-end-user-capped"),
|
||||
pytest.param(None, False, id="no-team-so-the-group-cap-does-not-apply"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_centralized_common_checks_applies_an_end_user_cap_on_the_tokens_team_access_group(
|
||||
team_id: str | None,
|
||||
must_raise: bool,
|
||||
) -> None:
|
||||
from starlette.datastructures import URL
|
||||
|
||||
import litellm.proxy.proxy_server as _proxy_server_mod
|
||||
from litellm.proxy.common_utils.user_api_key_cache import end_user_cache_key
|
||||
from litellm.proxy.utils import ProxyLogging
|
||||
|
||||
group_cap: Final = {"t1-group": {"budget_limit": 0.00004, "time_period": "1d"}}
|
||||
user_api_key_cache: Final = UserApiKeyCache()
|
||||
await user_api_key_cache.async_set_cache(
|
||||
key="team_id:t1",
|
||||
value=LiteLLM_TeamTableCachedObj(team_id="t1", models=["gpt-4o"]),
|
||||
model_type=LiteLLM_TeamTableCachedObj,
|
||||
)
|
||||
await user_api_key_cache.async_set_cache(
|
||||
key=end_user_cache_key("eu-1"),
|
||||
value=LiteLLM_EndUserTable(
|
||||
user_id="eu-1",
|
||||
blocked=False,
|
||||
budget_id="eu-group-budget",
|
||||
litellm_budget_table=LiteLLM_BudgetTable(budget_id="eu-group-budget", model_max_budget=group_cap),
|
||||
),
|
||||
model_type=LiteLLM_EndUserTable,
|
||||
)
|
||||
limiter: Final = PROXY_VirtualKeyModelMaxBudgetLimiter(
|
||||
dual_cache=DualCache(), model_access_groups=_access_groups_only_team_t1_puts_gpt_4o_in
|
||||
)
|
||||
await limiter.dual_cache.async_set_cache(
|
||||
key=model_budget_spend_cache_key(
|
||||
entity_type=Litellm_EntityType.END_USER,
|
||||
entity_id="eu-1",
|
||||
budget_model="t1-group",
|
||||
budget_duration="1d",
|
||||
),
|
||||
value=0.0001,
|
||||
)
|
||||
router: Final = litellm.Router(model_list=[{"model_name": "gpt-4o", "litellm_params": {"model": "openai/gpt-4o"}}])
|
||||
attrs: Final = {
|
||||
**_proxy_attrs_for_centralized_checks(),
|
||||
"general_settings": {"disable_budget_reservation": True},
|
||||
"llm_model_list": router.model_list,
|
||||
"llm_router": router,
|
||||
"prisma_client": MagicMock(),
|
||||
"user_api_key_cache": user_api_key_cache,
|
||||
"spend_counter_cache": DualCache(),
|
||||
"proxy_logging_obj": ProxyLogging(user_api_key_cache=user_api_key_cache),
|
||||
"model_max_budget_limiter": limiter,
|
||||
}
|
||||
originals: Final = {attribute: getattr(_proxy_server_mod, attribute, None) for attribute in attrs}
|
||||
request: Final = Request(scope={"type": "http", "path_params": {}, "headers": []})
|
||||
request._url = URL(url="/v1/chat/completions")
|
||||
try:
|
||||
for attribute, value in attrs.items():
|
||||
setattr(_proxy_server_mod, attribute, value)
|
||||
with pytest.raises(litellm.BudgetExceededError) if must_raise else nullcontext():
|
||||
await run_centralized_common_checks(
|
||||
user_api_key_auth_obj=UserAPIKeyAuth(
|
||||
api_key="sk-test", token="hashed", team_id=team_id, end_user_id="eu-1", models=["gpt-4o"]
|
||||
),
|
||||
request=request,
|
||||
request_data={"model": "gpt-4o", "user": "eu-1", "messages": [{"role": "user", "content": "hi"}]},
|
||||
route="/v1/chat/completions",
|
||||
)
|
||||
finally:
|
||||
for attribute, value in originals.items():
|
||||
setattr(_proxy_server_mod, attribute, value)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_centralized_common_checks_loads_member_budget_and_exempts_discovery_routes():
|
||||
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
|
||||
|
|
|
|||
|
|
@ -92,6 +92,8 @@ _STOCK_MODELS = {
|
|||
"base_instructions": "You are Codex, a coding agent based on GPT-5.6.",
|
||||
"apply_patch_tool_type": "freeform",
|
||||
"supports_parallel_tool_calls": True,
|
||||
"supports_search_tool": True,
|
||||
"tool_mode": "code_mode_only",
|
||||
"context_window": 272000,
|
||||
"comp_hash": "terra-hash",
|
||||
},
|
||||
|
|
@ -669,10 +671,16 @@ class TestCodexModelSync:
|
|||
def _row(model_id, **extra):
|
||||
return {"id": model_id, "object": "model", "created": 1, "owned_by": "openai", **extra}
|
||||
|
||||
def _sync(self, listing, codex_home, base_url="http://localhost:4000/", run=None):
|
||||
def _sync(self, listing, codex_home, base_url="http://localhost:4000/", run=None, model_info=None):
|
||||
captured = {}
|
||||
|
||||
def fake_get(url, headers, timeout):
|
||||
if headers.get("Authorization") != "Bearer sk-key":
|
||||
return _FakeResponse(401)
|
||||
if url.endswith("/v1/model/info"):
|
||||
if isinstance(model_info, Exception):
|
||||
raise model_info
|
||||
return model_info if model_info is not None else _FakeResponse(200, {"data": []})
|
||||
captured["url"] = url
|
||||
captured["headers"] = headers
|
||||
return _FakeResponse(200, listing)
|
||||
|
|
@ -737,6 +745,202 @@ class TestCodexModelSync:
|
|||
assert entries["gpt-5.4"]["upgrade"] == _STOCK_MODELS["gpt-5.4"]["upgrade"]
|
||||
assert [entries["gpt-5.4"]["priority"], entries["gpt-5.6-terra"]["priority"]] == [0, 1]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"deployment",
|
||||
[
|
||||
{"model_name": "api-terra", "litellm_params": {"model": "openai/gpt-5.6-terra"}},
|
||||
{"model_name": "api-terra", "litellm_params": {"model": "gpt-5.6-terra"}, "model_info": None},
|
||||
{
|
||||
"model_name": "api-terra",
|
||||
"litellm_params": {"model": "openai/gpt-5.6-terra"},
|
||||
"model_info": {"id": "d1", "team_public_model_name": ""},
|
||||
},
|
||||
{
|
||||
"model_name": "model_name_team-1_d1",
|
||||
"litellm_params": {"model": "openai/gpt-5.6-terra", "api_base": "https://api.openai.com/v1"},
|
||||
"model_info": {"id": "d1", "team_public_model_name": "api-terra"},
|
||||
},
|
||||
],
|
||||
ids=["provider_prefixed", "bare_without_model_info", "empty_team_public_name", "team_public_model_name"],
|
||||
)
|
||||
def test_alias_of_a_codex_model_gets_that_models_entry_under_its_own_name(self, tmp_path, deployment):
|
||||
listing = self._listing(self._row("api-terra"), self._row("m"))
|
||||
_, result = self._sync(listing, tmp_path, model_info=_FakeResponse(200, {"data": [deployment]}))
|
||||
|
||||
assert isinstance(result, ModelSyncArgs)
|
||||
entries = self._entries(tmp_path)
|
||||
assert entries["api-terra"] == {
|
||||
**_STOCK_MODELS["gpt-5.6-terra"],
|
||||
"slug": "api-terra",
|
||||
"display_name": "api-terra",
|
||||
"priority": 0,
|
||||
"service_tiers": [],
|
||||
"default_service_tier": None,
|
||||
"default_verbosity": None,
|
||||
"availability_nux": None,
|
||||
"default_reasoning_level": None,
|
||||
}
|
||||
assert "supports_search_tool" not in entries["m"]
|
||||
|
||||
def test_alias_gets_no_upgrade_nudge_away_from_its_own_name(self, tmp_path):
|
||||
listing = self._listing(self._row("gpt-5.6-terra"), self._row("api-54"))
|
||||
deployment = {"model_name": "api-54", "litellm_params": {"model": "openai/gpt-5.4"}}
|
||||
self._sync(listing, tmp_path, model_info=_FakeResponse(200, {"data": [deployment]}))
|
||||
|
||||
entry = self._entries(tmp_path)["api-54"]
|
||||
assert (entry["comp_hash"], entry["upgrade"]) == (_STOCK_MODELS["gpt-5.4"]["comp_hash"], None)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("pins", "expected"),
|
||||
[
|
||||
(("high",), "high"),
|
||||
(("high", "high"), "high"),
|
||||
(("high", None), None),
|
||||
(("high", "low"), None),
|
||||
(({"effort": "high"},), None),
|
||||
],
|
||||
ids=["one_deployment", "every_deployment_agrees", "one_unpinned", "pins_differ", "non_string_pin"],
|
||||
)
|
||||
def test_alias_starts_at_the_reasoning_effort_every_deployment_pins(self, tmp_path, pins, expected):
|
||||
deployments = [
|
||||
{
|
||||
"model_name": "api-terra",
|
||||
"litellm_params": {"model": "openai/gpt-5.6-terra", **({"reasoning_effort": p} if p else {})},
|
||||
}
|
||||
for p in pins
|
||||
]
|
||||
self._sync(
|
||||
self._listing(self._row("api-terra")), tmp_path, model_info=_FakeResponse(200, {"data": deployments})
|
||||
)
|
||||
|
||||
entry = self._entries(tmp_path)["api-terra"]
|
||||
assert (entry["comp_hash"], entry["default_reasoning_level"]) == ("terra-hash", expected)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"deployments",
|
||||
[
|
||||
[
|
||||
{"model_name": "api-terra", "litellm_params": {"model": "openai/gpt-5.6-terra"}},
|
||||
{"model_name": "api-terra", "litellm_params": {"model": "gpt-5.6-terra"}},
|
||||
],
|
||||
[
|
||||
{
|
||||
"model_name": "api-terra",
|
||||
"litellm_params": {"model": "gpt-5.6-terra", "custom_llm_provider": "openai"},
|
||||
}
|
||||
],
|
||||
[
|
||||
{
|
||||
"model_name": "api-terra",
|
||||
"litellm_params": {"model": "openai/gpt-5.5"},
|
||||
"model_info": {"id": "paused", "blocked": True},
|
||||
},
|
||||
{"model_name": "api-terra", "litellm_params": {"model": "openai/gpt-5.6-terra"}},
|
||||
],
|
||||
[
|
||||
{
|
||||
"model_name": "api-terra",
|
||||
"litellm_params": {"model": "openai/gpt-5.6-terra", "api_base": "https://eu.api.openai.com/v1"},
|
||||
}
|
||||
],
|
||||
[
|
||||
{
|
||||
"model_name": "api-terra",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-5.6-terra",
|
||||
"api_base": "https://api.openai.com/v1",
|
||||
"litellm_credential_name": "openai-key",
|
||||
},
|
||||
}
|
||||
],
|
||||
],
|
||||
ids=[
|
||||
"same_model_with_and_without_prefix",
|
||||
"openai_custom_llm_provider",
|
||||
"paused_deployment_ignored",
|
||||
"openai_regional_host",
|
||||
"credential_under_an_openai_host",
|
||||
],
|
||||
)
|
||||
def test_alias_whose_deployments_all_route_to_one_openai_model_gets_its_entry(self, tmp_path, deployments):
|
||||
self._sync(
|
||||
self._listing(self._row("api-terra")), tmp_path, model_info=_FakeResponse(200, {"data": deployments})
|
||||
)
|
||||
|
||||
assert self._entries(tmp_path)["api-terra"]["comp_hash"] == _STOCK_MODELS["gpt-5.6-terra"]["comp_hash"]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"deployments",
|
||||
[
|
||||
[
|
||||
{"model_name": "api-terra", "litellm_params": {"model": "openai/gpt-5.5"}},
|
||||
{
|
||||
"model_name": "model_name_team-1_d1",
|
||||
"litellm_params": {"model": "openai/gpt-5.6-terra"},
|
||||
"model_info": {"id": "d1", "team_public_model_name": "api-terra"},
|
||||
},
|
||||
],
|
||||
[
|
||||
{"model_name": "api-terra", "litellm_params": {"model": "openai/gpt-5.6-terra"}},
|
||||
{"model_name": "api-terra", "litellm_params": {"model": "anthropic/claude-opus-4-7"}},
|
||||
],
|
||||
[{"model_name": "api-terra", "litellm_params": {"model": "azure/gpt-5.6-terra"}}],
|
||||
[{"model_name": "api-terra", "litellm_params": {"model": "gpt-5.6-terra", "custom_llm_provider": "azure"}}],
|
||||
[
|
||||
{
|
||||
"model_name": "api-terra",
|
||||
"litellm_params": {"model": "openai/gpt-5.6-terra", "api_base": "http://vllm.example.invalid/v1"},
|
||||
}
|
||||
],
|
||||
[
|
||||
{
|
||||
"model_name": "api-terra",
|
||||
"litellm_params": {"model": "gpt-5.6-terra", "api_base": "https://api.openai.com.example.invalid"},
|
||||
}
|
||||
],
|
||||
[
|
||||
{
|
||||
"model_name": "api-terra",
|
||||
"litellm_params": {"model": "openai/gpt-5.6-terra", "litellm_credential_name": "gateway"},
|
||||
}
|
||||
],
|
||||
],
|
||||
ids=[
|
||||
"team_row_and_global_row_differ",
|
||||
"load_balanced_across_providers",
|
||||
"azure_prefix",
|
||||
"azure_provider",
|
||||
"openai_prefix_on_another_host",
|
||||
"openai_lookalike_host",
|
||||
"host_only_a_credential_knows",
|
||||
],
|
||||
)
|
||||
def test_alias_without_one_shared_openai_upstream_keeps_the_fallback_entry(self, tmp_path, deployments):
|
||||
listing = self._listing(self._row("api-terra"))
|
||||
self._sync(listing, tmp_path / "with", model_info=_FakeResponse(200, {"data": deployments}))
|
||||
self._sync(listing, tmp_path / "without")
|
||||
|
||||
assert self._entries(tmp_path / "with")["api-terra"] == self._entries(tmp_path / "without")["api-terra"]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_info",
|
||||
[
|
||||
requests.ConnectionError("refused"),
|
||||
_FakeResponse(500, {"data": [{"model_name": "api-terra", "litellm_params": {"model": "gpt-5.6-terra"}}]}),
|
||||
_FakeResponse(200, {"data": "nope"}),
|
||||
_FakeResponse(200, {"data": [{"model_name": "api-terra"}]}),
|
||||
],
|
||||
ids=["unreachable", "http_500", "unexpected_body", "deployment_without_litellm_params"],
|
||||
)
|
||||
def test_unavailable_model_info_keeps_the_alias_on_the_fallback_entry(self, tmp_path, model_info):
|
||||
_, result = self._sync(self._listing(self._row("api-terra")), tmp_path, model_info=model_info)
|
||||
|
||||
catalog = tmp_path / "litellm-models.json"
|
||||
assert result == ModelSyncArgs(("-c", f"model_catalog_json={json.dumps(str(catalog))}"))
|
||||
entry = self._entries(tmp_path)["api-terra"]
|
||||
assert entry["slug"] == "api-terra"
|
||||
assert "supports_search_tool" not in entry
|
||||
|
||||
def test_stock_catalog_is_decoded_as_utf8_regardless_of_locale(self, tmp_path):
|
||||
description = "Modelo equilibrado para el trabajo diario, con acentos y ñ."
|
||||
catalog = {"models": [{**_STOCK_MODELS["gpt-5.5"], "description": description}]}
|
||||
|
|
|
|||
|
|
@ -135,6 +135,84 @@ def test_alias_of_a_known_upstream_model_takes_stock_metadata_under_its_own_name
|
|||
assert (entry["slug"], entry["display_name"]) == ("team-55", "team-55")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("max_input_tokens", "expected"), [(100000, 100000), (922000, 272000), (None, 272000)])
|
||||
def test_alias_with_deployment_defaults_caps_codexs_window_at_its_listed_max_input_tokens(max_input_tokens, expected):
|
||||
stock = MappingProxyType({"gpt-5.5": _stock("gpt-5.5").model_copy(update={"context_window": 272000})})
|
||||
alias = _row("team-55", upstream_model="openai/gpt-5.5", max_input_tokens=max_input_tokens)
|
||||
|
||||
body = codex_models_response_json((alias,), stock=stock, instructions=_PROMPT, alias_defaults="deployment")
|
||||
|
||||
assert json.loads(body.json)["models"][0]["context_window"] == expected
|
||||
|
||||
|
||||
def test_alias_with_deployment_defaults_drops_codexs_tier_and_upgrade_while_the_native_name_keeps_them():
|
||||
rows = (_row("gpt-6-sol"), _row("team-sol", upstream_model="gpt-6-sol"), _row("team-55", upstream_model="gpt-5.5"))
|
||||
codex = {entry["slug"]: entry for entry in _models(*rows)}
|
||||
deployment = {entry["slug"]: entry for entry in _models(*rows, alias_defaults="deployment")}
|
||||
|
||||
assert deployment["gpt-6-sol"] == codex["gpt-6-sol"]
|
||||
assert (codex["team-sol"]["default_service_tier"], codex["team-55"]["upgrade"]) == (
|
||||
"priority",
|
||||
{"model": "gpt-6-sol"},
|
||||
)
|
||||
assert _tiers(codex["team-sol"]) == _tiers(codex["gpt-6-sol"]) != []
|
||||
assert [
|
||||
(deployment[slug]["service_tiers"], deployment[slug]["default_service_tier"])
|
||||
for slug in ("team-sol", "team-55")
|
||||
] == [
|
||||
([], None),
|
||||
([], None),
|
||||
]
|
||||
assert deployment["team-55"]["upgrade"] is None
|
||||
assert deployment["team-sol"]["model_messages"] == codex["team-sol"]["model_messages"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("pinned", "expected"), [("high", "high"), ("turbo", None), (None, None)])
|
||||
def test_alias_with_deployment_defaults_starts_only_at_a_reasoning_effort_its_deployments_pin(pinned, expected):
|
||||
stock = CodexStockModel.model_validate(
|
||||
{
|
||||
"slug": "gpt-5.5",
|
||||
"display_name": "GPT-5.5",
|
||||
"priority": 1,
|
||||
"visibility": "list",
|
||||
"default_reasoning_level": "medium",
|
||||
"supported_reasoning_levels": [{"effort": e, "description": e} for e in ("low", "medium", "high")],
|
||||
}
|
||||
)
|
||||
alias = _row("team-55", upstream_model="gpt-5.5", reasoning_effort=pinned)
|
||||
|
||||
body = codex_models_response_json(
|
||||
(alias,), stock=MappingProxyType({"gpt-5.5": stock}), instructions=_PROMPT, alias_defaults="deployment"
|
||||
)
|
||||
|
||||
assert json.loads(body.json)["models"][0]["default_reasoning_level"] == expected
|
||||
|
||||
|
||||
def test_alias_with_deployment_defaults_sends_no_verbosity_or_promo_of_codexs_while_the_native_name_keeps_them():
|
||||
stock = CodexStockModel.model_validate(
|
||||
{
|
||||
"slug": "gpt-6-sol",
|
||||
"display_name": "GPT-6 Sol",
|
||||
"priority": 1,
|
||||
"visibility": "list",
|
||||
"support_verbosity": True,
|
||||
"default_verbosity": "low",
|
||||
"availability_nux": {"message": "GPT-6 Sol is here"},
|
||||
}
|
||||
)
|
||||
rows = (_row("gpt-6-sol"), _row("team-sol", upstream_model="gpt-6-sol"))
|
||||
|
||||
body = codex_models_response_json(
|
||||
rows, stock=MappingProxyType({"gpt-6-sol": stock}), instructions=_PROMPT, alias_defaults="deployment"
|
||||
)
|
||||
native, alias = json.loads(body.json)["models"]
|
||||
|
||||
assert [(e["support_verbosity"], e["default_verbosity"], e["availability_nux"]) for e in (native, alias)] == [
|
||||
(True, "low", {"message": "GPT-6 Sol is here"}),
|
||||
(True, None, None),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("row", "expected"),
|
||||
[
|
||||
|
|
|
|||
|
|
@ -1,16 +1,26 @@
|
|||
import json
|
||||
import sys
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import prisma.errors
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
from prisma._async_http import Response
|
||||
from prisma.engine.utils import handle_response_errors
|
||||
|
||||
import litellm
|
||||
from litellm.proxy._types import ProxyErrorTypes, ProxyException
|
||||
from litellm.proxy.db.db_lookup_gate import DBLookupDeadlineExceeded
|
||||
from litellm.proxy.common_utils.openai_error_payload import (
|
||||
error_status_code,
|
||||
litellm_call_id_headers,
|
||||
openai_error_param,
|
||||
openai_error_type,
|
||||
proxy_exception_for,
|
||||
with_litellm_call_id,
|
||||
)
|
||||
from litellm.types.router import RouterErrors
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
@ -203,3 +213,287 @@ def test_a_proxy_error_keeps_the_call_id_it_was_raised_with():
|
|||
|
||||
assert with_litellm_call_id(raised_with_id, "second").headers == {"x-litellm-call-id": "first"}
|
||||
assert with_litellm_call_id(ProxyException(message="nope", type="None", param=None, code=400), None).headers == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raised_status, expected_type",
|
||||
[
|
||||
(400, "invalid_request_error"),
|
||||
(404, "invalid_request_error"),
|
||||
(401, "auth_error"),
|
||||
(403, "auth_error"),
|
||||
(422, "invalid_request_error"),
|
||||
(500, "internal_server_error"),
|
||||
],
|
||||
)
|
||||
def test_a_catch_all_handler_keeps_the_status_its_route_answered_with(raised_status: int, expected_type: str) -> None:
|
||||
wrapped: Final = proxy_exception_for(
|
||||
HTTPException(status_code=raised_status, detail={"message": "boom"}),
|
||||
default_status_code=400,
|
||||
)
|
||||
|
||||
assert (wrapped.code, wrapped.type) == (str(raised_status), expected_type)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"default_status_code, expected_type", [(400, "invalid_request_error"), (500, "internal_server_error")]
|
||||
)
|
||||
def test_an_exception_carrying_no_status_falls_back_to_the_routes_default(default_status_code: int, expected_type: str) -> None:
|
||||
wrapped: Final = proxy_exception_for(ValueError("boom"), default_status_code=default_status_code)
|
||||
|
||||
assert (wrapped.code, wrapped.type) == (str(default_status_code), expected_type)
|
||||
|
||||
|
||||
def test_a_proxy_exception_passes_through_with_the_type_it_already_named() -> None:
|
||||
raised: Final = ProxyException(
|
||||
message="Budget has been exceeded",
|
||||
type=ProxyErrorTypes.budget_exceeded.value,
|
||||
param="max_budget",
|
||||
code=400,
|
||||
)
|
||||
|
||||
assert proxy_exception_for(raised, default_status_code=500) is raised
|
||||
|
||||
|
||||
def test_the_message_is_what_the_route_raised_without_an_invented_prefix() -> None:
|
||||
assert proxy_exception_for(ValueError("boom"), default_status_code=400).message == "boom"
|
||||
|
||||
|
||||
def test_an_http_exceptions_detail_survives_the_wrapping() -> None:
|
||||
wrapped: Final = proxy_exception_for(
|
||||
HTTPException(status_code=400, detail={"error": "Invalid budget_duration 'not-a-duration'."}),
|
||||
default_status_code=400,
|
||||
)
|
||||
|
||||
assert "Invalid budget_duration" in wrapped.message
|
||||
|
||||
|
||||
def test_a_carried_type_survives_the_wrapping() -> None:
|
||||
class _Carrier(Exception):
|
||||
type: str = "context_window_exceeded"
|
||||
status_code: int = 400
|
||||
|
||||
assert proxy_exception_for(_Carrier("boom"), default_status_code=500).type == "context_window_exceeded"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("exc", [HTTPException(status_code=404, detail="nope"), ValueError("boom")])
|
||||
def test_the_wrapped_error_serializes_as_an_openai_error_object(exc: Exception) -> None:
|
||||
body: Final = json.loads(json.dumps({"error": proxy_exception_for(exc, default_status_code=400).to_dict()}))
|
||||
|
||||
assert body["error"]["param"] is None
|
||||
assert body["error"]["type"] == "invalid_request_error"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"provider_failure, expected",
|
||||
[
|
||||
(
|
||||
litellm.AuthenticationError(message="invalid x-api-key", llm_provider="anthropic", model="claude-haiku-5-5"),
|
||||
("401", "authentication_error"),
|
||||
),
|
||||
(
|
||||
litellm.PermissionDeniedError(
|
||||
message="model not allowed",
|
||||
llm_provider="anthropic",
|
||||
model="claude-haiku-5-5",
|
||||
response=httpx.Response(403, request=httpx.Request("POST", "https://api.anthropic.com/v1/messages")),
|
||||
),
|
||||
("403", "permission_error"),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_a_provider_credential_failure_is_labelled_the_way_chat_completions_labels_it(
|
||||
provider_failure: Exception, expected: tuple[str, str]
|
||||
) -> None:
|
||||
wrapped: Final = proxy_exception_for(provider_failure, default_status_code=400)
|
||||
|
||||
assert (wrapped.code, wrapped.type) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status_code", [401, 403])
|
||||
def test_a_rejection_the_route_raised_keeps_the_auth_label(status_code: int) -> None:
|
||||
wrapped: Final = proxy_exception_for(HTTPException(status_code=status_code, detail="no"), default_status_code=400)
|
||||
|
||||
assert (wrapped.code, wrapped.type) == (str(status_code), "auth_error")
|
||||
|
||||
|
||||
def test_an_exception_without_a_message_is_named_by_its_type() -> None:
|
||||
wrapped: Final = proxy_exception_for(TimeoutError(), default_status_code=500)
|
||||
|
||||
assert wrapped.message == "TimeoutError"
|
||||
|
||||
|
||||
def test_a_database_outage_answers_503_so_callers_retry_instead_of_fixing_their_request() -> None:
|
||||
wrapped: Final = proxy_exception_for(
|
||||
prisma.errors.PrismaError("Can't reach database server at `db:5432`"), default_status_code=400
|
||||
)
|
||||
|
||||
assert (wrapped.code, wrapped.type) == ("503", "no_db_connection")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"postgres_refusal",
|
||||
[
|
||||
'FATAL: database "litellm" is not currently accepting connections',
|
||||
"FATAL: the database system is starting up",
|
||||
"FATAL: the database system is shutting down",
|
||||
"FATAL: the database system is in recovery mode",
|
||||
"FATAL: the database system is not yet accepting connections",
|
||||
"FATAL: terminating connection due to administrator command",
|
||||
"ERROR: cannot execute UPDATE in a read-only transaction",
|
||||
],
|
||||
)
|
||||
def test_a_database_refusing_work_during_a_failover_answers_503(postgres_refusal: str) -> None:
|
||||
refused: Final = prisma.errors.DataError(
|
||||
{"user_facing_error": {"message": f"Error in connector: Error querying the database: {postgres_refusal}"}}
|
||||
)
|
||||
|
||||
wrapped: Final = proxy_exception_for(refused, default_status_code=400)
|
||||
|
||||
assert (wrapped.code, wrapped.type) == ("503", "no_db_connection")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"localized_refusal",
|
||||
[
|
||||
{
|
||||
"error_code": "P2010",
|
||||
"message": "Raw query failed. Message: `FEHLER: kann UPDATE nicht in einer Nur-Lese-Transaktion ausführen`",
|
||||
"meta": {"code": "25006"},
|
||||
},
|
||||
{
|
||||
"message": 'PostgresError { code: "57P01", message: "Verbindung wird abgebrochen", severity: "FATAL" }',
|
||||
},
|
||||
],
|
||||
)
|
||||
def test_a_failover_refusal_in_a_localized_postgres_answers_503(localized_refusal: dict[str, object]) -> None:
|
||||
refused: Final = prisma.errors.DataError({"user_facing_error": localized_refusal})
|
||||
|
||||
wrapped: Final = proxy_exception_for(refused, default_status_code=400)
|
||||
|
||||
assert (wrapped.code, wrapped.type) == ("503", "no_db_connection")
|
||||
|
||||
|
||||
def test_a_failover_phrase_outside_the_database_is_not_reported_as_a_database_outage() -> None:
|
||||
callback_down: Final = ValueError("FATAL: the database system is starting up")
|
||||
|
||||
wrapped: Final = proxy_exception_for(callback_down, default_status_code=400)
|
||||
|
||||
assert (wrapped.code, wrapped.type) == ("400", "invalid_request_error")
|
||||
|
||||
|
||||
def test_a_database_dropped_mid_query_answers_503_though_prisma_fails_to_name_the_error() -> None:
|
||||
engine_reply_with_null_meta: Final = [{"user_facing_error": {"error_code": "P1001", "message": "", "meta": None}}]
|
||||
with pytest.raises(AttributeError) as raised:
|
||||
handle_response_errors(Response(httpx.Response(500)), engine_reply_with_null_meta)
|
||||
|
||||
wrapped: Final = proxy_exception_for(raised.value, default_status_code=500)
|
||||
|
||||
assert (wrapped.code, wrapped.type) == ("503", "no_db_connection")
|
||||
|
||||
|
||||
def test_a_network_failure_outside_the_database_is_not_reported_as_a_database_outage() -> None:
|
||||
callback_unreachable: Final = httpx.ConnectError("[Errno 111] Connection refused")
|
||||
|
||||
wrapped: Final = proxy_exception_for(callback_unreachable, default_status_code=500)
|
||||
|
||||
assert (wrapped.code, wrapped.type) == ("500", "internal_server_error")
|
||||
|
||||
|
||||
def _add_member_failing_with(db_error: Exception) -> None:
|
||||
try:
|
||||
raise db_error
|
||||
except Exception as e:
|
||||
raise ValueError(f"Error adding member to organization: {e}")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"underlying_error, expected",
|
||||
[
|
||||
(prisma.errors.PrismaError("Can't reach database server at `db:5432`"), ("503", "no_db_connection")),
|
||||
(httpx.ConnectError("[Errno 111] Connection refused"), ("500", "internal_server_error")),
|
||||
],
|
||||
)
|
||||
def test_a_route_that_rewraps_its_error_keeps_the_database_outage_it_wrapped(
|
||||
underlying_error: Exception, expected: tuple[str, str]
|
||||
) -> None:
|
||||
with pytest.raises(ValueError, match="Error adding member") as raised:
|
||||
_add_member_failing_with(underlying_error)
|
||||
|
||||
wrapped: Final = proxy_exception_for(raised.value, default_status_code=500)
|
||||
|
||||
assert (wrapped.code, wrapped.type) == expected
|
||||
|
||||
|
||||
def test_a_proxy_without_prisma_installed_still_answers_with_an_error_object(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setitem(sys.modules, "prisma", None)
|
||||
|
||||
wrapped: Final = proxy_exception_for(httpx.ConnectError("[Errno 111] Connection refused"), default_status_code=500)
|
||||
|
||||
assert (wrapped.code, wrapped.type) == ("500", "internal_server_error")
|
||||
|
||||
|
||||
def _look_up_team_failing_with(lookup_error: Exception) -> None:
|
||||
try:
|
||||
raise lookup_error
|
||||
except Exception:
|
||||
raise HTTPException(status_code=404, detail="Team doesn't exist in db")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"lookup_error, expected",
|
||||
[
|
||||
(DBLookupDeadlineExceeded("team", 10.0), ("503", "no_db_connection")),
|
||||
(prisma.errors.PrismaError("Can't reach database server at `db:5432`"), ("503", "no_db_connection")),
|
||||
(KeyError("team_id"), ("404", "invalid_request_error")),
|
||||
],
|
||||
)
|
||||
def test_a_not_found_answer_raised_while_the_database_was_down_reports_the_outage(
|
||||
lookup_error: Exception, expected: tuple[str, str]
|
||||
) -> None:
|
||||
with pytest.raises(HTTPException) as raised:
|
||||
_look_up_team_failing_with(lookup_error)
|
||||
|
||||
wrapped: Final = proxy_exception_for(raised.value, default_status_code=400)
|
||||
|
||||
assert (wrapped.code, wrapped.type) == expected
|
||||
|
||||
|
||||
def test_a_lookup_that_missed_its_deadline_answers_503() -> None:
|
||||
wrapped: Final = proxy_exception_for(DBLookupDeadlineExceeded("team", 10.0), default_status_code=400)
|
||||
|
||||
assert (wrapped.code, wrapped.type) == ("503", "no_db_connection")
|
||||
|
||||
|
||||
def test_a_database_outage_answer_does_not_reveal_where_the_database_lives() -> None:
|
||||
wrapped: Final = proxy_exception_for(
|
||||
prisma.errors.PrismaError("Can't reach database server at `prod-db.internal:5432`"), default_status_code=400
|
||||
)
|
||||
|
||||
assert wrapped.code == "503"
|
||||
assert "prod-db.internal" not in wrapped.message
|
||||
|
||||
|
||||
def test_an_upstream_failure_answers_with_its_own_status_not_the_routes_default() -> None:
|
||||
upstream_overloaded: Final = litellm.ServiceUnavailableError(
|
||||
message="upstream overloaded", llm_provider="openai", model="gpt-5.4-mini"
|
||||
)
|
||||
|
||||
wrapped: Final = proxy_exception_for(upstream_overloaded, default_status_code=400)
|
||||
|
||||
assert (wrapped.code, wrapped.type) == ("503", "internal_server_error")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"router_rejection, expected",
|
||||
[
|
||||
(f"{RouterErrors.no_deployments_with_tag_routing.value}. Passed model=gpt-5.4-mini", ("401", "auth_error")),
|
||||
(f"{RouterErrors.no_deployments_available.value}. Passed model=gpt-5.4-mini", ("429", "rate_limit_error")),
|
||||
],
|
||||
)
|
||||
def test_a_router_rejection_is_labelled_by_the_status_it_answers_with(
|
||||
router_rejection: str, expected: tuple[str, str]
|
||||
) -> None:
|
||||
wrapped: Final = proxy_exception_for(ValueError(router_rejection), default_status_code=400)
|
||||
|
||||
assert (wrapped.code, wrapped.type) == expected
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ from typing import Final
|
|||
|
||||
import pytest
|
||||
from dotenv import load_dotenv
|
||||
from pydantic import JsonValue
|
||||
from pydantic import JsonValue, ValidationError
|
||||
from starlette.requests import Request
|
||||
|
||||
from litellm.proxy.google_endpoints.endpoints import _google_count_tokens_request, google_count_tokens
|
||||
|
|
@ -105,3 +105,91 @@ async def test_proxy_gemini_to_openai_like_model_token_counting():
|
|||
)
|
||||
|
||||
assert response.get("totalTokens") > 0
|
||||
|
||||
|
||||
_DECLARATION: Final = {"name": "lookup", "description": "Find a record"}
|
||||
_SYSTEM: Final = {"systemInstruction": {"parts": [{"text": "Follow these rules"}]}}
|
||||
_NO_SYSTEM_PARTS: Final = {"systemInstruction": {"parts": []}}
|
||||
_TOOLS: Final = {"tools": [{"functionDeclarations": [_DECLARATION]}]}
|
||||
_TURN: Final = {"role": "user", "parts": [{"text": "Count this request"}]}
|
||||
_CONTENTS: Final = {"contents": [_TURN]}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("spelled", "canonical"),
|
||||
(
|
||||
({"systemInstruction": {"parts": {"text": "Follow these rules"}}}, _SYSTEM),
|
||||
({"system_instruction": {"parts": {"text": "Follow these rules"}}}, _SYSTEM),
|
||||
({"systemInstruction": {"parts": [{"text": "Follow these rules"}, None]}}, _SYSTEM),
|
||||
({"systemInstruction": {"parts": None}}, _NO_SYSTEM_PARTS),
|
||||
({"systemInstruction": {"parts": [{"text": None}, {"text": "Follow these rules"}]}}, _SYSTEM),
|
||||
({"systemInstruction": {"parts": {"text": None}}}, _NO_SYSTEM_PARTS),
|
||||
({"systemInstruction": {"parts": {"text": None, "inlineData": None}}}, _NO_SYSTEM_PARTS),
|
||||
({"systemInstruction": {"parts": {"text": None, "thought": False}}}, _NO_SYSTEM_PARTS),
|
||||
({"systemInstruction": {"parts": [{"thoughtSignature": "c2ln"}, {"text": "Follow these rules"}]}}, _SYSTEM),
|
||||
({"systemInstruction": {"parts": [{}]}}, _NO_SYSTEM_PARTS),
|
||||
({"systemInstruction": {}}, _NO_SYSTEM_PARTS),
|
||||
({"tools": None}, {}),
|
||||
({"tools": [None]}, {}),
|
||||
({"tools": [{"functionDeclarations": None}]}, {}),
|
||||
({"tools": {"functionDeclarations": [_DECLARATION]}}, _TOOLS),
|
||||
({"tools": [{"functionDeclarations": _DECLARATION}]}, _TOOLS),
|
||||
({"tools": {"functionDeclarations": _DECLARATION}}, _TOOLS),
|
||||
({"contents": _TURN}, _CONTENTS),
|
||||
({"contents": [None, _TURN]}, _CONTENTS),
|
||||
({"contents": None}, {"contents": []}),
|
||||
({"contents": [{"role": "user", "parts": {"text": "Count this request"}}]}, _CONTENTS),
|
||||
({"contents": [{"role": "user", "parts": [None, {"text": "Count this request"}]}]}, _CONTENTS),
|
||||
({"contents": [{"role": "user", "parts": None}]}, {"contents": [{"role": "user", "parts": []}]}),
|
||||
),
|
||||
)
|
||||
def test_google_count_tokens_counts_every_google_spelling_of_a_repeated_field_like_its_list_form(
|
||||
spelled: dict[str, JsonValue], canonical: dict[str, JsonValue]
|
||||
) -> None:
|
||||
contents: Final = [{"role": "user", "parts": [{"text": "Count this request"}]}]
|
||||
|
||||
assert _google_count_tokens_request("counted", {"contents": contents, **spelled}) == _google_count_tokens_request(
|
||||
"counted", {"contents": contents, **canonical}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"malformed",
|
||||
(
|
||||
{"tools": "lookup"},
|
||||
{"tools": [{"functionDeclarations": "lookup"}]},
|
||||
{"systemInstruction": {"parts": 7}},
|
||||
{"contents": 5},
|
||||
{"contents": ["Count this request"]},
|
||||
),
|
||||
)
|
||||
def test_google_count_tokens_rejects_a_scalar_where_google_expects_an_object_or_list(
|
||||
malformed: dict[str, JsonValue],
|
||||
) -> None:
|
||||
with pytest.raises(ValidationError):
|
||||
_google_count_tokens_request("counted", {"contents": [_TURN], **malformed})
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"parts",
|
||||
(
|
||||
{"inlineData": {"mimeType": "image/png", "data": "aGVsbG8="}},
|
||||
[{"inlineData": {"mimeType": "image/png", "data": "aGVsbG8="}}],
|
||||
{"text": None, "inlineData": {"mimeType": "image/png", "data": "aGVsbG8="}},
|
||||
[{"text": "Follow these rules"}, {"text": None, "fileData": {"fileUri": "gs://bucket/rules.pdf"}}],
|
||||
),
|
||||
)
|
||||
def test_google_count_tokens_rejects_media_only_system_parts(parts: JsonValue) -> None:
|
||||
with pytest.raises(ValidationError):
|
||||
_google_count_tokens_request(
|
||||
"counted",
|
||||
{"contents": [{"role": "user", "parts": [{"text": "Hello"}]}], "systemInstruction": {"parts": parts}},
|
||||
)
|
||||
|
||||
|
||||
def test_google_count_tokens_keeps_counting_system_text_that_sits_beside_media() -> None:
|
||||
text_with_media: Final = {"text": "Follow these rules", "inlineData": {"mimeType": "image/png", "data": "aGVsbG8="}}
|
||||
|
||||
assert _google_count_tokens_request(
|
||||
"counted", {**_CONTENTS, "systemInstruction": {"parts": [text_with_media]}}
|
||||
) == _google_count_tokens_request("counted", {**_CONTENTS, **_SYSTEM})
|
||||
|
|
|
|||
|
|
@ -5480,6 +5480,15 @@ async def test_health_services_endpoint_langfuse_missing_keys_errors(monkeypatch
|
|||
assert "connection refused by lf.internal.example" in str(raised.value.message)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_health_services_reports_an_unknown_service_as_a_request_error_not_an_auth_error() -> None:
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await health_services_endpoint(service="totally_unknown_service_xyz")
|
||||
|
||||
assert (exc_info.value.code, exc_info.value.type) == ("400", "invalid_request_error")
|
||||
assert "Authentication Error" not in exc_info.value.message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("admin", [False, True])
|
||||
async def test_unsaved_chatgpt_account_probe_requires_proxy_admin(
|
||||
|
|
|
|||
|
|
@ -5211,3 +5211,18 @@ async def test_user_update_without_password_revokes_nothing(_admin_prisma, mocke
|
|||
await _update_single_user_helper(user_request=user_request, user_api_key_dict=admin_caller)
|
||||
|
||||
revoke_mock.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_user_update_reports_a_bad_budget_duration_as_a_request_error_not_an_auth_error(_admin_prisma: MagicMock) -> None:
|
||||
from litellm.proxy.management_endpoints.internal_user_endpoints import user_update
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await user_update(
|
||||
data=UpdateUserRequest(user_id="target-user", budget_duration="not-a-duration"),
|
||||
user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN),
|
||||
)
|
||||
|
||||
assert (exc_info.value.code, exc_info.value.type) == ("400", "invalid_request_error")
|
||||
assert "Invalid budget_duration" in exc_info.value.message
|
||||
assert "Authentication Error" not in exc_info.value.message
|
||||
|
|
|
|||
|
|
@ -6538,7 +6538,7 @@ async def test_list_keys_with_invalid_status():
|
|||
)
|
||||
|
||||
# Verify ProxyException properties
|
||||
assert exc_info.value.code == "400"
|
||||
assert (exc_info.value.code, exc_info.value.type) == ("400", "invalid_request_error")
|
||||
assert "Invalid status value" in str(exc_info.value.message)
|
||||
assert "deleted" in str(exc_info.value.message)
|
||||
|
||||
|
|
@ -23377,3 +23377,73 @@ def test_key_info_database_not_connected_sanitized_500(
|
|||
assert "Database not connected" in caplog.text
|
||||
finally:
|
||||
proxy_server.app.dependency_overrides.pop(user_api_key_auth, None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_key_reports_a_bad_budget_duration_as_a_request_error_not_an_auth_error(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from fastapi import Request
|
||||
|
||||
from litellm.proxy.management_endpoints.key_management_endpoints import update_key_fn
|
||||
|
||||
monkeypatch.setattr( # test-quality-ok: the endpoint reads proxy_server.prisma_client itself, no parameter to inject
|
||||
"litellm.proxy.proxy_server.prisma_client", MagicMock()
|
||||
)
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await update_key_fn(
|
||||
request=MagicMock(spec=Request),
|
||||
data=UpdateKeyRequest(key="sk-whatever", budget_duration="not-a-duration"),
|
||||
user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN),
|
||||
)
|
||||
|
||||
assert (exc_info.value.code, exc_info.value.type) == ("400", "invalid_request_error")
|
||||
assert "Authentication Error" not in exc_info.value.message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_key_passes_its_own_rejection_through_without_making_it_its_own_cause(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from fastapi import Request
|
||||
|
||||
from litellm.proxy.management_endpoints.key_management_endpoints import update_key_fn
|
||||
|
||||
prisma_client: Final = MagicMock()
|
||||
prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=None)
|
||||
monkeypatch.setattr( # test-quality-ok: the endpoint reads proxy_server.prisma_client itself, no parameter to inject
|
||||
"litellm.proxy.proxy_server.prisma_client", prisma_client
|
||||
)
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await update_key_fn(
|
||||
request=MagicMock(spec=Request),
|
||||
data=UpdateKeyRequest(key="sk-missing", max_budget=5),
|
||||
user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN),
|
||||
)
|
||||
|
||||
assert (exc_info.value.code, exc_info.value.type) == ("404", "not_found_error")
|
||||
assert exc_info.value.__cause__ is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_key_aliases_answers_503_when_the_database_is_unreachable(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import prisma.errors
|
||||
|
||||
from litellm.proxy.management_endpoints.key_management_endpoints import key_aliases
|
||||
|
||||
unreachable_db: Final = MagicMock()
|
||||
unreachable_db.db.query_raw = AsyncMock(
|
||||
side_effect=prisma.errors.PrismaError("Can't reach database server at `db:5432`")
|
||||
)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", unreachable_db)
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await key_aliases(
|
||||
user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN),
|
||||
page=1,
|
||||
size=50,
|
||||
search=None,
|
||||
team_id=None,
|
||||
)
|
||||
|
||||
assert (exc_info.value.code, exc_info.value.type) == ("503", "no_db_connection")
|
||||
|
|
|
|||
|
|
@ -311,7 +311,7 @@ async def test_model_creation_fails_closed_when_authoritative_policy_is_unavaila
|
|||
Deployment(model_name="cannot-create", litellm_params={"model": "openai/test-model"}),
|
||||
UserAPIKeyAuth(user_id="test_user", user_role=LitellmUserRoles.INTERNAL_USER),
|
||||
)
|
||||
assert error.value.code == "503"
|
||||
assert (error.value.code, error.value.type) == ("503", "internal_server_error")
|
||||
prisma.db.litellm_proxymodeltable.create.assert_not_called()
|
||||
prisma.db.litellm_teamtable.find_unique.assert_not_called()
|
||||
|
||||
|
|
@ -10913,3 +10913,76 @@ class TestNonAdminCannotSetPerUserOauthOnModel:
|
|||
user_api_key_dict=admin,
|
||||
)
|
||||
assert result is created_row
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_model_reports_an_unknown_model_id_as_a_request_error_not_an_auth_error(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy._types import ModelInfoDelete
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import delete_model
|
||||
|
||||
prisma: Final = MagicMock()
|
||||
prisma.db.litellm_proxymodeltable.find_unique = AsyncMock(return_value=None)
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await delete_model(
|
||||
model_info=ModelInfoDelete(id="no-such-model"),
|
||||
user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN),
|
||||
)
|
||||
|
||||
assert (exc_info.value.code, exc_info.value.type) == ("400", "invalid_request_error")
|
||||
assert "Authentication Error" not in exc_info.value.message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_model_reports_an_unknown_model_id_as_a_request_error_not_an_auth_error(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import update_model
|
||||
|
||||
prisma: Final = MagicMock()
|
||||
prisma.db.litellm_proxymodeltable.find_unique = AsyncMock(return_value=None)
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", None)
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await update_model(
|
||||
model_params=updateDeployment(model_info=ModelInfo(id="no-such-model")),
|
||||
user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN),
|
||||
)
|
||||
|
||||
assert (exc_info.value.code, exc_info.value.type) == ("400", "invalid_request_error")
|
||||
assert exc_info.value.message == "model not found"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_new_model_reports_a_database_outage_during_the_insert_as_503(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
import prisma.errors
|
||||
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import add_new_model
|
||||
|
||||
unreachable_db: Final = MagicMock()
|
||||
unreachable_db.db.query_raw = AsyncMock(return_value=[])
|
||||
unreachable_db.db.litellm_proxymodeltable.create = AsyncMock(
|
||||
side_effect=prisma.errors.PrismaError("Can't reach database server at `db:5432`")
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", unreachable_db)
|
||||
monkeypatch.setattr(proxy_server, "store_model_in_db", True)
|
||||
monkeypatch.setattr(proxy_server, "general_settings", {})
|
||||
monkeypatch.setattr(proxy_server, "premium_user", True)
|
||||
monkeypatch.setenv("LITELLM_SALT_KEY", "sk-outage-salt")
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await add_new_model(
|
||||
model_params=Deployment(
|
||||
model_name="outage-model",
|
||||
litellm_params=LiteLLM_Params(model="openai/gpt-5.4-mini", api_key="fake-key"),
|
||||
model_info={"id": "outage-model-1"},
|
||||
),
|
||||
user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN),
|
||||
)
|
||||
|
||||
assert (exc_info.value.code, exc_info.value.type) == ("503", "no_db_connection")
|
||||
|
|
|
|||
|
|
@ -1617,6 +1617,37 @@ async def test_delete_organization_is_refused_while_one_of_its_teams_holds_a_ptu
|
|||
prisma_client.db.litellm_organizationtable.delete.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_organization_member_add_reports_an_unknown_org_as_a_request_error_not_an_auth_error(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from fastapi import Request
|
||||
|
||||
from litellm.proxy._types import (
|
||||
LitellmUserRoles,
|
||||
OrganizationMemberAddRequest,
|
||||
OrgMember,
|
||||
ProxyException,
|
||||
UserAPIKeyAuth,
|
||||
)
|
||||
from litellm.proxy.management_endpoints.organization_endpoints import organization_member_add
|
||||
|
||||
mock_prisma_client: Final = MagicMock()
|
||||
mock_prisma_client.db.litellm_organizationtable.find_unique = AsyncMock(return_value=None)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await organization_member_add(
|
||||
data=OrganizationMemberAddRequest(
|
||||
organization_id="no-such-org",
|
||||
member=OrgMember(role="internal_user", user_id="someone"),
|
||||
),
|
||||
http_request=MagicMock(spec=Request),
|
||||
user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN),
|
||||
)
|
||||
|
||||
assert (exc_info.value.code, exc_info.value.type) == ("404", "invalid_request_error")
|
||||
assert "Authentication Error" not in exc_info.value.message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("budget_duration", ["30d", None])
|
||||
async def test_new_organization_schedules_budget_reset_from_duration(monkeypatch, budget_duration):
|
||||
|
|
|
|||
|
|
@ -19372,3 +19372,48 @@ async def test_default_budget_cache_eviction_does_not_fail_after_redis_error() -
|
|||
)
|
||||
|
||||
user_api_key_cache.async_delete_cache.assert_awaited_once_with(key="team_member_default_budget:budget_456")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_team_info_reports_a_missing_team_as_a_request_error_not_an_auth_error(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from fastapi import Request
|
||||
|
||||
from litellm.proxy._types import ProxyException
|
||||
from litellm.proxy.management_endpoints import team_endpoints
|
||||
|
||||
mock_prisma_client: Final = MagicMock()
|
||||
mock_prisma_client.db.litellm_teamtable.find_unique = AsyncMock(return_value=None)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await team_endpoints.team_info(
|
||||
http_request=MagicMock(spec=Request),
|
||||
team_id="no-such-team",
|
||||
user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN),
|
||||
)
|
||||
|
||||
assert (exc_info.value.code, exc_info.value.type) == ("404", "invalid_request_error")
|
||||
assert "Authentication Error" not in exc_info.value.message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_team_info_keeps_the_auth_label_for_a_caller_outside_the_team(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from fastapi import Request
|
||||
|
||||
from litellm.proxy._types import ProxyException
|
||||
from litellm.proxy.management_endpoints import team_endpoints
|
||||
|
||||
other_team: Final = MagicMock()
|
||||
other_team.model_dump.return_value = {"team_id": "team-a", "members_with_roles": []}
|
||||
mock_prisma_client: Final = MagicMock()
|
||||
mock_prisma_client.db.litellm_teamtable.find_unique = AsyncMock(return_value=other_team)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await team_endpoints.team_info(
|
||||
http_request=MagicMock(spec=Request),
|
||||
team_id="team-a",
|
||||
user_api_key_dict=UserAPIKeyAuth(user_id="outsider", user_role=LitellmUserRoles.INTERNAL_USER),
|
||||
)
|
||||
|
||||
assert (exc_info.value.code, exc_info.value.type) == ("403", "auth_error")
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import sys
|
|||
import traceback
|
||||
from collections.abc import AsyncGenerator, Awaitable, Callable, Coroutine, Mapping
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from typing import Final, Literal, NoReturn, cast
|
||||
from unittest import mock
|
||||
|
||||
|
|
@ -3817,6 +3818,48 @@ def test_live_session_preserves_pretty_printed_start_frame(
|
|||
assert routed_data["live_session_start"] == expected_start
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_config_reports_an_unsupported_router_setting_as_a_request_error_not_an_auth_error() -> None:
|
||||
from fastapi import Request
|
||||
|
||||
from litellm.proxy._types import ConfigYAML, ProxyException
|
||||
from litellm.proxy.proxy_server import update_config
|
||||
|
||||
body: Final = {"router_settings": {"not_a_router_setting": True}}
|
||||
request: Final = MagicMock(spec=Request)
|
||||
request.json = AsyncMock(return_value=body)
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await update_config(
|
||||
config_info=ConfigYAML(**body),
|
||||
request=request,
|
||||
user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN),
|
||||
)
|
||||
|
||||
assert (exc_info.value.code, exc_info.value.type) == ("400", "invalid_request_error")
|
||||
assert "Authentication Error" not in exc_info.value.message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_config_callbacks_reports_an_unreadable_config_as_a_server_error_not_an_auth_error(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy._types import ProxyException
|
||||
from litellm.proxy.proxy_server import get_config
|
||||
|
||||
empty_config: Final = tmp_path / "config.yaml"
|
||||
empty_config.write_text("")
|
||||
monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False)
|
||||
monkeypatch.setattr(proxy_server, "user_config_file_path", str(empty_config))
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await get_config(user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN))
|
||||
|
||||
assert (exc_info.value.code, exc_info.value.type) == ("500", "internal_server_error")
|
||||
assert exc_info.value.message == "Config cannot be None or Empty."
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scope", [None, "expand"])
|
||||
@pytest.mark.parametrize(
|
||||
"general_settings, wildcard_query, wildcard_listed",
|
||||
|
|
@ -3861,6 +3904,62 @@ async def test_model_list_return_wildcard_routes_defaults_to_general_setting(
|
|||
assert ("openai/*" in listed) is wildcard_listed, listed
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_queue_request_labels_an_upstream_failure_a_server_error_not_an_auth_error(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from starlette.requests import Request
|
||||
from starlette.responses import Response
|
||||
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy._types import ProxyException
|
||||
from litellm.proxy.proxy_server import async_queue_request
|
||||
|
||||
body: Final = json.dumps(
|
||||
{"model": "gpt-5.4-mini", "messages": [{"role": "user", "content": "hi"}], "priority": 0}
|
||||
).encode()
|
||||
|
||||
async def receive() -> dict[str, object]:
|
||||
return {"type": "http.request", "body": body, "more_body": False}
|
||||
|
||||
request: Final = Request(
|
||||
{
|
||||
"type": "http",
|
||||
"method": "POST",
|
||||
"path": "/queue/chat/completions",
|
||||
"headers": [(b"content-type", b"application/json")],
|
||||
"query_string": b"",
|
||||
"scheme": "http",
|
||||
"server": ("testserver", 80),
|
||||
"root_path": "",
|
||||
},
|
||||
receive,
|
||||
)
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-5.4-mini",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-5.4-mini",
|
||||
"api_key": "sk-test",
|
||||
"mock_response": "litellm.InternalServerError",
|
||||
},
|
||||
}
|
||||
],
|
||||
num_retries=0,
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", router)
|
||||
|
||||
with pytest.raises(ProxyException) as exc_info:
|
||||
await async_queue_request(
|
||||
request=request,
|
||||
fastapi_response=Response(),
|
||||
user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN),
|
||||
)
|
||||
|
||||
assert (exc_info.value.code, exc_info.value.type) == ("500", "internal_server_error")
|
||||
|
||||
|
||||
_DEBUG_LOG_SENTINEL: Final = "sentinel-request-body-7f3a"
|
||||
_LONG_PROMPT: Final = f"{_DEBUG_LOG_SENTINEL} " + "Keep edits minimal and explain them briefly. " * 20
|
||||
_TOOL_PARAMETERS: Final = {"type": "object", "properties": {"cmd": {"type": "string"}}}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue