Merge remote-tracking branch 'origin/main' into litellm_responses_comparison_response_id_decode

This commit is contained in:
mateo-berri 2026-10-28 21:50:59 -07:00
commit 19681de431
36 changed files with 2317 additions and 295 deletions

View file

@ -198,6 +198,11 @@ def _genai_message_part(part: object) -> _GenAIMessagePart | None:
return ChatCompletionTextObject(type="text", text=text) if isinstance(text, str) else _genai_media_part(fields)
def _genai_content_parts(value: object) -> Sequence[object]:
single_part: Final = _validated(_CONFIG_FIELDS, value)
return (single_part,) if single_part is not None else _validated(_GENAI_CONTENT_PARTS, value) or ()
def _genai_message_parts(parts: object) -> tuple[_GenAIMessagePart, ...]:
items: Final = _validated(_GENAI_CONTENT_PARTS, parts) or ()
return tuple(converted for part in items if (converted := _genai_message_part(part)) is not None)
@ -248,7 +253,7 @@ def _genai_content_messages(content: object) -> tuple[AllMessageValues, ...]:
fields: Final = _validated(_CONFIG_FIELDS, content)
if fields is None:
return ()
parts: Final = _validated(_GENAI_CONTENT_PARTS, fields.get("parts", ())) or ()
parts: Final = _genai_content_parts(fields.get("parts", ()))
converted: Final = _genai_message_parts(parts)
role: Final = fields.get("role", "user")
if role == "user":

View file

@ -31,6 +31,7 @@ from ..constants import (
)
from ..errors import AnthropicContextManagementError
from ..placeholders import build_cleared_tool_result_content
from .compact import count_system_tokens, history_after_client_compaction
if TYPE_CHECKING:
from litellm.types.llms.openai import ChatCompletionToolParam
@ -203,9 +204,10 @@ def _trigger_met(
model: str,
messages: list[dict[str, object]],
tools: list[dict[str, object]] | None,
system: str | Sequence[Mapping[str, object]] | None,
counted_tool_uses: int,
) -> tuple[bool, int | None]:
"""Return (trigger_met, input_tokens if counted for reuse)."""
"""Return (trigger_met, message and tool tokens for reuse); like the API, input_tokens adds the system prompt."""
trigger_type: Final = trigger.get("type", "input_tokens")
threshold = trigger.get("value")
@ -216,14 +218,17 @@ def _trigger_met(
if not isinstance(threshold, int):
threshold = DEFAULT_INPUT_TOKENS_TRIGGER
current_tokens: Final = litellm.token_counter(
message_tokens: Final = litellm.token_counter(
model=model,
messages=messages,
tools=cast("list[ChatCompletionToolParam] | None", tools),
)
if message_tokens > threshold:
return True, message_tokens
current_tokens: Final = message_tokens + count_system_tokens(model, system)
verbose_logger.debug("context_management polyfill: current_tokens: %s", current_tokens)
verbose_logger.debug("context_management polyfill: threshold: %s", threshold)
return current_tokens > threshold, current_tokens
return current_tokens > threshold, message_tokens
def _resolve_keep_count(keep: dict[str, object]) -> int:
@ -237,7 +242,7 @@ def _resolve_keep_count(keep: dict[str, object]) -> int:
def _last_completed_tool_use_id(
messages: list[dict[str, object]],
messages: Sequence[Mapping[str, object]],
) -> str | None:
"""Latest completed tool_result id; never cleared."""
last_id: str | None = None
@ -253,7 +258,7 @@ def _last_completed_tool_use_id(
def _clear_tool_results(
messages: list[dict[str, object]], ids_to_clear: set[str]
messages: list[dict[str, object]], ids_to_clear: Set[str]
) -> tuple[list[dict[str, object]], int]:
"""Clear matching tool_result content; return (messages, cleared_count)."""
cleared = 0
@ -311,7 +316,11 @@ def apply_clear_tool_uses_20250919(
system: str | list[dict[str, object]] | None,
edit_spec: ClearToolUsesEditSpec,
) -> tuple[list[dict[str, object]], AppliedEdit | None]:
"""Apply clear_tool_uses to an edit ``check_clear_tool_uses_spec`` returned; return (messages, AppliedEdit or None)."""
"""Apply clear_tool_uses to an edit ``check_clear_tool_uses_spec`` returned; return (messages, AppliedEdit or None).
Like the API, it reads only the history after a client compaction block and counts the block's summary as system.
"""
kept, counted_system = history_after_client_compaction(messages, system)
fields: Final = _ClearToolUsesFields.model_validate(edit_spec)
clear_at_least: Final = 0 if fields.clear_at_least is None else fields.clear_at_least.value
exclude_tools: Final = frozenset(fields.exclude_tools or ())
@ -326,7 +335,7 @@ def apply_clear_tool_uses_20250919(
"value": DEFAULT_KEEP_TOOL_USES,
}
blocks: Final = _all_blocks(messages)
blocks: Final = _all_blocks(kept)
results: Final = _blocks_by_id(blocks, "tool_result", "tool_use_id")
uses: Final = _blocks_by_id(blocks, "tool_use", "id")
tiny: Final = _tiny_result_ids(results)
@ -335,15 +344,15 @@ def apply_clear_tool_uses_20250919(
for tool_use_id, name in _tool_uses_in_order(blocks)
if name not in exclude_tools and (tool_use_id not in tiny or _clears_input(name, input_tools))
)
met, counted_tokens = _trigger_met(trigger, model, messages, tools, len(clearable))
if not met:
return messages, None
keep_count: Final = _resolve_keep_count(keep)
if len(clearable) <= keep_count:
return messages, None
last_completed_id: Final = _last_completed_tool_use_id(messages)
met, counted_tokens = _trigger_met(trigger, model, kept, tools, counted_system, len(clearable))
if not met:
return messages, None
last_completed_id: Final = _last_completed_tool_use_id(kept)
candidates: Final = tuple(
(tool_use_id, name)
for tool_use_id, name in clearable[: len(clearable) - keep_count]
@ -358,7 +367,7 @@ def apply_clear_tool_uses_20250919(
_select_in_batches(candidate_ids, tokens, clear_at_least) if clear_at_least > 0 else candidate_ids
)
edited, cleared_count = _clear_tool_results(messages, set(ids_to_clear))
edited, cleared_count = _clear_tool_results(messages, ids_to_clear)
verbose_logger.debug("context_management polyfill: edited: %s", edited)
if cleared_count == 0:
return messages, None
@ -370,11 +379,12 @@ def apply_clear_tool_uses_20250919(
counted_tokens
if counted_tokens is not None
else litellm.token_counter(
model=model, messages=messages, tools=cast("list[ChatCompletionToolParam] | None", tools)
model=model, messages=kept, tools=cast("list[ChatCompletionToolParam] | None", tools)
)
)
kept_after, _ = history_after_client_compaction(final_messages, system)
tokens_after: Final = litellm.token_counter(
model=model, messages=final_messages, tools=cast("list[ChatCompletionToolParam] | None", tools)
model=model, messages=kept_after, tools=cast("list[ChatCompletionToolParam] | None", tools)
)
cleared_input_tokens: Final = max(tokens_before - tokens_after, 0)

View file

@ -189,6 +189,21 @@ def _strip_compaction_blocks(
return cleaned
def history_after_client_compaction(
messages: Sequence[dict[str, object]],
system: str | Sequence[Mapping[str, object]] | None,
) -> tuple[list[dict[str, object]], str | Sequence[Mapping[str, object]] | None]:
"""Return the history the API keeps after the latest compaction block, without it, and the system with its summary."""
sliced, block = _slice_around_compaction_block(messages)
if block is None:
return list(messages), system
tail: Final = _strip_compaction_blocks(sliced)
summary: Final = block.get("content")
if not isinstance(summary, str) or not summary:
return tail, system
return tail, _augment_system_with_summary(_system_to_text(system) or None, summary)
def _augment_system_with_summary(
system: str | list[dict[str, object]] | None,
summary_text: str,
@ -320,14 +335,16 @@ def _count_effective_tokens(
content: Final = compaction_block.get("content") or ""
if content:
total += litellm.token_counter(model=model, text=content)
return total + count_system_tokens(model, system)
def count_system_tokens(model: str, system: str | Sequence[Mapping[str, object]] | None) -> int:
system_text: Final = _system_to_text(system)
if system_text:
total += litellm.token_counter(model=model, text=system_text)
return total
return litellm.token_counter(model=model, text=system_text) if system_text else 0
def _system_to_text(
system: str | list[dict[str, object]] | None,
system: str | Sequence[Mapping[str, object]] | None,
) -> str:
"""Flatten an Anthropic-style ``system`` value into a single string for
token counting. Returns ``""`` when ``system`` carries no text."""
@ -338,7 +355,7 @@ def _system_to_text(
return "\n".join(
text
for block in system
if isinstance(block, dict)
if isinstance(block, Mapping)
and block.get("type") == "text"
and isinstance(text := block.get("text"), str)
and text

View file

@ -3816,6 +3816,16 @@ class ManagementEndpointLoggingPayload(LiteLLMPydanticObjectBase):
end_time: datetime | None = None
def proxy_exception_status_code(message: str, code: str) -> str:
# Litellm router.py returns "No healthy deployment available" when there are no deployments available
# Should map to 429 errors https://github.com/BerriAI/litellm/issues/2487
if "No healthy deployment available" in message or "No deployments available" in message:
return "429"
if RouterErrors.no_deployments_with_tag_routing.value in message:
return "401"
return code
class ProxyException(Exception):
# NOTE: DO NOT MODIFY THIS
# This is used to map exactly to OPENAI Exceptions
@ -3844,13 +3854,7 @@ class ProxyException(Exception):
headers[k] = str(v)
self.headers = headers or {}
self.provider_specific_fields = provider_specific_fields
# rules for proxyExceptions
# Litellm router.py returns "No healthy deployment available" when there are no deployments available
# Should map to 429 errors https://github.com/BerriAI/litellm/issues/2487
if "No healthy deployment available" in self.message or "No deployments available" in self.message:
self.code = "429"
elif RouterErrors.no_deployments_with_tag_routing.value in self.message:
self.code = "401"
self.code = proxy_exception_status_code(self.message, self.code)
def to_dict(self) -> dict:
"""Converts the ProxyException instance to a dictionary."""

View file

@ -1063,6 +1063,15 @@ def is_mcp_discovery_request(route: str, request_body: Mapping[str, object]) ->
return isinstance(method, str) and method in MCP_ZERO_SPEND_JSONRPC_METHODS
class ModelMaxBudgetHolder(Protocol):
@property
def model_max_budget(self) -> Mapping[str, object] | None: ...
def model_max_budget_of(holder: ModelMaxBudgetHolder) -> Mapping[str, object] | None:
return holder.model_max_budget
def route_skips_budget_checks(route: str) -> bool:
return route not in BUDGET_ENFORCED_SIDE_EFFECT_ROUTES and (
route in MODEL_DISCOVERY_ROUTES or not RouteChecks.is_llm_api_route(route=route)

View file

@ -40,6 +40,7 @@ from litellm.integrations.otel.model.config import is_otel_v2_enabled
from litellm.integrations.otel.runtime import phase_event, phase_span, seed_request_identity
from litellm.litellm_core_utils.dd_tracing import tracer
from litellm.litellm_core_utils.dot_notation_indexing import get_nested_value
from litellm.models.end_user import LiteLLM_EndUserTable
from litellm.proxy._types import *
from litellm.proxy.agent_endpoints.auth.agent_caller import agent_caller_from_headers
from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module exports
@ -85,6 +86,7 @@ from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module expo
jwt_key_mapping_cache_key,
key_access_group_grants_model,
key_model_aliases_for_auth_check,
model_max_budget_of,
resolve_and_validate_end_user_id,
resolve_default_end_user_budget,
route_skips_budget_checks,
@ -384,6 +386,16 @@ class _TeamModelBudgetLimiter(Protocol):
) -> bool: ...
class _EndUserModelBudgetLimiter(Protocol):
async def is_end_user_within_model_budget(
self,
end_user_id: str,
end_user_model_max_budget: Mapping[str, object],
model: str,
team_id: str | None = None,
) -> bool: ...
class _EveryModelBudgetLimiter(Protocol):
async def is_key_within_model_budget(self, user_api_key_dict: UserAPIKeyAuth, model: str) -> bool: ...
@ -530,6 +542,49 @@ async def _check_team_model_budget(
)
async def _end_user_model_max_budget(
end_user_id: str | None,
end_user_object: LiteLLM_EndUserTable | None,
key_end_user_budget_id: str | None,
prisma_client: PrismaClient | None,
user_api_key_cache: UserApiKeyCache,
parent_otel_span: Span | None,
) -> Mapping[str, object] | None:
budget_table: Final = (
end_user_object.litellm_budget_table
if end_user_object is not None
else (
await resolve_default_end_user_budget(
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
key_end_user_budget_id=key_end_user_budget_id,
parent_otel_span=parent_otel_span,
)
if end_user_id and prisma_client is not None
else None
)
)
return model_max_budget_of(budget_table) if budget_table is not None else None
async def _check_end_user_model_budget(
end_user_id: str | None,
end_user_model_max_budget: Mapping[str, object] | None,
model_max_budget_limiter: _EndUserModelBudgetLimiter,
models: Sequence[str],
team_id: str | None,
) -> None:
if not end_user_id or not end_user_model_max_budget:
return
for model_name in models:
await model_max_budget_limiter.is_end_user_within_model_budget(
end_user_id=end_user_id,
end_user_model_max_budget=end_user_model_max_budget,
model=model_name,
team_id=team_id,
)
async def _is_within_every_model_budget(
valid_token: UserAPIKeyAuth,
model_max_budget_limiter: _EveryModelBudgetLimiter,
@ -2094,6 +2149,10 @@ async def _resolve_object_permission_for_unresolvable_team(
)
def _custom_auth_token_for_this_request(response: object) -> UserAPIKeyAuth:
return UserAPIKeyAuth.model_validate(response).model_copy()
async def user_api_key_auth_builder(
request: Request,
api_key: str,
@ -2172,7 +2231,7 @@ async def user_api_key_auth_builder(
request=request, api_key=api_key, user_custom_auth=user_custom_auth
)
if response is not None and isinstance(response, UserAPIKeyAuth):
validated = UserAPIKeyAuth.model_validate(response)
validated = _custom_auth_token_for_this_request(response)
if getattr(litellm, "enable_post_custom_auth_checks", False):
validated = await _run_post_custom_auth_checks(
valid_token=validated,
@ -2188,7 +2247,7 @@ async def user_api_key_auth_builder(
custom_auth_api_key = True
elif user_custom_auth is not None:
response = await user_custom_auth(request=request, api_key=api_key)
validated = UserAPIKeyAuth.model_validate(response)
validated = _custom_auth_token_for_this_request(response)
if getattr(litellm, "enable_post_custom_auth_checks", False):
validated = await _run_post_custom_auth_checks(
valid_token=validated,
@ -3129,7 +3188,7 @@ async def validate_resolved_virtual_key( # noqa: C901 # Preserve ordering of e
)
# Recompute after a potential budget-fallback rewrite so
# the end-user check below validates the final model
# the user check below validates the final model
current_model = _get_model_from_request_context(
request_data=request_data,
route=route,
@ -3147,23 +3206,6 @@ async def validate_resolved_virtual_key( # noqa: C901 # Preserve ordering of e
models=current_models,
)
# Check 5b. End-user model max budget
end_user_mmb: Final = valid_token.end_user_model_max_budget
if (
end_user_mmb is not None
and isinstance(end_user_mmb, dict)
and len(end_user_mmb) > 0
and current_models
and valid_token.end_user_id is not None
):
for model_name in current_models:
await model_max_budget_limiter.is_end_user_within_model_budget(
end_user_id=valid_token.end_user_id,
end_user_model_max_budget=end_user_mmb,
model=model_name,
team_id=valid_token.team_id,
)
# Check 6: Additional Common Checks across jwt + key auth
if valid_token.team_id is not None:
try:
@ -3733,9 +3775,7 @@ async def run_centralized_common_checks(
project_object: Final[LiteLLM_ProjectTableCachedObj | None] = (
None if isinstance(project_result, BaseException) else project_result
)
end_user_object: Final[LiteLLM_EndUserTable | None] = (
None if isinstance(end_user_result, BaseException) else end_user_result
)
end_user_object: Final = end_user_result if isinstance(end_user_result, LiteLLM_EndUserTable) else None
global_proxy_spend: float | None = None if isinstance(global_spend_result, BaseException) else global_spend_result
carry_team_and_user_budget_state(
valid_token=user_api_key_auth_obj,
@ -3795,6 +3835,21 @@ async def run_centralized_common_checks(
parent_otel_span=parent_otel_span,
keep_token_limits=user_custom_auth is not None,
)
end_user_model_max_budget: Final = await _end_user_model_max_budget(
end_user_id=end_user_id,
end_user_object=end_user_object,
key_end_user_budget_id=key_end_user_budget_id,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
parent_otel_span=parent_otel_span,
)
if end_user_id and (
not user_api_key_auth_obj.authenticated_by_custom_auth
or user_api_key_auth_obj.end_user_model_max_budget is None
):
user_api_key_auth_obj.end_user_model_max_budget = ( # rebind-ok: the spend hook charges from this token
dict(end_user_model_max_budget) if end_user_model_max_budget else None
)
skip_budget_checks: Final = _should_skip_budget_checks(
request_data=request_data,
@ -3887,6 +3942,21 @@ async def run_centralized_common_checks(
user_api_key_cache=user_api_key_cache,
proxy_logging_obj=proxy_logging_obj,
)
await _check_end_user_model_budget(
end_user_id=end_user_id,
end_user_model_max_budget=user_api_key_auth_obj.end_user_model_max_budget,
model_max_budget_limiter=model_max_budget_limiter,
team_id=user_api_key_auth_obj.team_id,
models=_get_model_names_for_budget_checks(
model=_get_model_from_request_context(
request_data=request_data,
route=route,
request=request,
llm_router=llm_router,
team_id=user_api_key_auth_obj.team_id,
)
),
)
if sub_call and RouteChecks.is_llm_api_route(route=route):
await virtual_key_max_budget_check(
valid_token=user_api_key_auth_obj,

View file

@ -5,16 +5,18 @@ import shutil
import subprocess
import sys
import tempfile
from collections.abc import Callable, Mapping, Sequence
from collections import Counter
from collections.abc import Callable, Hashable, Iterable, Mapping, Sequence
from dataclasses import dataclass
from itertools import chain
from pathlib import Path
from types import MappingProxyType
from typing import Final, TypeAlias, TypeVar
from urllib.parse import urlsplit
import click
import requests
from pydantic import JsonValue, TypeAdapter, ValidationError
from pydantic import ConfigDict, JsonValue, TypeAdapter, ValidationError
from litellm.proxy.common_utils.codex_model_catalog import (
CODEX_BASE_INSTRUCTIONS_PATH,
@ -497,6 +499,113 @@ def _fetch_model_listing(
return listing.data
class _DeploymentParams(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
model: str
custom_llm_provider: str | None = None
api_base: str | None = None
litellm_credential_name: str | None = None
reasoning_effort: JsonValue = None
class _DeploymentInfo(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
team_public_model_name: str | None = None
blocked: bool | None = None
class _Deployment(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
model_name: str
litellm_params: _DeploymentParams
model_info: _DeploymentInfo | None = None
class _DeploymentListing(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
data: tuple[_Deployment, ...]
@dataclass(frozen=True, slots=True)
class ModelUpstream:
"""The OpenAI model every deployment behind a public name routes to, and the reasoning effort they all pin."""
model: str
reasoning_effort: str | None = None
_NO_UPSTREAMS: Final[Mapping[str, ModelUpstream]] = MappingProxyType({})
_Pinned: Final = TypeVar("_Pinned", bound=Hashable)
def _public_name(deployment: _Deployment) -> str:
team_name: Final = deployment.model_info.team_public_model_name if deployment.model_info is not None else None
return team_name or deployment.model_name
def _is_paused(deployment: _Deployment) -> bool:
return deployment.model_info is not None and deployment.model_info.blocked is True
def _is_openai_host(params: _DeploymentParams) -> bool:
if params.api_base is None:
return not params.litellm_credential_name
host: Final = urlsplit(params.api_base).hostname or ""
return host == "api.openai.com" or host.endswith(".api.openai.com")
def _openai_model(params: _DeploymentParams) -> str | None:
if params.custom_llm_provider not in (None, "openai") or not _is_openai_host(params):
return None
provider, slash, name = params.model.partition("/")
if not slash:
return params.model
return name if provider == "openai" else None
def _pinned_effort(params: _DeploymentParams) -> str | None:
return params.reasoning_effort if isinstance(params.reasoning_effort, str) else None
def _shared_by_every_deployment(values: Iterable[tuple[str, _Pinned]]) -> Mapping[str, _Pinned]:
distinct: Final = frozenset(values)
values_per_name: Final = Counter(name for name, _ in distinct)
return MappingProxyType({name: value for name, value in distinct if values_per_name[name] == 1})
def _upstreams_by_public_name(deployments: Sequence[_Deployment]) -> Mapping[str, ModelUpstream]:
live: Final = tuple(d for d in deployments if not _is_paused(d))
models: Final = _shared_by_every_deployment((_public_name(d), _openai_model(d.litellm_params)) for d in live)
efforts: Final = _shared_by_every_deployment((_public_name(d), _pinned_effort(d.litellm_params)) for d in live)
return MappingProxyType(
{name: ModelUpstream(model, efforts.get(name)) for name, model in models.items() if model is not None}
)
def _fetch_model_upstreams(
base_url: str,
api_key: str,
*,
get: Callable[..., requests.Response],
) -> Mapping[str, ModelUpstream]:
url: Final = base_url.rstrip("/") + "/v1/model/info"
try:
resp: Final = get(url, headers=MappingProxyType({"Authorization": f"Bearer {api_key}"}), timeout=10)
except requests.RequestException:
return _NO_UPSTREAMS
if resp.status_code != 200:
return _NO_UPSTREAMS
try:
listing: Final = _DeploymentListing.model_validate_json(resp.content)
except ValidationError:
return _NO_UPSTREAMS
return _upstreams_by_public_name(listing.data)
class _OpenCodeLimit(LiteLLMBaseModel):
context: int
output: int
@ -576,8 +685,21 @@ def opencode_model_sync_env(
return MappingProxyType({OPENCODE_CONFIG_CONTENT_ENV: opencode_provider_config(base_url, listing)})
def _codex_row(model: ListedModel, upstream: ModelUpstream | None) -> CodexCatalogRow:
return CodexCatalogRow(
id=model.id,
mode=model.mode,
max_input_tokens=model.max_input_tokens,
upstream_model=upstream.model if upstream is not None else None,
reasoning_effort=upstream.reasoning_effort if upstream is not None else None,
)
def codex_model_catalog(
models: Sequence[ListedModel], stock: Sequence[CodexStockModel], instructions: str
models: Sequence[ListedModel],
stock: Sequence[CodexStockModel],
instructions: str,
upstreams: Mapping[str, ModelUpstream],
) -> str | None:
"""The `model_catalog_json` body listing the proxy's chat models, or None if there are none.
@ -586,14 +708,23 @@ def codex_model_catalog(
installed Codex knows keeps that Codex's own entry and the proxy only
decides its place in the picker: the listing orders it, lists it even when
Codex hides it or keeps it off the API, and keeps Codex's upgrade nudge only
when the model it points at is served too. A model Codex does not know gets the fallback
when the model it points at is served too. An alias of such a model, per
`upstreams`, takes that entry too, minus the defaults Codex picks for the
model: no service tier or upgrade nudge, the reasoning effort its
deployments pin, and a window no larger than its listed max_input_tokens.
A model Codex does not know gets the fallback
entry, with the same base instructions Codex itself uses so the agent never
runs without a system prompt. The catalog is the one `/v1/models?client_version=...`
serves, built from the installed Codex's own entries instead of the proxy's vendored ones.
runs without a system prompt. Apart from aliases, the catalog is the one
`/v1/models?client_version=...` serves, built from the installed Codex's own
entries instead of the proxy's vendored ones.
"""
rows: Final = tuple(CodexCatalogRow(id=m.id, mode=m.mode, max_input_tokens=m.max_input_tokens) for m in models)
rows: Final = tuple(_codex_row(m, upstreams.get(m.id)) for m in models)
body: Final = codex_models_response_json(
rows, stock=MappingProxyType({m.slug: m for m in stock}), instructions=instructions, byte_limit=None
rows,
stock=MappingProxyType({m.slug: m for m in stock}),
instructions=instructions,
byte_limit=None,
alias_defaults="deployment",
)
return body.json if body.listed else None
@ -678,12 +809,14 @@ def codex_model_sync_args(
Codex has no env or inline equivalent of OPENCODE_CONFIG_CONTENT: the catalog
must be a file, so it is written under $CODEX_HOME (default ~/.codex) and
atomically replaced on every launch. The Codex at `binary` first lists its
own models, so the ones the proxy serves keep that Codex's entries, and then
own models, so the ones the proxy serves keep that Codex's entries, an alias
included when every unpaused deployment /v1/model/info lists for it routes
to the same OpenAI model and Codex knows that model, and then
reads the file back once before it is handed over. The key never lands in
the file. A failed fetch, read, listing, write or read-back is reported
rather than raised: Codex still launches with its built-in catalog and takes
a proxy model by name via -m, and a rejected file stays on disk to be looked
at.
at. An unavailable /v1/model/info only leaves every alias on the fallback entry.
"""
listing: Final = _fetch_model_listing(base_url, api_key, get=get)
if isinstance(listing, ModelSyncSkipped):
@ -700,7 +833,8 @@ def codex_model_sync_args(
stock: Final = _stock_codex_models(binary, base_env, run=run)
if isinstance(stock, ModelSyncSkipped):
return stock
catalog: Final = codex_model_catalog(listing, stock, instructions)
upstreams: Final = _fetch_model_upstreams(base_url, api_key, get=get)
catalog: Final = codex_model_catalog(listing, stock, instructions, upstreams)
if catalog is None:
return ModelSyncSkipped(f"{base_url.rstrip('/')}/v1/models lists no chat models")
try:

View file

@ -24,7 +24,7 @@ from functools import cache
from itertools import accumulate, pairwise
from pathlib import Path
from types import MappingProxyType, NoneType
from typing import TYPE_CHECKING, Annotated, Final, Literal
from typing import TYPE_CHECKING, Annotated, Final, Literal, TypeAlias
from pydantic import ConfigDict, StringConstraints, TypeAdapter, ValidationError
@ -110,6 +110,12 @@ class CodexStockUpgrade(LiteLLMBaseModel):
model: str
class CodexReasoningLevel(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True, extra="allow")
effort: str
class CodexStockModel(LiteLLMBaseModel):
"""One `ModelInfo` entry as Codex ships it or prints it from `codex debug models`.
@ -128,6 +134,8 @@ class CodexStockModel(LiteLLMBaseModel):
upgrade: CodexStockUpgrade | None = None
service_tiers: tuple[CodexServiceTier, ...] = ()
default_service_tier: str | None = None
supported_reasoning_levels: tuple[CodexReasoningLevel, ...] = ()
context_window: int | None = None
class CodexStockCatalog(LiteLLMBaseModel):
@ -135,19 +143,21 @@ class CodexStockCatalog(LiteLLMBaseModel):
CodexCatalogEntry = CodexFallbackModel | CodexStockModel
CodexAliasDefaults: TypeAlias = Literal["codex", "deployment"]
@dataclass(frozen=True, slots=True)
class CodexCatalogRow:
"""What the catalog needs to know about one listed model: its public id, the listing's mode and
input limit, the upstream model behind it, and the `model_info` values the operator set;
`service_tiers` holds one raw `model_info.service_tiers` value per deployment behind the id, None
where a deployment sets none."""
input limit, the upstream model behind it, the reasoning effort its deployments pin, and the
`model_info` values the operator set; `service_tiers` holds one raw `model_info.service_tiers`
value per deployment behind the id, None where a deployment sets none."""
id: str
mode: str | None = None
max_input_tokens: int | None = None
upstream_model: str | None = None
reasoning_effort: str | None = None
display_name: str | None = None
service_tiers: tuple[object, ...] = ()
@ -254,12 +264,42 @@ def _stock_for(row: CodexCatalogRow, stock: Mapping[str, CodexStockModel]) -> Co
return upstream if upstream is not None else stock.get(row.upstream_model.partition("/")[2])
def _alias_context_window(row: CodexCatalogRow, stock_model: CodexStockModel) -> Mapping[str, int]:
if row.max_input_tokens is None or stock_model.context_window is None:
return MappingProxyType({})
return MappingProxyType({"context_window": min(stock_model.context_window, row.max_input_tokens)})
def _alias_reasoning_level(row: CodexCatalogRow, stock_model: CodexStockModel) -> str | None:
efforts: Final = frozenset(level.effort for level in stock_model.supported_reasoning_levels)
return row.reasoning_effort if row.reasoning_effort in efforts else None
def _deployment_defaults(row: CodexCatalogRow, stock_model: CodexStockModel) -> Mapping[str, object]:
"""What an alias's deployments decide in place of the defaults Codex picks for the model they route to:
no service tier, verbosity, upgrade nudge or startup promo of Codex's own, so what the deployment
pins (or OpenAI's API default) stays in force, the pinned reasoning effort when Codex offers it and
none otherwise, and a window no larger than the alias's own limit."""
return MappingProxyType(
{
"service_tiers": (),
"default_service_tier": None,
"default_verbosity": None,
"upgrade": None,
"availability_nux": None,
"default_reasoning_level": _alias_reasoning_level(row, stock_model),
**_alias_context_window(row, stock_model),
}
)
def _entry(
index: int,
row: CodexCatalogRow,
stock: Mapping[str, CodexStockModel],
served: frozenset[str],
instructions: str,
alias_defaults: CodexAliasDefaults,
) -> CodexCatalogEntry:
stock_model: Final = _stock_for(row, stock)
stock_tiers: Final = MappingProxyType(
@ -278,7 +318,8 @@ def _entry(
upgrade: Final = (
stock_model.upgrade if stock_model.upgrade is not None and stock_model.upgrade.model in served else None
)
stock_name: Final = stock_model.display_name if row.id == stock_model.slug else row.id
is_alias: Final = row.id != stock_model.slug
stock_name: Final = row.id if is_alias else stock_model.display_name
listing: Final = {
"slug": row.id,
"display_name": row.display_name if row.display_name is not None else stock_name,
@ -286,6 +327,7 @@ def _entry(
"visibility": "list",
"supported_in_api": True,
"upgrade": upgrade,
**(_deployment_defaults(row, stock_model) if is_alias and alias_defaults == "deployment" else {}),
}
if configured is None:
return stock_model.model_copy(update=listing)
@ -351,6 +393,7 @@ def codex_models_response_json(
stock: Mapping[str, CodexStockModel] | None = None,
instructions: str | None = None,
byte_limit: int | None = CODEX_CATALOG_BYTE_LIMIT,
alias_defaults: CodexAliasDefaults = "codex",
) -> CodexCatalogBody:
"""Codex's `ModelsResponse` body for `rows`, in listing order, cut to Codex's byte limit.
@ -359,13 +402,18 @@ def codex_models_response_json(
byte limit silently and keeps its bundled one, so the body holds only the entries that fit,
the ones offering a service tier taken first and the rest in listing order, an entry too large
for the bytes left passed over, every kept entry at its listing position; `byte_limit=None`
keeps every entry.
keeps every entry. A row listed under another name than the Codex model its `upstream_model`
names (an alias) takes that model's entry; `alias_defaults="deployment"` drops the defaults
Codex picks for that model in favor of what the alias's deployments pin, while `"codex"` keeps
them, as Codex would use them for the model's own name.
"""
known: Final = bundled_codex_models() if stock is None else stock
prompt: Final = codex_base_instructions() if instructions is None else instructions
drivable: Final = tuple(row for row in rows if _codex_can_drive(row))
served: Final = frozenset(row.id for row in drivable)
entries: Final = tuple(_entry(index, row, known, served, prompt) for index, row in enumerate(drivable))
entries: Final = tuple(
_entry(index, row, known, served, prompt, alias_defaults) for index, row in enumerate(drivable)
)
serialized: Final = tuple(_serialized(entry) for entry in entries)
kept: Final = _kept_positions(entries, serialized, byte_limit)
return CodexCatalogBody(

View file

@ -2,14 +2,17 @@
``type`` is a required string and ``param`` is nullable, neither of which the literal
string ``"None"`` satisfies."""
import sys
from collections.abc import Mapping
from types import MappingProxyType
from typing import Final
from fastapi import status
from fastapi import HTTPException, status
from litellm.constants import STRINGIFIED_NONE
from litellm.proxy._types import ProxyException
from litellm.proxy._types import ProxyErrorTypes, ProxyException, proxy_exception_status_code
from litellm.proxy.db.db_lookup_gate import DBLookupDeadlineExceeded
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler, exception_chain
LITELLM_CALL_ID_HEADER: Final = "x-litellm-call-id"
@ -21,6 +24,19 @@ _OPENAI_ERROR_TYPE_BY_STATUS: Final[Mapping[int, str]] = MappingProxyType(
}
)
_AUTH_FAILURE_STATUS_CODES: Final = frozenset({status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN})
_FAILOVER_SQLSTATES: Final = frozenset({"25006", "57P01", "57P02", "57P03"})
_FAILOVER_REFUSALS: Final = (
"is not currently accepting connections",
"the database system is starting up",
"the database system is shutting down",
"the database system is in recovery mode",
"the database system is not yet accepting connections",
"terminating connection due to administrator command",
)
def attribute_of(value: object, name: str, default: object = None) -> object:
return getattr(value, name, default)
@ -61,6 +77,82 @@ def openai_error_param(exc: object) -> str | None:
return carried if isinstance(carried, str) and carried != STRINGIFIED_NONE else None
def proxy_exception_for(exc: Exception, default_status_code: int) -> ProxyException:
match exc:
case ProxyException():
return exc
case _ if _is_database_outage(exc):
return ProxyException(
message=PrismaDBExceptionHandler.database_unavailable_message(exc),
type=ProxyErrorTypes.no_db_connection.value,
param=None,
code=status.HTTP_503_SERVICE_UNAVAILABLE,
)
case HTTPException(status_code=status_code, detail=detail):
return _openai_shaped_proxy_exception(exc, str(detail), status_code)
case _:
return _openai_shaped_proxy_exception(
exc, str(exc) or type(exc).__name__, error_status_code(exc, default_status_code)
)
def _is_database_outage(exc: Exception) -> bool:
return any(map(_is_database_outage_link, exception_chain(exc)))
def _is_database_outage_link(exc: BaseException) -> bool:
match exc:
case DBLookupDeadlineExceeded():
return True
case Exception() if _prisma_is_loaded():
return _is_prisma_outage(exc)
case _:
return False
def _prisma_is_loaded() -> bool:
return sys.modules.get("prisma") is not None
def _is_prisma_outage(exc: Exception) -> bool:
if PrismaDBExceptionHandler.is_prisma_engine_internal_error(exc):
return True
if type(exc).__module__.partition(".")[0] != "prisma":
return False
return PrismaDBExceptionHandler.is_database_service_unavailable_error(exc) or _is_failover_refusal(exc)
def _is_failover_refusal(exc: Exception) -> bool:
if PrismaDBExceptionHandler.is_read_only_transaction_error(exc):
return True
if PrismaDBExceptionHandler.postgres_sqlstate(exc) in _FAILOVER_SQLSTATES:
return True
message: Final = str(exc).lower()
return any(refusal in message for refusal in _FAILOVER_REFUSALS)
def _openai_shaped_proxy_exception(exc: Exception, message: str, status_code: int) -> ProxyException:
answered_status: Final = int(proxy_exception_status_code(message, str(status_code)))
return ProxyException(
message=message,
type=_catch_all_error_type(exc, answered_status),
param=openai_error_param(exc),
code=answered_status,
)
def _catch_all_error_type(exc: Exception, status_code: int) -> str:
if _rejects_the_caller(exc, status_code):
return ProxyErrorTypes.auth_error.value
return openai_error_type(exc, status_code)
def _rejects_the_caller(exc: Exception, status_code: int) -> bool:
raised_as_http_answer: Final = isinstance(exc, HTTPException)
status_from_upstream: Final = error_status_code(exc, 0) == status_code
return status_code in _AUTH_FAILURE_STATUS_CODES and (raised_as_http_answer or not status_from_upstream)
def litellm_call_id_headers(litellm_call_id: str | None) -> dict[str, str] | None: # mutable-ok: ProxyException.headers
if litellm_call_id is None:
return None

View file

@ -36,7 +36,7 @@ _LOCK_TIMEOUT_SQLSTATE: Final = "55P03"
_LOCK_TIMEOUT_PHRASE: Final = "canceling statement due to lock timeout"
def _exception_chain(e: BaseException) -> Iterator[BaseException]:
def exception_chain(e: BaseException) -> Iterator[BaseException]:
current = e # rebind-ok: advances one link per iteration of the bounded walk
for _ in range(_MAX_EXCEPTION_CHAIN_DEPTH):
yield current
@ -49,7 +49,7 @@ def _exception_chain(e: BaseException) -> Iterator[BaseException]:
def _database_service_unavailable_errors(e: BaseException) -> tuple[Exception, ...]:
return tuple(
link
for link in _exception_chain(e)
for link in exception_chain(e)
if isinstance(link, Exception) and PrismaDBExceptionHandler.is_database_service_unavailable_error(link)
)

View file

@ -1,11 +1,11 @@
from collections.abc import Mapping
from itertools import chain
from types import MappingProxyType
from typing import Final
from typing import Annotated, Final
from fastapi import APIRouter, Depends, HTTPException, Request, Response
from fastapi.responses import JSONResponse, ORJSONResponse
from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError
from pydantic import BaseModel, BeforeValidator, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError
from litellm.litellm_core_utils.json_validation_rule import normalize_json_schema_types
from litellm.proxy._types import *
@ -27,14 +27,67 @@ from litellm.proxy.common_utils.token_count_settings import count_source_fields
from litellm.types.llms.vertex_ai import TokenCountDetailsResponse
def _google_repeated_field(value: JsonValue) -> JsonValue:
match value:
case None:
return []
case dict():
return [value]
case list():
return [item for item in value if item is not None]
case _:
return value
def _google_content(value: JsonValue) -> JsonValue:
match value:
case {"parts": parts}:
return {**value, "parts": _google_repeated_field(parts)}
case _:
return value
def _google_contents(value: JsonValue) -> JsonValue:
match _google_repeated_field(value):
case list() as contents:
return [_google_content(content) for content in contents]
case other:
return other
_GOOGLE_PART_TEXT_AND_METADATA_FIELDS: Final = frozenset(
{"text", "thought", "thoughtSignature", "thought_signature", "partMetadata", "part_metadata"}
)
def _google_part_has_non_text_content(part: Mapping[str, JsonValue]) -> bool:
return any(field is not None for key, field in part.items() if key not in _GOOGLE_PART_TEXT_AND_METADATA_FIELDS)
def _google_system_part(value: JsonValue) -> JsonValue:
match value:
case dict() if value.get("text") is None and _google_part_has_non_text_content(value):
return {key: field for key, field in value.items() if key != "text"}
case dict():
return {"text": None, **value}
case _:
return value
_GOOGLE_REPEATED_FIELD: Final = BeforeValidator(_google_repeated_field)
class _GoogleCountTokensTextPart(BaseModel):
model_config = ConfigDict(frozen=True)
text: str
text: str | None
class _GoogleCountTokensSystemInstruction(BaseModel):
model_config = ConfigDict(frozen=True)
parts: tuple[_GoogleCountTokensTextPart, ...]
parts: Annotated[
tuple[Annotated[_GoogleCountTokensTextPart, BeforeValidator(_google_system_part)], ...],
_GOOGLE_REPEATED_FIELD,
] = ()
class _GoogleCountTokensFunctionDeclaration(BaseModel):
@ -47,16 +100,16 @@ class _GoogleCountTokensFunctionDeclaration(BaseModel):
class _GoogleCountTokensTool(BaseModel):
model_config = ConfigDict(frozen=True, populate_by_name=True)
function_declarations: tuple[_GoogleCountTokensFunctionDeclaration, ...] = Field(
function_declarations: Annotated[tuple[_GoogleCountTokensFunctionDeclaration, ...], _GOOGLE_REPEATED_FIELD] = Field(
default=(), alias="functionDeclarations"
)
class _GoogleCountTokensRequest(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore", populate_by_name=True)
contents: tuple[Mapping[str, JsonValue], ...] = ()
contents: Annotated[tuple[Mapping[str, JsonValue], ...], BeforeValidator(_google_contents)] = ()
system_instruction: _GoogleCountTokensSystemInstruction | None = Field(default=None, alias="systemInstruction")
tools: tuple[_GoogleCountTokensTool, ...] = ()
tools: Annotated[tuple[_GoogleCountTokensTool, ...], _GOOGLE_REPEATED_FIELD] = ()
router: Final = APIRouter(
@ -112,7 +165,11 @@ def _google_count_tokens_request(model_name: str, data: object) -> TokenCountReq
GoogleGenAIAdapter().translate_generate_content_to_completion(model_name, contents).get("messages", [])
)
system: Final = (
[{"type": "text", "text": part.text} for part in validated_data.system_instruction.parts]
[
{"type": "text", "text": part.text}
for part in validated_data.system_instruction.parts
if part.text is not None
]
if validated_data.system_instruction is not None
else None
)

View file

@ -32,7 +32,6 @@ from litellm.proxy._types import (
EnterpriseLicenseData,
Litellm_EntityType,
LitellmUserRoles,
ProxyErrorTypes,
ProxyException,
SpecialModelNames,
UserAPIKeyAuth,
@ -47,6 +46,7 @@ from litellm.proxy.auth.auth_utils import (
from litellm.proxy.auth.model_checks import get_key_models
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils import http_parsing_utils
from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for
from litellm.proxy.db.db_lookup_gate import db_lookup_stall_tracker
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
from litellm.proxy.db.health_check_latest import (
@ -683,21 +683,9 @@ async def health_services_endpoint(
except Exception as e:
verbose_proxy_logger.error("litellm.proxy.proxy_server.health_services_endpoint(): Exception occured - %s", e)
verbose_proxy_logger.debug(traceback.format_exc())
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "detail", f"Authentication Error({e})"),
type=ProxyErrorTypes.auth_error,
param=getattr(e, "param", "None"),
code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR),
)
elif isinstance(e, ProxyException):
raise e
raise ProxyException(
message="Authentication Error, " + str(e),
type=ProxyErrorTypes.auth_error,
param=getattr(e, "param", "None"),
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
if isinstance(e, ProxyException):
raise
raise proxy_exception_for(e, default_status_code=status.HTTP_500_INTERNAL_SERVER_ERROR) from e
def convert_health_check_to_dict(check) -> dict:

View file

@ -46,6 +46,7 @@ from litellm.proxy.auth.password_policy import (
)
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast
from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for
from litellm.proxy.common_utils.user_api_key_cache import (
AUTH_OBJECTS_TARGET,
object_permission_cache_key,
@ -1729,21 +1730,9 @@ async def user_update(
except Exception as e:
verbose_proxy_logger.exception("litellm.proxy.proxy_server.user_update(): Exception occured - %s", e)
verbose_proxy_logger.debug(traceback.format_exc())
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "detail", f"Authentication Error({e})"),
type=ProxyErrorTypes.auth_error,
param=getattr(e, "param", "None"),
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
)
elif isinstance(e, ProxyException):
raise e
raise ProxyException(
message="Authentication Error, " + str(e),
type=ProxyErrorTypes.auth_error,
param=getattr(e, "param", "None"),
code=status.HTTP_400_BAD_REQUEST,
)
if isinstance(e, ProxyException):
raise
raise proxy_exception_for(e, default_status_code=status.HTTP_400_BAD_REQUEST) from e
async def bulk_update_processed_users(

View file

@ -75,6 +75,7 @@ from litellm.proxy.common_utils.config_sync_pubsub import (
coordination_redis_cache,
publish_config_change,
)
from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for
from litellm.proxy.common_utils.rbac_utils import check_org_admin_can_generate_keys
from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time
from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET, UserApiKeyCache
@ -4087,21 +4088,9 @@ async def update_key_fn(
# update based on remaining passed in values
except Exception as e:
verbose_proxy_logger.exception("litellm.proxy.proxy_server.update_key_fn(): Exception occured - %s", e)
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "detail", f"Authentication Error({e})"),
type=ProxyErrorTypes.auth_error,
param=getattr(e, "param", "None"),
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
)
elif isinstance(e, ProxyException):
raise e
raise ProxyException(
message="Authentication Error, " + str(e),
type=ProxyErrorTypes.auth_error,
param=getattr(e, "param", "None"),
code=status.HTTP_400_BAD_REQUEST,
)
if isinstance(e, ProxyException):
raise
raise proxy_exception_for(e, default_status_code=status.HTTP_400_BAD_REQUEST) from e
@router.post(
@ -7244,21 +7233,9 @@ async def list_keys(
except Exception as e:
verbose_proxy_logger.exception("Error in list_keys: %s", e)
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "detail", f"error({e})"),
type=ProxyErrorTypes.internal_server_error,
param=getattr(e, "param", "None"),
code=getattr(e, "status_code", fastapi.status.HTTP_500_INTERNAL_SERVER_ERROR),
)
elif isinstance(e, ProxyException):
raise e
raise ProxyException(
message="Authentication Error, " + str(e),
type=ProxyErrorTypes.internal_server_error,
param=getattr(e, "param", "None"),
code=fastapi.status.HTTP_500_INTERNAL_SERVER_ERROR,
)
if isinstance(e, ProxyException):
raise
raise proxy_exception_for(e, default_status_code=fastapi.status.HTTP_500_INTERNAL_SERVER_ERROR) from e
async def _apply_non_admin_alias_scope(
@ -7402,21 +7379,9 @@ async def key_aliases(
except Exception as e:
verbose_proxy_logger.exception("Error in key_aliases: %s", e)
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "detail", f"error({e})"),
type=ProxyErrorTypes.internal_server_error,
param=getattr(e, "param", "None"),
code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR),
)
elif isinstance(e, ProxyException):
raise e
raise ProxyException(
message="Authentication Error, " + str(e),
type=ProxyErrorTypes.internal_server_error,
param=getattr(e, "param", "None"),
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
if isinstance(e, ProxyException):
raise
raise proxy_exception_for(e, default_status_code=status.HTTP_500_INTERNAL_SERVER_ERROR) from e
def _validate_sort_params(

View file

@ -77,6 +77,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import (
decrypt_value_helper,
encrypt_value_helper,
)
from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient
from litellm.proxy.management.teams.authz import TEAM_ADMIN_ONLY, is_team_admin
@ -2845,21 +2846,9 @@ async def delete_model(
except Exception as e:
verbose_proxy_logger.exception("Failed to delete model. Due to error - %s", e)
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "detail", f"Authentication Error({e})"),
type=ProxyErrorTypes.auth_error,
param=getattr(e, "param", "None"),
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
)
elif isinstance(e, ProxyException):
raise e
raise ProxyException(
message="Authentication Error, " + str(e),
type=ProxyErrorTypes.auth_error,
param=getattr(e, "param", "None"),
code=status.HTTP_400_BAD_REQUEST,
)
if isinstance(e, ProxyException):
raise
raise proxy_exception_for(e, default_status_code=status.HTTP_400_BAD_REQUEST) from e
async def delete_team_model_alias(
@ -2899,6 +2888,13 @@ async def delete_team_model_alias(
return removed_model_aliases
def _model_not_added_to_db() -> HTTPException:
return HTTPException(
status_code=500,
detail={"error": "Failed to add model to db. Check your server logs for more details."},
)
@router.post(
"/model/new",
description="Allows adding new models to the model list in the config.yaml",
@ -3080,6 +3076,8 @@ async def add_new_model(
if isinstance(e, HTTPException):
raise
verbose_proxy_logger.exception("Exception in add_new_model: %s", e)
if model_response is None:
raise _model_not_added_to_db() from e
else:
raise HTTPException(
@ -3088,10 +3086,7 @@ async def add_new_model(
)
if model_response is None:
raise HTTPException(
status_code=500,
detail={"error": "Failed to add model to db. Check your server logs for more details."},
)
raise _model_not_added_to_db()
## CREATE AUDIT LOG ##
asyncio.create_task(
@ -3121,21 +3116,9 @@ async def add_new_model(
except Exception as e:
verbose_proxy_logger.exception("litellm.proxy.proxy_server.add_new_model(): Exception occured - %s", e)
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "detail", f"Authentication Error({e})"),
type=ProxyErrorTypes.auth_error,
param=getattr(e, "param", "None"),
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
)
elif isinstance(e, ProxyException):
raise e
raise ProxyException(
message="Authentication Error, " + str(e),
type=ProxyErrorTypes.auth_error,
param=getattr(e, "param", "None"),
code=status.HTTP_400_BAD_REQUEST,
)
if isinstance(e, ProxyException):
raise
raise proxy_exception_for(e, default_status_code=status.HTTP_400_BAD_REQUEST) from e
#### MODEL MANAGEMENT ####
@ -3364,21 +3347,9 @@ async def update_model(
return None if model_response is None else _model_write_response(model_response, member_write)
except Exception as e:
verbose_proxy_logger.exception("litellm.proxy.proxy_server.update_model(): Exception occured - %s", e)
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "detail", f"Authentication Error({e})"),
type=ProxyErrorTypes.auth_error,
param=getattr(e, "param", "None"),
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
)
elif isinstance(e, ProxyException):
raise e
raise ProxyException(
message="Authentication Error, " + str(e),
type=ProxyErrorTypes.auth_error,
param=getattr(e, "param", "None"),
code=status.HTTP_400_BAD_REQUEST,
)
if isinstance(e, ProxyException):
raise
raise proxy_exception_for(e, default_status_code=status.HTTP_400_BAD_REQUEST) from e
@router.post(

View file

@ -41,6 +41,7 @@ from litellm.proxy.auth.auth_checks import (
)
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast
from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for
from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
from litellm.proxy.management_endpoints.budget_management_endpoints import (
@ -1396,21 +1397,9 @@ async def organization_member_add(
)
except Exception as e:
verbose_proxy_logger.exception("Error adding member to organization: %s", e)
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "detail", f"Authentication Error({e})"),
type=ProxyErrorTypes.auth_error,
param=getattr(e, "param", "None"),
code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR),
)
elif isinstance(e, ProxyException):
raise e
raise ProxyException(
message="Authentication Error, " + str(e),
type=ProxyErrorTypes.auth_error,
param=getattr(e, "param", "None"),
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
if isinstance(e, ProxyException):
raise
raise proxy_exception_for(e, default_status_code=status.HTTP_500_INTERNAL_SERVER_ERROR) from e
async def find_member_if_email(user_email: str, prisma_client: PrismaClient) -> LiteLLM_UserTable:

View file

@ -132,6 +132,7 @@ from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import (
)
from litellm.proxy.common_utils.callback_utils import encrypt_callback_vars
from litellm.proxy.common_utils.json_merge_patch import apply_json_merge_patch
from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for
from litellm.proxy.common_utils.user_api_key_cache import (
UserApiKeyCache,
team_membership_auth_cache_key,
@ -5526,21 +5527,9 @@ async def team_info(
e,
traceback.format_exc(),
)
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "detail", f"Authentication Error({e})"),
type=ProxyErrorTypes.auth_error,
param=getattr(e, "param", "None"),
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
)
elif isinstance(e, ProxyException):
raise e
raise ProxyException(
message="Authentication Error, " + str(e),
type=ProxyErrorTypes.auth_error,
param=getattr(e, "param", "None"),
code=status.HTTP_400_BAD_REQUEST,
)
if isinstance(e, ProxyException):
raise
raise proxy_exception_for(e, default_status_code=status.HTTP_400_BAD_REQUEST) from e
@router.get(

View file

@ -524,6 +524,7 @@ from litellm.proxy.common_utils.openai_error_payload import (
LITELLM_CALL_ID_HEADER,
headers_with_litellm_call_id,
litellm_call_id_headers,
proxy_exception_for,
with_litellm_call_id,
)
from litellm.proxy.common_utils.periodic_reload_schedule import (
@ -18536,25 +18537,13 @@ async def async_queue_request(
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
)
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "detail", f"Authentication Error({e})"),
type=ProxyErrorTypes.auth_error,
param=getattr(e, "param", "None"),
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
)
elif isinstance(e, ProxyException):
raise e
elif isinstance(e, litellm.BadRequestError):
if isinstance(e, litellm.BadRequestError):
raise ProxyException(
message=e.message, type="invalid_request_error", param=getattr(e, "param", None), code=e.status_code
)
raise ProxyException(
message="Authentication Error, " + str(e),
type=ProxyErrorTypes.auth_error,
param=getattr(e, "param", "None"),
code=status.HTTP_400_BAD_REQUEST,
)
) from e
if isinstance(e, ProxyException):
raise
raise proxy_exception_for(e, default_status_code=status.HTTP_400_BAD_REQUEST) from e
@app.get("/fallback/login", tags=["experimental"], include_in_schema=False)
@ -19833,21 +19822,9 @@ async def update_config(
except Exception as e:
verbose_proxy_logger.error("litellm.proxy.proxy_server.update_config(): Exception occured - %s", e)
verbose_proxy_logger.debug(traceback.format_exc())
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "detail", f"Authentication Error({e})"),
type=ProxyErrorTypes.auth_error,
param=getattr(e, "param", "None"),
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
)
elif isinstance(e, ProxyException):
raise e
raise ProxyException(
message="Authentication Error, " + str(e),
type=ProxyErrorTypes.auth_error,
param=getattr(e, "param", "None"),
code=status.HTTP_400_BAD_REQUEST,
)
if isinstance(e, ProxyException):
raise
raise proxy_exception_for(e, default_status_code=status.HTTP_400_BAD_REQUEST) from e
### CONFIG GENERAL SETTINGS
@ -21030,21 +21007,9 @@ async def get_config(
}
except Exception as e:
verbose_proxy_logger.exception("litellm.proxy.proxy_server.get_config(): Exception occured - %s", e)
if isinstance(e, HTTPException):
raise ProxyException(
message=getattr(e, "detail", f"Authentication Error({e})"),
type=ProxyErrorTypes.auth_error,
param=getattr(e, "param", "None"),
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
)
elif isinstance(e, ProxyException):
raise e
raise ProxyException(
message="Authentication Error, " + str(e),
type=ProxyErrorTypes.auth_error,
param=getattr(e, "param", "None"),
code=status.HTTP_400_BAD_REQUEST,
)
if isinstance(e, ProxyException):
raise
raise proxy_exception_for(e, default_status_code=status.HTTP_500_INTERNAL_SERVER_ERROR) from e
@router.get(

View file

@ -746,7 +746,7 @@ def test_internal_user_model_creation_prohibition_refuses_team_admin_but_not_pro
"error": {
"message": "Model creation is disabled for internal users by disable_model_add_for_internal_users.",
"type": "auth_error",
"param": "None",
"param": None,
"code": "403",
}
},

View file

@ -38,7 +38,7 @@ def _escalation_error(field: str) -> dict[str, JsonValue]:
"error": {
"message": f"{{'error': \"Non-admin users cannot modify '{field}' on their own record. Contact your proxy admin.\"}}",
"type": "auth_error",
"param": "None",
"param": None,
"code": "403",
}
}
@ -49,7 +49,7 @@ def _role_change_error() -> dict[str, JsonValue]:
"error": {
"message": "Only proxy admins can modify user roles.",
"type": "auth_error",
"param": "None",
"param": None,
"code": "403",
}
}

View file

@ -255,8 +255,8 @@ def _team_not_found_body(team_id: str) -> dict[str, JsonValue]:
return {
"error": {
"message": f"{{'message': 'Team not found, passed team id: {team_id}.'}}",
"type": "auth_error",
"param": "None",
"type": "invalid_request_error",
"param": None,
"code": "404",
}
}

View file

@ -158,6 +158,18 @@ def test_native_function_response_preserves_json_when_media_is_malformed(invalid
assert messages == [{"role": "tool", "tool_call_id": "call_lookup", "content": '{"ok": true}'}]
@pytest.mark.parametrize("role", ["user", "model"])
def test_one_object_content_parts_translate_like_their_list_form(role: str) -> None:
part: Final = {"text": "Reply with OK"}
def messages(parts: object) -> object:
return GoogleGenAIAdapter().translate_generate_content_to_completion(
model="anthropic/claude-haiku-5-5", contents=[{"role": role, "parts": parts}]
)["messages"]
assert messages(part) == messages([part]) != []
@pytest.mark.parametrize(
("args", "expected_arguments"),
[(None, "null"), ('{"query":"x"}', '"{\\"query\\":\\"x\\"}"')],

View file

@ -10,7 +10,7 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
import litellm
from litellm.llms.anthropic.messages.adapters.handler import _run_polyfill_if_enabled
from litellm.llms.anthropic.messages.adapters.handler import _run_polyfill_if_enabled, prepare_context_managed_request
from litellm.llms.anthropic.messages.context_management import (
AnthropicContextManagementError,
PolyfillResult,
@ -690,6 +690,115 @@ def test_tool_uses_trigger_does_not_count_excluded_tools(trigger_value: int, tri
assert (applied is not None) == triggers
@pytest.mark.parametrize(
"system",
["Reference notes. " * 400, [{"type": "text", "text": "Reference notes. " * 400}]],
ids=["string", "blocks"],
)
def test_input_tokens_trigger_counts_the_system_prompt(system: str | list[dict[str, object]]) -> None:
"""Like the API, the trigger compares the whole prompt: a system prompt can push it over."""
messages = _make_history(n_pairs=3)
history_tokens: Final = litellm.token_counter(model=MODEL, messages=messages)
edit_spec: Final = {
"type": "clear_tool_uses_20250919",
"trigger": {"type": "input_tokens", "value": history_tokens + 100},
"keep": {"type": "tool_uses", "value": 1},
}
_, without_system = apply_clear_tool_uses_20250919(
model=MODEL, messages=deepcopy(messages), tools=None, system=None, edit_spec=edit_spec
)
_, with_system = apply_clear_tool_uses_20250919(
model=MODEL, messages=deepcopy(messages), tools=None, system=system, edit_spec=edit_spec
)
_, met_without_system = apply_clear_tool_uses_20250919(
model=MODEL,
messages=deepcopy(messages),
tools=None,
system=None,
edit_spec={**edit_spec, "trigger": {"type": "input_tokens", "value": 0}},
)
assert without_system is None
assert with_system is not None
assert met_without_system is not None
assert with_system["cleared_tool_uses"] == 2
assert with_system["cleared_input_tokens"] == met_without_system["cleared_input_tokens"] > 0
@pytest.mark.parametrize("compact_position", [None, "after", "before"], ids=["alone", "before_compact", "after_compact"])
async def test_input_tokens_trigger_counts_a_client_compaction_summary(compact_position: str | None) -> None:
"""A client compaction summary counts toward the trigger and calls beside it stay clearable, in any edit order."""
tail: Final = _make_history(n_pairs=3)
edit_spec: Final = {
"type": "clear_tool_uses_20250919",
"trigger": {"type": "input_tokens", "value": litellm.token_counter(model=MODEL, messages=tail) + 100},
"keep": {"type": "tool_uses", "value": 1},
}
compaction: Final = {"type": "compaction", "content": "Earlier work. " * 400}
compacted_history: Final = [tail[0], {**tail[1], "content": [compaction, *tail[1]["content"]]}, *tail[2:]]
compact_spec: Final = {"type": "compact_20260112", "trigger": {"type": "input_tokens", "value": 50_000}}
edits: Final = {None: [edit_spec], "after": [edit_spec, compact_spec], "before": [compact_spec, edit_spec]}[
compact_position
]
async def prepare(messages: list[dict[str, object]]) -> PolyfillResult | None:
return await prepare_context_managed_request(
model=MODEL,
messages=messages,
tools=None,
system=None,
context_management_spec={"edits": edits},
litellm_metadata=None,
additional_drop_params=None,
llm_router=None,
)
def cleared(result: PolyfillResult | None) -> list[object]:
applied: Final = result.applied_edits if result is not None else []
return [edit.get("cleared_tool_uses") for edit in applied if edit.get("type") == "clear_tool_uses_20250919"]
without_summary: Final = await prepare(deepcopy(tail))
with_summary: Final = await prepare(deepcopy(compacted_history))
assert cleared(without_summary) == []
assert cleared(with_summary) == [2]
async def test_compact_after_clearing_replaces_the_client_compaction_summary(monkeypatch: pytest.MonkeyPatch) -> None:
from litellm.proxy.proxy_server import general_settings
monkeypatch.setitem(general_settings, COMPACT_SUMMARY_MODEL_SETTING_KEY, "summary-model")
summary: Final = MagicMock()
summary.choices = [MagicMock(message=MagicMock(content="<summary>Fresh summary</summary>"))]
summary.usage = MagicMock(prompt_tokens=50, completion_tokens=100)
router: Final = MagicMock()
router.acompletion = AsyncMock(return_value=summary)
tail: Final = _make_history(n_pairs=3)
result: Final = await prepare_context_managed_request(
model=MODEL,
messages=[{"role": "assistant", "content": [{"type": "compaction", "content": "Old summary. " * 20_000}]}, *tail],
tools=None,
system="You are a helpful assistant.",
context_management_spec={
"edits": [
{
"type": "clear_tool_uses_20250919",
"trigger": {"type": "input_tokens", "value": 1_000},
"keep": {"type": "tool_uses", "value": 1},
},
{"type": "compact_20260112", "trigger": {"type": "input_tokens", "value": 50_000}},
]
},
litellm_metadata=None,
additional_drop_params=None,
llm_router=router,
)
assert result is not None
assert isinstance(result.system, str)
assert "Fresh summary" in result.system
assert "Old summary" not in result.system
assert result.system.endswith("You are a helpful assistant.")
@pytest.mark.parametrize(("clear_tool_inputs", "triggers"), [(False, False), (True, True)])
def test_tool_uses_trigger_counts_tiny_results_only_when_their_inputs_are_cleared(
clear_tool_inputs: bool, triggers: bool

View file

@ -151,6 +151,7 @@ async def test_sync_editor_counts_tokens_off_the_event_loop():
{
"type": "clear_tool_uses_20250919",
"trigger": {"type": "input_tokens", "value": 10_000_000},
"keep": {"type": "tool_uses", "value": 0},
}
]
},

View file

@ -5,7 +5,7 @@ import os
import subprocess
import sys
import time
from collections.abc import Mapping, Sequence
from collections.abc import Awaitable, Callable, Mapping, Sequence
from contextlib import contextmanager, nullcontext
from datetime import datetime, timedelta, timezone
from functools import partial
@ -37,6 +37,7 @@ from litellm.proxy.hooks.model_max_budget_limiter import (
team_member_budget_entity_id,
)
from litellm.proxy._types import (
JWTAuthBuilderResult,
LiteLLMRoutes,
LiteLLM_JWTAuth,
LiteLLM_BudgetTable,
@ -55,7 +56,7 @@ from litellm.proxy._types import (
UserAPIKeyAuth,
JWTRoutingOverride,
)
from litellm.proxy.auth.handle_jwt import JWTHandler
from litellm.proxy.auth.handle_jwt import JWTAuthManager, JWTHandler
from litellm.proxy.auth.auth_checks import (
OrganizationNotFoundError,
TeamNotFoundError,
@ -724,6 +725,44 @@ async def test_user_custom_auth_runs_post_custom_auth_checks_when_opt_in():
litellm.enable_post_custom_auth_checks = original_flag
async def _authenticate_through_custom_auth() -> UserAPIKeyAuth:
from starlette.datastructures import URL
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth_builder
request: Final = Request(scope={"type": "http"})
request._url = URL(url="/chat/completions")
return await user_api_key_auth_builder(
request=request,
api_key="Bearer sk-custom-auth-shared",
azure_api_key_header="",
anthropic_api_key_header=None,
google_ai_studio_api_key_header=None,
azure_apim_header=None,
request_data={},
)
@pytest.mark.asyncio
async def test_user_custom_auth_returning_one_shared_token_keeps_each_requests_end_user_apart(
monkeypatch: pytest.MonkeyPatch,
) -> None:
import litellm.proxy.proxy_server as _proxy_server_mod
shared_token: Final = UserAPIKeyAuth(api_key="sk-custom-auth-shared", user_id="custom-user-123")
proxy_attrs: Final = _proxy_server_attrs_for_custom_auth(user_custom_auth=AsyncMock(return_value=shared_token))
for attr, val in proxy_attrs.items():
monkeypatch.setattr(_proxy_server_mod, attr, val, raising=False)
monkeypatch.setattr(litellm, "enable_post_custom_auth_checks", False, raising=False)
capped_end_users_request: Final = await _authenticate_through_custom_auth()
capped_end_users_request.end_user_id = "eu-capped"
capped_end_users_request.end_user_model_max_budget = {"gpt-4o": {"max_budget": 0.00001}}
next_end_users_request: Final = await _authenticate_through_custom_auth()
assert (next_end_users_request.end_user_id, next_end_users_request.end_user_model_max_budget) == (None, None)
@pytest.mark.asyncio
async def test_enterprise_custom_auth_skips_post_custom_auth_checks_by_default():
"""
@ -5293,6 +5332,559 @@ async def test_centralized_common_checks_enforces_team_model_max_budget_from_the
assert limiter.calls == [("t1", team_caps, key_caps, "gpt-4o")]
_END_USER_MODEL_CAPS: Final = {"gpt-4o": {"budget_limit": 0.00004, "time_period": "1d"}}
def _virtual_key_token_for_end_user() -> UserAPIKeyAuth:
return UserAPIKeyAuth(api_key="sk-test", token="hashed", end_user_id="eu-1")
def _master_key_token_cached_with_another_end_users_cap() -> UserAPIKeyAuth:
return UserAPIKeyAuth(
api_key=LITELLM_PROXY_MASTER_KEY_ALIAS,
user_id="admin",
user_role=LitellmUserRoles.PROXY_ADMIN,
end_user_id="eu-1",
end_user_model_max_budget={"gpt-4o": {"budget_limit": 1000.0, "time_period": "1d"}},
)
def _end_user_linked_to_the_capped_budget() -> LiteLLM_EndUserTable:
return LiteLLM_EndUserTable(
user_id="eu-1",
blocked=False,
budget_id="eu-budget",
litellm_budget_table=LiteLLM_BudgetTable(budget_id="eu-budget", model_max_budget=_END_USER_MODEL_CAPS),
)
def _end_user_on_the_proxy_default_budget() -> LiteLLM_EndUserTable:
return LiteLLM_EndUserTable(user_id="eu-1", blocked=False)
def _no_end_user_row_yet() -> None:
return None
def _jwt_token(end_user_id: str | None, end_user_object: LiteLLM_EndUserTable | None) -> UserAPIKeyAuth:
return JWTAuthManager.user_api_key_auth_from_result(
JWTAuthBuilderResult(
is_proxy_admin=False,
team_object=None,
user_object=None,
end_user_object=end_user_object,
org_object=None,
token="jwt",
team_id=None,
user_id=None,
user_email=None,
end_user_id=end_user_id,
org_id=None,
team_membership=None,
jwt_claims={},
agent_id=None,
)
)
def _jwt_token_with_an_end_user_claim() -> UserAPIKeyAuth:
return _jwt_token(end_user_id="eu-1", end_user_object=_end_user_linked_to_the_capped_budget())
def _jwt_token_naming_the_end_user_only_in_the_body() -> UserAPIKeyAuth:
return _jwt_token(end_user_id=None, end_user_object=None)
async def _run_centralized_checks_for_end_user(
monkeypatch: pytest.MonkeyPatch,
token: UserAPIKeyAuth,
end_user: LiteLLM_EndUserTable | None,
gpt_4o_spend: float,
model: str,
route: str,
default_budget_id: str | None,
user_custom_auth: Callable[[Request, str], Awaitable[UserAPIKeyAuth]] | None,
) -> None:
from starlette.datastructures import URL
import litellm.proxy.proxy_server as _proxy_server_mod
from litellm.proxy.common_utils.user_api_key_cache import end_user_cache_key
from litellm.proxy.utils import ProxyLogging
monkeypatch.setattr(litellm, "max_end_user_budget_id", default_budget_id)
user_api_key_cache: Final = UserApiKeyCache()
if end_user is not None:
await user_api_key_cache.async_set_cache(
key=end_user_cache_key("eu-1"),
value=end_user,
model_type=LiteLLM_EndUserTable,
)
for budget_id in ("eu-default", "eu-key-budget"):
await user_api_key_cache.async_set_cache(
key=f"default_end_user_budget:{budget_id}",
value=LiteLLM_BudgetTable(budget_id=budget_id, model_max_budget=_END_USER_MODEL_CAPS),
model_type=LiteLLM_BudgetTable,
)
await user_api_key_cache.async_set_cache(
key="admin",
value=LiteLLM_UserTable(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN),
model_type=LiteLLM_UserTable,
)
limiter: Final = PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=DualCache())
await limiter.dual_cache.async_set_cache(
key=model_budget_spend_cache_key(
entity_type=Litellm_EntityType.END_USER,
entity_id="eu-1",
budget_model="gpt-4o",
budget_duration="1d",
),
value=gpt_4o_spend,
)
router: Final = litellm.Router(
model_list=[
{"model_name": "gpt-4o", "litellm_params": {"model": "openai/gpt-4o"}},
{"model_name": "gpt-4o-mini", "litellm_params": {"model": "openai/gpt-4o-mini"}},
],
)
attrs: Final = {
**_proxy_attrs_for_centralized_checks(user_custom_auth=user_custom_auth),
"general_settings": {
"disable_budget_reservation": True,
"custom_auth_run_common_checks": user_custom_auth is not None,
},
"llm_model_list": router.model_list,
"llm_router": router,
"prisma_client": MagicMock(),
"user_api_key_cache": user_api_key_cache,
"spend_counter_cache": DualCache(),
"proxy_logging_obj": ProxyLogging(user_api_key_cache=user_api_key_cache),
"model_max_budget_limiter": limiter,
}
originals: Final = {attribute: getattr(_proxy_server_mod, attribute, None) for attribute in attrs}
request: Final = Request(scope={"type": "http"})
request._url = URL(url=route)
try:
for attribute, value in attrs.items():
setattr(_proxy_server_mod, attribute, value)
await run_centralized_common_checks(
user_api_key_auth_obj=token,
request=request,
request_data={"model": model, "user": "eu-1", "messages": [{"role": "user", "content": "hi"}]},
route=route,
)
finally:
for attribute, value in originals.items():
setattr(_proxy_server_mod, attribute, value)
@pytest.mark.parametrize(
("make_token", "make_end_user", "gpt_4o_spend", "model", "route", "must_raise"),
[
pytest.param(
_virtual_key_token_for_end_user,
_end_user_linked_to_the_capped_budget,
0.0001,
"gpt-4o",
"/v1/chat/completions",
True,
id="virtual-key-over-cap",
),
pytest.param(
_master_key_token_cached_with_another_end_users_cap,
_end_user_linked_to_the_capped_budget,
0.0001,
"gpt-4o",
"/v1/chat/completions",
True,
id="master-key-over-cap",
),
pytest.param(
_jwt_token_with_an_end_user_claim,
_end_user_linked_to_the_capped_budget,
0.0001,
"gpt-4o",
"/v1/chat/completions",
True,
id="jwt-claim-over-cap",
),
pytest.param(
_jwt_token_naming_the_end_user_only_in_the_body,
_end_user_linked_to_the_capped_budget,
0.0001,
"gpt-4o",
"/v1/chat/completions",
True,
id="jwt-body-user-over-cap",
),
pytest.param(
_virtual_key_token_for_end_user,
_no_end_user_row_yet,
0.0001,
"gpt-4o",
"/v1/chat/completions",
True,
id="new-end-user-on-the-proxy-default-budget-over-cap",
),
pytest.param(
_virtual_key_token_for_end_user,
_end_user_on_the_proxy_default_budget,
0.0001,
"gpt-4o",
"/v1/chat/completions",
True,
id="proxy-default-budget-over-cap",
),
pytest.param(
_virtual_key_token_for_end_user,
_end_user_linked_to_the_capped_budget,
0.00001,
"gpt-4o",
"/v1/chat/completions",
False,
id="under-cap",
),
pytest.param(
_virtual_key_token_for_end_user,
_end_user_linked_to_the_capped_budget,
0.0001,
"gpt-4o-mini",
"/v1/chat/completions",
False,
id="model-without-a-cap",
),
pytest.param(
_master_key_token_cached_with_another_end_users_cap,
_end_user_linked_to_the_capped_budget,
0.0001,
"gpt-4o",
"/spend/calculate",
False,
id="budget-exempt-route",
),
],
)
@pytest.mark.asyncio
async def test_centralized_common_checks_enforces_the_end_users_model_max_budget(
monkeypatch: pytest.MonkeyPatch,
make_token: Callable[[], UserAPIKeyAuth],
make_end_user: Callable[[], LiteLLM_EndUserTable | None],
gpt_4o_spend: float,
model: str,
route: str,
must_raise: bool,
) -> None:
token: Final = make_token()
with pytest.raises(litellm.BudgetExceededError) if must_raise else nullcontext():
await _run_centralized_checks_for_end_user(
monkeypatch=monkeypatch,
token=token,
end_user=make_end_user(),
gpt_4o_spend=gpt_4o_spend,
model=model,
route=route,
default_budget_id="eu-default",
user_custom_auth=None,
)
assert token.end_user_model_max_budget == _END_USER_MODEL_CAPS
def _virtual_key_token_assigning_end_users_a_budget() -> UserAPIKeyAuth:
return UserAPIKeyAuth(
api_key="sk-test", token="hashed", end_user_id="eu-1", metadata={"end_user_budget_id": "eu-key-budget"}
)
@pytest.mark.parametrize(
("make_token", "make_end_user"),
[
pytest.param(
_virtual_key_token_for_end_user,
_end_user_linked_to_the_capped_budget,
id="end-users-own-budget",
),
pytest.param(
_virtual_key_token_assigning_end_users_a_budget,
_end_user_on_the_proxy_default_budget,
id="keys-end-user-budget",
),
pytest.param(
_virtual_key_token_assigning_end_users_a_budget,
_no_end_user_row_yet,
id="keys-end-user-budget-for-a-new-end-user",
),
],
)
@pytest.mark.asyncio
async def test_centralized_common_checks_enforces_the_end_users_model_max_budget_without_a_proxy_default(
monkeypatch: pytest.MonkeyPatch,
make_token: Callable[[], UserAPIKeyAuth],
make_end_user: Callable[[], LiteLLM_EndUserTable | None],
) -> None:
token: Final = make_token()
with pytest.raises(litellm.BudgetExceededError):
await _run_centralized_checks_for_end_user(
monkeypatch=monkeypatch,
token=token,
end_user=make_end_user(),
gpt_4o_spend=0.0001,
model="gpt-4o",
route="/v1/chat/completions",
default_budget_id=None,
user_custom_auth=None,
)
assert token.end_user_model_max_budget == _END_USER_MODEL_CAPS
async def _custom_auth_that_is_never_called(request: Request, api_key: str) -> UserAPIKeyAuth:
raise AssertionError("the shared checks never call custom auth")
def _token_built_by_custom_auth_with_its_own_cap() -> UserAPIKeyAuth:
return UserAPIKeyAuth(
api_key="sk-custom", end_user_id="eu-1", end_user_model_max_budget=_END_USER_MODEL_CAPS
).model_copy(update={"authenticated_by_custom_auth": True})
def _master_key_token_custom_auth_handed_back_with_another_end_users_cap() -> UserAPIKeyAuth:
return UserAPIKeyAuth(
api_key=LITELLM_PROXY_MASTER_KEY_ALIAS,
user_id="admin",
user_role=LitellmUserRoles.PROXY_ADMIN,
end_user_id="eu-1",
end_user_model_max_budget=_END_USER_MODEL_CAPS,
)
@pytest.mark.parametrize(
("make_token", "must_raise", "expected_cap"),
[
pytest.param(
_token_built_by_custom_auth_with_its_own_cap,
True,
_END_USER_MODEL_CAPS,
id="custom-auth-token-keeps-its-own-cap",
),
pytest.param(
_master_key_token_custom_auth_handed_back_with_another_end_users_cap,
False,
None,
id="key-auth-token-behind-custom-auth-drops-another-end-users-cap",
),
],
)
@pytest.mark.asyncio
async def test_centralized_common_checks_keeps_only_a_cap_custom_auth_set(
monkeypatch: pytest.MonkeyPatch,
make_token: Callable[[], UserAPIKeyAuth],
must_raise: bool,
expected_cap: Mapping[str, object] | None,
) -> None:
token: Final = make_token()
with pytest.raises(litellm.BudgetExceededError) if must_raise else nullcontext():
await _run_centralized_checks_for_end_user(
monkeypatch=monkeypatch,
token=token,
end_user=_end_user_on_the_proxy_default_budget(),
gpt_4o_spend=0.0001,
model="gpt-4o",
route="/v1/chat/completions",
default_budget_id=None,
user_custom_auth=_custom_auth_that_is_never_called,
)
assert token.end_user_model_max_budget == expected_cap
@pytest.mark.asyncio
async def test_centralized_common_checks_holds_the_end_user_to_the_model_a_team_member_fallback_picked(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from starlette.datastructures import URL
import litellm.proxy.proxy_server as _proxy_server_mod
from litellm.proxy.common_utils.user_api_key_cache import end_user_cache_key
from litellm.proxy.utils import ProxyLogging
budget_id: Final = "member-budget-1"
user_api_key_cache: Final = UserApiKeyCache()
await user_api_key_cache.async_set_cache(
key=team_membership_reservation_cache_key(user_id="u1", team_id="t1"),
value=LiteLLM_TeamMembership(user_id="u1", team_id="t1", budget_id=budget_id),
model_type=LiteLLM_TeamMembership,
)
await user_api_key_cache.async_set_cache(
key="team_id:t1",
value=LiteLLM_TeamTableCachedObj(
team_id="t1",
models=["gpt-4o", "gpt-4o-mini"],
metadata={"team_member_budget_id": budget_id},
),
model_type=LiteLLM_TeamTableCachedObj,
)
await user_api_key_cache.async_set_cache(
key=f"team_member_default_budget:{budget_id}",
value=LiteLLM_BudgetTable(
budget_id=budget_id,
model_max_budget={
"gpt-4o": {"max_budget": 0.00004, "budget_duration": "1d"},
"gpt-4o-mini": {"max_budget": 100.0, "budget_duration": "1d"},
},
),
model_type=LiteLLM_BudgetTable,
)
await user_api_key_cache.async_set_cache(
key="u1",
value=LiteLLM_UserTable(user_id="u1", user_role=LitellmUserRoles.INTERNAL_USER),
model_type=LiteLLM_UserTable,
)
await user_api_key_cache.async_set_cache(
key=end_user_cache_key("eu-1"),
value=_end_user_linked_to_the_capped_budget(),
model_type=LiteLLM_EndUserTable,
)
limiter: Final = PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=DualCache())
for entity_type, entity_id in (
(Litellm_EntityType.TEAM_MEMBER, team_member_budget_entity_id(user_id="u1", team_id="t1")),
(Litellm_EntityType.END_USER, "eu-1"),
):
await limiter.dual_cache.async_set_cache(
key=model_budget_spend_cache_key(
entity_type=entity_type,
entity_id=entity_id,
budget_model="gpt-4o",
budget_duration="1d",
),
value=0.0001,
)
router: Final = litellm.Router(
model_list=[
{"model_name": "gpt-4o", "litellm_params": {"model": "openai/gpt-4o"}},
{"model_name": "gpt-4o-mini", "litellm_params": {"model": "openai/gpt-4o-mini"}},
],
fallbacks=[{"gpt-4o": ["gpt-4o-mini"]}],
)
attrs: Final = {
**_proxy_attrs_for_centralized_checks(),
"general_settings": {"disable_budget_reservation": True},
"llm_model_list": router.model_list,
"llm_router": router,
"prisma_client": MagicMock(),
"user_api_key_cache": user_api_key_cache,
"spend_counter_cache": DualCache(),
"proxy_logging_obj": ProxyLogging(user_api_key_cache=user_api_key_cache),
"model_max_budget_limiter": limiter,
}
originals: Final = {attribute: getattr(_proxy_server_mod, attribute, None) for attribute in attrs}
request: Final = Request(scope={"type": "http", "path_params": {}, "headers": []})
request._url = URL(url="/v1/chat/completions")
request_data: Final = {"model": "gpt-4o", "user": "eu-1", "messages": [{"role": "user", "content": "hi"}]}
try:
for attribute, value in attrs.items():
setattr(_proxy_server_mod, attribute, value)
await run_centralized_common_checks(
user_api_key_auth_obj=UserAPIKeyAuth(
api_key="sk-test",
token="hashed",
user_id="u1",
team_id="t1",
end_user_id="eu-1",
models=["gpt-4o", "gpt-4o-mini"],
user_role=LitellmUserRoles.INTERNAL_USER,
),
request=request,
request_data=request_data,
route="/v1/chat/completions",
)
finally:
for attribute, value in originals.items():
setattr(_proxy_server_mod, attribute, value)
assert request_data["model"] == "gpt-4o-mini"
def _access_groups_only_team_t1_puts_gpt_4o_in(model: str, team_id: str | None) -> frozenset[str]:
return frozenset({"t1-group"}) if (model, team_id) == ("gpt-4o", "t1") else frozenset()
@pytest.mark.parametrize(
("team_id", "must_raise"),
[
pytest.param("t1", True, id="team-whose-access-group-the-end-user-capped"),
pytest.param(None, False, id="no-team-so-the-group-cap-does-not-apply"),
],
)
@pytest.mark.asyncio
async def test_centralized_common_checks_applies_an_end_user_cap_on_the_tokens_team_access_group(
team_id: str | None,
must_raise: bool,
) -> None:
from starlette.datastructures import URL
import litellm.proxy.proxy_server as _proxy_server_mod
from litellm.proxy.common_utils.user_api_key_cache import end_user_cache_key
from litellm.proxy.utils import ProxyLogging
group_cap: Final = {"t1-group": {"budget_limit": 0.00004, "time_period": "1d"}}
user_api_key_cache: Final = UserApiKeyCache()
await user_api_key_cache.async_set_cache(
key="team_id:t1",
value=LiteLLM_TeamTableCachedObj(team_id="t1", models=["gpt-4o"]),
model_type=LiteLLM_TeamTableCachedObj,
)
await user_api_key_cache.async_set_cache(
key=end_user_cache_key("eu-1"),
value=LiteLLM_EndUserTable(
user_id="eu-1",
blocked=False,
budget_id="eu-group-budget",
litellm_budget_table=LiteLLM_BudgetTable(budget_id="eu-group-budget", model_max_budget=group_cap),
),
model_type=LiteLLM_EndUserTable,
)
limiter: Final = PROXY_VirtualKeyModelMaxBudgetLimiter(
dual_cache=DualCache(), model_access_groups=_access_groups_only_team_t1_puts_gpt_4o_in
)
await limiter.dual_cache.async_set_cache(
key=model_budget_spend_cache_key(
entity_type=Litellm_EntityType.END_USER,
entity_id="eu-1",
budget_model="t1-group",
budget_duration="1d",
),
value=0.0001,
)
router: Final = litellm.Router(model_list=[{"model_name": "gpt-4o", "litellm_params": {"model": "openai/gpt-4o"}}])
attrs: Final = {
**_proxy_attrs_for_centralized_checks(),
"general_settings": {"disable_budget_reservation": True},
"llm_model_list": router.model_list,
"llm_router": router,
"prisma_client": MagicMock(),
"user_api_key_cache": user_api_key_cache,
"spend_counter_cache": DualCache(),
"proxy_logging_obj": ProxyLogging(user_api_key_cache=user_api_key_cache),
"model_max_budget_limiter": limiter,
}
originals: Final = {attribute: getattr(_proxy_server_mod, attribute, None) for attribute in attrs}
request: Final = Request(scope={"type": "http", "path_params": {}, "headers": []})
request._url = URL(url="/v1/chat/completions")
try:
for attribute, value in attrs.items():
setattr(_proxy_server_mod, attribute, value)
with pytest.raises(litellm.BudgetExceededError) if must_raise else nullcontext():
await run_centralized_common_checks(
user_api_key_auth_obj=UserAPIKeyAuth(
api_key="sk-test", token="hashed", team_id=team_id, end_user_id="eu-1", models=["gpt-4o"]
),
request=request,
request_data={"model": "gpt-4o", "user": "eu-1", "messages": [{"role": "user", "content": "hi"}]},
route="/v1/chat/completions",
)
finally:
for attribute, value in originals.items():
setattr(_proxy_server_mod, attribute, value)
@pytest.mark.asyncio
async def test_centralized_common_checks_loads_member_budget_and_exempts_discovery_routes():
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache

View file

@ -92,6 +92,8 @@ _STOCK_MODELS = {
"base_instructions": "You are Codex, a coding agent based on GPT-5.6.",
"apply_patch_tool_type": "freeform",
"supports_parallel_tool_calls": True,
"supports_search_tool": True,
"tool_mode": "code_mode_only",
"context_window": 272000,
"comp_hash": "terra-hash",
},
@ -669,10 +671,16 @@ class TestCodexModelSync:
def _row(model_id, **extra):
return {"id": model_id, "object": "model", "created": 1, "owned_by": "openai", **extra}
def _sync(self, listing, codex_home, base_url="http://localhost:4000/", run=None):
def _sync(self, listing, codex_home, base_url="http://localhost:4000/", run=None, model_info=None):
captured = {}
def fake_get(url, headers, timeout):
if headers.get("Authorization") != "Bearer sk-key":
return _FakeResponse(401)
if url.endswith("/v1/model/info"):
if isinstance(model_info, Exception):
raise model_info
return model_info if model_info is not None else _FakeResponse(200, {"data": []})
captured["url"] = url
captured["headers"] = headers
return _FakeResponse(200, listing)
@ -737,6 +745,202 @@ class TestCodexModelSync:
assert entries["gpt-5.4"]["upgrade"] == _STOCK_MODELS["gpt-5.4"]["upgrade"]
assert [entries["gpt-5.4"]["priority"], entries["gpt-5.6-terra"]["priority"]] == [0, 1]
@pytest.mark.parametrize(
"deployment",
[
{"model_name": "api-terra", "litellm_params": {"model": "openai/gpt-5.6-terra"}},
{"model_name": "api-terra", "litellm_params": {"model": "gpt-5.6-terra"}, "model_info": None},
{
"model_name": "api-terra",
"litellm_params": {"model": "openai/gpt-5.6-terra"},
"model_info": {"id": "d1", "team_public_model_name": ""},
},
{
"model_name": "model_name_team-1_d1",
"litellm_params": {"model": "openai/gpt-5.6-terra", "api_base": "https://api.openai.com/v1"},
"model_info": {"id": "d1", "team_public_model_name": "api-terra"},
},
],
ids=["provider_prefixed", "bare_without_model_info", "empty_team_public_name", "team_public_model_name"],
)
def test_alias_of_a_codex_model_gets_that_models_entry_under_its_own_name(self, tmp_path, deployment):
listing = self._listing(self._row("api-terra"), self._row("m"))
_, result = self._sync(listing, tmp_path, model_info=_FakeResponse(200, {"data": [deployment]}))
assert isinstance(result, ModelSyncArgs)
entries = self._entries(tmp_path)
assert entries["api-terra"] == {
**_STOCK_MODELS["gpt-5.6-terra"],
"slug": "api-terra",
"display_name": "api-terra",
"priority": 0,
"service_tiers": [],
"default_service_tier": None,
"default_verbosity": None,
"availability_nux": None,
"default_reasoning_level": None,
}
assert "supports_search_tool" not in entries["m"]
def test_alias_gets_no_upgrade_nudge_away_from_its_own_name(self, tmp_path):
listing = self._listing(self._row("gpt-5.6-terra"), self._row("api-54"))
deployment = {"model_name": "api-54", "litellm_params": {"model": "openai/gpt-5.4"}}
self._sync(listing, tmp_path, model_info=_FakeResponse(200, {"data": [deployment]}))
entry = self._entries(tmp_path)["api-54"]
assert (entry["comp_hash"], entry["upgrade"]) == (_STOCK_MODELS["gpt-5.4"]["comp_hash"], None)
@pytest.mark.parametrize(
("pins", "expected"),
[
(("high",), "high"),
(("high", "high"), "high"),
(("high", None), None),
(("high", "low"), None),
(({"effort": "high"},), None),
],
ids=["one_deployment", "every_deployment_agrees", "one_unpinned", "pins_differ", "non_string_pin"],
)
def test_alias_starts_at_the_reasoning_effort_every_deployment_pins(self, tmp_path, pins, expected):
deployments = [
{
"model_name": "api-terra",
"litellm_params": {"model": "openai/gpt-5.6-terra", **({"reasoning_effort": p} if p else {})},
}
for p in pins
]
self._sync(
self._listing(self._row("api-terra")), tmp_path, model_info=_FakeResponse(200, {"data": deployments})
)
entry = self._entries(tmp_path)["api-terra"]
assert (entry["comp_hash"], entry["default_reasoning_level"]) == ("terra-hash", expected)
@pytest.mark.parametrize(
"deployments",
[
[
{"model_name": "api-terra", "litellm_params": {"model": "openai/gpt-5.6-terra"}},
{"model_name": "api-terra", "litellm_params": {"model": "gpt-5.6-terra"}},
],
[
{
"model_name": "api-terra",
"litellm_params": {"model": "gpt-5.6-terra", "custom_llm_provider": "openai"},
}
],
[
{
"model_name": "api-terra",
"litellm_params": {"model": "openai/gpt-5.5"},
"model_info": {"id": "paused", "blocked": True},
},
{"model_name": "api-terra", "litellm_params": {"model": "openai/gpt-5.6-terra"}},
],
[
{
"model_name": "api-terra",
"litellm_params": {"model": "openai/gpt-5.6-terra", "api_base": "https://eu.api.openai.com/v1"},
}
],
[
{
"model_name": "api-terra",
"litellm_params": {
"model": "openai/gpt-5.6-terra",
"api_base": "https://api.openai.com/v1",
"litellm_credential_name": "openai-key",
},
}
],
],
ids=[
"same_model_with_and_without_prefix",
"openai_custom_llm_provider",
"paused_deployment_ignored",
"openai_regional_host",
"credential_under_an_openai_host",
],
)
def test_alias_whose_deployments_all_route_to_one_openai_model_gets_its_entry(self, tmp_path, deployments):
self._sync(
self._listing(self._row("api-terra")), tmp_path, model_info=_FakeResponse(200, {"data": deployments})
)
assert self._entries(tmp_path)["api-terra"]["comp_hash"] == _STOCK_MODELS["gpt-5.6-terra"]["comp_hash"]
@pytest.mark.parametrize(
"deployments",
[
[
{"model_name": "api-terra", "litellm_params": {"model": "openai/gpt-5.5"}},
{
"model_name": "model_name_team-1_d1",
"litellm_params": {"model": "openai/gpt-5.6-terra"},
"model_info": {"id": "d1", "team_public_model_name": "api-terra"},
},
],
[
{"model_name": "api-terra", "litellm_params": {"model": "openai/gpt-5.6-terra"}},
{"model_name": "api-terra", "litellm_params": {"model": "anthropic/claude-opus-4-7"}},
],
[{"model_name": "api-terra", "litellm_params": {"model": "azure/gpt-5.6-terra"}}],
[{"model_name": "api-terra", "litellm_params": {"model": "gpt-5.6-terra", "custom_llm_provider": "azure"}}],
[
{
"model_name": "api-terra",
"litellm_params": {"model": "openai/gpt-5.6-terra", "api_base": "http://vllm.example.invalid/v1"},
}
],
[
{
"model_name": "api-terra",
"litellm_params": {"model": "gpt-5.6-terra", "api_base": "https://api.openai.com.example.invalid"},
}
],
[
{
"model_name": "api-terra",
"litellm_params": {"model": "openai/gpt-5.6-terra", "litellm_credential_name": "gateway"},
}
],
],
ids=[
"team_row_and_global_row_differ",
"load_balanced_across_providers",
"azure_prefix",
"azure_provider",
"openai_prefix_on_another_host",
"openai_lookalike_host",
"host_only_a_credential_knows",
],
)
def test_alias_without_one_shared_openai_upstream_keeps_the_fallback_entry(self, tmp_path, deployments):
listing = self._listing(self._row("api-terra"))
self._sync(listing, tmp_path / "with", model_info=_FakeResponse(200, {"data": deployments}))
self._sync(listing, tmp_path / "without")
assert self._entries(tmp_path / "with")["api-terra"] == self._entries(tmp_path / "without")["api-terra"]
@pytest.mark.parametrize(
"model_info",
[
requests.ConnectionError("refused"),
_FakeResponse(500, {"data": [{"model_name": "api-terra", "litellm_params": {"model": "gpt-5.6-terra"}}]}),
_FakeResponse(200, {"data": "nope"}),
_FakeResponse(200, {"data": [{"model_name": "api-terra"}]}),
],
ids=["unreachable", "http_500", "unexpected_body", "deployment_without_litellm_params"],
)
def test_unavailable_model_info_keeps_the_alias_on_the_fallback_entry(self, tmp_path, model_info):
_, result = self._sync(self._listing(self._row("api-terra")), tmp_path, model_info=model_info)
catalog = tmp_path / "litellm-models.json"
assert result == ModelSyncArgs(("-c", f"model_catalog_json={json.dumps(str(catalog))}"))
entry = self._entries(tmp_path)["api-terra"]
assert entry["slug"] == "api-terra"
assert "supports_search_tool" not in entry
def test_stock_catalog_is_decoded_as_utf8_regardless_of_locale(self, tmp_path):
description = "Modelo equilibrado para el trabajo diario, con acentos y ñ."
catalog = {"models": [{**_STOCK_MODELS["gpt-5.5"], "description": description}]}

View file

@ -135,6 +135,84 @@ def test_alias_of_a_known_upstream_model_takes_stock_metadata_under_its_own_name
assert (entry["slug"], entry["display_name"]) == ("team-55", "team-55")
@pytest.mark.parametrize(("max_input_tokens", "expected"), [(100000, 100000), (922000, 272000), (None, 272000)])
def test_alias_with_deployment_defaults_caps_codexs_window_at_its_listed_max_input_tokens(max_input_tokens, expected):
stock = MappingProxyType({"gpt-5.5": _stock("gpt-5.5").model_copy(update={"context_window": 272000})})
alias = _row("team-55", upstream_model="openai/gpt-5.5", max_input_tokens=max_input_tokens)
body = codex_models_response_json((alias,), stock=stock, instructions=_PROMPT, alias_defaults="deployment")
assert json.loads(body.json)["models"][0]["context_window"] == expected
def test_alias_with_deployment_defaults_drops_codexs_tier_and_upgrade_while_the_native_name_keeps_them():
rows = (_row("gpt-6-sol"), _row("team-sol", upstream_model="gpt-6-sol"), _row("team-55", upstream_model="gpt-5.5"))
codex = {entry["slug"]: entry for entry in _models(*rows)}
deployment = {entry["slug"]: entry for entry in _models(*rows, alias_defaults="deployment")}
assert deployment["gpt-6-sol"] == codex["gpt-6-sol"]
assert (codex["team-sol"]["default_service_tier"], codex["team-55"]["upgrade"]) == (
"priority",
{"model": "gpt-6-sol"},
)
assert _tiers(codex["team-sol"]) == _tiers(codex["gpt-6-sol"]) != []
assert [
(deployment[slug]["service_tiers"], deployment[slug]["default_service_tier"])
for slug in ("team-sol", "team-55")
] == [
([], None),
([], None),
]
assert deployment["team-55"]["upgrade"] is None
assert deployment["team-sol"]["model_messages"] == codex["team-sol"]["model_messages"]
@pytest.mark.parametrize(("pinned", "expected"), [("high", "high"), ("turbo", None), (None, None)])
def test_alias_with_deployment_defaults_starts_only_at_a_reasoning_effort_its_deployments_pin(pinned, expected):
stock = CodexStockModel.model_validate(
{
"slug": "gpt-5.5",
"display_name": "GPT-5.5",
"priority": 1,
"visibility": "list",
"default_reasoning_level": "medium",
"supported_reasoning_levels": [{"effort": e, "description": e} for e in ("low", "medium", "high")],
}
)
alias = _row("team-55", upstream_model="gpt-5.5", reasoning_effort=pinned)
body = codex_models_response_json(
(alias,), stock=MappingProxyType({"gpt-5.5": stock}), instructions=_PROMPT, alias_defaults="deployment"
)
assert json.loads(body.json)["models"][0]["default_reasoning_level"] == expected
def test_alias_with_deployment_defaults_sends_no_verbosity_or_promo_of_codexs_while_the_native_name_keeps_them():
stock = CodexStockModel.model_validate(
{
"slug": "gpt-6-sol",
"display_name": "GPT-6 Sol",
"priority": 1,
"visibility": "list",
"support_verbosity": True,
"default_verbosity": "low",
"availability_nux": {"message": "GPT-6 Sol is here"},
}
)
rows = (_row("gpt-6-sol"), _row("team-sol", upstream_model="gpt-6-sol"))
body = codex_models_response_json(
rows, stock=MappingProxyType({"gpt-6-sol": stock}), instructions=_PROMPT, alias_defaults="deployment"
)
native, alias = json.loads(body.json)["models"]
assert [(e["support_verbosity"], e["default_verbosity"], e["availability_nux"]) for e in (native, alias)] == [
(True, "low", {"message": "GPT-6 Sol is here"}),
(True, None, None),
]
@pytest.mark.parametrize(
("row", "expected"),
[

View file

@ -1,16 +1,26 @@
import json
import sys
from typing import Final
import httpx
import prisma.errors
import pytest
from fastapi import HTTPException
from prisma._async_http import Response
from prisma.engine.utils import handle_response_errors
import litellm
from litellm.proxy._types import ProxyErrorTypes, ProxyException
from litellm.proxy.db.db_lookup_gate import DBLookupDeadlineExceeded
from litellm.proxy.common_utils.openai_error_payload import (
error_status_code,
litellm_call_id_headers,
openai_error_param,
openai_error_type,
proxy_exception_for,
with_litellm_call_id,
)
from litellm.types.router import RouterErrors
@pytest.mark.parametrize(
@ -203,3 +213,287 @@ def test_a_proxy_error_keeps_the_call_id_it_was_raised_with():
assert with_litellm_call_id(raised_with_id, "second").headers == {"x-litellm-call-id": "first"}
assert with_litellm_call_id(ProxyException(message="nope", type="None", param=None, code=400), None).headers == {}
@pytest.mark.parametrize(
"raised_status, expected_type",
[
(400, "invalid_request_error"),
(404, "invalid_request_error"),
(401, "auth_error"),
(403, "auth_error"),
(422, "invalid_request_error"),
(500, "internal_server_error"),
],
)
def test_a_catch_all_handler_keeps_the_status_its_route_answered_with(raised_status: int, expected_type: str) -> None:
wrapped: Final = proxy_exception_for(
HTTPException(status_code=raised_status, detail={"message": "boom"}),
default_status_code=400,
)
assert (wrapped.code, wrapped.type) == (str(raised_status), expected_type)
@pytest.mark.parametrize(
"default_status_code, expected_type", [(400, "invalid_request_error"), (500, "internal_server_error")]
)
def test_an_exception_carrying_no_status_falls_back_to_the_routes_default(default_status_code: int, expected_type: str) -> None:
wrapped: Final = proxy_exception_for(ValueError("boom"), default_status_code=default_status_code)
assert (wrapped.code, wrapped.type) == (str(default_status_code), expected_type)
def test_a_proxy_exception_passes_through_with_the_type_it_already_named() -> None:
raised: Final = ProxyException(
message="Budget has been exceeded",
type=ProxyErrorTypes.budget_exceeded.value,
param="max_budget",
code=400,
)
assert proxy_exception_for(raised, default_status_code=500) is raised
def test_the_message_is_what_the_route_raised_without_an_invented_prefix() -> None:
assert proxy_exception_for(ValueError("boom"), default_status_code=400).message == "boom"
def test_an_http_exceptions_detail_survives_the_wrapping() -> None:
wrapped: Final = proxy_exception_for(
HTTPException(status_code=400, detail={"error": "Invalid budget_duration 'not-a-duration'."}),
default_status_code=400,
)
assert "Invalid budget_duration" in wrapped.message
def test_a_carried_type_survives_the_wrapping() -> None:
class _Carrier(Exception):
type: str = "context_window_exceeded"
status_code: int = 400
assert proxy_exception_for(_Carrier("boom"), default_status_code=500).type == "context_window_exceeded"
@pytest.mark.parametrize("exc", [HTTPException(status_code=404, detail="nope"), ValueError("boom")])
def test_the_wrapped_error_serializes_as_an_openai_error_object(exc: Exception) -> None:
body: Final = json.loads(json.dumps({"error": proxy_exception_for(exc, default_status_code=400).to_dict()}))
assert body["error"]["param"] is None
assert body["error"]["type"] == "invalid_request_error"
@pytest.mark.parametrize(
"provider_failure, expected",
[
(
litellm.AuthenticationError(message="invalid x-api-key", llm_provider="anthropic", model="claude-haiku-5-5"),
("401", "authentication_error"),
),
(
litellm.PermissionDeniedError(
message="model not allowed",
llm_provider="anthropic",
model="claude-haiku-5-5",
response=httpx.Response(403, request=httpx.Request("POST", "https://api.anthropic.com/v1/messages")),
),
("403", "permission_error"),
),
],
)
def test_a_provider_credential_failure_is_labelled_the_way_chat_completions_labels_it(
provider_failure: Exception, expected: tuple[str, str]
) -> None:
wrapped: Final = proxy_exception_for(provider_failure, default_status_code=400)
assert (wrapped.code, wrapped.type) == expected
@pytest.mark.parametrize("status_code", [401, 403])
def test_a_rejection_the_route_raised_keeps_the_auth_label(status_code: int) -> None:
wrapped: Final = proxy_exception_for(HTTPException(status_code=status_code, detail="no"), default_status_code=400)
assert (wrapped.code, wrapped.type) == (str(status_code), "auth_error")
def test_an_exception_without_a_message_is_named_by_its_type() -> None:
wrapped: Final = proxy_exception_for(TimeoutError(), default_status_code=500)
assert wrapped.message == "TimeoutError"
def test_a_database_outage_answers_503_so_callers_retry_instead_of_fixing_their_request() -> None:
wrapped: Final = proxy_exception_for(
prisma.errors.PrismaError("Can't reach database server at `db:5432`"), default_status_code=400
)
assert (wrapped.code, wrapped.type) == ("503", "no_db_connection")
@pytest.mark.parametrize(
"postgres_refusal",
[
'FATAL: database "litellm" is not currently accepting connections',
"FATAL: the database system is starting up",
"FATAL: the database system is shutting down",
"FATAL: the database system is in recovery mode",
"FATAL: the database system is not yet accepting connections",
"FATAL: terminating connection due to administrator command",
"ERROR: cannot execute UPDATE in a read-only transaction",
],
)
def test_a_database_refusing_work_during_a_failover_answers_503(postgres_refusal: str) -> None:
refused: Final = prisma.errors.DataError(
{"user_facing_error": {"message": f"Error in connector: Error querying the database: {postgres_refusal}"}}
)
wrapped: Final = proxy_exception_for(refused, default_status_code=400)
assert (wrapped.code, wrapped.type) == ("503", "no_db_connection")
@pytest.mark.parametrize(
"localized_refusal",
[
{
"error_code": "P2010",
"message": "Raw query failed. Message: `FEHLER: kann UPDATE nicht in einer Nur-Lese-Transaktion ausführen`",
"meta": {"code": "25006"},
},
{
"message": 'PostgresError { code: "57P01", message: "Verbindung wird abgebrochen", severity: "FATAL" }',
},
],
)
def test_a_failover_refusal_in_a_localized_postgres_answers_503(localized_refusal: dict[str, object]) -> None:
refused: Final = prisma.errors.DataError({"user_facing_error": localized_refusal})
wrapped: Final = proxy_exception_for(refused, default_status_code=400)
assert (wrapped.code, wrapped.type) == ("503", "no_db_connection")
def test_a_failover_phrase_outside_the_database_is_not_reported_as_a_database_outage() -> None:
callback_down: Final = ValueError("FATAL: the database system is starting up")
wrapped: Final = proxy_exception_for(callback_down, default_status_code=400)
assert (wrapped.code, wrapped.type) == ("400", "invalid_request_error")
def test_a_database_dropped_mid_query_answers_503_though_prisma_fails_to_name_the_error() -> None:
engine_reply_with_null_meta: Final = [{"user_facing_error": {"error_code": "P1001", "message": "", "meta": None}}]
with pytest.raises(AttributeError) as raised:
handle_response_errors(Response(httpx.Response(500)), engine_reply_with_null_meta)
wrapped: Final = proxy_exception_for(raised.value, default_status_code=500)
assert (wrapped.code, wrapped.type) == ("503", "no_db_connection")
def test_a_network_failure_outside_the_database_is_not_reported_as_a_database_outage() -> None:
callback_unreachable: Final = httpx.ConnectError("[Errno 111] Connection refused")
wrapped: Final = proxy_exception_for(callback_unreachable, default_status_code=500)
assert (wrapped.code, wrapped.type) == ("500", "internal_server_error")
def _add_member_failing_with(db_error: Exception) -> None:
try:
raise db_error
except Exception as e:
raise ValueError(f"Error adding member to organization: {e}")
@pytest.mark.parametrize(
"underlying_error, expected",
[
(prisma.errors.PrismaError("Can't reach database server at `db:5432`"), ("503", "no_db_connection")),
(httpx.ConnectError("[Errno 111] Connection refused"), ("500", "internal_server_error")),
],
)
def test_a_route_that_rewraps_its_error_keeps_the_database_outage_it_wrapped(
underlying_error: Exception, expected: tuple[str, str]
) -> None:
with pytest.raises(ValueError, match="Error adding member") as raised:
_add_member_failing_with(underlying_error)
wrapped: Final = proxy_exception_for(raised.value, default_status_code=500)
assert (wrapped.code, wrapped.type) == expected
def test_a_proxy_without_prisma_installed_still_answers_with_an_error_object(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setitem(sys.modules, "prisma", None)
wrapped: Final = proxy_exception_for(httpx.ConnectError("[Errno 111] Connection refused"), default_status_code=500)
assert (wrapped.code, wrapped.type) == ("500", "internal_server_error")
def _look_up_team_failing_with(lookup_error: Exception) -> None:
try:
raise lookup_error
except Exception:
raise HTTPException(status_code=404, detail="Team doesn't exist in db")
@pytest.mark.parametrize(
"lookup_error, expected",
[
(DBLookupDeadlineExceeded("team", 10.0), ("503", "no_db_connection")),
(prisma.errors.PrismaError("Can't reach database server at `db:5432`"), ("503", "no_db_connection")),
(KeyError("team_id"), ("404", "invalid_request_error")),
],
)
def test_a_not_found_answer_raised_while_the_database_was_down_reports_the_outage(
lookup_error: Exception, expected: tuple[str, str]
) -> None:
with pytest.raises(HTTPException) as raised:
_look_up_team_failing_with(lookup_error)
wrapped: Final = proxy_exception_for(raised.value, default_status_code=400)
assert (wrapped.code, wrapped.type) == expected
def test_a_lookup_that_missed_its_deadline_answers_503() -> None:
wrapped: Final = proxy_exception_for(DBLookupDeadlineExceeded("team", 10.0), default_status_code=400)
assert (wrapped.code, wrapped.type) == ("503", "no_db_connection")
def test_a_database_outage_answer_does_not_reveal_where_the_database_lives() -> None:
wrapped: Final = proxy_exception_for(
prisma.errors.PrismaError("Can't reach database server at `prod-db.internal:5432`"), default_status_code=400
)
assert wrapped.code == "503"
assert "prod-db.internal" not in wrapped.message
def test_an_upstream_failure_answers_with_its_own_status_not_the_routes_default() -> None:
upstream_overloaded: Final = litellm.ServiceUnavailableError(
message="upstream overloaded", llm_provider="openai", model="gpt-5.4-mini"
)
wrapped: Final = proxy_exception_for(upstream_overloaded, default_status_code=400)
assert (wrapped.code, wrapped.type) == ("503", "internal_server_error")
@pytest.mark.parametrize(
"router_rejection, expected",
[
(f"{RouterErrors.no_deployments_with_tag_routing.value}. Passed model=gpt-5.4-mini", ("401", "auth_error")),
(f"{RouterErrors.no_deployments_available.value}. Passed model=gpt-5.4-mini", ("429", "rate_limit_error")),
],
)
def test_a_router_rejection_is_labelled_by_the_status_it_answers_with(
router_rejection: str, expected: tuple[str, str]
) -> None:
wrapped: Final = proxy_exception_for(ValueError(router_rejection), default_status_code=400)
assert (wrapped.code, wrapped.type) == expected

View file

@ -6,7 +6,7 @@ from typing import Final
import pytest
from dotenv import load_dotenv
from pydantic import JsonValue
from pydantic import JsonValue, ValidationError
from starlette.requests import Request
from litellm.proxy.google_endpoints.endpoints import _google_count_tokens_request, google_count_tokens
@ -105,3 +105,91 @@ async def test_proxy_gemini_to_openai_like_model_token_counting():
)
assert response.get("totalTokens") > 0
_DECLARATION: Final = {"name": "lookup", "description": "Find a record"}
_SYSTEM: Final = {"systemInstruction": {"parts": [{"text": "Follow these rules"}]}}
_NO_SYSTEM_PARTS: Final = {"systemInstruction": {"parts": []}}
_TOOLS: Final = {"tools": [{"functionDeclarations": [_DECLARATION]}]}
_TURN: Final = {"role": "user", "parts": [{"text": "Count this request"}]}
_CONTENTS: Final = {"contents": [_TURN]}
@pytest.mark.parametrize(
("spelled", "canonical"),
(
({"systemInstruction": {"parts": {"text": "Follow these rules"}}}, _SYSTEM),
({"system_instruction": {"parts": {"text": "Follow these rules"}}}, _SYSTEM),
({"systemInstruction": {"parts": [{"text": "Follow these rules"}, None]}}, _SYSTEM),
({"systemInstruction": {"parts": None}}, _NO_SYSTEM_PARTS),
({"systemInstruction": {"parts": [{"text": None}, {"text": "Follow these rules"}]}}, _SYSTEM),
({"systemInstruction": {"parts": {"text": None}}}, _NO_SYSTEM_PARTS),
({"systemInstruction": {"parts": {"text": None, "inlineData": None}}}, _NO_SYSTEM_PARTS),
({"systemInstruction": {"parts": {"text": None, "thought": False}}}, _NO_SYSTEM_PARTS),
({"systemInstruction": {"parts": [{"thoughtSignature": "c2ln"}, {"text": "Follow these rules"}]}}, _SYSTEM),
({"systemInstruction": {"parts": [{}]}}, _NO_SYSTEM_PARTS),
({"systemInstruction": {}}, _NO_SYSTEM_PARTS),
({"tools": None}, {}),
({"tools": [None]}, {}),
({"tools": [{"functionDeclarations": None}]}, {}),
({"tools": {"functionDeclarations": [_DECLARATION]}}, _TOOLS),
({"tools": [{"functionDeclarations": _DECLARATION}]}, _TOOLS),
({"tools": {"functionDeclarations": _DECLARATION}}, _TOOLS),
({"contents": _TURN}, _CONTENTS),
({"contents": [None, _TURN]}, _CONTENTS),
({"contents": None}, {"contents": []}),
({"contents": [{"role": "user", "parts": {"text": "Count this request"}}]}, _CONTENTS),
({"contents": [{"role": "user", "parts": [None, {"text": "Count this request"}]}]}, _CONTENTS),
({"contents": [{"role": "user", "parts": None}]}, {"contents": [{"role": "user", "parts": []}]}),
),
)
def test_google_count_tokens_counts_every_google_spelling_of_a_repeated_field_like_its_list_form(
spelled: dict[str, JsonValue], canonical: dict[str, JsonValue]
) -> None:
contents: Final = [{"role": "user", "parts": [{"text": "Count this request"}]}]
assert _google_count_tokens_request("counted", {"contents": contents, **spelled}) == _google_count_tokens_request(
"counted", {"contents": contents, **canonical}
)
@pytest.mark.parametrize(
"malformed",
(
{"tools": "lookup"},
{"tools": [{"functionDeclarations": "lookup"}]},
{"systemInstruction": {"parts": 7}},
{"contents": 5},
{"contents": ["Count this request"]},
),
)
def test_google_count_tokens_rejects_a_scalar_where_google_expects_an_object_or_list(
malformed: dict[str, JsonValue],
) -> None:
with pytest.raises(ValidationError):
_google_count_tokens_request("counted", {"contents": [_TURN], **malformed})
@pytest.mark.parametrize(
"parts",
(
{"inlineData": {"mimeType": "image/png", "data": "aGVsbG8="}},
[{"inlineData": {"mimeType": "image/png", "data": "aGVsbG8="}}],
{"text": None, "inlineData": {"mimeType": "image/png", "data": "aGVsbG8="}},
[{"text": "Follow these rules"}, {"text": None, "fileData": {"fileUri": "gs://bucket/rules.pdf"}}],
),
)
def test_google_count_tokens_rejects_media_only_system_parts(parts: JsonValue) -> None:
with pytest.raises(ValidationError):
_google_count_tokens_request(
"counted",
{"contents": [{"role": "user", "parts": [{"text": "Hello"}]}], "systemInstruction": {"parts": parts}},
)
def test_google_count_tokens_keeps_counting_system_text_that_sits_beside_media() -> None:
text_with_media: Final = {"text": "Follow these rules", "inlineData": {"mimeType": "image/png", "data": "aGVsbG8="}}
assert _google_count_tokens_request(
"counted", {**_CONTENTS, "systemInstruction": {"parts": [text_with_media]}}
) == _google_count_tokens_request("counted", {**_CONTENTS, **_SYSTEM})

View file

@ -5480,6 +5480,15 @@ async def test_health_services_endpoint_langfuse_missing_keys_errors(monkeypatch
assert "connection refused by lf.internal.example" in str(raised.value.message)
@pytest.mark.asyncio
async def test_health_services_reports_an_unknown_service_as_a_request_error_not_an_auth_error() -> None:
with pytest.raises(ProxyException) as exc_info:
await health_services_endpoint(service="totally_unknown_service_xyz")
assert (exc_info.value.code, exc_info.value.type) == ("400", "invalid_request_error")
assert "Authentication Error" not in exc_info.value.message
@pytest.mark.asyncio
@pytest.mark.parametrize("admin", [False, True])
async def test_unsaved_chatgpt_account_probe_requires_proxy_admin(

View file

@ -5211,3 +5211,18 @@ async def test_user_update_without_password_revokes_nothing(_admin_prisma, mocke
await _update_single_user_helper(user_request=user_request, user_api_key_dict=admin_caller)
revoke_mock.assert_not_awaited()
@pytest.mark.asyncio
async def test_user_update_reports_a_bad_budget_duration_as_a_request_error_not_an_auth_error(_admin_prisma: MagicMock) -> None:
from litellm.proxy.management_endpoints.internal_user_endpoints import user_update
with pytest.raises(ProxyException) as exc_info:
await user_update(
data=UpdateUserRequest(user_id="target-user", budget_duration="not-a-duration"),
user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN),
)
assert (exc_info.value.code, exc_info.value.type) == ("400", "invalid_request_error")
assert "Invalid budget_duration" in exc_info.value.message
assert "Authentication Error" not in exc_info.value.message

View file

@ -6538,7 +6538,7 @@ async def test_list_keys_with_invalid_status():
)
# Verify ProxyException properties
assert exc_info.value.code == "400"
assert (exc_info.value.code, exc_info.value.type) == ("400", "invalid_request_error")
assert "Invalid status value" in str(exc_info.value.message)
assert "deleted" in str(exc_info.value.message)
@ -23377,3 +23377,73 @@ def test_key_info_database_not_connected_sanitized_500(
assert "Database not connected" in caplog.text
finally:
proxy_server.app.dependency_overrides.pop(user_api_key_auth, None)
@pytest.mark.asyncio
async def test_update_key_reports_a_bad_budget_duration_as_a_request_error_not_an_auth_error(monkeypatch: pytest.MonkeyPatch) -> None:
from fastapi import Request
from litellm.proxy.management_endpoints.key_management_endpoints import update_key_fn
monkeypatch.setattr( # test-quality-ok: the endpoint reads proxy_server.prisma_client itself, no parameter to inject
"litellm.proxy.proxy_server.prisma_client", MagicMock()
)
with pytest.raises(ProxyException) as exc_info:
await update_key_fn(
request=MagicMock(spec=Request),
data=UpdateKeyRequest(key="sk-whatever", budget_duration="not-a-duration"),
user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN),
)
assert (exc_info.value.code, exc_info.value.type) == ("400", "invalid_request_error")
assert "Authentication Error" not in exc_info.value.message
@pytest.mark.asyncio
async def test_update_key_passes_its_own_rejection_through_without_making_it_its_own_cause(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from fastapi import Request
from litellm.proxy.management_endpoints.key_management_endpoints import update_key_fn
prisma_client: Final = MagicMock()
prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=None)
monkeypatch.setattr( # test-quality-ok: the endpoint reads proxy_server.prisma_client itself, no parameter to inject
"litellm.proxy.proxy_server.prisma_client", prisma_client
)
with pytest.raises(ProxyException) as exc_info:
await update_key_fn(
request=MagicMock(spec=Request),
data=UpdateKeyRequest(key="sk-missing", max_budget=5),
user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN),
)
assert (exc_info.value.code, exc_info.value.type) == ("404", "not_found_error")
assert exc_info.value.__cause__ is None
@pytest.mark.asyncio
async def test_key_aliases_answers_503_when_the_database_is_unreachable(monkeypatch: pytest.MonkeyPatch) -> None:
import prisma.errors
from litellm.proxy.management_endpoints.key_management_endpoints import key_aliases
unreachable_db: Final = MagicMock()
unreachable_db.db.query_raw = AsyncMock(
side_effect=prisma.errors.PrismaError("Can't reach database server at `db:5432`")
)
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", unreachable_db)
with pytest.raises(ProxyException) as exc_info:
await key_aliases(
user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN),
page=1,
size=50,
search=None,
team_id=None,
)
assert (exc_info.value.code, exc_info.value.type) == ("503", "no_db_connection")

View file

@ -311,7 +311,7 @@ async def test_model_creation_fails_closed_when_authoritative_policy_is_unavaila
Deployment(model_name="cannot-create", litellm_params={"model": "openai/test-model"}),
UserAPIKeyAuth(user_id="test_user", user_role=LitellmUserRoles.INTERNAL_USER),
)
assert error.value.code == "503"
assert (error.value.code, error.value.type) == ("503", "internal_server_error")
prisma.db.litellm_proxymodeltable.create.assert_not_called()
prisma.db.litellm_teamtable.find_unique.assert_not_called()
@ -10913,3 +10913,76 @@ class TestNonAdminCannotSetPerUserOauthOnModel:
user_api_key_dict=admin,
)
assert result is created_row
@pytest.mark.asyncio
async def test_delete_model_reports_an_unknown_model_id_as_a_request_error_not_an_auth_error(monkeypatch: pytest.MonkeyPatch) -> None:
from litellm.proxy import proxy_server
from litellm.proxy._types import ModelInfoDelete
from litellm.proxy.management_endpoints.model_management_endpoints import delete_model
prisma: Final = MagicMock()
prisma.db.litellm_proxymodeltable.find_unique = AsyncMock(return_value=None)
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
with pytest.raises(ProxyException) as exc_info:
await delete_model(
model_info=ModelInfoDelete(id="no-such-model"),
user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN),
)
assert (exc_info.value.code, exc_info.value.type) == ("400", "invalid_request_error")
assert "Authentication Error" not in exc_info.value.message
@pytest.mark.asyncio
async def test_update_model_reports_an_unknown_model_id_as_a_request_error_not_an_auth_error(monkeypatch: pytest.MonkeyPatch) -> None:
from litellm.proxy import proxy_server
from litellm.proxy.management_endpoints.model_management_endpoints import update_model
prisma: Final = MagicMock()
prisma.db.litellm_proxymodeltable.find_unique = AsyncMock(return_value=None)
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
monkeypatch.setattr(proxy_server, "llm_router", None)
with pytest.raises(ProxyException) as exc_info:
await update_model(
model_params=updateDeployment(model_info=ModelInfo(id="no-such-model")),
user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN),
)
assert (exc_info.value.code, exc_info.value.type) == ("400", "invalid_request_error")
assert exc_info.value.message == "model not found"
@pytest.mark.asyncio
async def test_add_new_model_reports_a_database_outage_during_the_insert_as_503(
monkeypatch: pytest.MonkeyPatch,
) -> None:
import prisma.errors
from litellm.proxy import proxy_server
from litellm.proxy.management_endpoints.model_management_endpoints import add_new_model
unreachable_db: Final = MagicMock()
unreachable_db.db.query_raw = AsyncMock(return_value=[])
unreachable_db.db.litellm_proxymodeltable.create = AsyncMock(
side_effect=prisma.errors.PrismaError("Can't reach database server at `db:5432`")
)
monkeypatch.setattr(proxy_server, "prisma_client", unreachable_db)
monkeypatch.setattr(proxy_server, "store_model_in_db", True)
monkeypatch.setattr(proxy_server, "general_settings", {})
monkeypatch.setattr(proxy_server, "premium_user", True)
monkeypatch.setenv("LITELLM_SALT_KEY", "sk-outage-salt")
with pytest.raises(ProxyException) as exc_info:
await add_new_model(
model_params=Deployment(
model_name="outage-model",
litellm_params=LiteLLM_Params(model="openai/gpt-5.4-mini", api_key="fake-key"),
model_info={"id": "outage-model-1"},
),
user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN),
)
assert (exc_info.value.code, exc_info.value.type) == ("503", "no_db_connection")

View file

@ -1617,6 +1617,37 @@ async def test_delete_organization_is_refused_while_one_of_its_teams_holds_a_ptu
prisma_client.db.litellm_organizationtable.delete.assert_not_called()
@pytest.mark.asyncio
async def test_organization_member_add_reports_an_unknown_org_as_a_request_error_not_an_auth_error(monkeypatch: pytest.MonkeyPatch) -> None:
from fastapi import Request
from litellm.proxy._types import (
LitellmUserRoles,
OrganizationMemberAddRequest,
OrgMember,
ProxyException,
UserAPIKeyAuth,
)
from litellm.proxy.management_endpoints.organization_endpoints import organization_member_add
mock_prisma_client: Final = MagicMock()
mock_prisma_client.db.litellm_organizationtable.find_unique = AsyncMock(return_value=None)
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
with pytest.raises(ProxyException) as exc_info:
await organization_member_add(
data=OrganizationMemberAddRequest(
organization_id="no-such-org",
member=OrgMember(role="internal_user", user_id="someone"),
),
http_request=MagicMock(spec=Request),
user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN),
)
assert (exc_info.value.code, exc_info.value.type) == ("404", "invalid_request_error")
assert "Authentication Error" not in exc_info.value.message
@pytest.mark.asyncio
@pytest.mark.parametrize("budget_duration", ["30d", None])
async def test_new_organization_schedules_budget_reset_from_duration(monkeypatch, budget_duration):

View file

@ -19372,3 +19372,48 @@ async def test_default_budget_cache_eviction_does_not_fail_after_redis_error() -
)
user_api_key_cache.async_delete_cache.assert_awaited_once_with(key="team_member_default_budget:budget_456")
@pytest.mark.asyncio
async def test_team_info_reports_a_missing_team_as_a_request_error_not_an_auth_error(monkeypatch: pytest.MonkeyPatch) -> None:
from fastapi import Request
from litellm.proxy._types import ProxyException
from litellm.proxy.management_endpoints import team_endpoints
mock_prisma_client: Final = MagicMock()
mock_prisma_client.db.litellm_teamtable.find_unique = AsyncMock(return_value=None)
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
with pytest.raises(ProxyException) as exc_info:
await team_endpoints.team_info(
http_request=MagicMock(spec=Request),
team_id="no-such-team",
user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN),
)
assert (exc_info.value.code, exc_info.value.type) == ("404", "invalid_request_error")
assert "Authentication Error" not in exc_info.value.message
@pytest.mark.asyncio
async def test_team_info_keeps_the_auth_label_for_a_caller_outside_the_team(monkeypatch: pytest.MonkeyPatch) -> None:
from fastapi import Request
from litellm.proxy._types import ProxyException
from litellm.proxy.management_endpoints import team_endpoints
other_team: Final = MagicMock()
other_team.model_dump.return_value = {"team_id": "team-a", "members_with_roles": []}
mock_prisma_client: Final = MagicMock()
mock_prisma_client.db.litellm_teamtable.find_unique = AsyncMock(return_value=other_team)
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)
with pytest.raises(ProxyException) as exc_info:
await team_endpoints.team_info(
http_request=MagicMock(spec=Request),
team_id="team-a",
user_api_key_dict=UserAPIKeyAuth(user_id="outsider", user_role=LitellmUserRoles.INTERNAL_USER),
)
assert (exc_info.value.code, exc_info.value.type) == ("403", "auth_error")

View file

@ -3,6 +3,7 @@ import sys
import traceback
from collections.abc import AsyncGenerator, Awaitable, Callable, Coroutine, Mapping
from contextlib import asynccontextmanager
from pathlib import Path
from typing import Final, Literal, NoReturn, cast
from unittest import mock
@ -3817,6 +3818,48 @@ def test_live_session_preserves_pretty_printed_start_frame(
assert routed_data["live_session_start"] == expected_start
@pytest.mark.asyncio
async def test_update_config_reports_an_unsupported_router_setting_as_a_request_error_not_an_auth_error() -> None:
from fastapi import Request
from litellm.proxy._types import ConfigYAML, ProxyException
from litellm.proxy.proxy_server import update_config
body: Final = {"router_settings": {"not_a_router_setting": True}}
request: Final = MagicMock(spec=Request)
request.json = AsyncMock(return_value=body)
with pytest.raises(ProxyException) as exc_info:
await update_config(
config_info=ConfigYAML(**body),
request=request,
user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN),
)
assert (exc_info.value.code, exc_info.value.type) == ("400", "invalid_request_error")
assert "Authentication Error" not in exc_info.value.message
@pytest.mark.asyncio
async def test_get_config_callbacks_reports_an_unreadable_config_as_a_server_error_not_an_auth_error(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
from litellm.proxy import proxy_server
from litellm.proxy._types import ProxyException
from litellm.proxy.proxy_server import get_config
empty_config: Final = tmp_path / "config.yaml"
empty_config.write_text("")
monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False)
monkeypatch.setattr(proxy_server, "user_config_file_path", str(empty_config))
with pytest.raises(ProxyException) as exc_info:
await get_config(user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN))
assert (exc_info.value.code, exc_info.value.type) == ("500", "internal_server_error")
assert exc_info.value.message == "Config cannot be None or Empty."
@pytest.mark.parametrize("scope", [None, "expand"])
@pytest.mark.parametrize(
"general_settings, wildcard_query, wildcard_listed",
@ -3861,6 +3904,62 @@ async def test_model_list_return_wildcard_routes_defaults_to_general_setting(
assert ("openai/*" in listed) is wildcard_listed, listed
@pytest.mark.asyncio
async def test_queue_request_labels_an_upstream_failure_a_server_error_not_an_auth_error(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from starlette.requests import Request
from starlette.responses import Response
from litellm.proxy import proxy_server
from litellm.proxy._types import ProxyException
from litellm.proxy.proxy_server import async_queue_request
body: Final = json.dumps(
{"model": "gpt-5.4-mini", "messages": [{"role": "user", "content": "hi"}], "priority": 0}
).encode()
async def receive() -> dict[str, object]:
return {"type": "http.request", "body": body, "more_body": False}
request: Final = Request(
{
"type": "http",
"method": "POST",
"path": "/queue/chat/completions",
"headers": [(b"content-type", b"application/json")],
"query_string": b"",
"scheme": "http",
"server": ("testserver", 80),
"root_path": "",
},
receive,
)
router: Final = litellm.Router(
model_list=[
{
"model_name": "gpt-5.4-mini",
"litellm_params": {
"model": "openai/gpt-5.4-mini",
"api_key": "sk-test",
"mock_response": "litellm.InternalServerError",
},
}
],
num_retries=0,
)
monkeypatch.setattr(proxy_server, "llm_router", router)
with pytest.raises(ProxyException) as exc_info:
await async_queue_request(
request=request,
fastapi_response=Response(),
user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN),
)
assert (exc_info.value.code, exc_info.value.type) == ("500", "internal_server_error")
_DEBUG_LOG_SENTINEL: Final = "sentinel-request-body-7f3a"
_LONG_PROMPT: Final = f"{_DEBUG_LOG_SENTINEL} " + "Keep edits minimal and explain them briefly. " * 20
_TOOL_PARAMETERS: Final = {"type": "object", "properties": {"cmd": {"type": "string"}}}