scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-08-28 05:25:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 36

Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_redis_pubsub_config_sync

Browse source 
# Conflicts:
#	litellm/proxy/proxy_server.py
This commit is contained in:
mateo-berri 2026-08-01 09:29:24 -07:00
commit b2fd79f487
38 changed files with 834 additions and 227 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
basedpyright-code-budget.json
View file

@ -1,6 +1,6 @@
{
"reportAny": {
"limit": 31903
"limit": 31256
},
"reportArgumentType": {
"limit": 2645
@ -24,7 +24,7 @@
"limit": 42
},
"reportExplicitAny": {
"limit": 10214
"limit": 10208
},
"reportFunctionMemberAccess": {
"limit": 11
@ -33,7 +33,7 @@
"limit": 227
},
"reportIncompatibleMethodOverride": {
"limit": 78
"limit": 77
},
"reportIncompatibleVariableOverride": {
"limit": 12
@ -99,7 +99,7 @@
"limit": 0
},
"reportUnknownArgumentType": {
"limit": 45366
"limit": 45357
},
"reportUnknownLambdaType": {
"limit": 113

8
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -215,7 +215,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
)
if result:
return LiteLLM_ManagedFileTable(**result)
return LiteLLM_ManagedFileTable.model_validate(result)
## CHECK DB
db_object = await self.prisma_client.db.litellm_managedfiletable.find_first(
@ -223,7 +223,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
)
if db_object:
return LiteLLM_ManagedFileTable(**db_object.model_dump())
return LiteLLM_ManagedFileTable.model_validate(db_object.model_dump())
return None
async def delete_unified_file_id(
@ -349,7 +349,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
if isinstance(batch.file_object, str)
else batch.file_object
)
batch_obj = LiteLLMBatch(**batch_data)
batch_obj = LiteLLMBatch.model_validate(batch_data)
batch_obj.id = batch.unified_object_id
batch_objects.append(batch_obj)
@ -382,7 +382,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
"flat_model_file_ids": {"hasSome": model_object_ids},
}
)
return [OpenAIFileObject(**file_object.file_object) for file_object in file_ids]
return [OpenAIFileObject.model_validate(file_object.file_object) for file_object in file_ids]
async def check_managed_file_id_access(
self, data: Dict, user_api_key_dict: UserAPIKeyAuth

4
litellm/litellm_core_utils/streaming_handler.py
View file

@ -131,8 +131,8 @@ class CustomStreamWrapper:
self.sent_last_chunk = False
self._stream_created_time: float = time.time()
litellm_params: GenericLiteLLMParams = GenericLiteLLMParams(
**self.logging_obj.model_call_details.get("litellm_params", {})
litellm_params: GenericLiteLLMParams = GenericLiteLLMParams.model_validate(
dict(**self.logging_obj.model_call_details.get("litellm_params", {}))
)
self.merge_reasoning_content_in_choices: bool = litellm_params.merge_reasoning_content_in_choices or False
self.sent_first_thinking_block = False

4
litellm/llms/azure/responses/transformation.py
View file

@ -66,7 +66,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
)
# Create ResponseReasoningItem object from the item data
reasoning_item = ResponseReasoningItem(**item_data)
reasoning_item = ResponseReasoningItem.model_validate(item_data)
# Convert back to dict with exclude_none=True to exclude None fields
dict_reasoning_item = reasoning_item.model_dump(exclude_none=True)
@ -346,4 +346,4 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
from litellm.llms.azure.chat.gpt_transformation import AzureOpenAIError
raise AzureOpenAIError(message=raw_response.text, status_code=raw_response.status_code)
return ResponsesAPIResponse(**raw_response_json)
return ResponsesAPIResponse.model_validate(raw_response_json)

10
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -6184,10 +6184,12 @@ class BaseLLMHTTPHandler:
import websockets
from websockets.asyncio.client import ClientConnection
litellm_params = GenericLiteLLMParams(
api_base=api_base,
api_key=api_key,
**kwargs,
litellm_params = GenericLiteLLMParams.model_validate(
{
"api_base": api_base,
"api_key": api_key,
**kwargs,
}
)
headers = responses_api_provider_config.validate_environment(
headers={},

4
litellm/llms/manus/responses/transformation.py
View file

@ -217,7 +217,7 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig):
raw_response_json["id"] = f"unknown-{uuid.uuid4().hex[:8]}"
try:
response = ResponsesAPIResponse(**raw_response_json)
response = ResponsesAPIResponse.model_validate(raw_response_json)
except Exception:
verbose_logger.debug(f"Error constructing ResponsesAPIResponse: {raw_response_json}, using model_construct")
response = ResponsesAPIResponse.model_construct(**raw_response_json)
@ -305,7 +305,7 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig):
raw_response_json["id"] = f"unknown-{uuid.uuid4().hex[:8]}"
try:
response = ResponsesAPIResponse(**raw_response_json)
response = ResponsesAPIResponse.model_validate(raw_response_json)
except Exception:
verbose_logger.debug(f"Error constructing ResponsesAPIResponse: {raw_response_json}, using model_construct")
response = ResponsesAPIResponse.model_construct(**raw_response_json)

17
litellm/proxy/hooks/user_management_event_hooks.py
View file

@ -6,8 +6,6 @@ import asyncio
from datetime import datetime, timezone
from typing import Optional
from pydantic import BaseModel
import litellm
from litellm._logging import verbose_proxy_logger
from litellm._uuid import uuid
@ -16,7 +14,6 @@ from litellm.proxy._types import (
CommonProxyErrors,
LiteLLM_AuditLogs,
Litellm_EntityType,
LiteLLM_UserTable,
LitellmTableNames,
NewUserRequest,
NewUserResponse,
@ -58,20 +55,20 @@ class UserManagementEventHooks:
try:
if prisma_client is None:
raise Exception(CommonProxyErrors.db_not_connected_error.value)
user_row: BaseModel = await UserRepository(prisma_client).table.find_first(
where={"user_id": response.user_id}
)
user_row_litellm_typed = LiteLLM_UserTable(**user_row.model_dump(exclude_none=True))
if response.user_id is None:
raise Exception("no user_id returned for the newly created user")
user_row = await UserRepository(prisma_client).find_by_id(response.user_id)
if user_row is None:
raise Exception(f"no user row found for user_id={response.user_id}")
asyncio.create_task(
UserManagementEventHooks.create_internal_user_audit_log(
user_id=user_row_litellm_typed.user_id,
user_id=user_row.user_id,
action="created",
litellm_changed_by=user_api_key_dict.user_id,
user_api_key_dict=user_api_key_dict,
litellm_proxy_admin_name=litellm_proxy_admin_name,
before_value=None,
after_value=user_row_litellm_typed.model_dump_json(exclude_none=True),
after_value=user_row.model_dump_json(exclude_none=True),
)
)
except Exception as e:

48
litellm/proxy/management_endpoints/model_management_endpoints.py
View file

@ -328,7 +328,7 @@ async def patch_model(
# Clear cache and reload models (uses config setting or defaults to preserving config models for DB updates)
live_before_reload = live_model_ids_snapshot()
await clear_cache()
still_desired_ids = await clear_cache()
## CREATE AUDIT LOG ##
asyncio.create_task(
@ -348,6 +348,7 @@ async def patch_model(
before=live_before_reload,
written_models=[(model_id, getattr(updated_model, "model_info", None))],
action="update",
still_desired=still_desired_ids,
)
return updated_model
@ -433,7 +434,7 @@ async def _set_model_blocked_status(
)
live_before_reload = live_model_ids_snapshot()
await clear_cache()
still_desired_ids = await clear_cache()
asyncio.create_task(
create_object_audit_log(
@ -454,6 +455,7 @@ async def _set_model_blocked_status(
before=live_before_reload,
written_models=[(data.model_id, getattr(updated_model, "model_info", None))],
action=action,
still_desired=still_desired_ids,
)
return updated_model
@ -1362,6 +1364,7 @@ async def add_new_model(
"""
live_before_reload = live_model_ids_snapshot()
still_desired_ids: frozenset[str] | None = None
try:
_original_litellm_model_name = model_params.model_name
if model_params.model_info.team_id is None:
@ -1376,7 +1379,9 @@ async def add_new_model(
user_api_key_dict=user_api_key_dict,
prisma_client=prisma_client,
)
await proxy_config.add_deployment(prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj)
still_desired_ids = await proxy_config.add_deployment(
prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj
)
# don't let failed slack alert block the /model/new response
_alerting = general_settings.get("alerting", []) or []
if "slack" in _alerting:
@ -1421,6 +1426,7 @@ async def add_new_model(
before=live_before_reload,
written_models=[(model_response.model_id, getattr(model_response, "model_info", None))],
action="create",
still_desired=still_desired_ids,
)
return model_response
@ -1549,7 +1555,7 @@ async def update_model(
# Clear cache and reload models (uses config setting or defaults to preserving config models for DB updates)
live_before_reload = live_model_ids_snapshot()
await clear_cache()
still_desired_ids = await clear_cache()
## CREATE AUDIT LOG ##
asyncio.create_task(
create_object_audit_log(
@ -1576,6 +1582,7 @@ async def update_model(
before=live_before_reload,
written_models=[(_model_id, getattr(model_response, "model_info", None))],
action="update",
still_desired=still_desired_ids,
)
return model_response
@ -1821,6 +1828,7 @@ def reload_serving_verdict(
before: frozenset[str],
written_models: Sequence[tuple[str, object]],
written_must_serve: bool,
still_desired: frozenset[str] | None = None,
) -> tuple[tuple[str, ...], tuple[str, ...]]:
"""Judge a write-triggered reload by diffing the router's serving state instead of
trusting any layer of the reload stack to report its own failure.
@ -1835,10 +1843,14 @@ def reload_serving_verdict(
this write and blaming it would block unrelated metadata fixes
- not written but live before and gone now: collateral degradation of this pod
caused by the reload this request triggered (a wholesale re-add failure, or a
newly introduced conflict), always reported
newly introduced conflict), reported only when the db still wants that id
``still_desired`` is the db + config id set the reload just reconciled against. An
id absent from it was deleted on purpose, most often by another pod this one had not
yet polled, so the reload dropping it is the reconcile working rather than damage.
Without it (no reconcile ran) every drop is reported, which is the safe direction.
Returns (written ids violating their obligation, collateral ids no longer served).
Best effort under concurrent admin writes: the snapshot spans only this request.
"""
now = live_model_ids_snapshot()
written_ids = frozenset(model_id for model_id, _ in written_models)
@ -1850,7 +1862,8 @@ def reload_serving_verdict(
)
else:
missing = tuple(model_id for model_id, _ in written_models if model_id in before and model_id not in now)
collateral = tuple(sorted(before - now - written_ids))
dropped = before - now - written_ids
collateral = tuple(sorted(dropped if still_desired is None else dropped & still_desired))
return (missing, collateral)
@ -1858,12 +1871,18 @@ def raise_if_reload_degraded_serving(
before: frozenset[str],
written_models: Sequence[tuple[str, object]],
action: str,
still_desired: frozenset[str] | None = None,
) -> None:
"""The caller-visible error this pod's model-write endpoints owe their caller when
the model they wrote is not being served after the reload they triggered. The DB
write is durable either way and every other pod reloads on its own interval; this
speaks only for the handling pod."""
missing, collateral = reload_serving_verdict(before=before, written_models=written_models, written_must_serve=True)
missing, collateral = reload_serving_verdict(
before=before,
written_models=written_models,
written_must_serve=True,
still_desired=still_desired,
)
if not missing and not collateral:
return
missing_clause = (
@ -1889,9 +1908,12 @@ def raise_if_reload_degraded_serving(
)
async def clear_cache():
async def clear_cache() -> frozenset[str] | None:
"""
Clear router caches and reload models.
Returns the db + config id set the reload reconciled against, or None when no
reload ran, so callers can pass it to raise_if_reload_degraded_serving.
"""
from litellm.proxy.proxy_server import (
llm_router,
@ -1903,7 +1925,7 @@ async def clear_cache():
if llm_router is None or prisma_client is None:
verbose_proxy_logger.debug("llm_router or prisma_client is None, skipping cache clear")
return
return None
try:
# Only clear DB models, preserve config models
@ -1950,10 +1972,14 @@ async def clear_cache():
llm_router.quality_routers.pop(model_name, None)
# Reload only DB models
await proxy_config.add_deployment(prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj)
still_desired_ids = await proxy_config.add_deployment(
prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj
)
verbose_proxy_logger.debug(
f"Cleared {len(db_model_ids)} DB models, preserved {len(config_models)} config models"
)
return still_desired_ids
except Exception as e:
verbose_proxy_logger.exception(f"Failed to clear cache and reload models. Due to error - {str(e)}")
return None

13
litellm/proxy/management_endpoints/team_endpoints.py
View file

@ -131,6 +131,7 @@ from litellm.types.proxy.management_endpoints.team_endpoints import (
BulkUpdateTeamMemberPermissionsRequest,
BulkUpdateTeamMemberPermissionsResponse,
GetTeamMemberPermissionsResponse,
TeamIdSearchMatch,
TeamListItem,
TeamListResponse,
TeamMemberAddResult,
@ -3973,6 +3974,7 @@ async def _build_team_list_where_conditions(
user_id: Optional[str],
use_deleted_table: bool,
search: Optional[str] = None,
search_team_id_match: TeamIdSearchMatch = "exact",
org_admin_org_ids: Optional[List[str]] = None,
user_api_key_cache: Optional[Any] = None,
proxy_logging_obj: Optional[Any] = None,
@ -3996,7 +3998,7 @@ async def _build_team_list_where_conditions(
if search:
where_conditions["OR"] = [
{"team_id": search},
({"team_id": {"startsWith": search}} if search_team_id_match == "prefix" else {"team_id": search}),
{"team_alias": {"contains": search, "mode": "insensitive"}},
]
@ -4230,8 +4232,14 @@ async def list_team_v2(
),
search: Optional[str] = fastapi.Query(
default=None,
description="Combined search: matches teams whose 'team_id' equals the value OR whose 'team_alias' contains it (case-insensitive).",
description="Combined search: matches teams whose 'team_id' matches the value OR whose 'team_alias' contains it (case-insensitive).",
),
search_team_id_match: Annotated[
TeamIdSearchMatch,
fastapi.Query(
description="How 'search' matches 'team_id': 'exact' (default) or 'prefix' for a case-sensitive prefix match."
),
] = "exact",
page: int = fastapi.Query(default=1, description="Page number for pagination", ge=1),
page_size: int = fastapi.Query(default=10, description="Number of teams per page", ge=1, le=100),
sort_by: Optional[str] = fastapi.Query(
@ -4308,6 +4316,7 @@ async def list_team_v2(
user_id=user_id,
use_deleted_table=use_deleted_table,
search=search,
search_team_id_match=search_team_id_match,
org_admin_org_ids=org_admin_org_ids,
user_api_key_cache=user_api_key_cache,
proxy_logging_obj=proxy_logging_obj,

26
litellm/proxy/management_endpoints/ui_sso.py
View file

@ -3137,7 +3137,7 @@ class SSOAuthenticationHandler:
_default_team_params = deepcopy(litellm.default_team_params)
_new_team_request = team_request.model_dump()
_new_team_request.update(_default_team_params)
team_request = NewTeamRequest(**_new_team_request)
team_request = NewTeamRequest.model_validate(_new_team_request)
return team_request
@staticmethod
@ -3271,14 +3271,6 @@ class SSOAuthenticationHandler:
# User might not be already created on first generation of key
# But if it is, we want their models preferences
default_ui_key_values: Dict[str, Any] = {
"duration": LITELLM_UI_SESSION_DURATION,
"key_max_budget": litellm.max_ui_session_budget,
"aliases": {},
"config": {},
"spend": 0,
"team_id": "litellm-dashboard",
}
user_defined_values: Optional[SSOUserDefinedValues] = None
if user_custom_sso is not None:
@ -3338,10 +3330,20 @@ class SSOAuthenticationHandler:
verbose_proxy_logger.info(f"user_defined_values for creating ui key: {user_defined_values}")
default_ui_key_values.update(user_defined_values)
default_ui_key_values["request_type"] = "key"
response = await generate_key_helper_fn(
**default_ui_key_values, # type: ignore
request_type="key",
duration=LITELLM_UI_SESSION_DURATION,
key_max_budget=litellm.max_ui_session_budget,
aliases={},
config={},
spend=0,
team_id="litellm-dashboard",
models=user_defined_values["models"],
user_id=user_defined_values["user_id"],
user_email=user_defined_values["user_email"],
user_role=user_defined_values["user_role"],
max_budget=user_defined_values["max_budget"],
budget_duration=user_defined_values["budget_duration"],
table_name="key",
)

2
litellm/proxy/openai_files_endpoints/common_utils.py
View file

@ -1035,7 +1035,7 @@ async def get_batch_from_database(
if isinstance(db_batch_object.file_object, str)
else db_batch_object.file_object
)
response = LiteLLMBatch(**batch_data)
response = LiteLLMBatch.model_validate(batch_data)
response.id = batch_id
# The stored batch object has the raw provider input_file_id. Resolve to unified ID.

72
litellm/proxy/proxy_server.py
View file

@ -1043,6 +1043,8 @@ async def proxy_startup_event(app: FastAPI):
redis_usage_cache=transaction_buffer_redis_cache,
)
ProxyStartupEvent._warn_if_mock_testing_params_enabled(general_settings=general_settings)
## SEMANTIC TOOL FILTER ##
# Read litellm_settings from config for semantic filter initialization
try:
@ -5305,7 +5307,7 @@ class ProxyConfig:
_model_info = RouterModelInfo(id=model.model_id, db_model=db_model)
return _model_info
async def _delete_deployment(self, db_models: list) -> int:
async def _delete_deployment(self, db_models: list) -> frozenset[str] | None:
"""
(Helper function of add deployment) -> combined to reduce prisma db calls
@ -5314,14 +5316,16 @@ class ProxyConfig:
- Remove any that are missing
Return:
- int - returns number of deleted deployments
- frozenset[str] - the ids the db + config say should be served after this
reconcile, so a caller can tell an id this evicted on purpose from one that
went missing. None when no reconcile ran and that set is therefore unknown.
"""
global user_config_file_path, llm_router
combined_id_list = []
## BASE CASES ##
if llm_router is None:
return 0
return None
# NOTE: db_models may be legitimately empty when all DB models have been deleted.
# Do NOT short-circuit on len(db_models) == 0 — we must still evict any
# DB-sourced deployments that are no longer in the DB. The caller
@ -5342,7 +5346,7 @@ class ProxyConfig:
"Skipping deployment cleanup to avoid removing valid models.",
str(e),
)
return 0
return None
model_list = config.get("model_list", None)
if model_list:
for model in model_list:
@ -5366,13 +5370,10 @@ class ProxyConfig:
router_model_ids = llm_router.get_model_ids()
# Check for model IDs in llm_router not present in combined_id_list and delete them
deleted_deployments = 0
for model_id in router_model_ids:
if model_id not in combined_id_list:
is_deleted = llm_router.delete_deployment(id=model_id)
if is_deleted is not None:
deleted_deployments += 1
return deleted_deployments
llm_router.delete_deployment(id=model_id)
return frozenset(combined_id_list)
def _resolve_db_litellm_param(self, key: str, value: object) -> object:
if not isinstance(value, str):
@ -5457,9 +5458,11 @@ class ProxyConfig:
self,
new_models: Optional[Json],
proxy_logging_obj: ProxyLogging,
):
) -> frozenset[str] | None:
global llm_router, llm_model_list, master_key, general_settings
still_desired_ids: frozenset[str] | None = None
# Load config separately so a timeout here doesn't block model loading
config_data: dict = {}
search_tools = None
@ -5506,7 +5509,7 @@ class ProxyConfig:
if search_tools is not None and llm_router is not None:
llm_router.search_tools = search_tools
## DELETE MODEL LOGIC
await self._delete_deployment(db_models=models_list)
still_desired_ids = await self._delete_deployment(db_models=models_list)
## ADD MODEL LOGIC
self._add_deployment(db_models=models_list)
@ -5532,6 +5535,8 @@ class ProxyConfig:
proxy_logging_obj=proxy_logging_obj,
)
return still_desired_ids
def _add_callback_from_db_to_in_memory_litellm_callbacks(
self,
callback: str,
@ -6164,14 +6169,20 @@ class ProxyConfig:
self,
prisma_client: PrismaClient,
proxy_logging_obj: ProxyLogging,
):
) -> frozenset[str] | None:
"""
- Check db for new models
- Check if model id's in router already
- If not, add to router
Returns the ids the db + config say should be served after the reconcile, or
None when no reconcile ran. Callers that judge their own reload need it to tell
a deliberate eviction from a deployment that went missing.
"""
global llm_router, llm_model_list, master_key, general_settings
still_desired_ids: frozenset[str] | None = None
try:
# warm the config cache so the per-param reads below all hit
await prefetch_config_params(
@ -6191,7 +6202,9 @@ class ProxyConfig:
new_models = await self._get_models_from_db(prisma_client=prisma_client)
# update llm router
await self._update_llm_router(new_models=new_models, proxy_logging_obj=proxy_logging_obj)
still_desired_ids = await self._update_llm_router(
new_models=new_models, proxy_logging_obj=proxy_logging_obj
)
db_general_settings = await get_config_param(prisma_client, "general_settings")
@ -6209,6 +6222,8 @@ class ProxyConfig:
"litellm.proxy.proxy_server.py::ProxyConfig:add_deployment - {}".format(str(e))
)
return still_desired_ids
def start_config_sync_subscriber(
self,
prisma_client: PrismaClient,
@ -7749,6 +7764,37 @@ class ProxyStartupEvent:
proxy_logging_obj.startup_event(llm_router=llm_router, redis_usage_cache=redis_usage_cache)
@staticmethod
def _warn_if_mock_testing_params_enabled(general_settings: dict) -> None:
"""Announce, loudly, that any caller may inject synthetic failures."""
from litellm.proxy.route_llm_request import (
GATED_MOCK_PARAM_NAMES,
MOCK_TESTING_CONFIG_KEY,
)
if general_settings.get(MOCK_TESTING_CONFIG_KEY, False) is not True:
return
verbose_proxy_logger.warning(
"\n%s\n"
" DANGEROUS SETTING ENABLED\n"
" general_settings.%s = true\n"
"\n"
" Any caller with a valid key on this proxy can now inject synthetic\n"
" failures and latency into their own requests using these body params:\n"
"%s\n"
"\n"
" A request using them consumes a connection and a concurrency slot\n"
" without reaching a provider, and returns an error the caller chose.\n"
"\n"
" Intended for testing fallback chains. Do not leave enabled.\n"
"%s",
"=" * 72,
MOCK_TESTING_CONFIG_KEY,
"\n".join(f" {name}" for name in GATED_MOCK_PARAM_NAMES),
"=" * 72,
)
@staticmethod
def _validate_redis_transaction_buffer_config(
general_settings: dict,

58
litellm/proxy/route_llm_request.py
View file

@ -8,18 +8,24 @@ import litellm
from litellm.proxy._types import UserAPIKeyAuth
from litellm.router_utils.common_utils import _is_proxy_admin_request
# Router-internal mock_testing_* flag names — kept in sync with
# ``litellm.types.router.MockRouterTestingParams`` by the test
# ``test_mock_testing_kwarg_names_matches_dataclass``. Hardcoding (rather
# than deriving via ``dataclasses.fields(MockRouterTestingParams)`` at
# Client-supplied params that make the router or the call path fabricate a
# failure or a delay instead of calling the provider. The ``mock_testing_*``
# names are kept in sync with ``litellm.types.router.MockRouterTestingParams``
# by ``test_gated_mock_params_cover_mock_router_testing_params``. Hardcoding
# (rather than deriving via ``dataclasses.fields(MockRouterTestingParams)`` at
# import time) avoids a cyclic import: ``litellm.types.router`` imports
# back into proxy modules before this module finishes loading.
_MOCK_TESTING_KWARG_NAMES: tuple = (
GATED_MOCK_PARAM_NAMES: tuple[str, ...] = (
"mock_testing_fallbacks",
"mock_testing_context_fallbacks",
"mock_testing_content_policy_fallbacks",
"mock_testing_rate_limit_error",
"mock_timeout",
"mock_delay",
)
MOCK_TESTING_CONFIG_KEY = "dangerously_allow_mock_testing_request_params"
if TYPE_CHECKING:
from litellm.router import Router as _Router
@ -169,6 +175,41 @@ def raise_if_required_body_param_missing(route_type: str, data: Mapping[str, obj
)
class MockTestingParamsDisabledError(HTTPException):
def __init__(self, params: tuple[str, ...]):
super().__init__(
status_code=status.HTTP_400_BAD_REQUEST,
detail={ # mutable-ok: HTTPException.detail has no immutable form; same shape as the sibling errors here
"error": (
f"Mock testing request params are disabled on this proxy: {', '.join(params)}. "
f"An admin can enable them by setting `general_settings.{MOCK_TESTING_CONFIG_KEY}: true` "
"in config.yaml. This setting cannot be changed from the Admin UI or the API."
)
},
)
def raise_if_mock_testing_params_disallowed(data: Mapping[str, object], *, allowed: bool) -> None:
"""Reject client-supplied mock testing params unless an admin opted in.
Rejecting (rather than silently dropping) keeps a request that asked for a
synthetic failure from returning a normal success, which reads as a passing
fallback test that never ran.
"""
if allowed:
return
present = tuple(name for name in GATED_MOCK_PARAM_NAMES if name in data)
if present:
raise MockTestingParamsDisabledError(params=present)
def mock_testing_params_allowed() -> bool:
"""Read the opt-in from the running proxy's ``general_settings``."""
import litellm.proxy.proxy_server as proxy_server
return proxy_server.general_settings.get(MOCK_TESTING_CONFIG_KEY, False) is True
def get_team_id_from_data(data: dict) -> Optional[str]:
"""
Get the team id from the data's metadata or litellm_metadata params.
@ -381,12 +422,7 @@ async def route_request(
await add_shared_session_to_data(data)
# Strip router-internal mock_testing_* flags. Combined with an
# unauthorized fallback in ``router_settings_override`` they let a
# caller deterministically execute requests against restricted
# models. VERIA-44.
for _key in _MOCK_TESTING_KWARG_NAMES:
data.pop(_key, None)
raise_if_mock_testing_params_disallowed(data, allowed=mock_testing_params_allowed())
data.pop("enable_tag_filtering", None)

57
litellm/repositories/user_repository.py
View file

@ -3,61 +3,56 @@ User repository for database operations on LiteLLM_UserTable.
"""
import json
from typing import Any, Dict, List, Optional, Type
from typing import Any, Dict, List, Mapping, Optional, Type
from litellm.models.user import LiteLLM_UserTable
from litellm.repositories.base_repository import BaseRepository
from litellm.repositories.base_repository import BaseRepository, DbRecord, record_to_dict
_JSON_ENCODED_COLUMNS = frozenset({"metadata", "model_spend", "model_max_budget"})
class UserRepository(BaseRepository[LiteLLM_UserTable]):
"""Repository for user database operations."""
@property
def table(self) -> Any:
def table(self) -> Any: # any-ok: Prisma table actions are reached through the untyped client wrapper
return self.prisma_client.db.litellm_usertable
@property
def model_class(self) -> Type[LiteLLM_UserTable]:
return LiteLLM_UserTable
def _to_model(self, record: Any) -> Optional[LiteLLM_UserTable]:
def _to_model(self, record: Optional[DbRecord]) -> Optional[LiteLLM_UserTable]:
"""Convert a database record to a User model."""
if record is None:
return None
data = record.dict() if hasattr(record, "dict") else dict(record)
return LiteLLM_UserTable.model_validate(
{
column: json.loads(value) if column in _JSON_ENCODED_COLUMNS and isinstance(value, str) else value
for column, value in record_to_dict(record).items()
}
)
json_fields = ["metadata", "model_spend", "model_max_budget"]
for field in json_fields:
if isinstance(data.get(field), str):
data[field] = json.loads(data[field])
return LiteLLM_UserTable(**data)
async def find_by_id(self, user_id: str, id_field: str = "user_id") -> Optional[LiteLLM_UserTable]:
return await super().find_by_id(user_id, id_field)
async def find_by_id(self, id_value: str, id_field: str = "user_id") -> Optional[LiteLLM_UserTable]:
return await super().find_by_id(id_value, id_field)
async def find_by_email(self, user_email: str) -> Optional[LiteLLM_UserTable]:
"""Find a user by email."""
records = await self.table.find_many(where={"user_email": user_email})
if records:
return self._to_model(records[0])
return None
records = await self.find_many(where={"user_email": user_email})
return records[0] if records else None
async def find_by_sso_id(self, sso_user_id: str) -> Optional[LiteLLM_UserTable]:
"""Find a user by SSO ID."""
record = await self.table.find_unique(where={"sso_user_id": sso_user_id})
return self._to_model(record)
return await self.find_by_id(sso_user_id, id_field="sso_user_id")
async def find_by_organization_id(self, organization_id: str) -> List[LiteLLM_UserTable]:
"""Find all users in an organization."""
records = await self.table.find_many(where={"organization_id": organization_id})
return self._to_model_list(records)
return await self.find_many(where={"organization_id": organization_id})
async def find_by_team_id(self, team_id: str) -> List[LiteLLM_UserTable]:
"""Find all users in a team."""
records = await self.table.find_many(where={"teams": {"has": team_id}})
return self._to_model_list(records)
return await self.find_many(where={"teams": {"has": team_id}})
async def count_billable_users(self) -> int:
"""Number of users that count toward the license seat limit.
@ -86,7 +81,7 @@ class UserRepository(BaseRepository[LiteLLM_UserTable]):
max_budget: Optional[float] = None,
user_email: Optional[str] = None,
models: Optional[List[str]] = None,
metadata: Optional[Dict[str, Any]] = None,
metadata: Optional[Mapping[str, object]] = None,
max_parallel_requests: Optional[int] = None,
tpm_limit: Optional[int] = None,
rpm_limit: Optional[int] = None,
@ -96,7 +91,7 @@ class UserRepository(BaseRepository[LiteLLM_UserTable]):
object_permission_id: Optional[str] = None,
) -> LiteLLM_UserTable:
"""Create a new user."""
data: Dict[str, Any] = {"user_id": user_id}
data: Dict[str, object] = {"user_id": user_id}
if user_alias is not None:
data["user_alias"] = user_alias
if team_id is not None:
@ -149,7 +144,7 @@ class UserRepository(BaseRepository[LiteLLM_UserTable]):
max_budget: Optional[float] = None,
user_email: Optional[str] = None,
models: Optional[List[str]] = None,
metadata: Optional[Dict[str, Any]] = None,
metadata: Optional[Mapping[str, object]] = None,
max_parallel_requests: Optional[int] = None,
tpm_limit: Optional[int] = None,
rpm_limit: Optional[int] = None,
@ -159,7 +154,7 @@ class UserRepository(BaseRepository[LiteLLM_UserTable]):
object_permission_id: Optional[str] = None,
) -> Optional[LiteLLM_UserTable]:
"""Update a user."""
data: Dict[str, Any] = {}
data: Dict[str, object] = {}
if user_alias is not None:
data["user_alias"] = user_alias
if team_id is not None:
@ -212,11 +207,7 @@ class UserRepository(BaseRepository[LiteLLM_UserTable]):
if not await self.exists(user_id, id_field="user_id"):
return None
record = await self.table.update(
where={"user_id": user_id},
data={"teams": {"push": team_id}},
)
return self._to_model(record)
return await self.update(user_id, {"teams": {"push": team_id}}, id_field="user_id")
async def remove_from_team(self, user_id: str, team_id: str) -> Optional[LiteLLM_UserTable]:
"""Remove a user from a team.

12
litellm/router.py
View file

@ -8613,9 +8613,9 @@ class Router:
deployment = self.get_deployment(model_id=model_id)
if deployment is None or self._is_deployment_blocked(deployment):
return None
return CredentialLiteLLMParams(**deployment.litellm_params.model_dump(exclude_none=True)).model_dump(
exclude_none=True
)
return CredentialLiteLLMParams.model_validate(
deployment.litellm_params.model_dump(exclude_none=True)
).model_dump(exclude_none=True)
def get_deployment_by_model_group_name(self, model_group_name: str) -> Optional[Deployment]:
"""
@ -8755,9 +8755,9 @@ class Router:
return None
# Get basic credentials
credentials = CredentialLiteLLMParams(**deployment.litellm_params.model_dump(exclude_none=True)).model_dump(
exclude_none=True
)
credentials = CredentialLiteLLMParams.model_validate(
deployment.litellm_params.model_dump(exclude_none=True)
).model_dump(exclude_none=True)
# Resolve litellm_credential_name to actual credentials
if deployment.litellm_params.litellm_credential_name is not None:

2
litellm/router_strategy/complexity_router/complexity_router.py
View file

@ -329,7 +329,7 @@ class ComplexityRouter(CustomLogger):
# Parse config - always create a new instance to avoid singleton mutation
if complexity_router_config:
self.config = ComplexityRouterConfig(**complexity_router_config)
self.config = ComplexityRouterConfig.model_validate(complexity_router_config)
else:
self.config = ComplexityRouterConfig()

4
litellm/types/proxy/management_endpoints/team_endpoints.py
View file

@ -1,4 +1,4 @@
from typing import Any, Dict, List, Optional, Union
from typing import Any, Dict, List, Literal, Optional, Union
from pydantic import BaseModel
@ -11,6 +11,8 @@ from litellm.proxy._types import (
Member,
)
TeamIdSearchMatch = Literal["exact", "prefix"]
class GetTeamMemberPermissionsRequest(BaseModel):
"""Request to get the team member permissions for a team"""

4
ruff-strict-budget.json
View file

@ -24,7 +24,7 @@
"limit": 130
},
"ANN401": {
"limit": 2010
"limit": 2009
},
"ASYNC230": {
"limit": 14
@ -324,7 +324,7 @@
"limit": 879
},
"UP006": {
"limit": 12138
"limit": 12135
},
"UP007": {
"limit": 2526

99
tests/test_litellm/proxy/hooks/test_user_management_event_hooks.py Normal file
View file

@ -0,0 +1,99 @@
import asyncio
import json
import sys
from types import SimpleNamespace
from typing import Any, Dict, List, Optional
from unittest.mock import AsyncMock, patch
import pytest
import litellm
from litellm.proxy._types import NewUserRequest, NewUserResponse, UserAPIKeyAuth
from litellm.proxy.hooks.user_management_event_hooks import UserManagementEventHooks
class FakeUserTable:
def __init__(self, rows: List[Dict[str, Any]]):
self._rows = rows
async def find_unique(self, where: Dict[str, Any]) -> Optional[Dict[str, Any]]:
return next(
(row for row in self._rows if all(row.get(key) == value for key, value in where.items())),
None,
)
class FakePrismaClient:
def __init__(self, rows: List[Dict[str, Any]]):
self.db = SimpleNamespace(litellm_usertable=FakeUserTable(rows))
async def _run_created_hook(prisma_client: FakePrismaClient, audit_log: AsyncMock) -> None:
proxy_server = SimpleNamespace(
prisma_client=prisma_client,
litellm_proxy_admin_name="admin-user",
)
with (
patch.dict(sys.modules, {"litellm.proxy.proxy_server": proxy_server}),
patch.object(litellm, "store_audit_logs", True),
patch(
"litellm.proxy.hooks.user_management_event_hooks.create_audit_log_for_update",
audit_log,
),
patch.object(
UserManagementEventHooks,
"async_send_user_invitation_email",
AsyncMock(),
),
):
await UserManagementEventHooks.async_user_created_hook(
data=NewUserRequest(user_email="new@example.com", send_invite_email=False),
response=NewUserResponse(
user_id="user-1",
user_email="new@example.com",
key="sk-test",
),
user_api_key_dict=UserAPIKeyAuth(user_id="admin-user", api_key="sk-admin"),
)
await asyncio.sleep(0)
@pytest.mark.asyncio
async def test_created_hook_audit_logs_the_row_read_back_from_the_database():
"""The audit entry must describe the persisted user row, not the /user/new response."""
prisma_client = FakePrismaClient(
[
{
"user_id": "user-1",
"user_email": "new@example.com",
"user_role": "proxy_admin",
"models": ["gpt-4"],
"teams": ["team-a"],
"metadata": '{"source": "api"}',
}
]
)
audit_log = AsyncMock()
await _run_created_hook(prisma_client, audit_log)
audit_log.assert_awaited_once()
request_data = audit_log.await_args.kwargs["request_data"]
assert request_data.object_id == "user-1"
assert request_data.action == "created"
updated_values = json.loads(request_data.updated_values)
assert updated_values["user_role"] == "proxy_admin"
assert updated_values["models"] == ["gpt-4"]
assert updated_values["teams"] == ["team-a"]
assert updated_values["metadata"] == {"source": "api"}
@pytest.mark.asyncio
async def test_created_hook_skips_the_audit_log_when_no_user_row_exists():
"""A user id that resolves to nothing must not produce an audit entry."""
audit_log = AsyncMock()
await _run_created_hook(FakePrismaClient([]), audit_log)
audit_log.assert_not_awaited()

70
tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py
View file

@ -839,7 +839,7 @@ class TestUpdateModel:
),
patch(
"litellm.proxy.management_endpoints.model_management_endpoints.clear_cache",
new=AsyncMock(return_value=True),
new=AsyncMock(return_value=None),
) as mock_clear_cache,
):
await update_model(
@ -3223,7 +3223,7 @@ class TestPatchModelBlockedAuthGate:
),
patch(
"litellm.proxy.management_endpoints.model_management_endpoints.clear_cache",
new=AsyncMock(return_value=True),
new=AsyncMock(return_value=None),
),
):
result = await patch_model(
@ -3314,6 +3314,72 @@ class TestWriteSurfacesReloadDrop:
before=frozenset({"m-live", "m-collateral"}), written_models=[("m-live", None)], action="update"
)
def test_a_model_the_db_no_longer_has_is_not_collateral(self, monkeypatch):
"""Another pod deleting a model is not this pod's reload breaking.
A pod that has not yet polled the delete still lists the id when the write
snapshots `before`; the reload it triggers then evicts the id because the db no
longer has it. That eviction is the reconcile working, so it must not fail the
write. `still_desired` is the db + config set the reload reconciled against, so
an id missing from it drops out of the collateral diff.
The cases below, in order: an id the db no longer wants is not collateral and the
write succeeds; an id the db still wants that stopped serving is real degradation
and still raises, so a genuinely broken reload is caught; and with no reconcile at
all the desired set is unknown, so every drop is reported.
"""
import litellm
from litellm.proxy._types import ProxyException
from litellm.proxy.management_endpoints.model_management_endpoints import (
raise_if_reload_degraded_serving,
reload_serving_verdict,
)
live_router = litellm.Router(
model_list=[
{
"model_name": "gpt-4o",
"litellm_params": {"model": "gpt-4o"},
"model_info": {"id": "m-live", "db_model": True},
}
]
)
monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", live_router)
_, collateral = reload_serving_verdict(
before=frozenset({"m-live", "m-deleted-elsewhere"}),
written_models=[("m-live", None)],
written_must_serve=True,
still_desired=frozenset({"m-live"}),
)
assert collateral == ()
assert (
raise_if_reload_degraded_serving(
before=frozenset({"m-live", "m-deleted-elsewhere"}),
written_models=[("m-live", None)],
action="create",
still_desired=frozenset({"m-live"}),
)
is None
)
with pytest.raises(ProxyException, match="m-should-be-serving"):
raise_if_reload_degraded_serving(
before=frozenset({"m-live", "m-should-be-serving"}),
written_models=[("m-live", None)],
action="create",
still_desired=frozenset({"m-live", "m-should-be-serving"}),
)
with pytest.raises(ProxyException, match="m-deleted-elsewhere"):
raise_if_reload_degraded_serving(
before=frozenset({"m-live", "m-deleted-elsewhere"}),
written_models=[("m-live", None)],
action="create",
still_desired=None,
)
class TestModelInfoAsMapping:
"""The model_info column reaches consumers as a dict or as its JSON string; this is

57
tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py
View file

@ -3627,9 +3627,9 @@ async def test_list_team_v2_with_invalid_status():
@pytest.mark.asyncio
async def test_list_team_v2_search_builds_or_clause():
"""
`search` should be passed as a Prisma OR across team_id (exact) and
team_alias (case-insensitive contains), so the UI can hit a single
backend filter with either a UUID or a name fragment.
`search` should be passed as a Prisma OR across an exact team_id match and a
case-insensitive team_alias contains, so the UI needs one backend filter.
Exact id matching is the documented default and must not change.
"""
from unittest.mock import AsyncMock, Mock, patch
@ -3671,6 +3671,54 @@ async def test_list_team_v2_search_builds_or_clause():
}
@pytest.mark.asyncio
async def test_list_team_v2_search_team_id_match_prefix():
"""
Opting into `search_team_id_match="prefix"` should widen the team_id side of
the search OR to an index-friendly prefix match, so the first characters of a
team id quoted in a proxy error find the team.
"""
from unittest.mock import AsyncMock, Mock, patch
from fastapi import Request
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy.management_endpoints.team_endpoints import list_team_v2
mock_request = Mock(spec=Request)
mock_admin = UserAPIKeyAuth(
user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin_user"
)
with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma_client:
mock_db = Mock()
mock_prisma_client.db = mock_db
mock_db.litellm_teamtable.find_many = AsyncMock(return_value=[])
mock_db.litellm_teamtable.count = AsyncMock(return_value=0)
await list_team_v2(
http_request=mock_request,
user_id=None,
organization_id=None,
team_id=None,
team_alias=None,
search="66c432fa",
search_team_id_match="prefix",
user_api_key_dict=mock_admin,
page=1,
page_size=10,
status=None,
)
find_many_kwargs = mock_db.litellm_teamtable.find_many.call_args.kwargs
assert find_many_kwargs["where"] == {
"OR": [
{"team_id": {"startsWith": "66c432fa"}},
{"team_alias": {"contains": "66c432fa", "mode": "insensitive"}},
]
}
@pytest.mark.asyncio
async def test_list_team_v2_search_composes_with_user_id_filter():
"""
@ -3724,6 +3772,7 @@ async def test_list_team_v2_search_composes_with_user_id_filter():
team_id=None,
team_alias=None,
search="team_a",
search_team_id_match="prefix",
user_api_key_dict=mock_user_api_key_dict,
page=1,
page_size=10,
@ -3733,7 +3782,7 @@ async def test_list_team_v2_search_composes_with_user_id_filter():
find_many_kwargs = mock_db.litellm_teamtable.find_many.call_args.kwargs
where = find_many_kwargs["where"]
assert where["OR"] == [
{"team_id": "team_a"},
{"team_id": {"startsWith": "team_a"}},
{"team_alias": {"contains": "team_a", "mode": "insensitive"}},
]
assert where["team_id"] == {"in": ["team_a", "team_b"]}

6
tests/test_litellm/proxy/proxy_server/test_proxy_config.py
View file

@ -1404,12 +1404,12 @@ def test_ProxyConfig_get_model_info_with_id_missing_model_id_raises(monkeypatch)
@pytest.mark.asyncio
async def test_ProxyConfig__delete_deployment_empty_returns_zero(monkeypatch):
async def test_ProxyConfig__delete_deployment_no_router_returns_none(monkeypatch):
monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None)
pc = ProxyConfig()
result = await pc._delete_deployment(db_models=[])
snapshot = {"deleted": result, "router_was": "none", "empty_db_models": True}
assert snapshot == {"deleted": 0, "router_was": "none", "empty_db_models": True}
snapshot = {"still_desired": result, "router_was": "none", "empty_db_models": True}
assert snapshot == {"still_desired": None, "router_was": "none", "empty_db_models": True}
@pytest.mark.asyncio

99
tests/test_litellm/proxy/test_proxy_server.py
View file

@ -2459,13 +2459,11 @@ async def test_delete_deployment_type_mismatch():
patch("litellm.proxy.proxy_server.user_config_file_path", "test_config.yaml"),
):
# Call the function under test
deleted_count = await pc._delete_deployment(db_models=[])
still_desired = await pc._delete_deployment(db_models=[])
# The two SHA-hash models have no corresponding entry in combined_id_list
# and must be evicted.
assert (
deleted_count == 2
), f"Expected 2 deletions (SHA-hash models), got {deleted_count}"
assert len(deleted_ids) == 2, f"Expected 2 deletions (SHA-hash models), got {deleted_ids}"
assert (
"a96e12e76b36a57cfae57a41288eb41567629cac89b4828c6f7074afc3534695"
in deleted_ids
@ -2485,6 +2483,12 @@ async def test_delete_deployment_type_mismatch():
"12345679" not in deleted_ids
), f"Model 12345679 should NOT be deleted. Deleted IDs: {deleted_ids}"
assert still_desired is not None
assert {"12345678", "12345679"} <= still_desired, (
"the int-keyed config models must come back as strings in the desired set, so a "
f"caller judging its own reload reads them as wanted rather than evicted; got {still_desired}"
)
@pytest.mark.asyncio
async def test_get_config_from_file(tmp_path, monkeypatch):
@ -9119,10 +9123,13 @@ class TestDeleteDeploymentSync:
with patch.object(
proxy_config, "get_config", AsyncMock(return_value={"model_list": []})
):
count = await proxy_config._delete_deployment(db_models=[])
still_desired = await proxy_config._delete_deployment(db_models=[])
mock_router.delete_deployment.assert_called_once_with(id="model-id-to-evict")
assert count == 1
assert still_desired == frozenset(), (
"an empty db and an empty config want nothing, which must stay distinct from "
f"the None returned when no reconcile ran at all; got {still_desired}"
)
@pytest.mark.asyncio
async def test_update_llm_router_skips_update_on_db_fetch_failure(self):
@ -10678,3 +10685,83 @@ async def test_async_data_generator_forwards_usage_chunk_without_strip_marker():
assert len(data_frames) == 4
assert any('"usage"' in frame and '"completion_tokens":188' in frame.replace(" ", "") for frame in data_frames)
assert frames[-1] == "data: [DONE]\n\n"
@pytest.mark.asyncio
async def test_config_field_update_rejects_mock_testing_flag():
"""The mock-testing opt-in is deliberately absent from
``ConfigGeneralSettings`` so that ``/config/field/update`` refuses it. If
someone later adds the field for tidiness, this test fails and tells them
they have just opened an API write path into a config-file-only setting."""
from fastapi import HTTPException
from litellm.proxy._types import ConfigFieldUpdate
from litellm.proxy.proxy_server import update_config_general_settings
from litellm.proxy.route_llm_request import MOCK_TESTING_CONFIG_KEY
admin = UserAPIKeyAuth(
user_role=LitellmUserRoles.PROXY_ADMIN,
api_key="sk-test",
)
with patch.object(proxy_server_module, "prisma_client", MagicMock()):
with pytest.raises(HTTPException) as exc_info:
await update_config_general_settings(
data=ConfigFieldUpdate(
field_name=MOCK_TESTING_CONFIG_KEY,
field_value=True,
config_type="general_settings",
),
user_api_key_dict=admin,
)
assert exc_info.value.status_code == 400
def test_config_update_body_drops_mock_testing_flag():
"""``/config/update`` parses its body as ``ConfigYAML``, whose
``general_settings`` is a ``ConfigGeneralSettings``. Undeclared keys are
dropped on parse, so the flag never reaches the DB by that route either."""
from litellm.proxy._types import ConfigYAML
from litellm.proxy.route_llm_request import MOCK_TESTING_CONFIG_KEY
parsed = ConfigYAML.model_validate({"general_settings": {MOCK_TESTING_CONFIG_KEY: True}})
assert parsed.general_settings is not None
assert MOCK_TESTING_CONFIG_KEY not in parsed.general_settings.model_dump(exclude_none=True)
def test_startup_warns_when_mock_testing_params_enabled(caplog):
"""Enabling the opt-in must announce itself, naming every param it
unlocks — the config key says ``mock_testing`` but the gate also covers
``mock_timeout`` and ``mock_delay``, so coverage cannot be inferred from
the name alone."""
import logging
from litellm.proxy.proxy_server import ProxyStartupEvent
from litellm.proxy.route_llm_request import (
GATED_MOCK_PARAM_NAMES,
MOCK_TESTING_CONFIG_KEY,
)
with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
ProxyStartupEvent._warn_if_mock_testing_params_enabled(
general_settings={MOCK_TESTING_CONFIG_KEY: True}
)
assert MOCK_TESTING_CONFIG_KEY in caplog.text
for param_name in GATED_MOCK_PARAM_NAMES:
assert param_name in caplog.text
def test_startup_is_silent_when_mock_testing_params_disabled(caplog):
"""A proxy that never set the opt-in must not emit the warning."""
import logging
from litellm.proxy.proxy_server import ProxyStartupEvent
from litellm.proxy.route_llm_request import MOCK_TESTING_CONFIG_KEY
with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
ProxyStartupEvent._warn_if_mock_testing_params_enabled(general_settings={})
assert MOCK_TESTING_CONFIG_KEY not in caplog.text

148
tests/test_litellm/proxy/test_route_llm_request.py
View file

@ -8,6 +8,8 @@ sys.path.insert(0, os.path.abspath("../../..")) # Adds the parent directory to
from unittest.mock import MagicMock
from fastapi import HTTPException
from litellm.proxy.route_llm_request import ProxyModelNotFoundError, route_request
@ -471,38 +473,147 @@ async def test_route_request_with_router_settings_override_preserves_existing():
assert call_kwargs["timeout"] == 30
def test_mock_testing_kwarg_names_matches_dataclass():
"""``_MOCK_TESTING_KWARG_NAMES`` is hardcoded to avoid a cyclic import
against ``litellm.types.router``. This test guards against drift —
if a new ``mock_testing_*`` field is added to ``MockRouterTestingParams``
the strip list must be updated to keep covering it."""
def test_gated_mock_params_cover_mock_router_testing_params():
"""``GATED_MOCK_PARAM_NAMES`` is hardcoded to avoid a cyclic import
against ``litellm.types.router``. This test guards against drift — if a
new ``mock_testing_*`` field is added to ``MockRouterTestingParams`` the
gate must be updated to keep covering it. The gate is a superset: it also
covers params consumed outside that dataclass."""
from dataclasses import fields
from litellm.proxy.route_llm_request import _MOCK_TESTING_KWARG_NAMES
from litellm.proxy.route_llm_request import GATED_MOCK_PARAM_NAMES
from litellm.types.router import MockRouterTestingParams
assert set(_MOCK_TESTING_KWARG_NAMES) == {f.name for f in fields(MockRouterTestingParams)}
assert {f.name for f in fields(MockRouterTestingParams)} <= set(GATED_MOCK_PARAM_NAMES)
assert {"mock_testing_rate_limit_error", "mock_timeout", "mock_delay"} <= set(GATED_MOCK_PARAM_NAMES)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"mock_flag",
"mock_param",
[
"mock_testing_fallbacks",
"mock_testing_context_fallbacks",
"mock_testing_content_policy_fallbacks",
"mock_testing_rate_limit_error",
"mock_timeout",
"mock_delay",
],
)
async def test_route_request_strips_mock_testing_flags(mock_flag):
"""VERIA-44: router-internal testing flags must not survive a
user-supplied request body. Without this strip, an attacker can
combine ``mock_testing_fallbacks=true`` with an unauthorized fallback
in ``router_settings_override`` to deterministically execute requests
against restricted models."""
def test_mock_params_rejected_when_not_allowed(mock_param):
"""Every gated param must be rejected by name when the proxy has not
opted in, and the error must point the caller at the config key."""
from litellm.proxy.route_llm_request import (
MOCK_TESTING_CONFIG_KEY,
raise_if_mock_testing_params_disallowed,
)
data = {"model": "gpt-3.5-turbo", mock_param: True}
with pytest.raises(HTTPException) as exc_info:
raise_if_mock_testing_params_disallowed(data, allowed=False)
assert exc_info.value.status_code == 400
error_message = exc_info.value.detail["error"]
assert mock_param in error_message
assert MOCK_TESTING_CONFIG_KEY in error_message
@pytest.mark.parametrize(
"mock_param",
[
"mock_testing_fallbacks",
"mock_testing_context_fallbacks",
"mock_testing_content_policy_fallbacks",
"mock_testing_rate_limit_error",
"mock_timeout",
"mock_delay",
],
)
def test_mock_params_pass_through_when_allowed(mock_param):
"""With the opt-in set, gated params must survive untouched — a gate that
rejects correctly but strips anyway would leave the feature unusable."""
from litellm.proxy.route_llm_request import raise_if_mock_testing_params_disallowed
data = {"model": "gpt-3.5-turbo", mock_param: True}
raise_if_mock_testing_params_disallowed(data, allowed=True)
assert data[mock_param] is True
def test_mock_param_gate_reports_every_param_present():
"""A request carrying several gated params must name all of them, so a
caller fixing one is not surprised by the next."""
from litellm.proxy.route_llm_request import raise_if_mock_testing_params_disallowed
data = {
"model": "gpt-3.5-turbo",
"mock_testing_fallbacks": True,
"mock_delay": 30,
}
with pytest.raises(HTTPException) as exc_info:
raise_if_mock_testing_params_disallowed(data, allowed=False)
error_message = exc_info.value.detail["error"]
assert "mock_testing_fallbacks" in error_message
assert "mock_delay" in error_message
def test_ordinary_request_is_not_rejected_by_the_mock_param_gate():
"""The gate must not fire on a request that carries no gated param."""
from litellm.proxy.route_llm_request import raise_if_mock_testing_params_disallowed
data = {
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "Hello"}],
mock_flag: True,
"mock_response": "hi",
}
raise_if_mock_testing_params_disallowed(data, allowed=False)
@pytest.mark.asyncio
async def test_route_request_rejects_mock_params_by_default(monkeypatch):
"""End-to-end through ``route_request``: with no opt-in configured the
request is rejected before it ever reaches the router."""
import litellm.proxy.proxy_server as proxy_server
monkeypatch.setattr(proxy_server, "general_settings", {}, raising=False)
data = {
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "Hello"}],
"mock_testing_fallbacks": True,
}
llm_router = MagicMock()
with pytest.raises(HTTPException) as exc_info:
await route_request(data, llm_router, None, "acompletion")
assert exc_info.value.status_code == 400
llm_router.acompletion.assert_not_called()
@pytest.mark.asyncio
async def test_route_request_forwards_mock_params_when_opted_in(monkeypatch):
"""End-to-end through ``route_request``: with the opt-in set the param
reaches the router, which is what makes a fallback drill possible."""
import litellm.proxy.proxy_server as proxy_server
from litellm.proxy.route_llm_request import MOCK_TESTING_CONFIG_KEY
monkeypatch.setattr(
proxy_server,
"general_settings",
{MOCK_TESTING_CONFIG_KEY: True},
raising=False,
)
data = {
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "Hello"}],
"mock_testing_fallbacks": True,
}
llm_router = MagicMock()
llm_router.acompletion.return_value = "ok"
@ -510,10 +621,7 @@ async def test_route_request_strips_mock_testing_flags(mock_flag):
await route_request(data, llm_router, None, "acompletion")
call_kwargs = llm_router.acompletion.call_args[1]
assert mock_flag not in call_kwargs
# The flag is also gone from the original data dict so any subsequent
# processing (e.g. logging) doesn't see it either.
assert mock_flag not in data
assert call_kwargs["mock_testing_fallbacks"] is True
@pytest.mark.parametrize("route_type", ["agenerate_content", "agenerate_content_stream"])

15
tests/test_litellm/proxy/test_update_llm_router_resilience.py
View file

@ -115,8 +115,10 @@ class TestDeleteDeploymentResilience:
"""Test _delete_deployment handles get_config failures gracefully."""
@pytest.mark.asyncio
async def test_returns_zero_when_get_config_times_out(self):
"""Should return 0 (no deletions) when get_config fails, not raise."""
async def test_returns_none_when_get_config_times_out(self):
"""Should return None (no reconcile ran, desired set unknown) when get_config
fails, not raise. A caller judging its own reload must not read that as "the db
wants nothing" and blame the reload for every model it serves."""
proxy_config = ProxyConfig()
db_models = [_make_db_model("gpt-5.1", "db-id-1")]
@ -136,8 +138,8 @@ class TestDeleteDeploymentResilience:
):
result = await proxy_config._delete_deployment(db_models=db_models)
# Should safely return 0 instead of raising
assert result == 0
# Should safely return None instead of raising
assert result is None
# Should NOT have deleted any deployments
mock_router.delete_deployment.assert_not_called()
@ -175,5 +177,8 @@ class TestDeleteDeploymentResilience:
result = await proxy_config._delete_deployment(db_models=db_models)
# "stale-id" should have been deleted (not in db_models or config)
assert result == 1
mock_router.delete_deployment.assert_called_once_with(id="stale-id")
assert result == frozenset({"db-id-1", "config-id-1"}), (
"the returned set must be what the db + config still want, so a caller can "
f"tell that eviction apart from a deployment that went missing; got {result}"
)

22
tests/test_litellm/repositories/test_repositories.py
View file

@ -1923,6 +1923,28 @@ class TestUserRepositoryExtended:
)
assert updated.user_email == "new@example.com"
@pytest.mark.asyncio
async def test_find_by_id_decodes_only_the_json_encoded_columns(self, repo):
repo._prisma_client.db.litellm_usertable._records["user-json"] = {
"user_id": "user-json",
"user_email": "json@example.com",
"user_role": '{"not": "json"}',
"teams": [],
"models": [],
"metadata": '{"department": "engineering"}',
"model_spend": '{"gpt-4": 10.5}',
"model_max_budget": '{"gpt-4": 100.0}',
}
user = await repo.find_by_id("user-json")
assert user is not None
assert user.metadata == {"department": "engineering"}
assert user.model_spend == {"gpt-4": 10.5}
assert user.model_max_budget == {"gpt-4": 100.0}
assert user.user_email == "json@example.com"
assert user.user_role == '{"not": "json"}'
class TestProjectRepositoryExtended:
@pytest.fixture

2
tests/test_litellm/test_model_block_unblock.py
View file

@ -36,7 +36,7 @@ def _setup_model_block_mocks(monkeypatch, *, updated_blocked: bool):
mock_router = MagicMock()
mock_router.get_model_ids.return_value = [model_id]
mock_clear_cache = AsyncMock(return_value=True)
mock_clear_cache = AsyncMock(return_value=None)
mock_audit_log = AsyncMock(return_value=None)
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)

6
type-discipline-budget.json
View file

@ -1,9 +1,9 @@
{
"LIT001": {
"limit": 23253
"limit": 23250
},
"LIT002": {
"limit": 27280
"limit": 27277
},
"LIT003": {
"limit": 292
@ -24,6 +24,6 @@
"limit": 1004
},
"LIT009": {
"limit": 2474
"limit": 2473
}
}

3
ui/litellm-dashboard/src/app/(dashboard)/hooks/teams/useTeams.ts
View file

@ -24,6 +24,7 @@ export interface TeamListCallOptions {
teamID?: string | null;
team_alias?: string | null;
search?: string | null;
searchTeamIdMatch?: "exact" | "prefix" | null;
userID?: string | null;
sortBy?: string | null;
sortOrder?: string | null;
@ -48,6 +49,7 @@ export const teamListCall = async (
organization_id: options.organizationID,
team_alias: options.team_alias,
search: options.search,
search_team_id_match: options.searchTeamIdMatch,
user_id: options.userID,
page,
page_size: pageSize,
@ -213,6 +215,7 @@ const deletedTeamListCall = async (
organization_id: options.organizationID,
team_alias: options.team_alias,
search: options.search,
search_team_id_match: options.searchTeamIdMatch,
user_id: options.userID,
page,
page_size: pageSize,

36
ui/litellm-dashboard/src/components/Teams.tsx
View file

@ -957,25 +957,23 @@ const Teams: React.FC<TeamProps> = ({ accessToken, userID, userRole, premiumUser
placeholder="Select vector stores (optional)"
/>
</Form.Item>
<Form.Item label="Allowed Pass Through Routes" name="allowed_passthrough_routes" className="mt-8">
<Tooltip
title={
!premiumUser
? "Premium feature - Upgrade to set allowed pass through routes"
: !isProxyAdminRole(userRole || "")
? "Only proxy admins can set allowed pass through routes"
: ""
}
placement="top"
>
<PassThroughRoutesSelector
onChange={(values: string[]) => form.setFieldValue("allowed_passthrough_routes", values)}
value={form.getFieldValue("allowed_passthrough_routes")}
accessToken={accessToken || ""}
placeholder="Select pass through routes (optional)"
disabled={!premiumUser || !isProxyAdminRole(userRole || "")}
/>
</Tooltip>
<Form.Item
label="Allowed Pass Through Routes"
name="allowed_passthrough_routes"
className="mt-8"
tooltip={
!premiumUser
? "Premium feature - Upgrade to set allowed pass through routes"
: !isProxyAdminRole(userRole || "")
? "Only proxy admins can set allowed pass through routes"
: undefined
}
>
<PassThroughRoutesSelector
accessToken={accessToken || ""}
placeholder="Select pass through routes (optional)"
disabled={!premiumUser || !isProxyAdminRole(userRole || "")}
/>
</Form.Item>
</AccordionBody>
</Accordion>

13
ui/litellm-dashboard/src/components/TeamsPage/TeamsTable.test.tsx
View file

@ -196,6 +196,19 @@ describe("server-side filtering maps controls to the right query params", () =>
expect(mockUseTeamsTable).toHaveBeenLastCalledWith(1, 50, expect.objectContaining({ search: "platform" }));
});
});
it("opts into team id prefix matching so a partial id from a proxy error finds the team", async () => {
renderTable();
fireEvent.change(screen.getByTestId("datatable-search"), { target: { value: "66c432fa" } });
await waitFor(() => {
expect(mockUseTeamsTable).toHaveBeenLastCalledWith(
1,
50,
expect.objectContaining({ search: "66c432fa", searchTeamIdMatch: "prefix" }),
);
});
});
});
describe("non-admin scoping", () => {

1
ui/litellm-dashboard/src/components/TeamsPage/TeamsTable.tsx
View file

@ -66,6 +66,7 @@ export function TeamsTable({ userRole, userID, onSelectTeam, onEditTeam, onDelet
team_alias: getFilterValue("alias"),
teamID: getFilterValue("team_id"),
search: searchQuery.trim() || undefined,
searchTeamIdMatch: "prefix" as const,
userID: isAdminView ? undefined : userID ?? undefined,
sortBy: sorting[0]?.id,
sortOrder: toSortOrder(sorting),

2
ui/litellm-dashboard/src/components/common_components/PassThroughRoutesSelector.tsx
View file

@ -3,7 +3,7 @@ import { Select } from "antd";
import { getPassThroughEndpointsCall } from "../networking";
interface PassThroughRoutesSelectorProps {
onChange: (selectedRoutes: string[]) => void;
onChange?: (selectedRoutes: string[]) => void;
value?: string[];
className?: string;
accessToken: string;

2
ui/litellm-dashboard/src/components/organisms/create_key_button.tsx
View file

@ -1385,8 +1385,6 @@ const CreateKey: React.FC<CreateKeyProps> = ({ team, teams, data, addKey, autoOp
}
>
<PassThroughRoutesSelector
onChange={(values: string[]) => form.setFieldValue("allowed_passthrough_routes", values)}
value={form.getFieldValue("allowed_passthrough_routes")}
accessToken={accessToken}
placeholder={
!premiumUser

51
ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx
View file

@ -18,6 +18,7 @@ vi.mock("@/components/networking", () => ({
getTeamPermissionsCall: vi.fn(),
organizationInfoCall: vi.fn(),
getRouterSettingsCall: vi.fn().mockResolvedValue({ fields: [] }),
getPassThroughEndpointsCall: vi.fn(),
}));
vi.mock("@/components/utils/dataUtils", () => ({
@ -1142,4 +1143,54 @@ describe("TeamInfoView", () => {
expect(within(dropdown).getByTitle("opt-in")).toBeInTheDocument();
});
});
describe("allowed pass through routes", () => {
beforeEach(() => {
testQueryClient.clear();
vi.mocked(networking.teamInfoCall).mockResolvedValue(createMockTeamData({ models: ["gpt-4"] }));
vi.mocked(networking.teamUpdateCall).mockResolvedValue({ data: {}, team_id: "123" } as any);
vi.mocked(networking.getPassThroughEndpointsCall).mockResolvedValue({
endpoints: [{ path: "/bedrock-passthrough", methods: ["POST"] }],
});
});
it("should show a route picked from the dropdown in the field and save it", async () => {
const user = userEvent.setup({ delay: null });
renderWithProviders(<TeamInfoView {...defaultProps} premiumUser={true} />);
await waitFor(() => {
expect(screen.queryAllByText("Test Team").length).toBeGreaterThan(0);
});
await user.click(screen.getByRole("tab", { name: "Settings" }));
await user.click(await screen.findByRole("button", { name: /edit settings/i }));
const routesLabel = await screen.findByText("Allowed Pass Through Routes");
const routesFormItem = routesLabel.closest(".ant-form-item") as HTMLElement;
await user.click(within(routesFormItem).getByRole("combobox"));
const option = await screen.findByTitle("POST /bedrock-passthrough");
await user.click(option);
await waitFor(() => {
expect(within(routesFormItem).getByText(/\/bedrock-passthrough/)).toBeInTheDocument();
});
await user.click(screen.getByRole("button", { name: /save changes/i }));
await waitFor(() => {
expect(networking.teamUpdateCall).toHaveBeenCalledWith(
"test-token",
expect.objectContaining({
team_id: "123",
metadata: expect.objectContaining({
allowed_passthrough_routes: ["/bedrock-passthrough"],
}),
}),
);
});
});
});
});

35
ui/litellm-dashboard/src/components/team/TeamInfo.tsx
View file

@ -1361,25 +1361,22 @@ const TeamInfoView: React.FC<TeamInfoProps> = ({
/>
</Form.Item>
<Form.Item label="Allowed Pass Through Routes" name="allowed_passthrough_routes">
<Tooltip
title={
!premiumUser
? "Premium feature - Upgrade to set allowed pass through routes"
: !is_proxy_admin
? "Only proxy admins can set allowed pass through routes"
: ""
}
placement="top"
>
<PassThroughRoutesSelector
onChange={(values: string[]) => form.setFieldValue("allowed_passthrough_routes", values)}
value={form.getFieldValue("allowed_passthrough_routes")}
accessToken={accessToken || ""}
placeholder="Select pass through routes"
disabled={!premiumUser || !is_proxy_admin}
/>
</Tooltip>
<Form.Item
label="Allowed Pass Through Routes"
name="allowed_passthrough_routes"
tooltip={
!premiumUser
? "Premium feature - Upgrade to set allowed pass through routes"
: !is_proxy_admin
? "Only proxy admins can set allowed pass through routes"
: undefined
}
>
<PassThroughRoutesSelector
accessToken={accessToken || ""}
placeholder="Select pass through routes"
disabled={!premiumUser || !is_proxy_admin}
/>
</Form.Item>
<Form.Item label="MCP Servers / Access Groups" name="mcp_servers_and_groups">

37
ui/litellm-dashboard/src/components/templates/key_edit_view.tsx
View file

@ -689,26 +689,23 @@ export function KeyEditView({
<AccessGroupSelector placeholder="Select access groups (optional)" />
</Form.Item>
<Form.Item label="Allowed Pass Through Routes" name="allowed_passthrough_routes">
<Tooltip
title={!premiumUser ? "Setting allowed pass through routes by key is a premium feature" : ""}
placement="top"
>
<PassThroughRoutesSelector
onChange={(values: string[]) => form.setFieldValue("allowed_passthrough_routes", values)}
value={form.getFieldValue("allowed_passthrough_routes")}
accessToken={accessToken || ""}
placeholder={
!premiumUser
? "Premium feature - Upgrade to set allowed pass through routes by key"
: Array.isArray(keyData.metadata?.allowed_passthrough_routes) &&
keyData.metadata.allowed_passthrough_routes.length > 0
? `Current: ${keyData.metadata.allowed_passthrough_routes.join(", ")}`
: "Select or enter allowed pass through routes"
}
disabled={!premiumUser}
/>
</Tooltip>
<Form.Item
label="Allowed Pass Through Routes"
name="allowed_passthrough_routes"
tooltip={!premiumUser ? "Setting allowed pass through routes by key is a premium feature" : undefined}
>
<PassThroughRoutesSelector
accessToken={accessToken || ""}
placeholder={
!premiumUser
? "Premium feature - Upgrade to set allowed pass through routes by key"
: Array.isArray(keyData.metadata?.allowed_passthrough_routes) &&
keyData.metadata.allowed_passthrough_routes.length > 0
? `Current: ${keyData.metadata.allowed_passthrough_routes.join(", ")}`
: "Select or enter allowed pass through routes"
}
disabled={!premiumUser}
/>
</Form.Item>
<Form.Item label="Vector Stores" name="vector_stores">

4
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -58372,8 +58372,10 @@ export interface operations {
team_id?: string | null;
/** @description Only return teams which this 'team_alias' belongs to. Supports partial matching. */
team_alias?: string | null;
/** @description Combined search: matches teams whose 'team_id' equals the value OR whose 'team_alias' contains it (case-insensitive). */
/** @description Combined search: matches teams whose 'team_id' matches the value OR whose 'team_alias' contains it (case-insensitive). */
search?: string | null;
/** @description How 'search' matches 'team_id': 'exact' (default) or 'prefix' for a case-sensitive prefix match. */
search_team_id_match?: "exact" | "prefix";
/** @description Page number for pagination */
page?: number;
/** @description Number of teams per page */