From 7015bf37bb3830dab79b3efd886eee9b5b7ffad6 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Sat, 5 Sep 2026 17:32:17 -0700 Subject: [PATCH 01/11] fix(proxy): apply team model aliases on the JWT auth path Team reads on the auth path never loaded the team's alias table, and the team-based JWT branch copied a hand-picked subset of team fields onto UserAPIKeyAuth, so aliases (and a few other team grants) never reached JWT callers: restricted teams 403'd alias requests and open teams 400'd them Load the alias relation where the team row is read and cached, project the team onto every team_* token field through one shared team_grants helper used by both JWT returns, and keep the relation when team model add/delete rewrites the cached team. ui_sso reuses the shared alias table model Resolves LIT-5858 Claude-Session: https://claude.ai/code/session_01EX13mWex6RaBo9PYnkAtFW --- basedpyright-code-budget.json | 2 +- litellm/proxy/auth/auth_checks.py | 9 +- litellm/proxy/auth/handle_jwt.py | 5 +- litellm/proxy/auth/team_grants.py | 122 +++++++++++++++++ litellm/proxy/auth/user_api_key_auth.py | 22 +-- .../management_endpoints/team_endpoints.py | 4 +- litellm/proxy/management_endpoints/ui_sso.py | 23 +--- .../proxy/auth/test_auth_checks.py | 64 +++++++++ .../proxy/auth/test_handle_jwt.py | 52 +++++++ .../proxy/auth/test_team_grants.py | 129 ++++++++++++++++++ .../proxy/auth/test_user_api_key_auth.py | 116 ++++++++++++++++ .../test_team_endpoints.py | 44 ++++++ type-discipline-budget.json | 2 +- 13 files changed, 547 insertions(+), 47 deletions(-) create mode 100644 litellm/proxy/auth/team_grants.py create mode 100644 tests/test_litellm/proxy/auth/test_team_grants.py diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index 0b0a61192e6..8bdc251c684 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -105,7 +105,7 @@ "limit": 109 }, "reportUnknownMemberType": { - "limit": 38271 + "limit": 38269 }, "reportUnknownParameterType": { "limit": 19584 diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index dc693317de0..a71a1993064 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -475,6 +475,7 @@ def _is_model_cost_zero(model: str | list[str] | None, llm_router: Router | None _NO_MODEL_INFO: Final[Mapping[str, object]] = MappingProxyType({}) +_TEAM_GRANT_RELATIONS: Final[Mapping[str, object]] = MappingProxyType({"litellm_model_table": True}) def _has_ptu_flat_cost(model: str, llm_router: "Router") -> bool: @@ -2858,7 +2859,9 @@ class TeamNotFoundError(HTTPException): async def _get_team_db_check( team_id: str, prisma_client: PrismaClient, team_id_upsert: bool | None = None ) -> "_PrismaTeamRow | None": - response = await _team_table(TeamRepository(prisma_client)).find_unique(where={"team_id": team_id}) + response = await _team_table(TeamRepository(prisma_client)).find_unique( + where={"team_id": team_id}, include=_TEAM_GRANT_RELATIONS + ) if response is None and team_id_upsert: from litellm.proxy.management_endpoints.team_endpoints import new_team @@ -3158,7 +3161,9 @@ async def get_team_object_by_alias( # Query database by team_alias try: - teams: Final = await _team_table(TeamRepository(prisma_client)).find_many(where={"team_alias": team_alias}) + teams: Final = await _team_table(TeamRepository(prisma_client)).find_many( + where={"team_alias": team_alias}, include=_TEAM_GRANT_RELATIONS + ) if not teams: raise HTTPException( diff --git a/litellm/proxy/auth/handle_jwt.py b/litellm/proxy/auth/handle_jwt.py index 0795cee7409..69091ee8344 100644 --- a/litellm/proxy/auth/handle_jwt.py +++ b/litellm/proxy/auth/handle_jwt.py @@ -53,6 +53,7 @@ from litellm.proxy._types import ( ) from litellm.proxy.auth.auth_checks import can_team_access_model from litellm.proxy.auth.route_checks import RouteChecks +from litellm.proxy.auth.team_grants import team_model_aliases from litellm.proxy.common_utils.user_api_key_cache import ( UserApiKeyCache, get_management_object_ttl, @@ -1595,7 +1596,7 @@ class JWTAuthManager: model=requested_model, team_object=team_object, llm_router=llm_router, - team_model_aliases=None, + team_model_aliases=team_model_aliases(team_object), ) ): is_allowed = allowed_routes_check( @@ -2132,7 +2133,7 @@ class JWTAuthManager: model=requested_model, team_object=team_object, llm_router=llm_router, - team_model_aliases=None, + team_model_aliases=team_model_aliases(team_object), ) except ProxyException: continue diff --git a/litellm/proxy/auth/team_grants.py b/litellm/proxy/auth/team_grants.py new file mode 100644 index 00000000000..1196011dcdd --- /dev/null +++ b/litellm/proxy/auth/team_grants.py @@ -0,0 +1,122 @@ +"""Project a team row (plus the caller's membership in it) onto the ``team_*`` fields of ``UserAPIKeyAuth``. + +The virtual-key path gets these fields for free from the combined-view SQL join. Every other auth path +starts from a ``LiteLLM_TeamTable`` object instead and has to copy them over by hand, which is how JWT +callers kept losing grants (aliases, permissions, limits) one field at a time. Build the badge through +``team_grants`` and the two paths cannot drift. +""" + +from collections.abc import Mapping, Sequence +from types import MappingProxyType +from typing import Annotated, Final + +from pydantic import BaseModel, BeforeValidator, ConfigDict, TypeAdapter, ValidationError +from pydantic.main import IncEx +from typing_extensions import ReadOnly, TypedDict + +from litellm.proxy._types import ( + LiteLLM_ObjectPermissionTable, + LiteLLM_TeamMembership, + LiteLLM_TeamTable, + Member, +) + +_MODEL_ALIASES_ADAPTER: Final = TypeAdapter(dict[str, str]) +_JSON_COLUMNS: Final[Mapping[str, IncEx | bool]] = MappingProxyType( + {"metadata": True, "litellm_model_table": MappingProxyType({"model_aliases": True})} +) + + +def _decode_model_aliases(value: object) -> object: + """``LiteLLM_ModelTable.model_aliases`` is typed ``str | dict``; writers hand Prisma ``json.dumps(...)``, so take both.""" + if not isinstance(value, str): + return value + try: + return _MODEL_ALIASES_ADAPTER.validate_json(value) + except ValidationError: + return None + + +class TeamModelAliasTable(BaseModel): + model_config = ConfigDict(protected_namespaces=()) + + model_aliases: Annotated[Mapping[str, str] | None, BeforeValidator(_decode_model_aliases)] = None + + +class _TeamJsonColumns(BaseModel): + """The two loosely typed columns on ``LiteLLM_TeamTable``, re-read with the shape the badge needs.""" + + metadata: Mapping[str, object] | None = None + litellm_model_table: TeamModelAliasTable | None = None + + +class TeamGrants(TypedDict, total=False): + """Keyword arguments for ``UserAPIKeyAuth``. Empty when the caller has no team, so the model's own defaults apply.""" + + team_alias: ReadOnly[str | None] + team_tpm_limit: ReadOnly[int | None] + team_rpm_limit: ReadOnly[int | None] + team_max_budget: ReadOnly[float | None] + team_soft_budget: ReadOnly[float | None] + team_spend: ReadOnly[float | None] + team_models: ReadOnly[Sequence[str]] + team_blocked: ReadOnly[bool] + team_metadata: ReadOnly[Mapping[str, object] | None] + team_model_aliases: ReadOnly[Mapping[str, str] | None] + team_object_permission_id: ReadOnly[str | None] + team_object_permission: ReadOnly[LiteLLM_ObjectPermissionTable | None] + team_member: ReadOnly[Member | None] + team_member_spend: ReadOnly[float | None] + team_member_tpm_limit: ReadOnly[int | None] + team_member_rpm_limit: ReadOnly[int | None] + + +def _json_columns(team_object: LiteLLM_TeamTable) -> _TeamJsonColumns: + try: + return _TeamJsonColumns.model_validate(team_object.model_dump(include=_JSON_COLUMNS)) + except ValidationError: + return _TeamJsonColumns() + + +def team_model_aliases(team_object: LiteLLM_TeamTable | None) -> Mapping[str, str] | None: + if team_object is None: + return None + alias_table: Final = _json_columns(team_object).litellm_model_table + return alias_table.model_aliases if alias_table is not None else None + + +def team_grants( + team_object: LiteLLM_TeamTable | None, + team_membership: LiteLLM_TeamMembership | None, + user_id: str | None, +) -> TeamGrants: + if team_object is None: + return TeamGrants() + json_columns: Final = _json_columns(team_object) + return TeamGrants( + team_alias=team_object.team_alias, + team_tpm_limit=team_object.tpm_limit, + team_rpm_limit=team_object.rpm_limit, + team_max_budget=team_object.max_budget, + team_soft_budget=team_object.soft_budget, + team_spend=team_object.spend, + team_models=tuple(team_object.models), + team_blocked=team_object.blocked, + team_metadata=json_columns.metadata, + team_model_aliases=( + json_columns.litellm_model_table.model_aliases if json_columns.litellm_model_table is not None else None + ), + team_object_permission_id=team_object.object_permission_id, + team_object_permission=team_object.object_permission, + team_member=next( + (m for m in team_object.members_with_roles if user_id is not None and m.user_id == user_id), + None, + ), + team_member_spend=team_membership.spend if team_membership is not None else None, + team_member_tpm_limit=( + team_membership.safe_get_team_member_tpm_limit() if team_membership is not None else None + ), + team_member_rpm_limit=( + team_membership.safe_get_team_member_rpm_limit() if team_membership is not None else None + ), + ) diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 93293db24c6..08dbe2508ec 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -82,6 +82,7 @@ from litellm.proxy.auth.oauth2_proxy_hook import handle_oauth2_proxy_request from litellm.proxy.auth.resolvers import CredentialRef, Principal from litellm.proxy.auth.resolvers.store import IdentityStore from litellm.proxy.auth.route_checks import RouteChecks +from litellm.proxy.auth.team_grants import team_grants from litellm.proxy.auth.trusted_proxy_utils import get_trusted_proxy_cidrs from litellm.proxy.common_utils.cache_coordinator import EventDrivenCacheCoordinator from litellm.proxy.common_utils.http_parsing_utils import ( @@ -1476,24 +1477,16 @@ async def _user_api_key_auth_builder( user_id=user_id, user_email=user_email, team_id=team_id, - team_alias=(team_object.team_alias if team_object is not None else None), - team_tpm_limit=(team_object.tpm_limit if team_object is not None else None), - team_rpm_limit=(team_object.rpm_limit if team_object is not None else None), - team_models=(team_object.models if team_object is not None else []), - team_metadata=(team_object.metadata if team_object is not None else None), org_id=org_id, end_user_id=end_user_id, parent_otel_span=parent_otel_span, jwt_claims=jwt_claims, + **team_grants(team_object=team_object, team_membership=team_membership, user_id=user_id), ) valid_token = UserAPIKeyAuth( api_key=None, team_id=team_id, - team_alias=(team_object.team_alias if team_object is not None else None), - team_tpm_limit=(team_object.tpm_limit if team_object is not None else None), - team_rpm_limit=(team_object.rpm_limit if team_object is not None else None), - team_models=(team_object.models if team_object is not None else []), user_role=( LitellmUserRoles(user_object.user_role) if user_object is not None and user_object.user_role is not None @@ -1507,17 +1500,8 @@ async def _user_api_key_auth_builder( user_tpm_limit=(user_object.tpm_limit if user_object is not None else None), user_rpm_limit=(user_object.rpm_limit if user_object is not None else None), user_model_max_budget=(user_object.model_max_budget if user_object is not None else None), - team_member_rpm_limit=( - team_membership.safe_get_team_member_rpm_limit() if team_membership is not None else None - ), - team_member_tpm_limit=( - team_membership.safe_get_team_member_tpm_limit() if team_membership is not None else None - ), - team_metadata=(team_object.metadata if team_object is not None else None), jwt_claims=jwt_claims, - ) - valid_token.team_object_permission = ( - team_object.object_permission if team_object is not None else None + **team_grants(team_object=team_object, team_membership=team_membership, user_id=user_id), ) # AUTO_REGISTER deferred from _resolve_jwt_to_virtual_key. diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 2ec68a10f65..c050368b3fe 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -5601,7 +5601,7 @@ async def team_model_add( updated_team: Final = await _team_db(prisma_client).update( where={"team_id": data.team_id}, data={"updated_at": datetime.now(timezone.utc)}, - include={"object_permission": True}, + include={"litellm_model_table": True, "object_permission": True}, ) if updated_team is None: raise HTTPException( @@ -5688,7 +5688,7 @@ async def team_model_delete( updated_team: Final = await _team_db(prisma_client).update( where={"team_id": data.team_id}, data={"models": updated_models}, - include={"object_permission": True}, + include={"litellm_model_table": True, "object_permission": True}, ) if updated_team is None: raise HTTPException( diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 3e6434a5afd..c60888e298f 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -22,7 +22,6 @@ from html import escape from types import MappingProxyType from typing import ( TYPE_CHECKING, - Annotated, Any, Final, Literal, @@ -42,7 +41,7 @@ if TYPE_CHECKING: import jwt from fastapi import APIRouter, Depends, Header, HTTPException, Request, Response, status from fastapi.responses import RedirectResponse -from pydantic import BaseModel, BeforeValidator, ConfigDict, TypeAdapter, ValidationError +from pydantic import BaseModel, TypeAdapter, ValidationError import litellm from litellm._logging import verbose_proxy_logger @@ -95,6 +94,7 @@ from litellm.proxy.auth.auth_utils import ( ) from litellm.proxy.auth.handle_jwt import JWTHandler from litellm.proxy.auth.ip_address_utils import IPAddressUtils +from litellm.proxy.auth.team_grants import TeamModelAliasTable from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.admin_ui_utils import ( admin_ui_disabled, @@ -209,31 +209,14 @@ def _team_detail_db(repo: TeamRepository) -> "TableActions[_TeamDetailRow]": return repo.table -_MODEL_ALIASES_ADAPTER: Final = TypeAdapter(dict[str, str]) _SSO_TOKEN_CLAIMS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) -def _decode_model_aliases(value: object) -> object: - """``/team/new`` stores team model aliases as a JSON-encoded string in the Json column.""" - if not isinstance(value, str): - return value - try: - return _MODEL_ALIASES_ADAPTER.validate_json(value) - except ValidationError: - return None - - -class _TeamModelAliasTable(BaseModel): - model_config = ConfigDict(protected_namespaces=()) - - model_aliases: Annotated[Mapping[str, str] | None, BeforeValidator(_decode_model_aliases)] = None - - class _TeamRowGrants(BaseModel): team_id: str team_alias: str | None = None models: tuple[str, ...] = () - litellm_model_table: _TeamModelAliasTable | None = None + litellm_model_table: TeamModelAliasTable | None = None class CliSsoTeamDetail(BaseModel): diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py index 2284a05b2e9..5bfef2b6445 100644 --- a/tests/test_litellm/proxy/auth/test_auth_checks.py +++ b/tests/test_litellm/proxy/auth/test_auth_checks.py @@ -2374,6 +2374,44 @@ def _mock_prisma_for_team_lookup(find_unique): return mock_prisma_client +_TEAM_ALIAS_TABLE_ROW = {"id": 1, "model_aliases": '{"fast": "gpt-4o"}', "created_by": "admin", "updated_by": "admin"} + + +def _prisma_team_row(include): + """Mimics Prisma: the `litellm_model_table` relation rides on the row only when the query `include`s it.""" + columns = {"team_id": "team-aliases", "team_alias": "aliases", "models": ["gpt-4o"]} + row = ( + {**columns, "litellm_model_table": _TEAM_ALIAS_TABLE_ROW} + if (include or {}).get("litellm_model_table") + else columns + ) + return SimpleNamespace(dict=lambda: row, model_dump=lambda: row) + + +@pytest.mark.asyncio +async def test_get_team_object_loads_model_aliases_relation(): + """LIT-5858: the auth path read teams without `include`ing `litellm_model_table`, so every JWT + team came back with `model_aliases=None` and alias requests 403'd.""" + from litellm.proxy.auth.auth_checks import get_team_object + from litellm.proxy.auth.team_grants import team_model_aliases + + async def find_unique(where, include=None): + return _prisma_team_row(include) + + mock_cache = MagicMock() + mock_cache.async_get_cache = AsyncMock(return_value=None) + mock_cache.async_set_cache = AsyncMock() + + team = await get_team_object( + team_id="team-aliases", + prisma_client=_mock_prisma_for_team_lookup(AsyncMock(side_effect=find_unique)), + user_api_key_cache=mock_cache, + check_db_only=True, + ) + + assert team_model_aliases(team) == {"fast": "gpt-4o"} + + @pytest.mark.asyncio async def test_get_team_object_distinguishes_absent_team_from_unreadable_row(): """A deleted team and a database that would not answer both surface as a 404, @@ -6195,6 +6233,32 @@ async def test_get_team_object_by_alias_db_fetch_returns_cached_obj(): assert result.models == ["gpt-4"] +@pytest.mark.asyncio +async def test_get_team_object_by_alias_loads_model_aliases_relation(): + """LIT-5858: same regression as `test_get_team_object_loads_model_aliases_relation`, for the + `team_alias_jwt_field` lookup.""" + from litellm.proxy.auth.auth_checks import get_team_object_by_alias + from litellm.proxy.auth.team_grants import team_model_aliases + + async def find_many(where, include=None): + return [_prisma_team_row(include)] + + mock_prisma_client = MagicMock() + mock_prisma_client.db.litellm_teamtable.find_many = AsyncMock(side_effect=find_many) + + mock_cache = MagicMock() + mock_cache.async_get_cache = AsyncMock(return_value=None) + mock_cache.async_set_cache = AsyncMock() + + team = await get_team_object_by_alias( + team_alias="aliases", + prisma_client=mock_prisma_client, + user_api_key_cache=mock_cache, + ) + + assert team_model_aliases(team) == {"fast": "gpt-4o"} + + @pytest.mark.asyncio async def test_get_org_object_by_alias_db_fetch_returns_validated_org(): from litellm.proxy._types import LiteLLM_OrganizationTable diff --git a/tests/test_litellm/proxy/auth/test_handle_jwt.py b/tests/test_litellm/proxy/auth/test_handle_jwt.py index 99a0a4c0a8b..94226b5404d 100644 --- a/tests/test_litellm/proxy/auth/test_handle_jwt.py +++ b/tests/test_litellm/proxy/auth/test_handle_jwt.py @@ -13,6 +13,7 @@ from litellm.proxy._types import ( DEFAULT_JWKS_STALE_TTL, JWTLiteLLMRoleMap, LiteLLM_JWTAuth, + LiteLLM_ModelTable, LiteLLM_TeamMembership, LiteLLM_TeamTable, LiteLLM_UserTable, @@ -1255,6 +1256,57 @@ async def test_find_team_with_model_access_model_group(monkeypatch): assert team_obj.team_id == "team-1" +@pytest.mark.asyncio +@pytest.mark.parametrize( + "model_aliases", + ['{"fast": "gpt-4o"}', {"fast": "gpt-4o"}], + ids=["json-string", "dict"], +) +async def test_find_team_with_model_access_resolves_team_model_alias(monkeypatch, model_aliases): + """LIT-5858: a JWT team that grants `gpt-4o` under the alias `fast` must resolve a request + for `fast`. The JWT path used to pass `team_model_aliases=None`, so every alias request 403'd.""" + import sys + import types + + from litellm.caching import DualCache + from litellm.proxy.utils import ProxyLogging + from litellm.router import Router + + router = Router(model_list=[{"model_name": "gpt-4o", "litellm_params": {"model": "gpt-4o"}}]) + proxy_server_module = types.ModuleType("proxy_server") + proxy_server_module.llm_router = router + monkeypatch.setitem(sys.modules, "litellm.proxy.proxy_server", proxy_server_module) + + team = LiteLLM_TeamTable( + team_id="team-aliases", + models=["gpt-4o"], + litellm_model_table=LiteLLM_ModelTable(model_aliases=model_aliases, created_by="admin", updated_by="admin"), + ) + + async def mock_get_team_object(*args, **kwargs): + return team + + monkeypatch.setattr("litellm.proxy.auth.handle_jwt.get_team_object", mock_get_team_object) + + jwt_handler = JWTHandler() + jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth() + user_api_key_cache = DualCache() + + team_id, team_obj = await JWTAuthManager.find_team_with_model_access( + team_ids={"team-aliases"}, + requested_model="fast", + route="/chat/completions", + jwt_handler=jwt_handler, + prisma_client=None, + user_api_key_cache=user_api_key_cache, + parent_otel_span=None, + proxy_logging_obj=ProxyLogging(user_api_key_cache=user_api_key_cache), + ) + + assert team_id == "team-aliases" + assert team_obj is team + + @pytest.mark.asyncio async def test_find_team_with_model_access_v1_messages_default_routes(monkeypatch): """Regression for #31189: a single-team JWT that grants the requested model diff --git a/tests/test_litellm/proxy/auth/test_team_grants.py b/tests/test_litellm/proxy/auth/test_team_grants.py new file mode 100644 index 00000000000..447fc1c93a1 --- /dev/null +++ b/tests/test_litellm/proxy/auth/test_team_grants.py @@ -0,0 +1,129 @@ +import pytest + +from litellm.proxy._types import ( + LiteLLM_BudgetTable, + LiteLLM_ObjectPermissionTable, + LiteLLM_TeamMembership, + LiteLLM_TeamTable, + LiteLLM_VerificationTokenView, + Member, + UserAPIKeyAuth, +) +from litellm.models.team import LiteLLM_ModelTable +from litellm.proxy.auth.team_grants import team_grants, team_model_aliases + +TEAM_ID = "team-grants" +USER_ID = "user-in-team" +ALIASES = {"fast": "gpt-4o-mini", "smart": "gpt-4o"} + + +def _alias_table(model_aliases) -> LiteLLM_ModelTable: + return LiteLLM_ModelTable(model_aliases=model_aliases, created_by="admin", updated_by="admin") + + +def _full_team(model_aliases=ALIASES) -> LiteLLM_TeamTable: + return LiteLLM_TeamTable( + team_id=TEAM_ID, + team_alias="grants-team", + tpm_limit=1000, + rpm_limit=10, + max_budget=50.0, + soft_budget=25.0, + spend=12.5, + models=["gpt-4o", "gpt-4o-mini"], + blocked=True, + metadata={"tier": "gold"}, + litellm_model_table=_alias_table(model_aliases), + object_permission_id="op-1", + object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="op-1", mcp_servers=["mcp-a"]), + members_with_roles=[ + Member(user_id="someone-else", role="user"), + Member(user_id=USER_ID, role="admin"), + ], + ) + + +def _membership() -> LiteLLM_TeamMembership: + return LiteLLM_TeamMembership( + user_id=USER_ID, + team_id=TEAM_ID, + spend=3.25, + litellm_budget_table=LiteLLM_BudgetTable(tpm_limit=500, rpm_limit=5), + ) + + +def test_team_grants_cover_every_team_field_the_key_path_gets(): + """Class guard for LIT-5858 and its siblings: every ``team_*`` column the combined-view SQL hands the + virtual-key path must come out of the projection too, with the team's actual value, so adding a column + to ``LiteLLM_VerificationTokenView`` without teaching ``team_grants`` fails here instead of in prod.""" + team = _full_team() + grants = team_grants(team_object=team, team_membership=_membership(), user_id=USER_ID) + token = UserAPIKeyAuth(team_id=TEAM_ID, **grants) + + view_team_fields = {name for name in LiteLLM_VerificationTokenView.model_fields if name.startswith("team_")} + assert view_team_fields - {"team_id"} <= set(grants) + assert all(grants[name] is not None for name in view_team_fields - {"team_id"}) + + assert token.team_alias == "grants-team" + assert token.team_tpm_limit == 1000 + assert token.team_rpm_limit == 10 + assert token.team_max_budget == 50.0 + assert token.team_soft_budget == 25.0 + assert token.team_spend == 12.5 + assert token.team_models == ["gpt-4o", "gpt-4o-mini"] + assert token.team_blocked is True + assert token.team_metadata == {"tier": "gold"} + assert token.team_model_aliases == ALIASES + assert token.team_object_permission_id == "op-1" + assert token.team_object_permission is not None + assert token.team_object_permission.mcp_servers == ["mcp-a"] + assert token.team_member == Member(user_id=USER_ID, role="admin") + assert token.team_member_spend == 3.25 + assert token.team_member_tpm_limit == 500 + assert token.team_member_rpm_limit == 5 + + +def test_team_grants_without_team_leave_token_defaults(): + token = UserAPIKeyAuth(**team_grants(team_object=None, team_membership=None, user_id=USER_ID)) + assert token == UserAPIKeyAuth() + + +@pytest.mark.parametrize( + "stored_aliases", + [ALIASES, '{"fast": "gpt-4o-mini", "smart": "gpt-4o"}'], + ids=["json-object", "json-string-as-written-by-team-new"], +) +def test_team_model_aliases_decode_both_storage_shapes(stored_aliases): + team = _full_team(model_aliases=stored_aliases) + assert team_model_aliases(team) == ALIASES + assert team_grants(team_object=team, team_membership=None, user_id=None)["team_model_aliases"] == ALIASES + + +@pytest.mark.parametrize("stored_aliases", [None, "not json", '["a", "b"]', {"fast": 3}], ids=str) +def test_team_model_aliases_treat_unusable_column_as_no_aliases(stored_aliases): + team = _full_team(model_aliases=stored_aliases) + assert team_model_aliases(team) is None + assert team_grants(team_object=team, team_membership=None, user_id=None)["team_model_aliases"] is None + + +def test_team_model_aliases_none_without_relation_loaded(): + team = _full_team() + team.litellm_model_table = None + assert team_model_aliases(team) is None + assert team_model_aliases(None) is None + + +def test_team_member_is_the_callers_row_only(): + team = _full_team() + assert team_grants(team_object=team, team_membership=None, user_id="someone-else")["team_member"] == Member( + user_id="someone-else", role="user" + ) + assert team_grants(team_object=team, team_membership=None, user_id="stranger")["team_member"] is None + assert team_grants(team_object=team, team_membership=None, user_id=None)["team_member"] is None + + +def test_membership_limits_absent_without_membership_row(): + grants = team_grants(team_object=_full_team(), team_membership=None, user_id=USER_ID) + assert grants["team_member_spend"] is None + assert grants["team_member_tpm_limit"] is None + assert grants["team_member_rpm_limit"] is None diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index d44f96d95bf..78ffbb0db23 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -6872,3 +6872,119 @@ class TestLitellmReceivedAtStamping: assert result == earlier assert request.state.litellm_received_at == earlier + + +@pytest.mark.asyncio +@pytest.mark.parametrize("is_proxy_admin", [False, True], ids=["standard-return", "proxy-admin-return"]) +async def test_jwt_builder_returns_every_team_grant_the_key_path_gets(is_proxy_admin): + """LIT-5858: the team-based JWT path hand-built ``UserAPIKeyAuth`` from a short list of team fields, so the + team's model aliases (and on the admin return, its object permission) never reached the token and alias + requests 403'd. Both returns now go through ``team_grants``; pin the fields that used to be dropped.""" + import litellm.proxy.proxy_server as _proxy_server_mod + from fastapi import Request + from starlette.datastructures import URL + + from litellm.models.team import LiteLLM_ModelTable + from litellm.proxy._types import ( + LiteLLM_ObjectPermissionTable, + LiteLLM_TeamMembership, + LiteLLM_TeamTable, + Member, + ) + + class _AcceptEveryJwt(JWTHandler): + def is_jwt(self, token: str) -> bool: + return True + + jwt_handler = _AcceptEveryJwt() + jwt_handler.litellm_jwtauth = LiteLLM_JWTAuth() + + team = LiteLLM_TeamTable( + team_id="team-jwt-aliases", + team_alias="jwt-aliases", + models=["gpt-4o"], + max_budget=40.0, + spend=4.0, + blocked=False, + metadata={"tier": "gold"}, + litellm_model_table=LiteLLM_ModelTable( + model_aliases='{"fast": "gpt-4o"}', created_by="admin", updated_by="admin" + ), + object_permission_id="op-jwt", + object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="op-jwt", mcp_servers=["mcp-a"]), + members_with_roles=[Member(user_id="jwt-user", role="admin")], + ) + membership = LiteLLM_TeamMembership(user_id="jwt-user", team_id="team-jwt-aliases", spend=1.5) + builder_result = { + "is_proxy_admin": is_proxy_admin, + "team_object": team, + "user_object": None, + "end_user_object": None, + "org_object": None, + "token": "jwt", + "team_id": "team-jwt-aliases", + "user_id": "jwt-user", + "user_email": "jwt-user@example.com", + "end_user_id": None, + "org_id": None, + "team_membership": membership, + "jwt_claims": {"sub": "jwt-user"}, + } + + mock_proxy_logging_obj = MagicMock() + mock_proxy_logging_obj.internal_usage_cache = MagicMock() + mock_proxy_logging_obj.internal_usage_cache.dual_cache = AsyncMock() + mock_proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None) + attrs = { + "prisma_client": MagicMock(), + "user_api_key_cache": DualCache(), + "proxy_logging_obj": mock_proxy_logging_obj, + "master_key": "sk-master-key", + "general_settings": {"enable_jwt_auth": True}, + "llm_model_list": [], + "llm_router": None, + "open_telemetry_logger": None, + "model_max_budget_limiter": MagicMock(), + "user_custom_auth": None, + "jwt_handler": jwt_handler, + "premium_user": True, + "litellm_proxy_admin_name": "admin", + } + originals = {a: getattr(_proxy_server_mod, a, None) for a in attrs} + try: + for k, v in attrs.items(): + setattr(_proxy_server_mod, k, v) + request = Request(scope={"type": "http", "headers": [], "method": "POST"}) + request._url = URL(url="/chat/completions") + with patch( # test-quality-ok: auth_builder is the claim-resolution seam; the regression is how its result is projected onto the token + "litellm.proxy.auth.user_api_key_auth.JWTAuthManager.auth_builder", + new_callable=AsyncMock, + return_value=builder_result, + ): + token = await _user_api_key_auth_builder( + request=request, + api_key="Bearer header.payload.signature", + azure_api_key_header="", + anthropic_api_key_header=None, + google_ai_studio_api_key_header=None, + azure_apim_header=None, + request_data={}, + ) + finally: + for k, v in originals.items(): + setattr(_proxy_server_mod, k, v) + + assert token.team_id == "team-jwt-aliases" + assert token.user_role == (LitellmUserRoles.PROXY_ADMIN if is_proxy_admin else LitellmUserRoles.INTERNAL_USER) + assert token.team_model_aliases == {"fast": "gpt-4o"} + assert token.team_object_permission is not None + assert token.team_object_permission.mcp_servers == ["mcp-a"] + assert token.team_object_permission_id == "op-jwt" + assert token.team_alias == "jwt-aliases" + assert token.team_models == ["gpt-4o"] + assert token.team_max_budget == 40.0 + assert token.team_spend == 4.0 + assert token.team_metadata == {"tier": "gold"} + assert token.team_member == Member(user_id="jwt-user", role="admin") + assert token.team_member_spend == 1.5 + assert token.jwt_claims == {"sub": "jwt-user"} diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index 051e6bed4fd..2f6561046b1 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -2205,6 +2205,50 @@ async def test_team_model_add_delete_refresh_team_cache(endpoint_name): assert update_call_kwargs.get("include", {}).get("object_permission") is True +@pytest.mark.asyncio +@pytest.mark.parametrize("endpoint_name", ["team_model_add", "team_model_delete"]) +async def test_team_model_add_delete_keep_model_aliases_in_team_cache(endpoint_name, monkeypatch): + """LIT-5858: Prisma only returns `litellm_model_table` when the `update` asks for it, so the refreshed + cache entry lost the team's model aliases and JWT alias requests 403'd until the next DB read.""" + from litellm.proxy._types import TeamModelAddRequest, TeamModelDeleteRequest + from litellm.proxy.auth.team_grants import team_model_aliases + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.management_endpoints.team_endpoints import team_model_add, team_model_delete + + columns = {"team_id": "team-1234", "models": ["gpt-4o", "openai/*"]} + alias_table = {"id": 1, "model_aliases": '{"fast": "gpt-4o"}', "created_by": "admin", "updated_by": "admin"} + + async def update(where, data, include=None): + row = {**columns, "litellm_model_table": alias_table} if (include or {}).get("litellm_model_table") else columns + return SimpleNamespace(team_id="team-1234", model_dump=lambda: row) + + prisma_client = MagicMock() + prisma_client.db.litellm_teamtable.find_unique = AsyncMock(return_value=SimpleNamespace(model_dump=lambda: columns)) + prisma_client.db.litellm_teamtable.update = AsyncMock(side_effect=update) + prisma_client.db.execute_raw = AsyncMock(return_value=None) + cache = UserApiKeyCache() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma_client) + monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", cache) + monkeypatch.setattr("litellm.proxy.proxy_server.proxy_logging_obj", None) + + admin = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin") + if endpoint_name == "team_model_add": + await team_model_add( + data=TeamModelAddRequest(team_id="team-1234", models=["team-byok-1"]), + http_request=MagicMock(), + user_api_key_dict=admin, + ) + else: + await team_model_delete( + data=TeamModelDeleteRequest(team_id="team-1234", models=["openai/*"]), + http_request=MagicMock(), + user_api_key_dict=admin, + ) + + cached_team = await cache.async_get_cache(key="team_id:team-1234", model_type=LiteLLM_TeamTableCachedObj) + assert team_model_aliases(cached_team) == {"fast": "gpt-4o"} + + @pytest.mark.asyncio @pytest.mark.parametrize( "endpoint_name", diff --git a/type-discipline-budget.json b/type-discipline-budget.json index e7186dfe186..7db8ed501f8 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -3,7 +3,7 @@ "limit": 22180 }, "LIT002": { - "limit": 26729 + "limit": 26727 }, "LIT003": { "limit": 261 From 0086b62b4575fb5bb69653c2ad3d196822bd3f13 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 9 Sep 2026 02:05:08 +0000 Subject: [PATCH 02/11] fix(cost-map): keep first fetch blocking, run retries in background Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/__init__.py | 4 +- .../litellm_core_utils/get_model_cost_map.py | 132 +++++++++- litellm/proxy/proxy_server.py | 23 +- .../test_get_model_cost_map.py | 235 +++++++++++++----- 4 files changed, 296 insertions(+), 98 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 42c0ea881fd..4e3754399da 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -541,7 +541,7 @@ _key_management_system: Optional["KeyManagementSystem"] = None #### PII MASKING #### output_parse_pii: bool = False ############################################# -from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map +from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map, mark_litellm_import_complete model_cost = get_model_cost_map(url=model_cost_map_url) cost_discount_config: Dict[str, float] = {} # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount @@ -2397,3 +2397,5 @@ def __getattr__(name: str) -> Any: # ALL_LITELLM_RESPONSE_TYPES is lazy-loaded via __getattr__ to avoid loading utils at import time + +mark_litellm_import_complete() diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index 9cba5db8ab7..4118cd3420e 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -12,6 +12,7 @@ import asyncio import json import os import random +import threading import time from collections.abc import Awaitable, Callable from dataclasses import dataclass @@ -159,6 +160,15 @@ class GetModelCostMap: RETRYABLE_FETCH_STATUS_CODES: Final = frozenset({429, 500, 502, 503, 504}) MODEL_COST_MAP_FETCH_MAX_ATTEMPTS: Final = 3 MODEL_COST_MAP_FETCH_MAX_WAIT_SECONDS: Final = 30.0 +_litellm_import_complete = threading.Event() + + +def mark_litellm_import_complete() -> None: + _litellm_import_complete.set() + + +def _start_daemon_thread(fn: Callable[[], None]) -> None: + threading.Thread(target=fn, name="litellm-model-cost-map-retry", daemon=True).start() @dataclass(frozen=True, slots=True) @@ -297,9 +307,15 @@ def _fetch_remote_model_cost_map_with_retry_sync( sleep: Callable[[float], None], rng: random.Random, client: _SyncGetClient, + starting_attempt: int = 1, + initial_outcome: _FetchAttemptRetryable | None = None, ) -> ModelCostMapReloadResult: - for attempt in range(1, max_attempts + 1): - outcome = _attempt_fetch_sync(client=client, url=url, timeout=timeout) + for attempt in range(starting_attempt, max_attempts + 1): + outcome = ( + initial_outcome + if initial_outcome is not None and attempt == starting_attempt + else _attempt_fetch_sync(client=client, url=url, timeout=timeout) + ) if not isinstance(outcome, _FetchAttemptRetryable): return outcome wait_seconds = _next_retry_wait(outcome=outcome, attempt=attempt, max_attempts=max_attempts, rng=rng) @@ -464,6 +480,70 @@ def _finalize_model_cost_map(model_cost: dict) -> dict: return _expand_model_aliases(model_cost) +def adopt_model_cost_map( + new_model_cost_map: dict, # mutable-ok: public API preserves the mutable cost-map contract +) -> int: + import litellm + from litellm import utils + + litellm.model_cost = new_model_cost_map + utils._invalidate_model_cost_lowercase_map() # pyright: ignore[reportPrivateUsage] # required cache invalidation + litellm.add_known_models(model_cost_map=new_model_cost_map) + fetched_model_count: Final = len(new_model_cost_map) if new_model_cost_map else 0 + utils.reapply_runtime_model_cost_registrations() + return fetched_model_count + + +def _continue_remote_fetch_in_background( + url: str, + timeout: int, + max_attempts: int, + sleep: Callable[[float], None], + rng: random.Random, + client: _SyncGetClient, + first_outcome: _FetchAttemptRetryable, + apply: Callable[[dict], object], # mutable-ok: injected callback receives the mutable cost-map dict +) -> None: + try: + result: Final = _fetch_remote_model_cost_map_with_retry_sync( + url=url, + timeout=timeout, + max_attempts=max_attempts, + sleep=sleep, + rng=rng, + client=client, + initial_outcome=first_outcome, + ) + if isinstance(result, ModelCostMapReloadUnavailable): + verbose_logger.warning( + "LiteLLM: Failed to fetch remote model cost map from %s after %d attempts; keeping local backup", + url, + max_attempts, + ) + return + backup_model_count: Final = GetModelCostMap._get_backup_model_count() # pyright: ignore[reportPrivateUsage] # integrity cache + if not GetModelCostMap.validate_model_cost_map( + fetched_map=result.model_cost_map, + backup_model_count=backup_model_count, + ): + verbose_logger.warning( + "LiteLLM: Fetched model cost map failed integrity check. Keeping local backup. url=%s", + url, + ) + _cost_map_source_info.fallback_reason = "Remote data failed integrity validation" + return + finalized_map: Final = _finalize_model_cost_map(result.model_cost_map) + _litellm_import_complete.wait() + apply(finalized_map) + _cost_map_source_info.source = "remote" + _cost_map_source_info.url = url + _cost_map_source_info.is_env_forced = False + _cost_map_source_info.fallback_reason = None + _cost_map_source_info.loaded_at = datetime.now(timezone.utc) + except Exception as e: + verbose_logger.warning("LiteLLM: Background model cost map retry failed: %s", e) + + def get_model_cost_map( url: str, timeout: int = 5, @@ -471,14 +551,19 @@ def get_model_cost_map( sleep: Callable[[float], None] = time.sleep, rng: random.Random | None = None, client: "_SyncGetClient | None" = None, + start_background: Callable[[Callable[[], None]], None] = _start_daemon_thread, + apply: Callable[ # mutable-ok: injected callback receives the mutable cost-map dict + [dict], + object, + ] = adopt_model_cost_map, ) -> dict: """ Public entry point — returns the model cost map dict. 1. If ``LITELLM_LOCAL_MODEL_COST_MAP`` is set, uses the local backup only. - 2. Otherwise fetches from ``url``, retrying transient HTTP errors - (429/5xx/transport) with Retry-After-aware backoff, validates - integrity, and falls back to the local backup on any failure. + 2. Otherwise fetches from ``url``, validates the first response, and falls + back to the local backup while retrying transient HTTP errors in the + background. Only the backup model count is cached (a single int) for validation. The full backup dict is only parsed when it must be *returned* as a @@ -497,14 +582,35 @@ def get_model_cost_map( _cost_map_source_info.url = url _cost_map_source_info.is_env_forced = False - result: Final = _fetch_remote_model_cost_map_with_retry_sync( - url=url, - timeout=timeout, - max_attempts=max_attempts, - sleep=sleep, - rng=rng if rng is not None else random.Random(), - client=client if client is not None else httpx, - ) + fetch_client: Final = client if client is not None else httpx + fetch_rng: Final = rng if rng is not None else random.Random() + first_outcome: Final = _attempt_fetch_sync(client=fetch_client, url=url, timeout=timeout) + if isinstance(first_outcome, _FetchAttemptRetryable): + verbose_logger.warning( + "LiteLLM: model cost map fetch attempt 1/%d failed: %s; " + "using local backup while retrying in the background", + max_attempts, + first_outcome.reason, + ) + _cost_map_source_info.source = "local" + _cost_map_source_info.fallback_reason = f"Remote fetch failed: {first_outcome.reason}" + local_map: Final = _finalize_model_cost_map(GetModelCostMap.load_local_model_cost_map()) + if max_attempts > 1: + start_background( + lambda: _continue_remote_fetch_in_background( + url=url, + timeout=timeout, + max_attempts=max_attempts, + sleep=sleep, + rng=fetch_rng, + client=fetch_client, + first_outcome=first_outcome, + apply=apply, + ) + ) + return local_map + + result: Final = first_outcome if isinstance(result, ModelCostMapReloadUnavailable): verbose_logger.warning( "LiteLLM: Failed to fetch remote model cost map from %s: %s. Falling back to local backup.", diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 1cd08fe27c0..efe38376353 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -132,11 +132,7 @@ from litellm.types.utils import ( TextCompletionResponse, TokenCountResponse, ) -from litellm.utils import ( - _invalidate_model_cost_lowercase_map, - load_credentials_from_list, - reapply_runtime_model_cost_registrations, -) +from litellm.utils import load_credentials_from_list if TYPE_CHECKING: from aiohttp import ClientSession @@ -4411,20 +4407,9 @@ def resolve_classifier_plugin( def _swap_in_model_cost_map(new_model_cost_map: dict) -> int: - """Adopt a freshly fetched cost map into this process's litellm state, return the model count""" - litellm.model_cost = new_model_cost_map - # Invalidate case-insensitive lookup map since model_cost was replaced - _invalidate_model_cost_lowercase_map() - # Repopulate provider model sets (e.g. litellm.anthropic_models) so that - # wildcard patterns like "anthropic/*" include any newly added models. - litellm.add_known_models(model_cost_map=new_model_cost_map) - # Counted before the re-apply below, which writes into this same dict, so the - # number reported describes the fetched price data alone. - fetched_model_count: Final = len(new_model_cost_map) if new_model_cost_map else 0 - # The swap discards everything registered at runtime (deployment model_info, - # register_model overrides), so put it back on top of the fresh catalog. - reapply_runtime_model_cost_registrations() - return fetched_model_count + from litellm.litellm_core_utils.get_model_cost_map import adopt_model_cost_map + + return adopt_model_cost_map(new_model_cost_map) def should_load_db_object(object_type: str | SupportedDBObjectType) -> bool: diff --git a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py index 18185126775..dce1a431f13 100644 --- a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py +++ b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py @@ -20,21 +20,18 @@ from litellm.litellm_core_utils.get_model_cost_map import ( GetModelCostMap, _count_model_entries, _finalize_model_cost_map, + adopt_model_cost_map, ) def _load_root_cost_map() -> dict: - path = os.path.join( - os.path.dirname(__file__), "../../../model_prices_and_context_window.json" - ) + path = os.path.join(os.path.dirname(__file__), "../../../model_prices_and_context_window.json") with open(path) as f: return json.load(f) def _make_models(n: int) -> dict: - return { - f"model-{i}": {"litellm_provider": "openai", "mode": "chat"} for i in range(n) - } + return {f"model-{i}": {"litellm_provider": "openai", "mode": "chat"} for i in range(n)} def test_count_model_entries_excludes_reserved_keys(): @@ -117,9 +114,7 @@ def test_finalize_pops_key_and_installs_rules(): def test_finalize_with_no_block_clears_rules(): previous = list(get_fallback_generalization_rules()) try: - set_fallback_generalizations( - [{"name": "stale", "pattern": r"^x", "model_info": {"a": 1}}] - ) + set_fallback_generalizations([{"name": "stale", "pattern": r"^x", "model_info": {"a": 1}}]) _finalize_model_cost_map(_make_models(2)) assert match_capability_generalizations("x-1") is None finally: @@ -306,9 +301,7 @@ def test_get_model_cost_map_stamps_loaded_at(monkeypatch): from litellm.litellm_core_utils import get_model_cost_map as module monkeypatch.setattr(module._cost_map_source_info, "loaded_at", None) - client, _calls = _mock_client( - [httpx.Response(200, content=_real_map_bytes())], client_cls=httpx.Client - ) + client, _calls = _mock_client([httpx.Response(200, content=_real_map_bytes())], client_cls=httpx.Client) before = datetime.now(timezone.utc) module.get_model_cost_map(url="https://example.invalid/cost_map.json", client=client) @@ -317,6 +310,7 @@ def test_get_model_cost_map_stamps_loaded_at(monkeypatch): assert loaded_at is not None assert before <= loaded_at <= datetime.now(timezone.utc) + # --------------------------------------------------------------------------- # refetch_model_cost_map: retry/backoff behavior for runtime reloads # --------------------------------------------------------------------------- @@ -382,9 +376,7 @@ async def test_refetch_retries_429_honoring_retry_after(): ] ) sleeper = _SleepRecorder() - result = await refetch_model_cost_map( - url=_URL, sleep=sleeper, rng=random.Random(0), client=client - ) + result = await refetch_model_cost_map(url=_URL, sleep=sleeper, rng=random.Random(0), client=client) assert isinstance(result, ModelCostMapReloaded) assert len(result.model_cost_map) > 100 assert calls["count"] == 3 @@ -396,9 +388,7 @@ async def test_refetch_gives_up_after_max_attempts_with_exponential_backoff(): """All 429 without Retry-After: exponential backoff waits, then a failure value.""" client, calls = _mock_client([httpx.Response(429)]) sleeper = _SleepRecorder() - result = await refetch_model_cost_map( - url=_URL, sleep=sleeper, rng=random.Random(0), client=client - ) + result = await refetch_model_cost_map(url=_URL, sleep=sleeper, rng=random.Random(0), client=client) assert isinstance(result, ModelCostMapReloadUnavailable) assert "429" in result.reason assert "after 3 attempts" in result.reason @@ -418,9 +408,7 @@ async def test_refetch_caps_retry_after_wait(): ] ) sleeper = _SleepRecorder() - result = await refetch_model_cost_map( - url=_URL, sleep=sleeper, rng=random.Random(0), client=client - ) + result = await refetch_model_cost_map(url=_URL, sleep=sleeper, rng=random.Random(0), client=client) assert isinstance(result, ModelCostMapReloaded) assert sleeper.waits == [30.0] @@ -435,9 +423,7 @@ async def test_refetch_retries_transport_errors(): ] ) sleeper = _SleepRecorder() - result = await refetch_model_cost_map( - url=_URL, sleep=sleeper, rng=random.Random(0), client=client - ) + result = await refetch_model_cost_map(url=_URL, sleep=sleeper, rng=random.Random(0), client=client) assert isinstance(result, ModelCostMapReloaded) assert calls["count"] == 2 assert len(sleeper.waits) == 1 @@ -448,9 +434,7 @@ async def test_refetch_non_retryable_status_fails_immediately(): """A 404 is permanent: one attempt, no sleeps, failure value.""" client, calls = _mock_client([httpx.Response(404)]) sleeper = _SleepRecorder() - result = await refetch_model_cost_map( - url=_URL, sleep=sleeper, rng=random.Random(0), client=client - ) + result = await refetch_model_cost_map(url=_URL, sleep=sleeper, rng=random.Random(0), client=client) assert isinstance(result, ModelCostMapReloadUnavailable) assert "404" in result.reason assert calls["count"] == 1 @@ -461,9 +445,7 @@ async def test_refetch_non_retryable_status_fails_immediately(): async def test_refetch_invalid_json_fails_immediately(): client, calls = _mock_client([httpx.Response(200, content=b"not json")]) sleeper = _SleepRecorder() - result = await refetch_model_cost_map( - url=_URL, sleep=sleeper, rng=random.Random(0), client=client - ) + result = await refetch_model_cost_map(url=_URL, sleep=sleeper, rng=random.Random(0), client=client) assert isinstance(result, ModelCostMapReloadUnavailable) assert "invalid JSON" in result.reason assert calls["count"] == 1 @@ -475,9 +457,7 @@ async def test_refetch_shrunk_map_fails_integrity_not_swapped_in(): """A drastically shrunk upstream file is rejected instead of being adopted.""" tiny = json.dumps(_make_models(60)).encode() client, _calls = _mock_client([httpx.Response(200, content=tiny)]) - result = await refetch_model_cost_map( - url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=client - ) + result = await refetch_model_cost_map(url=_URL, sleep=_SleepRecorder(), rng=random.Random(0), client=client) assert isinstance(result, ModelCostMapReloadUnavailable) assert "integrity validation" in result.reason @@ -520,62 +500,187 @@ class _SyncSleepRecorder: self.waits.append(seconds) -def test_boot_load_retries_transient_failures_instead_of_falling_back(): - """A refused connection then a 503 at pod boot used to pin the process to the bundled - backup for its lifetime; both are transient and must be retried before giving up.""" +class _BackgroundRecorder: + def __init__(self): + self.callbacks = [] + + def __call__(self, callback): + self.callbacks.append(callback) + + +def test_boot_load_returns_local_map_and_schedules_transient_retry(): + client, calls = _mock_client( + [ + httpx.ConnectError("connection refused"), + ], + client_cls=httpx.Client, + ) + sleeper = _SyncSleepRecorder() + background = _BackgroundRecorder() + + cost_map = get_model_cost_map( + url=_URL, + sleep=sleeper, + rng=random.Random(0), + client=client, + start_background=background, + ) + assert calls["count"] == 1 + assert sleeper.waits == [] + assert len(background.callbacks) == 1 + assert len(cost_map) > 100 + source = get_model_cost_map_source_info() + assert source["source"] == "local" + assert source["fallback_reason"] is not None + + +def test_background_retry_adopts_valid_remote_map(): client, calls = _mock_client( [ httpx.ConnectError("connection refused"), - httpx.Response(503), httpx.Response(200, content=_real_map_bytes()), ], client_cls=httpx.Client, ) sleeper = _SyncSleepRecorder() + background = _BackgroundRecorder() + applied = [] - cost_map = get_model_cost_map(url=_URL, sleep=sleeper, rng=random.Random(0), client=client) + get_model_cost_map( + url=_URL, + sleep=sleeper, + rng=random.Random(0), + client=client, + start_background=background, + apply=applied.append, + ) + assert calls["count"] == 1 + assert sleeper.waits == [] + assert len(background.callbacks) == 1 + + background.callbacks[0]() + + assert calls["count"] == 2 + assert len(sleeper.waits) == 1 + assert 2.0 <= sleeper.waits[0] < 3.0 + assert len(applied) == 1 + assert applied[0].keys() >= _load_root_cost_map().keys() - {"sample_spec", FALLBACK_GENERALIZATIONS_KEY} + source = get_model_cost_map_source_info() + assert source["source"] == "remote" + assert source["fallback_reason"] is None + + +def test_background_retry_keeps_local_map_after_remaining_failures(): + client, calls = _mock_client( + [httpx.ConnectError("connection refused"), httpx.Response(503)], + client_cls=httpx.Client, + ) + sleeper = _SyncSleepRecorder() + background = _BackgroundRecorder() + applied = [] + + get_model_cost_map( + url=_URL, + sleep=sleeper, + rng=random.Random(0), + client=client, + start_background=background, + apply=applied.append, + ) + background.callbacks[0]() assert calls["count"] == 3 assert len(sleeper.waits) == 2 assert 2.0 <= sleeper.waits[0] < 3.0 assert 4.0 <= sleeper.waits[1] < 5.0 + assert applied == [] + assert get_model_cost_map_source_info()["source"] == "local" + + +def test_boot_load_does_not_schedule_non_retryable_failure(): + client, calls = _mock_client([httpx.Response(404)], client_cls=httpx.Client) + sleeper = _SyncSleepRecorder() + background = _BackgroundRecorder() + + get_model_cost_map( + url=_URL, + sleep=sleeper, + rng=random.Random(0), + client=client, + start_background=background, + ) + + assert calls["count"] == 1 + assert sleeper.waits == [] + assert background.callbacks == [] source = get_model_cost_map_source_info() - assert source["source"] == "remote" - assert source["fallback_reason"] is None + assert source["source"] == "local" + assert source["fallback_reason"] is not None + + +def test_boot_load_success_does_not_schedule_background_retry(): + client, calls = _mock_client([httpx.Response(200, content=_real_map_bytes())], client_cls=httpx.Client) + sleeper = _SyncSleepRecorder() + background = _BackgroundRecorder() + + cost_map = get_model_cost_map( + url=_URL, + sleep=sleeper, + rng=random.Random(0), + client=client, + start_background=background, + ) + assert calls["count"] == 1 + assert sleeper.waits == [] + assert background.callbacks == [] + assert get_model_cost_map_source_info()["source"] == "remote" assert cost_map.keys() >= _load_root_cost_map().keys() - {"sample_spec", FALLBACK_GENERALIZATIONS_KEY} -def test_boot_load_honors_retry_after_then_falls_back_after_max_attempts(): - """An outage longer than the retry budget still ends on the bundled backup, and the - recorded fallback reason says how many attempts were spent so operators can tell.""" - client, calls = _mock_client( - [httpx.Response(429, headers={"Retry-After": "7"})], client_cls=httpx.Client +def test_boot_load_with_one_attempt_does_not_schedule_background_retry(): + client, calls = _mock_client([httpx.ConnectError("connection refused")], client_cls=httpx.Client) + sleeper = _SyncSleepRecorder() + background = _BackgroundRecorder() + + get_model_cost_map( + url=_URL, + max_attempts=1, + sleep=sleeper, + rng=random.Random(0), + client=client, + start_background=background, ) - sleeper = _SyncSleepRecorder() - cost_map = get_model_cost_map(url=_URL, sleep=sleeper, rng=random.Random(0), client=client) - - assert calls["count"] == 3 - assert sleeper.waits == [7.0, 7.0] - source = get_model_cost_map_source_info() - assert source["source"] == "local" - assert "after 3 attempts" in source["fallback_reason"] - assert len(cost_map) > 100 - - -def test_boot_load_does_not_retry_permanent_failures(): - """A 404 or a malformed URL cannot heal by waiting: one attempt, no sleeps, backup.""" - client, calls = _mock_client([httpx.Response(404)], client_cls=httpx.Client) - sleeper = _SyncSleepRecorder() - - get_model_cost_map(url=_URL, sleep=sleeper, rng=random.Random(0), client=client) assert calls["count"] == 1 assert sleeper.waits == [] + assert background.callbacks == [] assert get_model_cost_map_source_info()["source"] == "local" - get_model_cost_map(url="not a url", sleep=sleeper, rng=random.Random(0)) - assert sleeper.waits == [] - assert get_model_cost_map_source_info()["source"] == "local" + +def test_adopt_model_cost_map_replays_runtime_registration_and_provider_models(): + import litellm + from litellm import utils as litellm_utils + + original_model_cost = litellm.model_cost + original_registry = dict(litellm_utils._runtime_registered_model_cost) + original_anthropic_models = set(litellm.anthropic_models) + try: + litellm.register_model( + model_cost={"custom/deployment-model": {"litellm_provider": "custom", "max_input_tokens": 4321}} + ) + + models_count = adopt_model_cost_map({"anthropic/new-model": {"litellm_provider": "anthropic", "mode": "chat"}}) + + assert models_count == 1 + assert "anthropic/new-model" in litellm.anthropic_models + assert litellm.model_cost["custom/deployment-model"]["max_input_tokens"] == 4321 + finally: + litellm.model_cost = original_model_cost # test-quality-ok: restore the module state changed by adoption + litellm_utils._runtime_registered_model_cost.clear() + litellm_utils._runtime_registered_model_cost.update(original_registry) + litellm.anthropic_models.clear() + litellm.anthropic_models.update(original_anthropic_models) + litellm_utils._invalidate_model_cost_lowercase_map() def test_boot_load_respects_local_env_override(monkeypatch): From aa0a9ab3ea4dff53a22cbc60fbc0195c4ab6098c Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 9 Sep 2026 02:29:56 +0000 Subject: [PATCH 03/11] refactor(cost-map): drop initial_outcome flag from retry loop Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_core_utils/get_model_cost_map.py | 50 ++++++++++--------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index 90ff696f926..d9b5a492539 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -324,19 +324,14 @@ async def _fetch_remote_model_cost_map_with_retry( def _fetch_remote_model_cost_map_with_retry_sync( url: str, timeout: int, - max_attempts: int, + attempts: range, sleep: Callable[[float], None], rng: random.Random, client: _SyncGetClient, - starting_attempt: int = 1, - initial_outcome: _FetchAttemptRetryable | None = None, ) -> ModelCostMapReloadResult: - for attempt in range(starting_attempt, max_attempts + 1): - outcome = ( - initial_outcome - if initial_outcome is not None and attempt == starting_attempt - else _attempt_fetch_sync(client=client, url=url, timeout=timeout) - ) + max_attempts: Final = attempts.stop - 1 + for attempt in attempts: + outcome = _attempt_fetch_sync(client=client, url=url, timeout=timeout) if not isinstance(outcome, _FetchAttemptRetryable): return outcome wait_seconds = _next_retry_wait(outcome=outcome, attempt=attempt, max_attempts=max_attempts, rng=rng) @@ -557,18 +552,18 @@ def _continue_remote_fetch_in_background( sleep: Callable[[float], None], rng: random.Random, client: _SyncGetClient, - first_outcome: _FetchAttemptRetryable, + first_wait: float, apply: Callable[[dict], object], # mutable-ok: injected callback receives the mutable cost-map dict ) -> None: try: + sleep(first_wait) result: Final = _fetch_remote_model_cost_map_with_retry_sync( url=url, timeout=timeout, - max_attempts=max_attempts, + attempts=range(2, max_attempts + 1), sleep=sleep, rng=rng, client=client, - initial_outcome=first_outcome, ) if isinstance(result, ModelCostMapReloadUnavailable): verbose_logger.warning( @@ -652,19 +647,26 @@ def get_model_cost_map( local_map: Final = _finalize_loaded_model_cost_map( GetModelCostMap.load_local_model_cost_map_with_revision() ).model_cost_map - if max_attempts > 1: - start_background( - lambda: _continue_remote_fetch_in_background( - url=url, - timeout=timeout, - max_attempts=max_attempts, - sleep=sleep, - rng=fetch_rng, - client=fetch_client, - first_outcome=first_outcome, - apply=apply, - ) + first_wait: Final = _next_retry_wait( + outcome=first_outcome, + attempt=1, + max_attempts=max_attempts, + rng=fetch_rng, + ) + if isinstance(first_wait, ModelCostMapReloadUnavailable): + return local_map + start_background( + lambda: _continue_remote_fetch_in_background( + url=url, + timeout=timeout, + max_attempts=max_attempts, + sleep=sleep, + rng=fetch_rng, + client=fetch_client, + first_wait=first_wait, + apply=apply, ) + ) return local_map result: Final = first_outcome From 536a85b42967fd0c9c6b49d2df9298232034427a Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 9 Sep 2026 02:48:17 +0000 Subject: [PATCH 04/11] fix(cost-map): keep register_model url fetch to a single attempt Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/utils.py | 2 +- tests/test_litellm/test_utils.py | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/litellm/utils.py b/litellm/utils.py index 36b48d3b8d8..8df28870544 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3079,7 +3079,7 @@ def register_model( # Convert stringified numbers to appropriate numeric types loaded_model_cost = model_cost elif isinstance(model_cost, str): - loaded_model_cost = litellm.get_model_cost_map(url=model_cost) + loaded_model_cost = litellm.get_model_cost_map(url=model_cost, max_attempts=1) if persist_across_reloads: _registrations: Final[Mapping[str, Mapping[str, object]]] = loaded_model_cost diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index fee5e3a2e4c..e90372141bd 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -2,10 +2,12 @@ import asyncio import json import logging import os +import threading from datetime import datetime, timedelta, timezone from typing import Final from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest import respx from jsonschema import validate @@ -2382,6 +2384,27 @@ def test_register_model_with_scientific_notation(): _invalidate_model_cost_lowercase_map() +@respx.mock +def test_register_model_url_fetch_uses_single_attempt(monkeypatch): + monkeypatch.setattr(litellm, "model_cost", dict(litellm.model_cost)) + before = dict(litellm.model_cost) + threads_before = {thread.name for thread in threading.enumerate()} + route = respx.get("https://example.invalid/custom_pricing.json").mock( + return_value=httpx.Response(503) + ) + + litellm.register_model(model_cost="https://example.invalid/custom_pricing.json") + + threads_after = {thread.name for thread in threading.enumerate()} + assert route.call_count == 1 + assert not (threads_after - threads_before) & {"litellm-model-cost-map-retry"} + assert not any( + thread.name == "litellm-model-cost-map-retry" and thread.is_alive() + for thread in threading.enumerate() + ) + assert litellm.model_cost.keys() >= before.keys() + + def test_register_model_openrouter_without_slash(): """ Test that register_model handles openrouter models without '/' in the name. From 4179f086e0196107dfc83da2a4e0bb1d15568edd Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 9 Sep 2026 02:58:51 +0000 Subject: [PATCH 05/11] refactor(cost-map): share local-fallback and remote-accept paths Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_core_utils/get_model_cost_map.py | 111 ++++++++---------- 1 file changed, 47 insertions(+), 64 deletions(-) diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index d9b5a492539..67d06a0758e 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -531,6 +531,32 @@ def _finalize_loaded_model_cost_map(loaded: ModelCostMapReloaded) -> ModelCostMa return replace(loaded, model_cost_map=_finalize_model_cost_map(loaded.model_cost_map)) +def _use_local_backup(reason: str | None) -> dict: # mutable-ok: returns the mutable model-cost map contract + _cost_map_source_info.source = "local" + _cost_map_source_info.fallback_reason = reason + return _finalize_loaded_model_cost_map(GetModelCostMap.load_local_model_cost_map_with_revision()).model_cost_map + + +def _accept_remote( + result: ModelCostMapReloaded, url: str +) -> dict | None: # mutable-ok: returns the mutable model-cost map contract + if not GetModelCostMap.validate_model_cost_map( + fetched_map=result.model_cost_map, + backup_model_count=GetModelCostMap._get_backup_model_count(), # pyright: ignore[reportPrivateUsage] # integrity cache + ): + verbose_logger.warning( + "LiteLLM: Fetched model cost map failed integrity check. Using local backup instead. url=%s", + url, + ) + return None + finalized: Final = _finalize_loaded_model_cost_map(result).model_cost_map + _cost_map_source_info.source = "remote" + _cost_map_source_info.url = url + _cost_map_source_info.is_env_forced = False + _cost_map_source_info.fallback_reason = None + return finalized + + def adopt_model_cost_map( new_model_cost_map: dict, # mutable-ok: public API preserves the mutable cost-map contract ) -> int: @@ -545,17 +571,20 @@ def adopt_model_cost_map( return fetched_model_count -def _continue_remote_fetch_in_background( +def _retry_remote_fetch_in_background( url: str, timeout: int, max_attempts: int, sleep: Callable[[float], None], rng: random.Random, client: _SyncGetClient, - first_wait: float, + first_outcome: _FetchAttemptRetryable, apply: Callable[[dict], object], # mutable-ok: injected callback receives the mutable cost-map dict ) -> None: try: + first_wait: Final = _next_retry_wait(outcome=first_outcome, attempt=1, max_attempts=max_attempts, rng=rng) + if isinstance(first_wait, ModelCostMapReloadUnavailable): + return sleep(first_wait) result: Final = _fetch_remote_model_cost_map_with_retry_sync( url=url, @@ -572,23 +601,12 @@ def _continue_remote_fetch_in_background( max_attempts, ) return - backup_model_count: Final = GetModelCostMap._get_backup_model_count() # pyright: ignore[reportPrivateUsage] # integrity cache - if not GetModelCostMap.validate_model_cost_map( - fetched_map=result.model_cost_map, - backup_model_count=backup_model_count, - ): - verbose_logger.warning( - "LiteLLM: Fetched model cost map failed integrity check. Keeping local backup. url=%s", - url, - ) + _litellm_import_complete.wait() + accepted: Final = _accept_remote(result, url) + if accepted is None: _cost_map_source_info.fallback_reason = "Remote data failed integrity validation" return - _litellm_import_complete.wait() - apply(_finalize_loaded_model_cost_map(result).model_cost_map) - _cost_map_source_info.source = "remote" - _cost_map_source_info.url = url - _cost_map_source_info.is_env_forced = False - _cost_map_source_info.fallback_reason = None + apply(accepted) _cost_map_source_info.loaded_at = datetime.now(timezone.utc) except Exception as e: verbose_logger.warning("LiteLLM: Background model cost map retry failed: %s", e) @@ -623,77 +641,42 @@ def get_model_cost_map( # Note: can't use get_secret_bool here — this runs during litellm.__init__ # before litellm._key_management_settings is set. if os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true": - _cost_map_source_info.source = "local" _cost_map_source_info.url = None _cost_map_source_info.is_env_forced = True - _cost_map_source_info.fallback_reason = None - return _finalize_loaded_model_cost_map(GetModelCostMap.load_local_model_cost_map_with_revision()).model_cost_map + return _use_local_backup(None) _cost_map_source_info.url = url _cost_map_source_info.is_env_forced = False fetch_client: Final = client if client is not None else httpx fetch_rng: Final = rng if rng is not None else random.Random() - first_outcome: Final = _attempt_fetch_sync(client=fetch_client, url=url, timeout=timeout) - if isinstance(first_outcome, _FetchAttemptRetryable): + outcome: Final = _attempt_fetch_sync(client=fetch_client, url=url, timeout=timeout) + if isinstance(outcome, ModelCostMapReloaded): + accepted: Final = _accept_remote(outcome, url) + return accepted if accepted is not None else _use_local_backup("Remote data failed integrity validation") + if isinstance(outcome, _FetchAttemptRetryable) and max_attempts > 1: verbose_logger.warning( "LiteLLM: model cost map fetch attempt 1/%d failed: %s; " "using local backup while retrying in the background", max_attempts, - first_outcome.reason, + outcome.reason, ) - _cost_map_source_info.source = "local" - _cost_map_source_info.fallback_reason = f"Remote fetch failed: {first_outcome.reason}" - local_map: Final = _finalize_loaded_model_cost_map( - GetModelCostMap.load_local_model_cost_map_with_revision() - ).model_cost_map - first_wait: Final = _next_retry_wait( - outcome=first_outcome, - attempt=1, - max_attempts=max_attempts, - rng=fetch_rng, - ) - if isinstance(first_wait, ModelCostMapReloadUnavailable): - return local_map start_background( - lambda: _continue_remote_fetch_in_background( + lambda: _retry_remote_fetch_in_background( url=url, timeout=timeout, max_attempts=max_attempts, sleep=sleep, rng=fetch_rng, client=fetch_client, - first_wait=first_wait, + first_outcome=outcome, apply=apply, ) ) - return local_map - - result: Final = first_outcome - if isinstance(result, ModelCostMapReloadUnavailable): + else: verbose_logger.warning( "LiteLLM: Failed to fetch remote model cost map from %s: %s. Falling back to local backup.", url, - result.reason, + outcome.reason, ) - _cost_map_source_info.source = "local" - _cost_map_source_info.fallback_reason = f"Remote fetch failed: {result.reason}" - return _finalize_loaded_model_cost_map(GetModelCostMap.load_local_model_cost_map_with_revision()).model_cost_map - content: Final = result.model_cost_map - - # Validate using cached count (cheap int comparison, no file I/O) - if not GetModelCostMap.validate_model_cost_map( - fetched_map=content, - backup_model_count=GetModelCostMap._get_backup_model_count(), - ): - verbose_logger.warning( - "LiteLLM: Fetched model cost map failed integrity check. Using local backup instead. url=%s", - url, - ) - _cost_map_source_info.source = "local" - _cost_map_source_info.fallback_reason = "Remote data failed integrity validation" - return _finalize_loaded_model_cost_map(GetModelCostMap.load_local_model_cost_map_with_revision()).model_cost_map - - _cost_map_source_info.source = "remote" - _cost_map_source_info.fallback_reason = None - return _finalize_loaded_model_cost_map(result).model_cost_map + return _use_local_backup(f"Remote fetch failed: {outcome.reason}") From 9a721abf0d098f40caa6ba67d343e99de74f7fc2 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 9 Sep 2026 03:50:49 +0000 Subject: [PATCH 06/11] test(cost-map): clear LITELLM_LOCAL_MODEL_COST_MAP in register_model url test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/test_litellm/test_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index e90372141bd..e42608c9904 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -2386,6 +2386,7 @@ def test_register_model_with_scientific_notation(): @respx.mock def test_register_model_url_fetch_uses_single_attempt(monkeypatch): + monkeypatch.delenv("LITELLM_LOCAL_MODEL_COST_MAP", raising=False) monkeypatch.setattr(litellm, "model_cost", dict(litellm.model_cost)) before = dict(litellm.model_cost) threads_before = {thread.name for thread in threading.enumerate()} From 884f90c72715b0cfe0d5e2ea09f3744cb8376a78 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 9 Sep 2026 05:15:53 +0000 Subject: [PATCH 07/11] refactor(cost-map): inline background retry, trim tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_core_utils/get_model_cost_map.py | 119 ++++----- .../test_get_model_cost_map.py | 243 +++++++----------- 2 files changed, 139 insertions(+), 223 deletions(-) diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index 67d06a0758e..f81ddbfee2e 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -184,10 +184,6 @@ def mark_litellm_import_complete() -> None: _litellm_import_complete.set() -def _start_daemon_thread(fn: Callable[[], None]) -> None: - threading.Thread(target=fn, name="litellm-model-cost-map-retry", daemon=True).start() - - @dataclass(frozen=True, slots=True) class ModelCostMapReloaded: model_cost_map: dict # mutable-ok: adopted as litellm.model_cost, whose consumer contract is a plain mutable dict @@ -531,32 +527,6 @@ def _finalize_loaded_model_cost_map(loaded: ModelCostMapReloaded) -> ModelCostMa return replace(loaded, model_cost_map=_finalize_model_cost_map(loaded.model_cost_map)) -def _use_local_backup(reason: str | None) -> dict: # mutable-ok: returns the mutable model-cost map contract - _cost_map_source_info.source = "local" - _cost_map_source_info.fallback_reason = reason - return _finalize_loaded_model_cost_map(GetModelCostMap.load_local_model_cost_map_with_revision()).model_cost_map - - -def _accept_remote( - result: ModelCostMapReloaded, url: str -) -> dict | None: # mutable-ok: returns the mutable model-cost map contract - if not GetModelCostMap.validate_model_cost_map( - fetched_map=result.model_cost_map, - backup_model_count=GetModelCostMap._get_backup_model_count(), # pyright: ignore[reportPrivateUsage] # integrity cache - ): - verbose_logger.warning( - "LiteLLM: Fetched model cost map failed integrity check. Using local backup instead. url=%s", - url, - ) - return None - finalized: Final = _finalize_loaded_model_cost_map(result).model_cost_map - _cost_map_source_info.source = "remote" - _cost_map_source_info.url = url - _cost_map_source_info.is_env_forced = False - _cost_map_source_info.fallback_reason = None - return finalized - - def adopt_model_cost_map( new_model_cost_map: dict, # mutable-ok: public API preserves the mutable cost-map contract ) -> int: @@ -579,7 +549,6 @@ def _retry_remote_fetch_in_background( rng: random.Random, client: _SyncGetClient, first_outcome: _FetchAttemptRetryable, - apply: Callable[[dict], object], # mutable-ok: injected callback receives the mutable cost-map dict ) -> None: try: first_wait: Final = _next_retry_wait(outcome=first_outcome, attempt=1, max_attempts=max_attempts, rng=rng) @@ -602,12 +571,20 @@ def _retry_remote_fetch_in_background( ) return _litellm_import_complete.wait() - accepted: Final = _accept_remote(result, url) - if accepted is None: - _cost_map_source_info.fallback_reason = "Remote data failed integrity validation" + if not GetModelCostMap.validate_model_cost_map( + fetched_map=result.model_cost_map, + backup_model_count=GetModelCostMap._get_backup_model_count(), # pyright: ignore[reportPrivateUsage] # integrity cache + ): + verbose_logger.warning( + "LiteLLM: Fetched model cost map failed integrity check. Using local backup instead. url=%s", + url, + ) return - apply(accepted) + finalized: Final = _finalize_loaded_model_cost_map(result).model_cost_map + _cost_map_source_info.source = "remote" + _cost_map_source_info.fallback_reason = None _cost_map_source_info.loaded_at = datetime.now(timezone.utc) + adopt_model_cost_map(finalized) except Exception as e: verbose_logger.warning("LiteLLM: Background model cost map retry failed: %s", e) @@ -619,19 +596,12 @@ def get_model_cost_map( sleep: Callable[[float], None] = time.sleep, rng: random.Random | None = None, client: "_SyncGetClient | None" = None, - start_background: Callable[[Callable[[], None]], None] = _start_daemon_thread, - apply: Callable[ # mutable-ok: injected callback receives the mutable cost-map dict - [dict], - object, - ] = adopt_model_cost_map, ) -> dict: """ Public entry point — returns the model cost map dict. 1. If ``LITELLM_LOCAL_MODEL_COST_MAP`` is set, uses the local backup only. - 2. Otherwise fetches from ``url``, validates the first response, and falls - back to the local backup while retrying transient HTTP errors in the - background. + 2. Otherwise fetches from ``url``, retrying transient errors in a background thread. Only the backup model count is cached (a single int) for validation. The full backup dict is only parsed when it must be *returned* as a @@ -641,9 +611,11 @@ def get_model_cost_map( # Note: can't use get_secret_bool here — this runs during litellm.__init__ # before litellm._key_management_settings is set. if os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true": + _cost_map_source_info.source = "local" _cost_map_source_info.url = None _cost_map_source_info.is_env_forced = True - return _use_local_backup(None) + _cost_map_source_info.fallback_reason = None + return _finalize_loaded_model_cost_map(GetModelCostMap.load_local_model_cost_map_with_revision()).model_cost_map _cost_map_source_info.url = url _cost_map_source_info.is_env_forced = False @@ -651,32 +623,45 @@ def get_model_cost_map( fetch_client: Final = client if client is not None else httpx fetch_rng: Final = rng if rng is not None else random.Random() outcome: Final = _attempt_fetch_sync(client=fetch_client, url=url, timeout=timeout) - if isinstance(outcome, ModelCostMapReloaded): - accepted: Final = _accept_remote(outcome, url) - return accepted if accepted is not None else _use_local_backup("Remote data failed integrity validation") if isinstance(outcome, _FetchAttemptRetryable) and max_attempts > 1: - verbose_logger.warning( - "LiteLLM: model cost map fetch attempt 1/%d failed: %s; " - "using local backup while retrying in the background", - max_attempts, - outcome.reason, - ) - start_background( - lambda: _retry_remote_fetch_in_background( - url=url, - timeout=timeout, - max_attempts=max_attempts, - sleep=sleep, - rng=fetch_rng, - client=fetch_client, - first_outcome=outcome, - apply=apply, - ) - ) - else: + threading.Thread( + target=_retry_remote_fetch_in_background, + kwargs={ # mutable-ok: threading requires a mutable keyword-arguments mapping + "url": url, + "timeout": timeout, + "max_attempts": max_attempts, + "sleep": sleep, + "rng": fetch_rng, + "client": fetch_client, + "first_outcome": outcome, + }, + name="litellm-model-cost-map-retry", + daemon=True, + ).start() + if not isinstance(outcome, ModelCostMapReloaded): verbose_logger.warning( "LiteLLM: Failed to fetch remote model cost map from %s: %s. Falling back to local backup.", url, outcome.reason, ) - return _use_local_backup(f"Remote fetch failed: {outcome.reason}") + _cost_map_source_info.source = "local" + _cost_map_source_info.fallback_reason = f"Remote fetch failed: {outcome.reason}" + return _finalize_loaded_model_cost_map(GetModelCostMap.load_local_model_cost_map_with_revision()).model_cost_map + content: Final = outcome.model_cost_map + + # Validate using cached count (cheap int comparison, no file I/O) + if not GetModelCostMap.validate_model_cost_map( + fetched_map=content, + backup_model_count=GetModelCostMap._get_backup_model_count(), + ): + verbose_logger.warning( + "LiteLLM: Fetched model cost map failed integrity check. Using local backup instead. url=%s", + url, + ) + _cost_map_source_info.source = "local" + _cost_map_source_info.fallback_reason = "Remote data failed integrity validation" + return _finalize_loaded_model_cost_map(GetModelCostMap.load_local_model_cost_map_with_revision()).model_cost_map + + _cost_map_source_info.source = "remote" + _cost_map_source_info.fallback_reason = None + return _finalize_loaded_model_cost_map(outcome).model_cost_map diff --git a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py index e39658a8f37..266c2ca1465 100644 --- a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py +++ b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py @@ -6,6 +6,7 @@ count actual model entries, not reserved meta keys) and the extraction of the import json import os +import threading import pytest @@ -20,7 +21,6 @@ from litellm.litellm_core_utils.get_model_cost_map import ( GetModelCostMap, _count_model_entries, _finalize_model_cost_map, - adopt_model_cost_map, get_model_cost_map_provenance, git_blob_id, ) @@ -566,194 +566,125 @@ from litellm.litellm_core_utils.get_model_cost_map import ( class _SyncSleepRecorder: """Injected in place of time.sleep so the boot path's waits are asserted without delay.""" - def __init__(self): + def __init__(self, block=False): self.waits = [] + self.block = block + self.started = threading.Event() + self.release = threading.Event() def __call__(self, seconds: float) -> None: + if self.block: + self.started.set() + self.release.wait(timeout=10) self.waits.append(seconds) -class _BackgroundRecorder: - def __init__(self): - self.callbacks = [] - - def __call__(self, callback): - self.callbacks.append(callback) +def _retry_threads(): + return [thread for thread in threading.enumerate() if thread.name == "litellm-model-cost-map-retry"] -def test_boot_load_returns_local_map_and_schedules_transient_retry(): - client, calls = _mock_client( - [ - httpx.ConnectError("connection refused"), - ], - client_cls=httpx.Client, - ) - sleeper = _SyncSleepRecorder() - background = _BackgroundRecorder() - - cost_map = get_model_cost_map( - url=_URL, - sleep=sleeper, - rng=random.Random(0), - client=client, - start_background=background, - ) - assert calls["count"] == 1 - assert sleeper.waits == [] - assert len(background.callbacks) == 1 - assert len(cost_map) > 100 - source = get_model_cost_map_source_info() - assert source["source"] == "local" - assert source["fallback_reason"] is not None - - -def test_background_retry_adopts_valid_remote_map(): - client, calls = _mock_client( - [ - httpx.ConnectError("connection refused"), - httpx.Response(200, content=_real_map_bytes()), - ], - client_cls=httpx.Client, - ) - sleeper = _SyncSleepRecorder() - background = _BackgroundRecorder() - applied = [] - - get_model_cost_map( - url=_URL, - sleep=sleeper, - rng=random.Random(0), - client=client, - start_background=background, - apply=applied.append, - ) - assert calls["count"] == 1 - assert sleeper.waits == [] - assert len(background.callbacks) == 1 - - background.callbacks[0]() - - assert calls["count"] == 2 - assert len(sleeper.waits) == 1 - assert 2.0 <= sleeper.waits[0] < 3.0 - assert len(applied) == 1 - assert applied[0].keys() >= _load_root_cost_map().keys() - {"sample_spec", FALLBACK_GENERALIZATIONS_KEY} - source = get_model_cost_map_source_info() - assert source["source"] == "remote" - assert source["fallback_reason"] is None - - -def test_background_retry_keeps_local_map_after_remaining_failures(): - client, calls = _mock_client( - [httpx.ConnectError("connection refused"), httpx.Response(503)], - client_cls=httpx.Client, - ) - sleeper = _SyncSleepRecorder() - background = _BackgroundRecorder() - applied = [] - - get_model_cost_map( - url=_URL, - sleep=sleeper, - rng=random.Random(0), - client=client, - start_background=background, - apply=applied.append, - ) - background.callbacks[0]() - - assert calls["count"] == 3 - assert len(sleeper.waits) == 2 - assert 2.0 <= sleeper.waits[0] < 3.0 - assert 4.0 <= sleeper.waits[1] < 5.0 - assert applied == [] - assert get_model_cost_map_source_info()["source"] == "local" - - -def test_boot_load_does_not_schedule_non_retryable_failure(): - client, calls = _mock_client([httpx.Response(404)], client_cls=httpx.Client) - sleeper = _SyncSleepRecorder() - background = _BackgroundRecorder() - - get_model_cost_map( - url=_URL, - sleep=sleeper, - rng=random.Random(0), - client=client, - start_background=background, - ) - - assert calls["count"] == 1 - assert sleeper.waits == [] - assert background.callbacks == [] - source = get_model_cost_map_source_info() - assert source["source"] == "local" - assert source["fallback_reason"] is not None - - -def test_boot_load_success_does_not_schedule_background_retry(): +def test_boot_load_success_does_not_start_background_retry(): client, calls = _mock_client([httpx.Response(200, content=_real_map_bytes())], client_cls=httpx.Client) sleeper = _SyncSleepRecorder() - background = _BackgroundRecorder() cost_map = get_model_cost_map( url=_URL, sleep=sleeper, rng=random.Random(0), client=client, - start_background=background, ) assert calls["count"] == 1 assert sleeper.waits == [] - assert background.callbacks == [] - assert get_model_cost_map_source_info()["source"] == "remote" + assert _retry_threads() == [] assert cost_map.keys() >= _load_root_cost_map().keys() - {"sample_spec", FALLBACK_GENERALIZATIONS_KEY} + assert get_model_cost_map_source_info()["source"] == "remote" -def test_boot_load_with_one_attempt_does_not_schedule_background_retry(): - client, calls = _mock_client([httpx.ConnectError("connection refused")], client_cls=httpx.Client) - sleeper = _SyncSleepRecorder() - background = _BackgroundRecorder() +def test_boot_load_transient_failure_returns_local_then_background_retry_adopts_remote(monkeypatch): + import litellm + from litellm import utils as litellm_utils + from litellm.litellm_core_utils import get_model_cost_map as module - get_model_cost_map( + original_model_cost = litellm.model_cost + monkeypatch.setattr(litellm, "model_cost", dict(original_model_cost)) + for name, provider_models in tuple(vars(litellm).items()): + if name.endswith("_models") and isinstance(provider_models, set): + monkeypatch.setattr(litellm, name, set(provider_models)) + monkeypatch.setattr(litellm, "models_by_provider", dict(litellm.models_by_provider)) + monkeypatch.setattr( + litellm_utils, + "_runtime_registered_model_cost", + dict(litellm_utils._runtime_registered_model_cost), + ) + source_info = module._cost_map_source_info + for name in ("source", "url", "is_env_forced", "fallback_reason", "loaded_at", "source_revision", "etag"): + monkeypatch.setattr(source_info, name, getattr(source_info, name)) + + remote_map = _load_root_cost_map() + remote_map["claude-remote-only-test"] = {"litellm_provider": "anthropic", "mode": "chat"} + client, calls = _mock_client( + [ + httpx.ConnectError("connection refused"), + httpx.Response(200, content=json.dumps(remote_map).encode()), + ], + client_cls=httpx.Client, + ) + sleeper = _SyncSleepRecorder(block=True) + litellm.register_model({"my-runtime-model": {"litellm_provider": "custom", "max_input_tokens": 4321}}) + + cost_map = get_model_cost_map( url=_URL, - max_attempts=1, + max_attempts=3, sleep=sleeper, rng=random.Random(0), client=client, - start_background=background, ) assert calls["count"] == 1 assert sleeper.waits == [] - assert background.callbacks == [] - assert get_model_cost_map_source_info()["source"] == "local" - - -def test_adopt_model_cost_map_replays_runtime_registration_and_provider_models(): - import litellm - from litellm import utils as litellm_utils - - original_model_cost = litellm.model_cost - original_registry = dict(litellm_utils._runtime_registered_model_cost) - original_anthropic_models = set(litellm.anthropic_models) + assert sleeper.started.wait(timeout=10) + threads = _retry_threads() try: - litellm.register_model( - model_cost={"custom/deployment-model": {"litellm_provider": "custom", "max_input_tokens": 4321}} - ) - - models_count = adopt_model_cost_map({"anthropic/new-model": {"litellm_provider": "anthropic", "mode": "chat"}}) - - assert models_count == 1 - assert "anthropic/new-model" in litellm.anthropic_models - assert litellm.model_cost["custom/deployment-model"]["max_input_tokens"] == 4321 + assert len(threads) == 1 + assert "claude-remote-only-test" not in cost_map + assert cost_map.keys() == _finalize_model_cost_map(GetModelCostMap.load_local_model_cost_map()).keys() + source = get_model_cost_map_source_info() + assert source["source"] == "local" + assert source["fallback_reason"].startswith("Remote fetch failed:") + sleeper.release.set() + for thread in threads: + thread.join(timeout=10) + assert all(not thread.is_alive() for thread in threads) + assert sleeper.waits and 2.0 <= sleeper.waits[0] < 3.0 + assert calls["count"] == 2 + assert "claude-remote-only-test" in litellm.model_cost + assert "claude-remote-only-test" in litellm.anthropic_models + assert "my-runtime-model" in litellm.model_cost + source = get_model_cost_map_source_info() + assert source["source"] == "remote" + assert source["fallback_reason"] is None finally: - litellm.model_cost = original_model_cost # test-quality-ok: restore the module state changed by adoption - litellm_utils._runtime_registered_model_cost.clear() - litellm_utils._runtime_registered_model_cost.update(original_registry) - litellm.anthropic_models.clear() - litellm.anthropic_models.update(original_anthropic_models) - litellm_utils._invalidate_model_cost_lowercase_map() + sleeper.release.set() + for thread in _retry_threads(): + thread.join(timeout=10) + + +def test_boot_load_does_not_retry_non_retryable_failure(): + client, calls = _mock_client([httpx.Response(404)], client_cls=httpx.Client) + sleeper = _SyncSleepRecorder() + + get_model_cost_map( + url=_URL, + sleep=sleeper, + rng=random.Random(0), + client=client, + ) + assert calls["count"] == 1 + assert sleeper.waits == [] + assert _retry_threads() == [] + source = get_model_cost_map_source_info() + assert source["source"] == "local" + assert source["fallback_reason"] is not None def test_boot_load_respects_local_env_override(monkeypatch): From e8e3172d7d70558929f32f057ebe4c7471c8c352 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 8 Sep 2026 23:10:18 -0700 Subject: [PATCH 08/11] fix(model-management): honor an explicit null as a clear on model update (#40047) * fix(model-management): honor an explicit null as a clear on model update PATCH /model/{model_id}/update merged the patch with exclude_none and then popped explicit nulls only for the mirrored pricing fields, so a null sent for max_input_tokens, mode, supports_vision or any other key was dropped and a value pinned by an earlier save could never be removed. The route now follows JSON Merge Patch over both blobs: a key absent from the body is unchanged, a key sent as null is removed from the stored row, and a key sent with a value is set. Ownership and identity keys keep ignoring a null, as do the fields the stored models require, since clearing one writes a row no reload can rebuild. Mirrored pricing keys still clear from both blobs. Clearing a price also needed the router to stop merging a deployment's cost-map entry onto its previous registration, which left the old rate in place and kept billing at a price the deployment no longer carried. Adds a create, read, partial-update, clear, enforce, delete lifecycle e2e that reads back on every replica, and a harness helper for that read-back. * fix(router): keep a deployment id that names a real model from evicting its catalog entry Deployments are keyed into litellm.model_cost alongside the built-in catalog, so evicting a deployment's stale entry by id could take a real model's entry with it: registering a deployment whose model_info.id is "gpt-4o" stripped that model's pricing, context window and capability flags process-wide, for every other deployment of it, until the next price-map reload. Only evict an entry this registration owns. A colliding id keeps the previous merge, which pollutes the catalog entry rather than emptying it. Also pins the Admin UI round trip: the model edit form echoes the whole /model/info row back on save, and that read reports every key the deployment never stored as an explicit null, so the clear path has to leave those keys alone. * fix(router): decide cost-map eviction by what this registrar created The previous guard read a catalog entry off `litellm_provider`, so a deployment that declares its own provider in model_info was treated as one and kept billing at a price it no longer carried. It also only held for a single registration: a second one under a colliding id saw the id the first merge left behind and evicted the catalog entry anyway. Track the cost-map keys this registrar creates instead. A key it created is evicted before re-registration; one it did not is left to merge, which is what a deployment id colliding with a catalog model name needs. Also folds the required-fields comment into the docstring that already gives the reason. * fix(router): release a deployment's cost-map key when it is deleted The ownership ledger only grew. A deleted deployment kept its claim, so if a later catalog refresh started publishing a model under that same name, the next registration would treat the catalog entry as the deployment's own and evict it. Deleting a deployment now gives the key back, which also stops the ledger growing for the life of the process. * fix(router): hold a cost-map key while another live router still serves it The claim is process-wide but the release was per-deletion, so with two routers serving one deployment id, the first deletion put the survivor back on merging and the price it had just cleared would keep billing. Release the key only once no live router still serves that id. * fix(router): register a router in the live set when it gains a deployment _live_routers was only joined when a router was constructed with a model_list, but a router built empty is populated through add_deployment, and the empty branch exists for exactly that. Such a router was invisible to the live-router scan, so deleting the deployment from another router released the shared cost-map key while it was still serving that id. Joining the set where a deployment enters the list covers every path, and it also lets a price reload rebuild what a dynamically built router serves. * fix(e2e): read the stored model row from the control plane, not each gateway The lifecycle suite polled /model/info on every URL in PROXY_REPLICA_URLS. Those URLs are the stack's gateways, and gateway/routes/allowlist.py trims them to the LLM data-plane surface, so /model/info answers only on the backend and 404s on every replica. All five tests failed at their first read-back in CI while passing against a monolith, where one process serves both planes. The stored row has one answer behind it, so it is read through the shared transport, which routes control-plane paths to the backend. What every gateway must agree on is which models it serves, so the create and delete steps poll /v1/models per replica instead, a route the gateway does serve. read_back_everywhere now rejects a control-plane path outright rather than timing out on it. Two things surfaced behind that. /public/ was missing from the transport's control-plane prefixes, so model_cost_map() was routed to a gateway and 404'd, and the billing steps needed a data-plane wait: a PATCH lands on the backend and each gateway picks it up on its own config reload, measured here at 12-24s, so they now drive calls until the new rate reaches the spend row and let the deadline fail them. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01C1S92J8gSxxKVe1JBzxWBF * test(models): keep polling outcomes immutable and document shared ownership * test: validate opaque stream IDs and hide log-reader credentials * test: isolate auto-router scenarios and clean partial setup --------- Co-authored-by: Claude Opus 5 --- .../model_management_endpoints.py | 70 +++- litellm/router.py | 26 +- tests/e2e/coverage_registry/mgmt.yaml | 2 + .../management/test_model_lifecycle_e2e.py | 365 ++++++++++++++++++ tests/e2e/models.py | 74 +++- tests/e2e/proxy_client.py | 199 +++++++++- tests/e2e/test_proxy_client.py | 57 ++- tests/e2e/transport.py | 1 + .../test_model_management_endpoints.py | 178 ++++++++- .../test_router_model_cost_isolation.py | 192 +++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 5 +- 11 files changed, 1130 insertions(+), 39 deletions(-) create mode 100644 tests/e2e/management/test_model_lifecycle_e2e.py diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 0f19e9ce149..742b9d9817f 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -119,6 +119,7 @@ from litellm.types.router import ( SPECIAL_MODEL_INFO_PARAMS, Deployment, GenericLiteLLMParams, + LiteLLM_Params, ModelInfo, updateDeployment, ) @@ -728,6 +729,44 @@ def _ptu_priced_deployment(model_params: Deployment) -> Deployment: ) +_OWNERSHIP_FIELDS: Final = frozenset( + { + "db_model", + "team_id", + "team_public_model_name", + "access_groups", + "created_at", + "created_by", + "updated_at", + "updated_by", + "blocked", + } +) + +_STORED_REQUIRED_FIELDS: Final = frozenset( + name for model in (LiteLLM_Params, ModelInfo) for name, field in model.model_fields.items() if field.is_required() +) + +_NULL_CLEAR_IGNORED_FIELDS: Final = _OWNERSHIP_FIELDS | _STORED_REQUIRED_FIELDS | frozenset(PTU_MODEL_INFO_FIELDS) + + +def _explicitly_cleared_fields(patch: BaseModel | None) -> frozenset[str]: + """The keys a patch sends as an explicit null, which update_db_model removes from the + stored blob (JSON Merge Patch). Ownership keys are left alone, as are the keys the stored + models require, since clearing one writes a row no reload can rebuild through + LiteLLM_Params / ModelInfo. The PTU keys are handled by _explicitly_cleared_ptu_fields, + whose clear is gated on the feature flag. Applied after both blobs merge, so a model_info blob + the UI echoes back cannot resurrect a pricing key the litellm_params patch clears. + """ + if patch is None: + return frozenset() + return frozenset( + field + for field in patch.model_fields_set + if field not in _NULL_CLEAR_IGNORED_FIELDS and getattr(patch, field) is None + ) + + def update_db_model(db_model: Deployment, updated_patch: updateDeployment) -> PrismaCompatibleUpdateDBModel: if updated_patch.model_info is not None: _raise_if_ptu_cost_attribution_disabled(updated_patch.model_info.model_dump(exclude_none=True)) @@ -748,25 +787,15 @@ def update_db_model(db_model: Deployment, updated_patch: updateDeployment) -> Pr if updated_patch.model_info: merged_model_info.update(updated_patch.model_info.model_dump(exclude_none=True)) - # Honor explicit-null clears LAST, after both merges, so a model_info blob the UI - # passes through (which today re-sends the OLD pricing on every save) cannot - # silently undo a litellm_params clear via .update(). - # - # Restricted to SPECIAL_MODEL_INFO_PARAMS (input/output cost per token/character - # and cache read/write costs) so this path cannot be used to null out privileged - # model_info fields like team_id or access groups. SPECIAL_MODEL_INFO_PARAMS are - # mirrored between litellm_params and model_info by Deployment.__init__, so the - # clear propagates to both blobs. - if updated_patch.litellm_params: - for field in updated_patch.litellm_params.model_fields_set: - if field in SPECIAL_MODEL_INFO_PARAMS and getattr(updated_patch.litellm_params, field) is None: - merged_litellm_params.pop(field, None) - merged_model_info.pop(field, None) + for field in _explicitly_cleared_fields(updated_patch.litellm_params): + merged_litellm_params.pop(field, None) + if field in SPECIAL_MODEL_INFO_PARAMS: + merged_model_info.pop(field, None) + for field in _explicitly_cleared_fields(updated_patch.model_info): + merged_model_info.pop(field, None) + if field in SPECIAL_MODEL_INFO_PARAMS: + merged_litellm_params.pop(field, None) if updated_patch.model_info: - for field in updated_patch.model_info.model_fields_set: - if field in SPECIAL_MODEL_INFO_PARAMS and getattr(updated_patch.model_info, field) is None: - merged_model_info.pop(field, None) - merged_litellm_params.pop(field, None) for field in _explicitly_cleared_ptu_fields(updated_patch.model_info): merged_model_info.pop(field, None) @@ -816,8 +845,9 @@ async def patch_model( """ PATCH Endpoint for partial model updates. - Only updates the fields specified in the request while preserving other existing values. - Follows proper PATCH semantics by only modifying provided fields. + JSON Merge Patch semantics over `litellm_params` and `model_info`: a key absent from the + body is unchanged, a key sent as null is removed from the stored row, and a key sent with a + value is set (identity and ownership keys such as `id` and `team_id` ignore a null). Args: model_id: The ID of the model to update diff --git a/litellm/router.py b/litellm/router.py index 934a4ac86a9..1252d7e7487 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -628,6 +628,15 @@ RETRY_BREADCRUMB_EXCLUDED_KWARGS: Final = frozenset( RETRY_BREADCRUMB_LIMIT: Final = 4 +# Cost-map keys created by _register_deployment_in_model_cost, which shares one flat +# namespace with the built-in model catalog. Only a key it created may be evicted, or a +# deployment whose id names a real model would strip that model's pricing and +# capabilities for every other deployment of it. delete_deployment gives a key back once no +# live router still serves that id, so a later catalog refresh that starts serving the name +# is not treated as a deployment's own. +_DEPLOYMENT_COST_MAP_KEYS: Final[set[str]] = set() # mutable-ok: ownership of shared cost-map keys + + class FallbackAwareStreamWrapper(CustomStreamWrapper): """Base for the Router's chat-completion stream wrappers, which are built around the attempt the Router picked first and have to repoint themselves when a fallback takes over.""" @@ -9761,6 +9770,7 @@ class Router: """ idx: Final = len(self.model_list) self.model_list.append(model) + _live_routers.add(self) # mutable-ok: track dynamic routers without extending their lifetimes self._invalidate_model_group_info_cache() self._invalidate_access_groups_cache() @@ -9929,7 +9939,12 @@ class Router: """Write a deployment's metadata into ``litellm.model_cost``. Runs when a deployment is added and again after a price data reload, so - the entries a refresh rebuilds are the ones a fresh boot would produce. + the entries a refresh rebuilds are the ones a fresh boot would produce. An + entry this function created is replaced rather than merged, so a price cleared + from the deployment does not linger from an earlier registration and keep + billing at the old rate. An entry it did not create is left to merge, because + a deployment id that collides with a catalog model name shares that model's + entry with every other deployment of it. Nothing is recorded for replay: a refresh walks the live routers instead, so a deleted, repointed or never-added deployment, and a discarded router, drop out of the rebuild on their own. @@ -9946,6 +9961,10 @@ class Router: } if model_id is not None: + if model_id in _DEPLOYMENT_COST_MAP_KEYS: + litellm.model_cost.pop(model_id, None) # mutable-ok: remove cleared prices from the shared entry + elif model_id not in litellm.model_cost: + _DEPLOYMENT_COST_MAP_KEYS.add(model_id) # mutable-ok: retain shared ownership across reloads litellm.register_model( model_cost={model_id: model_info}, persist_across_reloads=False, @@ -10042,6 +10061,11 @@ class Router: _budget_limiter: Final = self._get_router_deployment_budget_limiter() if _budget_limiter is not None: _budget_limiter.unregister_deployment_budget(model_id=id) + if not any( + router is not self and id in router.model_id_to_deployment_index_map + for router in tuple(_live_routers) + ): + _DEPLOYMENT_COST_MAP_KEYS.discard(id) # mutable-ok: the last owning router released this key try: self._unregister_pre_routing_strategy_for_deployment( deployment=item if isinstance(item, Deployment) else Deployment(**item) diff --git a/tests/e2e/coverage_registry/mgmt.yaml b/tests/e2e/coverage_registry/mgmt.yaml index c8d7037d2fd..83f1711a245 100644 --- a/tests/e2e/coverage_registry/mgmt.yaml +++ b/tests/e2e/coverage_registry/mgmt.yaml @@ -76,6 +76,8 @@ - {id: mgmt.credential_migration.check.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "key_management_endpoints.py:4252", rationale: "Encryption migration (smoke)"} - {id: mgmt.credential.new.serves_request, module: mgmt, tier: P1, surface: api, assertions: [serves_request], source: "credential_endpoints/endpoints.py:42", rationale: "Stored credential resolves into a deployment and serves a live /messages request"} - {id: mgmt.model.test_connection.happy_path, module: mgmt, tier: P0, surface: api, assertions: [happy_path], source: "_health_endpoints.py:1785", rationale: "Test Connection for a responses-mode Bedrock Mantle deployment reaches the live provider and reports success; this exact shape 500ed on an acompletion partial before v1.91.0", fail_before_fix: proven} +- {id: mgmt.model.update.preserves_unrelated_fields, module: mgmt, tier: P0, surface: api, assertions: [preserves_unrelated_fields], source: "model_management_endpoints.py:731", rationale: "A partial PATCH changes only the keys it names; every other stored key reads back byte-for-byte, and the new rate reaches billing"} +- {id: mgmt.model.update.clear_persists, module: mgmt, tier: P0, surface: api, assertions: [clear_persists], source: "model_management_endpoints.py:731", fail_before_fix: proven, rationale: "An explicit null on PATCH removes the key from the stored row and billing falls back to the cost map; before the fix only mirrored pricing keys could be cleared"} - {id: mgmt.mcp_server.new.persists, module: mgmt, tier: P0, surface: api, assertions: [persists], source: "mcp_management_endpoints.py:1577", rationale: "Every field of an admin-created MCP server reads back verbatim, by id and in the list, on every replica"} - {id: mgmt.mcp_server.list.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "mcp_management_endpoints.py:1112", rationale: "The MCP page grid lists a created server with the same field values its detail view reports"} - {id: mgmt.mcp_server.update.preserves_unrelated_fields, module: mgmt, tier: P0, surface: api, assertions: [preserves_unrelated_fields], source: "mcp_management_endpoints.py:2665", rationale: "A dashboard edit of one field leaves the others intact and is visible on every replica after one save; edits that took several saves to stick were a customer defect"} diff --git a/tests/e2e/management/test_model_lifecycle_e2e.py b/tests/e2e/management/test_model_lifecycle_e2e.py new file mode 100644 index 00000000000..99044efeb46 --- /dev/null +++ b/tests/e2e/management/test_model_lifecycle_e2e.py @@ -0,0 +1,365 @@ +"""Live e2e: the lifecycle of a DB-stored deployment through the model management +routes, read back on every gateway replica. + +Each test registers its own gpt-4o-mini mock deployment through /model/new (deleted +on teardown) with non-default pricing, context window, mode, and api_base pinned, then +walks the lifecycle up to the step it proves: the create reads back field for field, +a partial PATCH changes only the key it names, an explicit null on PATCH removes the +key from the stored row (JSON Merge Patch), a call after the price clear is billed at +the cost map's rate rather than the cleared override, and a delete removes the +deployment from /model/info and makes the model name unknown to /chat/completions. + +The stored row is read back from /model/info, a control-plane route with one answer +behind it. What every gateway must agree on is which models it serves, so the create +and delete steps poll /v1/models on every URL in PROXY_REPLICA_URLS through +ProxyClient.read_model_back_everywhere, failing by name on the gateway that never converged. +""" + +from __future__ import annotations + +import math +import time +from collections.abc import Callable +from dataclasses import dataclass +from typing import Final + +import pytest +from pydantic import BaseModel + +from e2e_config import unique_marker +from e2e_http import unwrap +from lifecycle import ResourceManager +from management_client import ManagementClient +from models import ( + ChatBody, + ChatMessage, + Clear, + LiteLLMParamsBody, + LiteLLMParamsPatch, + ModelInfoBody, + ModelInfoEntry, + ModelInfoResponse, + ModelNewBody, + ModelPatchBody, + ModelsListResponse, + SpendLogRow, +) + +pytestmark = pytest.mark.e2e + +BACKEND_MODEL: Final = "gpt-4o-mini" +PINNED_API_BASE: Final = "https://pinned.example.invalid/v1" +PINNED_MAX_INPUT_TOKENS: Final = 4096 +PINNED_INPUT_RATE: Final = 1e-05 +UPDATED_INPUT_RATE: Final = 2e-05 +PINNED_OUTPUT_RATE: Final = 3e-05 + +# A PATCH lands on the control plane, and each gateway picks it up on its own config +# reload, so the first call after the write can still be billed at the old rate. There +# is no price on the gateway's data-plane surface to poll, so the billing steps drive +# calls until the new rate shows up in the spend row and let the deadline be what fails. +BILLING_CONVERGENCE_TIMEOUT: Final = 90.0 +BILLING_CONVERGENCE_INTERVAL: Final = 5.0 + + +@dataclass(frozen=True, slots=True) +class Registered: + model_name: str + model_id: str + + +class _ErrorDetail(BaseModel): + message: str + + +class _ErrorEnvelope(BaseModel): + error: _ErrorDetail + + +def _register(client: ManagementClient, resources: ResourceManager) -> Registered: + """Register a mock gpt-4o-mini deployment with every field under test pinned to a + non-default value, deleted on teardown. max_input_tokens is pinned in + litellm_params only: a value in model_info is copied into the shared cost-map + entry for the backend model, which would leak into every other gpt-4o-mini + deployment on the proxy.""" + model_name: Final = f"e2e-lifecycle-{unique_marker()}" + model_id: Final = client.proxy.register_model( + ModelNewBody( + model_name=model_name, + litellm_params=LiteLLMParamsBody( + model=BACKEND_MODEL, + mock_response="ok", + api_base=PINNED_API_BASE, + input_cost_per_token=PINNED_INPUT_RATE, + output_cost_per_token=PINNED_OUTPUT_RATE, + max_input_tokens=PINNED_MAX_INPUT_TOKENS, + ), + model_info=ModelInfoBody(mode="chat"), + ) + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + return Registered(model_name=model_name, model_id=model_id) + + +def _entry(body: ModelInfoResponse, model_name: str) -> ModelInfoEntry | None: + return next((entry for entry in body.data if entry.model_name == model_name), None) + + +def _stored_entry( + client: ManagementClient, + model_name: str, + *, + converged: Callable[[ModelInfoEntry], bool], +) -> ModelInfoEntry: + """The stored /model/info row for `model_name`, once it satisfies `converged`. + + /model/info is a control-plane route: the gateways named in PROXY_REPLICA_URLS + serve the LLM surface only, so the stored row has one answer, not one per + gateway. What every gateway must agree on is which models it serves, and + `_assert_served_everywhere` / `_assert_absent_everywhere` poll /v1/models for + that.""" + + def has_converged(body: ModelInfoResponse) -> bool: + entry: Final = _entry(body, model_name) + return entry is not None and converged(entry) + + body: Final = client.proxy.read_model_back("/model/info", ModelInfoResponse, predicate=has_converged) + entry: Final = _entry(body, model_name) + assert entry is not None, f"/model/info stopped listing {model_name!r} between the poll and the read" + return entry + + +def _serves(body: ModelsListResponse, model_name: str) -> bool: + return any(entry.id == model_name for entry in body.data) + + +def _assert_served_everywhere(client: ManagementClient, model_name: str) -> None: + _ = client.proxy.read_model_back_everywhere( + "/v1/models", ModelsListResponse, predicate=lambda body: _serves(body, model_name) + ) + + +def _assert_absent_everywhere(client: ManagementClient, model_name: str) -> None: + _ = client.proxy.read_model_back_everywhere( + "/v1/models", ModelsListResponse, predicate=lambda body: not _serves(body, model_name) + ) + + +def _assert_untouched_keys_as_created(entry: ModelInfoEntry, replica: str) -> None: + """The keys no later step names read back byte-for-byte as /model/new wrote them.""" + params: Final = entry.litellm_params + assert params.model == BACKEND_MODEL, f"{replica}: litellm_params.model {params.model!r} != {BACKEND_MODEL!r}" + assert params.api_base == PINNED_API_BASE, f"{replica}: api_base {params.api_base!r} != {PINNED_API_BASE!r}" + assert params.output_cost_per_token == PINNED_OUTPUT_RATE, ( + f"{replica}: output_cost_per_token {params.output_cost_per_token} != {PINNED_OUTPUT_RATE}" + ) + assert entry.model_info.mode == "chat", f"{replica}: model_info.mode {entry.model_info.mode!r} != 'chat'" + + +def _approx_equal(actual: float, expected: float) -> bool: + return math.isclose(actual, expected, rel_tol=1e-2, abs_tol=1e-9) + + +def _priced(rows: list[SpendLogRow]) -> bool: + return any(row.metadata and row.metadata.cost_breakdown and row.metadata.cost_breakdown.input_cost for row in rows) + + +def _billed_input_cost(client: ManagementClient, model_name: str, key: str) -> tuple[int, float]: + """Drive one chat completion through `model_name` and return the prompt tokens and + input cost its spend row recorded, so a test can assert the rate the gateway actually + billed rather than only the rate it stored.""" + chat: Final = unwrap( + client.proxy.chat( + key, + ChatBody( + model=model_name, + messages=[ChatMessage(role="user", content=f"reply with one word {unique_marker()}")], + max_tokens=16, + ), + ) + ) + assert chat.id is not None, f"chat completion carried no id to find its spend row by: {chat}" + + rows: Final = client.proxy.poll_logs_for_request_id(chat.id, predicate=_priced) + row: Final = next((row for row in rows if row.request_id == chat.id), None) + assert row is not None and row.metadata and row.metadata.cost_breakdown, ( + f"no priced spend row for request {chat.id} before the deadline: {rows}" + ) + prompt_tokens: Final = row.prompt_tokens or 0 + input_cost: Final = row.metadata.cost_breakdown.input_cost + assert prompt_tokens > 0 and input_cost is not None, f"spend row logged no prompt tokens or input cost: {row}" + return prompt_tokens, input_cost + + +def _await_billed_input_cost( + client: ManagementClient, model_name: str, key: str, *, expected_rate: float +) -> tuple[int, float]: + """Drive calls through `model_name` until one is billed at `expected_rate`, and + return the prompt tokens and input cost of the last spend row either way. + + Only the deadline ends the wait unsatisfied: a rate that never reaches the gateway + comes back as the stale cost for the caller to assert on, so the rate the caller + expects is still what decides the test.""" + deadline: Final = time.monotonic() + BILLING_CONVERGENCE_TIMEOUT + while True: + prompt_tokens, input_cost = _billed_input_cost(client, model_name, key) + if _approx_equal(input_cost, prompt_tokens * expected_rate) or time.monotonic() >= deadline: + return prompt_tokens, input_cost + time.sleep(BILLING_CONVERGENCE_INTERVAL) + + +class TestModelLifecycle: + @pytest.mark.covers("mgmt.model.add.persists") + def test_create_reads_back_every_field_and_serves_on_every_replica( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + registered = _register(client, resources) + + entry = _stored_entry(client, registered.model_name, converged=lambda _entry: True) + stored = "/model/info" + + _assert_untouched_keys_as_created(entry, stored) + assert entry.litellm_params.input_cost_per_token == PINNED_INPUT_RATE, ( + f"{stored}: input_cost_per_token {entry.litellm_params.input_cost_per_token} != {PINNED_INPUT_RATE}" + ) + assert entry.litellm_params.max_input_tokens == PINNED_MAX_INPUT_TOKENS, ( + f"{stored}: max_input_tokens {entry.litellm_params.max_input_tokens} != {PINNED_MAX_INPUT_TOKENS}" + ) + assert entry.model_info.id == registered.model_id, ( + f"{stored}: model_info.id {entry.model_info.id!r} != {registered.model_id!r}" + ) + + _assert_served_everywhere(client, registered.model_name) + + @pytest.mark.covers("mgmt.model.update.preserves_unrelated_fields") + def test_partial_update_changes_only_the_named_key( + self, client: ManagementClient, resources: ResourceManager, scoped_key: str + ) -> None: + registered = _register(client, resources) + + stored = client.proxy.patch_model( + registered.model_id, + ModelPatchBody(litellm_params=LiteLLMParamsPatch(input_cost_per_token=UPDATED_INPUT_RATE)), + ) + assert stored.litellm_params.input_cost_per_token == UPDATED_INPUT_RATE, ( + f"PATCH response stores input_cost_per_token {stored.litellm_params.input_cost_per_token}, " + f"sent {UPDATED_INPUT_RATE}" + ) + + entry = _stored_entry( + client, + registered.model_name, + converged=lambda entry: entry.litellm_params.input_cost_per_token == UPDATED_INPUT_RATE, + ) + stored = "/model/info" + + _assert_untouched_keys_as_created(entry, stored) + assert entry.litellm_params.max_input_tokens == PINNED_MAX_INPUT_TOKENS, ( + f"{stored}: max_input_tokens {entry.litellm_params.max_input_tokens} != {PINNED_MAX_INPUT_TOKENS}" + ) + assert entry.model_info.input_cost_per_token == UPDATED_INPUT_RATE, ( + f"{stored}: model_info.input_cost_per_token {entry.model_info.input_cost_per_token} " + f"did not mirror the updated {UPDATED_INPUT_RATE}" + ) + + prompt_tokens, input_cost = _await_billed_input_cost( + client, registered.model_name, scoped_key, expected_rate=UPDATED_INPUT_RATE + ) + assert _approx_equal(input_cost, prompt_tokens * UPDATED_INPUT_RATE), ( + f"input_cost {input_cost} != {prompt_tokens} tokens * updated rate {UPDATED_INPUT_RATE} " + f"= {prompt_tokens * UPDATED_INPUT_RATE}; the partial update did not reach billing" + ) + + @pytest.mark.covers("mgmt.model.update.clear_persists") + def test_explicit_null_removes_the_key_from_the_stored_row( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + registered = _register(client, resources) + + stored = client.proxy.patch_model( + registered.model_id, + ModelPatchBody(litellm_params=LiteLLMParamsPatch(max_input_tokens=Clear(), input_cost_per_token=Clear())), + ) + stored_params = stored.litellm_params.model_fields_set + assert "max_input_tokens" not in stored_params, ( + f"stored litellm_params still carries max_input_tokens " + f"{stored.litellm_params.max_input_tokens} after an explicit null" + ) + assert "input_cost_per_token" not in stored_params, ( + f"stored litellm_params still carries input_cost_per_token " + f"{stored.litellm_params.input_cost_per_token} after an explicit null" + ) + assert "max_input_tokens" not in stored.model_info.model_fields_set, ( + f"stored model_info carries max_input_tokens {stored.model_info.max_input_tokens} after the clear" + ) + assert "input_cost_per_token" not in stored.model_info.model_fields_set, ( + f"stored model_info still mirrors input_cost_per_token {stored.model_info.input_cost_per_token}" + ) + + cost_map_input_rate = client.proxy.model_cost_map()[BACKEND_MODEL].input_cost_per_token + assert cost_map_input_rate is not None, f"cost map has no input rate for {BACKEND_MODEL}" + entry = _stored_entry( + client, + registered.model_name, + converged=lambda entry: "max_input_tokens" not in entry.litellm_params.model_fields_set, + ) + stored = "/model/info" + + _assert_untouched_keys_as_created(entry, stored) + served = entry.litellm_params.model_fields_set + assert "max_input_tokens" not in served, ( + f"{stored}: litellm_params still serves max_input_tokens {entry.litellm_params.max_input_tokens}" + ) + assert "input_cost_per_token" not in served, ( + f"{stored}: litellm_params still serves input_cost_per_token {entry.litellm_params.input_cost_per_token}" + ) + assert entry.model_info.input_cost_per_token == cost_map_input_rate, ( + f"{stored}: model_info.input_cost_per_token {entry.model_info.input_cost_per_token} is not the " + f"cost map's {cost_map_input_rate}; the cleared override {PINNED_INPUT_RATE} still resolves" + ) + + @pytest.mark.covers("mgmt.model.update.clear_persists") + def test_cleared_price_is_billed_at_the_cost_map_rate( + self, client: ManagementClient, resources: ResourceManager, scoped_key: str + ) -> None: + registered = _register(client, resources) + _ = client.proxy.patch_model( + registered.model_id, + ModelPatchBody(litellm_params=LiteLLMParamsPatch(max_input_tokens=Clear(), input_cost_per_token=Clear())), + ) + _ = _stored_entry( + client, + registered.model_name, + converged=lambda entry: "input_cost_per_token" not in entry.litellm_params.model_fields_set, + ) + cost_map_input_rate = client.proxy.model_cost_map()[BACKEND_MODEL].input_cost_per_token + assert cost_map_input_rate is not None, f"cost map has no input rate for {BACKEND_MODEL}" + + prompt_tokens, input_cost = _await_billed_input_cost( + client, registered.model_name, scoped_key, expected_rate=cost_map_input_rate + ) + + assert _approx_equal(input_cost, prompt_tokens * cost_map_input_rate), ( + f"input_cost {input_cost} != {prompt_tokens} tokens * cost map rate {cost_map_input_rate} " + f"= {prompt_tokens * cost_map_input_rate}" + ) + assert not _approx_equal(input_cost, prompt_tokens * PINNED_INPUT_RATE), ( + f"input_cost {input_cost} is still billed at the cleared override {PINNED_INPUT_RATE}" + ) + + @pytest.mark.covers("mgmt.model.delete.persists") + def test_delete_removes_the_deployment_everywhere( + self, client: ManagementClient, resources: ResourceManager, scoped_key: str + ) -> None: + registered = _register(client, resources) + _ = _stored_entry(client, registered.model_name, converged=lambda _entry: True) + + client.delete_model_strict(registered.model_id) + + _assert_absent_everywhere(client, registered.model_name) + refused = client.chat_status(scoped_key, registered.model_name, "hi this is a test") + assert refused.status_code == 400, ( + f"chat against the deleted model must be rejected 400, got {refused.status_code}: {refused.body[:300]}" + ) + envelope = _ErrorEnvelope.model_validate_json(refused.body) + assert envelope.error.message, f"400 body must carry an error message: {refused.body[:300]}" diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 62810e6cfd9..8db37bd25a5 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -655,6 +655,11 @@ class OcrResponse(BaseModel): # ---------- spend logs ---------- +class CostBreakdown(BaseModel): + input_cost: float | None = None + output_cost: float | None = None + + class GuardrailEntityMatch(BaseModel): entity_type: str score: float @@ -672,6 +677,7 @@ class GuardrailRunRecord(BaseModel): class SpendLogMetadata(BaseModel): + cost_breakdown: CostBreakdown | None = None applied_guardrails: list[str] | None = None guardrail_information: list[GuardrailRunRecord] | None = None @@ -815,15 +821,42 @@ class CustomPricing(BaseModel): return prompt_tokens * self.input_cost_per_token + completion_tokens * self.output_cost_per_token +class DeploymentParams(CustomPricing): + """The litellm_params half of a /model/info row: the stored deployment as written, + credentials scrubbed. Unlike model_info it is never back-filled from the cost map, + so a key the store dropped is absent here (check `model_fields_set`).""" + + model: str | None = None + api_base: str | None = None + max_input_tokens: int | None = None + + +class DeploymentModelInfo(CustomPricing): + id: str | None = None + max_input_tokens: int | None = None + + class ModelInfoEntry(BaseModel): """One /model/info row. `litellm_params` is the configured deployment (carries any custom-pricing override); `model_info` is the price the proxy resolved for - it - the override merged over the cost-map defaults.""" + it - the override merged over the cost-map defaults, so a key cleared from the + stored blob reads as the cost-map default here.""" model_config = ConfigDict(protected_namespaces=()) model_name: str - litellm_params: CustomPricing = CustomPricing() - model_info: CustomPricing = CustomPricing() + litellm_params: DeploymentParams = DeploymentParams() + model_info: DeploymentModelInfo = DeploymentModelInfo() + + +class StoredDeployment(BaseModel): + """PATCH /model/{model_id}/update answers with the row as stored: both blobs raw, + nothing back-filled, so a cleared key is absent from `model_fields_set` of the + blob it was cleared from.""" + + model_config = ConfigDict(protected_namespaces=()) + model_name: str + litellm_params: DeploymentParams + model_info: DeploymentModelInfo class ModelInfoResponse(BaseModel): @@ -924,9 +957,10 @@ class LiteLLMParamsBody(BaseModel): timeout: float | None = None tpm: int | None = None weight: int | None = None + max_input_tokens: int | None = None -ModelMode = Literal["batch", "realtime", "image_generation"] +ModelMode = Literal["chat", "batch", "realtime", "image_generation"] class ModelInfoBody(BaseModel): @@ -936,6 +970,7 @@ class ModelInfoBody(BaseModel): # constraint when a prior run's teardown had not removed the row. id: str | None = None mode: ModelMode | None = None + max_input_tokens: int | None = None access_groups: list[str] | None = None team_id: str | None = None allowed_fails_policy: dict[str, int] | None = None @@ -964,6 +999,37 @@ class ModelUpdateBody(BaseModel): model_info: ModelInfoBody +class Clear(BaseModel): + """Serializes to JSON null. The transport dumps every body with exclude_none, so a + field set to this is how a patch carries the explicit null that removes a stored key.""" + + @model_serializer + def _as_null(self) -> None: + return None + + +class LiteLLMParamsPatch(BaseModel): + api_base: str | Clear | None = None + max_input_tokens: int | Clear | None = None + input_cost_per_token: float | Clear | None = None + output_cost_per_token: float | Clear | None = None + + +class ModelInfoPatch(BaseModel): + mode: ModelMode | Clear | None = None + max_input_tokens: int | Clear | None = None + + +class ModelPatchBody(BaseModel): + """PATCH /model/{model_id}/update body, JSON Merge Patch over the stored deployment: + a field left None is dropped from the body and unchanged, a field set to `Clear()` + is sent as null and removed, a field with a value is set.""" + + model_config = ConfigDict(protected_namespaces=()) + litellm_params: LiteLLMParamsPatch | None = None + model_info: ModelInfoPatch | None = None + + class ModelListEntry(BaseModel): id: str diff --git a/tests/e2e/proxy_client.py b/tests/e2e/proxy_client.py index 1bac5116a9d..fa1b06fe7ed 100644 --- a/tests/e2e/proxy_client.py +++ b/tests/e2e/proxy_client.py @@ -10,10 +10,10 @@ from __future__ import annotations import time import warnings -from collections.abc import Callable, Mapping +from collections.abc import Callable, Iterator, Mapping from dataclasses import dataclass -from functools import reduce from datetime import datetime +from functools import reduce from types import MappingProxyType from typing import Final @@ -62,6 +62,7 @@ from models import ( ModelMode, ModelNewBody, ModelNewResponse, + ModelPatchBody, ModelsListParams, ModelsListResponse, ModelUpdateBody, @@ -72,6 +73,7 @@ from models import ( SpendLogsPage, SpendLogsPageParams, SpendLogsParams, + StoredDeployment, ToolsetCreateBody, ToolsetRow, ToolsetUpdateBody, @@ -132,6 +134,103 @@ class NotServableOn: last_result: Result[ModelsListResponse] | None +type BodyReader[R: BaseModel] = Callable[[float], Result[R]] + + +@dataclass(frozen=True, slots=True) +class BodyNotConverged[R: BaseModel]: + """The deadline passed without a read the predicate accepted; `last_result` is the + final read, so the caller can tell a body that never matched from a read that + failed.""" + + last_result: Result[R] | None + + +@dataclass(frozen=True, slots=True) +class BodyConverged[R: BaseModel]: + """Every replica answered a body the predicate accepted; `bodies` is the last read + per replica.""" + + bodies: Mapping[str, R] + + +@dataclass(frozen=True, slots=True) +class BodyNeverConvergedOn[R: BaseModel]: + """`BodyNotConverged` labeled with the replica whose reads never satisfied the predicate.""" + + replica: str + last_result: Result[R] | None + + +def await_body_converged[R: BaseModel]( + read: BodyReader[R], + *, + predicate: Callable[[R], bool], + timeout: float, + interval: float, + request_timeout: float, + now: Callable[[], float], + sleep: Callable[[float], None], +) -> Success[R] | BodyNotConverged[R]: + """Poll `read` until it answers a body `predicate` accepts, or `timeout` passes. + + Each read's request timeout is clamped to the remaining budget, and the sleep + between reads to the time left, so the last read before the deadline is never + skipped. Clock and sleep are injected.""" + deadline: Final = now() + timeout + + def reads() -> Iterator[Result[R]]: + while (remaining := deadline - now()) > 0: + yield read(min(request_timeout, remaining)) + sleep(min(interval, max(deadline - now(), 0.0))) + + def attempts() -> Iterator[Success[R] | BodyNotConverged[R]]: + for result in reads(): + if isinstance(result, Success) and predicate(result.data): + yield result + return + yield BodyNotConverged(last_result=result) + + initial: Final[Success[R] | BodyNotConverged[R]] = BodyNotConverged(last_result=None) + return reduce(lambda _previous, result: result, attempts(), initial) + + +def await_body_converged_everywhere[R: BaseModel]( + readers: Mapping[str, BodyReader[R]], + *, + predicate: Callable[[R], bool], + timeout: float, + interval: float, + request_timeout: float, + now: Callable[[], float], + sleep: Callable[[float], None], +) -> BodyConverged[R] | BodyNeverConvergedOn[R]: + """`await_body_converged` against every replica in turn, each with the full budget, so a + write counts as landed only once every replica serves it.""" + def read_replica( + outcome: BodyConverged[R] | BodyNeverConvergedOn[R], + item: tuple[str, BodyReader[R]], + ) -> BodyConverged[R] | BodyNeverConvergedOn[R]: + if isinstance(outcome, BodyNeverConvergedOn): + return outcome + replica, read = item + match await_body_converged( + read, + predicate=predicate, + timeout=timeout, + interval=interval, + request_timeout=request_timeout, + now=now, + sleep=sleep, + ): + case Success(data=data): + return BodyConverged(bodies=MappingProxyType({**outcome.bodies, replica: data})) + case BodyNotConverged(last_result=last_result): + return BodyNeverConvergedOn(replica=replica, last_result=last_result) + initial: Final[BodyConverged[R] | BodyNeverConvergedOn[R]] = BodyConverged(bodies=MappingProxyType({})) + return reduce(read_replica, readers.items(), initial) + + def await_servable( list_models: Callable[[float], Result[ModelsListResponse]], *, @@ -658,6 +757,102 @@ class ProxyClient: ) ) + def patch_model(self, model_id: str, body: ModelPatchBody) -> StoredDeployment: + """JSON Merge Patch the deployment `model_id` via PATCH /model/{model_id}/update: + a field the body omits is unchanged, one sent as null is removed from the stored + row, one sent with a value is set. See ModelPatchBody for how a null is sent. + Returns the row as stored after the write.""" + return unwrap( + self.transport.patch( + f"/model/{model_id}/update", + headers=self.transport.master, + json=body, + response_type=StoredDeployment, + ) + ) + + def read_model_back_everywhere[R: BaseModel]( + self, path: str, response_type: type[R], *, predicate: Callable[[R], bool] + ) -> Mapping[str, R]: + """GET `path` on every replica until each answers a body `predicate` accepts, + polling to poll_timeout, and return the last body per replica. + + Fails naming the replica that never converged, so a write that reached one + gateway but not the others is caught instead of passing on whichever gateway + the balancer answered from. Falls back to the single proxy address when no + replica list is configured. + + `path` must be a data-plane route. The replicas are gateways, which serve only + the LLM surface, so a control-plane path answers on exactly one service and + 404s on every replica in a split deployment: asking each replica for one is + never the question the caller means. Read those through `self.transport` + instead, which routes them to the control plane.""" + if is_control_plane_path(path): + raise AssertionError( + f"read_model_back_everywhere({path!r}) asks every data-plane replica for a control-plane route. " + "The replicas are gateways and do not serve it; poll a data-plane path such as /v1/models " + "here, and read the control plane through the shared transport." + ) + readers: Final = { + url: self._body_reader(transport, path, response_type) + for url, transport in self._read_back_replicas().items() + } + outcome: Final = await_body_converged_everywhere( + readers, + predicate=predicate, + timeout=self.poll_timeout, + interval=self.poll_interval, + request_timeout=REQUEST_TIMEOUT, + now=time.monotonic, + sleep=time.sleep, + ) + match outcome: + case BodyConverged(bodies=bodies): + return bodies + case BodyNeverConvergedOn(replica=replica, last_result=last_result): + raise AssertionError( + f"GET {path} on {replica} never answered the expected body within " + f"{self.poll_timeout}s; last read: {last_result}" + ) + + def read_model_back[R: BaseModel](self, path: str, response_type: type[R], *, predicate: Callable[[R], bool]) -> R: + """GET `path` through the shared transport until the body satisfies `predicate`, + polling to poll_timeout, and return that body. + + The counterpart to `read_model_back_everywhere` for a control-plane route such as + /model/info: the stored row lives in one database behind one control plane, so + there is a single answer to converge on rather than one per gateway.""" + outcome: Final = await_body_converged_everywhere( + {CONTROL_PLANE_BASE_URL: self._body_reader(self.transport, path, response_type)}, + predicate=predicate, + timeout=self.poll_timeout, + interval=self.poll_interval, + request_timeout=REQUEST_TIMEOUT, + now=time.monotonic, + sleep=time.sleep, + ) + match outcome: + case BodyConverged(bodies=bodies): + return bodies[CONTROL_PLANE_BASE_URL] + case BodyNeverConvergedOn(last_result=last_result): + raise AssertionError( + f"GET {path} never answered the expected body within " + f"{self.poll_timeout}s; last read: {last_result}" + ) + + def _read_back_replicas(self) -> Mapping[str, Transport]: + return self.replicas or MappingProxyType({CONTROL_PLANE_BASE_URL: self.transport}) + + @staticmethod + def _body_reader[R: BaseModel](transport: Transport, path: str, response_type: type[R]) -> BodyReader[R]: + return lambda timeout: transport.get( + path, + headers=transport.master, + params=NoBody(), + response_type=response_type, + timeout=timeout, + ) + def delete_model(self, model_id: str) -> None: result = self.transport.post( "/model/delete", diff --git a/tests/e2e/test_proxy_client.py b/tests/e2e/test_proxy_client.py index 3b84a47e3cc..cbf7f5648d4 100644 --- a/tests/e2e/test_proxy_client.py +++ b/tests/e2e/test_proxy_client.py @@ -20,8 +20,12 @@ from typing import Final, cast import pytest from e2e_config import parse_replica_urls from e2e_http import Result, Success -from models import KeyInfo, KeyInfoResponse, ModelListEntry, ModelsListResponse +from models import KeyInfo, KeyInfoResponse, ModelInfoEntry, ModelInfoResponse, ModelListEntry, ModelsListResponse from proxy_client import ( + BodyReader, + BodyConverged, + BodyNeverConvergedOn, + await_body_converged_everywhere, ConvergeOutcome, Converged, EverywhereConverged, @@ -274,3 +278,54 @@ class TestReplicasFor: client: Final = ProxyClient(transport=_NO_TRANSPORTS, replicas={}, control_replicas={}) with pytest.raises(AssertionError, match="no replica is configured"): _ = client.replicas_for("/v1/models") + + +def _info(*model_names: str) -> Success[ModelInfoResponse]: + entries: Final = [ModelInfoEntry(model_name=model_name) for model_name in model_names] + return Success(status_code=200, data=ModelInfoResponse(data=entries)) + + +def _reader(results: Iterable[Success[ModelInfoResponse]]) -> BodyReader[ModelInfoResponse]: + it: Final = iter(results) + return lambda _timeout: next(it) + + +def _lists_model(body: ModelInfoResponse) -> bool: + return any(entry.model_name == MODEL for entry in body.data) + + +def _read_back( + readers: Mapping[str, BodyReader[ModelInfoResponse]], +) -> tuple[BodyConverged[ModelInfoResponse] | BodyNeverConvergedOn[ModelInfoResponse], FakeClock]: + clock: Final = FakeClock() + outcome: Final = await_body_converged_everywhere( + readers, + predicate=_lists_model, + timeout=TIMEOUT, + interval=INTERVAL, + request_timeout=5.0, + now=clock.now, + sleep=clock.sleep, + ) + return outcome, clock + + +class TestAwaitBodyConvergedEverywhere: + def test_waits_for_the_lagging_replica_and_returns_every_body(self) -> None: + readers: Final = { + "gateway-1": _reader(repeat(_info(MODEL))), + "gateway-2": _reader(chain(repeat(_info(), 2), repeat(_info(MODEL)))), + } + outcome, clock = _read_back(readers) + assert outcome == BodyConverged(bodies={"gateway-1": _info(MODEL).data, "gateway-2": _info(MODEL).data}) + assert clock.elapsed == 2 * INTERVAL + + @pytest.mark.parametrize("lagging", ["gateway-1", "gateway-2"]) + def test_fails_naming_the_replica_that_never_converges(self, lagging: str) -> None: + readers: Final = { + "gateway-1": _reader(repeat(_info(MODEL))), + "gateway-2": _reader(repeat(_info(MODEL))), + } | {lagging: _reader(repeat(_info()))} + outcome, clock = _read_back(readers) + assert outcome == BodyNeverConvergedOn(replica=lagging, last_result=_info()) + assert clock.elapsed >= TIMEOUT diff --git a/tests/e2e/transport.py b/tests/e2e/transport.py index 44fdbaa3e41..804e073a4a0 100644 --- a/tests/e2e/transport.py +++ b/tests/e2e/transport.py @@ -306,6 +306,7 @@ CONTROL_PLANE_PREFIXES: tuple[str, ...] = ( "/config", "/guardrails", "/openapi.json", + "/public/", ) diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index c02f886fc31..1376727e296 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -3115,9 +3115,6 @@ class TestUpdateDBModelClearPricing: """Sending an explicit `null` for a pricing field must remove it from both `litellm_params` and `model_info` (SPECIAL_MODEL_INFO_PARAMS are mirrored between the two by Deployment.__init__). - - Restricted to SPECIAL_MODEL_INFO_PARAMS so non-pricing fields (e.g. team_id) - cannot be cleared via this path. """ def test_clear_input_cost_removes_from_both_blobs(self): @@ -3193,10 +3190,10 @@ class TestUpdateDBModelClearPricing: assert params["input_cost_per_token"] == 0.000001 assert params["output_cost_per_token"] == 0.000007 - def test_null_on_non_pricing_field_does_not_clear(self): - """Security guard: only SPECIAL_MODEL_INFO_PARAMS can be cleared via null. - Privileged or unrelated model_info fields (e.g. team_id) must be unaffected - by the null-clearing path so a team admin can't ungate a team-scoped model. + def test_null_on_one_field_leaves_other_fields_alone(self): + """A null clears only the key it names: pricing the patch never mentions and + the ownership key team_id stay put, so a team admin can't ungate a + team-scoped model through the clear path. """ from litellm.proxy.management_endpoints.model_management_endpoints import ( update_db_model, @@ -3217,8 +3214,6 @@ class TestUpdateDBModelClearPricing: model_info=ModelInfo(id="dep-pricing-1", team_id="team-keep-me"), ) - # Patch sends a null for api_base (non-SPECIAL field). Must NOT clear team_id - # or any other non-pricing field from the merged dict. result = update_db_model( db_model=db_model, updated_patch=updateDeployment( @@ -3394,6 +3389,171 @@ class TestUpdateDBModelClearPricing: assert info["cache_creation_input_token_cost"] == 0.000003 +_PROTECTED_MODEL_INFO_VALUES = { + "team_id": "team-keep-me", + "team_public_model_name": "team-facing-name", + "access_groups": ["group-a"], + "created_at": "2026-01-01T00:00:00+00:00", + "created_by": "creator", + "updated_at": "2026-01-02T00:00:00+00:00", + "updated_by": "updater", + "blocked": True, +} + + +def _build_db_model_with_pinned_model_info(): + """Deployment whose stored blobs pin non-pricing keys an earlier save wrote, next to a + pricing override, so a clear can be checked key by key.""" + from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo + + return Deployment( + model_name="pinned-gpt-4o-mini", + litellm_params=LiteLLM_Params( + model="gpt-4o-mini", input_cost_per_token=0.000001, max_input_tokens=4096 + ), + model_info=ModelInfo( + id="dep-pinned-0", + max_input_tokens=4096, + mode="chat", + supports_vision=True, + **_PROTECTED_MODEL_INFO_VALUES, + ), + ) + + +class TestUpdateDBModelNullClearsAnyKey: + """JSON Merge Patch on PATCH /model/{id}/update: a key sent as null is removed from the + stored blob it was sent in, whatever the key, except the identity and ownership keys, + whose nulls are ignored.""" + + def test_model_info_nulls_remove_pinned_non_pricing_keys(self): + from litellm.proxy.management_endpoints.model_management_endpoints import ( + update_db_model, + ) + + result = update_db_model( + db_model=_build_db_model_with_pinned_model_info(), + updated_patch=updateDeployment.model_validate( + {"model_info": {"max_input_tokens": None, "mode": None}} + ), + ) + + info = json.loads(result["model_info"]) + assert "max_input_tokens" not in info + assert "mode" not in info + assert info["supports_vision"] is True + assert info["input_cost_per_token"] == 0.000001 + + def test_litellm_params_null_removes_pinned_non_pricing_key(self): + from litellm.proxy.management_endpoints.model_management_endpoints import ( + update_db_model, + ) + + result = update_db_model( + db_model=_build_db_model_with_pinned_model_info(), + updated_patch=updateDeployment.model_validate( + {"litellm_params": {"max_input_tokens": None}} + ), + ) + + params = json.loads(result["litellm_params"]) + info = json.loads(result["model_info"]) + assert "max_input_tokens" not in params + assert params["model"] == "gpt-4o-mini" + assert params["input_cost_per_token"] == 0.000001 + assert info["max_input_tokens"] == 4096 + + def test_omitted_key_is_untouched_by_a_null_elsewhere(self): + from litellm.proxy.management_endpoints.model_management_endpoints import ( + update_db_model, + ) + + result = update_db_model( + db_model=_build_db_model_with_pinned_model_info(), + updated_patch=updateDeployment.model_validate( + {"model_info": {"mode": None, "supports_vision": False}} + ), + ) + + info = json.loads(result["model_info"]) + assert "mode" not in info + assert info["supports_vision"] is False + assert info["max_input_tokens"] == 4096 + + @pytest.mark.parametrize("field", sorted(_PROTECTED_MODEL_INFO_VALUES)) + def test_null_on_protected_key_is_ignored(self, field): + from litellm.proxy.management_endpoints.model_management_endpoints import ( + update_db_model, + ) + + result = update_db_model( + db_model=_build_db_model_with_pinned_model_info(), + updated_patch=updateDeployment.model_validate({"model_info": {field: None}}), + ) + + info = json.loads(result["model_info"]) + assert info[field] == _PROTECTED_MODEL_INFO_VALUES[field] + assert info["max_input_tokens"] == 4096 + + def test_echoing_the_read_back_blob_preserves_every_stored_key(self): + """The Admin UI edit form submits the whole /model/info row back, and that read reports + every key the deployment never stored as an explicit null. Those nulls have to stay + no-ops: a write drops None before storing, so a null in the echoed blob always names a + key the stored row does not carry. + """ + from litellm.proxy.management_endpoints.model_management_endpoints import ( + update_db_model, + ) + + db_model = _build_db_model_with_pinned_model_info() + echoed = { + "id": "dep-pinned-0", + "max_input_tokens": 4096, + "mode": "chat", + "supports_vision": True, + "input_cost_per_token": 0.000001, + "team_id": "team-keep-me", + "base_model": None, + "tier": None, + "max_output_tokens": None, + "supports_function_calling": None, + "cache_read_input_token_cost": None, + } + + result = update_db_model( + db_model=db_model, + updated_patch=updateDeployment.model_validate({"model_info": echoed}), + ) + + info = json.loads(result["model_info"]) + assert info["max_input_tokens"] == 4096 + assert info["mode"] == "chat" + assert info["supports_vision"] is True + assert info["input_cost_per_token"] == 0.000001 + assert info["team_id"] == "team-keep-me" + for never_stored in ("base_model", "tier", "max_output_tokens", "supports_function_calling"): + assert never_stored not in info + + def test_null_on_pricing_key_still_clears_both_blobs(self): + from litellm.proxy.management_endpoints.model_management_endpoints import ( + update_db_model, + ) + + result = update_db_model( + db_model=_build_db_model_with_pinned_model_info(), + updated_patch=updateDeployment.model_validate( + {"model_info": {"input_cost_per_token": None}} + ), + ) + + params = json.loads(result["litellm_params"]) + info = json.loads(result["model_info"]) + assert "input_cost_per_token" not in params + assert "input_cost_per_token" not in info + assert params["max_input_tokens"] == 4096 + assert info["max_input_tokens"] == 4096 + + class TestGetModelInfoWithIdBlocked: """`ProxyConfig.get_model_info_with_id` must propagate the DB-level `blocked` column into the in-memory `model_info` dict so the router filter can read it.""" diff --git a/tests/test_litellm/test_router_model_cost_isolation.py b/tests/test_litellm/test_router_model_cost_isolation.py index 30b265905f3..d22ec60e61a 100644 --- a/tests/test_litellm/test_router_model_cost_isolation.py +++ b/tests/test_litellm/test_router_model_cost_isolation.py @@ -220,6 +220,198 @@ def test_should_store_full_pricing_under_deployment_model_id(): assert entry["output_cost_per_token"] == 0.0 +def test_should_drop_a_price_the_deployment_no_longer_carries(): + """Re-registering a deployment must replace its model_id entry, not merge onto it. + + A merge left the old rate in the cost map after an operator cleared the override, so + the deployment kept billing at a price its config no longer had. + """ + backend_model = "vertex_ai/gemini-2.5-flash" + model_id = "deployment-cleared-price" + original = {model_id: litellm.model_cost.get(model_id)} + + try: + Router._register_deployment_in_model_cost( + model_id=model_id, + model_info={"input_cost_per_token": 0.005, "output_cost_per_token": 0.01}, + model=backend_model, + custom_llm_provider="vertex_ai", + ) + assert litellm.model_cost[model_id]["input_cost_per_token"] == 0.005 + + Router._register_deployment_in_model_cost( + model_id=model_id, + model_info={"mode": "chat"}, + model=backend_model, + custom_llm_provider="vertex_ai", + ) + + entry = litellm.model_cost[model_id] + assert entry.get("input_cost_per_token") != 0.005, ( + "the cleared override survived re-registration, so the deployment still bills at it" + ) + assert entry.get("output_cost_per_token") != 0.01 + finally: + _restore_model_cost_entries(original) + + +def test_should_not_strip_a_builtin_entry_when_a_deployment_id_collides_with_it(): + """Deployments are keyed into the same cost map as the built-in catalog, so a deployment + whose id happens to name a real model must not evict that model's entry. + + Stripping it would take the pricing and capability flags every other deployment of that + model reads, process-wide, until the next price-map reload. Registering twice, because + the first registration is what would mark the entry as this deployment's own. + """ + colliding_id = "gpt-4o" + original = {colliding_id: litellm.model_cost.get(colliding_id)} + builtin_max_tokens = litellm.model_cost[colliding_id]["max_tokens"] + + try: + for _ in range(2): + Router._register_deployment_in_model_cost( + model_id=colliding_id, + model_info={"id": colliding_id, "db_model": True, "mode": "chat"}, + model="gpt-4o-mini", + custom_llm_provider="openai", + ) + + entry = litellm.model_cost[colliding_id] + assert entry["max_tokens"] == builtin_max_tokens, ( + "registering a deployment under a catalog model's name wiped that model's context window" + ) + assert entry["litellm_provider"] == "openai" + assert entry["supports_vision"] is True + finally: + _restore_model_cost_entries(original) + + +def test_should_drop_a_stale_price_even_when_the_deployment_declares_a_provider(): + """A deployment may carry `litellm_provider` in its own model_info, which must not be + read as "this is a catalog entry" and stop the stale price from being dropped.""" + model_id = "deployment-provider-tagged" + original = {model_id: litellm.model_cost.get(model_id)} + + try: + Router._register_deployment_in_model_cost( + model_id=model_id, + model_info={"id": model_id, "litellm_provider": "openai", "input_cost_per_token": 0.005}, + model="gpt-4o-mini", + custom_llm_provider="openai", + ) + assert litellm.model_cost[model_id]["input_cost_per_token"] == 0.005 + + Router._register_deployment_in_model_cost( + model_id=model_id, + model_info={"id": model_id, "litellm_provider": "openai", "mode": "chat"}, + model="gpt-4o-mini", + custom_llm_provider="openai", + ) + + assert litellm.model_cost[model_id].get("input_cost_per_token") != 0.005, ( + "a deployment that declares its provider kept billing at the price it no longer carries" + ) + finally: + _restore_model_cost_entries(original) + + +def test_should_give_a_cost_map_key_back_when_the_deployment_is_deleted(): + """Deleting a deployment releases its claim on the shared cost-map key. + + Held forever, a later catalog refresh that starts publishing a model under that same + name would be treated as the deleted deployment's own entry and evicted. + """ + from litellm.router import _DEPLOYMENT_COST_MAP_KEYS + + model_id = "deployment-to-delete" + original = {model_id: litellm.model_cost.get(model_id)} + router = Router( + model_list=[ + { + "model_name": "to-delete", + "litellm_params": {"model": "gpt-4o-mini", "mock_response": "ok"}, + "model_info": {"id": model_id, "input_cost_per_token": 0.005}, + } + ] + ) + + try: + assert model_id in _DEPLOYMENT_COST_MAP_KEYS + + assert router.delete_deployment(id=model_id) is not None + + assert model_id not in _DEPLOYMENT_COST_MAP_KEYS, ( + "a deleted deployment kept its claim on the shared cost-map key" + ) + finally: + _DEPLOYMENT_COST_MAP_KEYS.discard(model_id) + _restore_model_cost_entries(original) + + +def test_should_keep_the_cost_map_key_while_another_router_still_serves_it(): + """Two live routers can serve the same deployment id, and the claim is process-wide. + + Releasing it when only one of them drops the deployment would put the survivor back on + merging, so the price it just cleared would keep billing. + """ + from litellm.router import _DEPLOYMENT_COST_MAP_KEYS + + model_id = "deployment-served-twice" + original = {model_id: litellm.model_cost.get(model_id)} + entry = { + "model_name": "served-twice", + "litellm_params": {"model": "gpt-4o-mini", "mock_response": "ok"}, + "model_info": {"id": model_id, "input_cost_per_token": 0.005}, + } + first = Router(model_list=[entry]) + second = Router(model_list=[entry]) + + try: + assert model_id in _DEPLOYMENT_COST_MAP_KEYS + + assert first.delete_deployment(id=model_id) is not None + + assert model_id in _DEPLOYMENT_COST_MAP_KEYS, ( + "the claim was released while another router still served the deployment" + ) + + assert second.delete_deployment(id=model_id) is not None + assert model_id not in _DEPLOYMENT_COST_MAP_KEYS + finally: + _DEPLOYMENT_COST_MAP_KEYS.discard(model_id) + _restore_model_cost_entries(original) + + +def test_should_keep_the_cost_map_key_while_a_dynamically_built_router_serves_it(): + """A router built with no model_list still serves whatever add_deployment gives it, so it + counts when deciding whether the shared cost-map claim can be released.""" + from litellm.router import _DEPLOYMENT_COST_MAP_KEYS + + model_id = "deployment-added-dynamically" + original = {model_id: litellm.model_cost.get(model_id)} + entry = { + "model_name": "added-dynamically", + "litellm_params": {"model": "gpt-4o-mini", "mock_response": "ok"}, + "model_info": {"id": model_id, "input_cost_per_token": 0.005}, + } + configured = Router(model_list=[entry]) + dynamic = Router() + dynamic.add_deployment(deployment=Deployment(**entry)) + + try: + assert configured.delete_deployment(id=model_id) is not None + + assert model_id in _DEPLOYMENT_COST_MAP_KEYS, ( + "the claim was released while a dynamically built router still served the deployment" + ) + + assert dynamic.delete_deployment(id=model_id) is not None + assert model_id not in _DEPLOYMENT_COST_MAP_KEYS + finally: + _DEPLOYMENT_COST_MAP_KEYS.discard(model_id) + _restore_model_cost_entries(original) + + def test_should_preserve_builtin_pricing_regardless_of_deployment_order(): """ The built-in pricing should be preserved no matter which deployment diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 83b0d58f2b2..93aaf3ca58c 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -9064,8 +9064,9 @@ export interface paths { * Patch Model * @description PATCH Endpoint for partial model updates. * - * Only updates the fields specified in the request while preserving other existing values. - * Follows proper PATCH semantics by only modifying provided fields. + * JSON Merge Patch semantics over `litellm_params` and `model_info`: a key absent from the + * body is unchanged, a key sent as null is removed from the stored row, and a key sent with a + * value is set (identity and ownership keys such as `id` and `team_id` ignore a null). * * Args: * model_id: The ID of the model to update From ef3a3c16ae02bc6d83e14b09c437ef36081c0498 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2026 23:32:31 -0700 Subject: [PATCH 09/11] feat(guardrails): map each guardrail scan id to its guardrail, stage and provider (#40327) * feat(guardrails): map each guardrail scan id to its guardrail, stage and provider Adds the x-litellm-guardrail-scan-metadata response header, a JSON list of {guardrail, stage, provider, scan_id} entries, next to the existing comma-separated x-litellm-guardrail-scan-id header. Prisma AIRS records the execution stage for every scan and OpenAI Moderation now records its moderation id too. The new metadata key is internal: client-supplied values are stripped and it is exposed through the UI CORS allow list. Resolves LIT-6018 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(guardrails): cap the scan metadata response header at a configurable length Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(guardrails): hardcode the scan metadata header cap Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yucheng Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 2 + litellm/proxy/common_utils/callback_utils.py | 62 ++++++++- .../guardrail_hooks/openai/moderations.py | 10 +- .../panw_prisma_airs/panw_prisma_airs.py | 34 +++-- litellm/proxy/litellm_pre_call_utils.py | 2 + .../proxy/common_utils/test_callback_utils.py | 122 +++++++++++++++--- .../openai/test_moderations.py | 32 +++++ .../guardrail_hooks/test_panw_prisma_airs.py | 32 ++++- 8 files changed, 265 insertions(+), 31 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 6f2384c8c6c..108a914e9c1 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -143,6 +143,7 @@ DEFAULT_MCP_SEMANTIC_FILTER_SIMILARITY_THRESHOLD: Final = float( os.getenv("DEFAULT_MCP_SEMANTIC_FILTER_SIMILARITY_THRESHOLD", 0.3) ) MAX_MCP_SEMANTIC_FILTER_TOOLS_HEADER_LENGTH: Final = int(os.getenv("MAX_MCP_SEMANTIC_FILTER_TOOLS_HEADER_LENGTH", 150)) +MAX_GUARDRAIL_SCAN_METADATA_HEADER_LENGTH: Final = 2048 DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS: Final = 2000 @@ -197,6 +198,7 @@ LITELLM_UI_ALLOW_HEADERS: Final = [ "x-litellm-adaptive-router-model", "x-litellm-applied-guardrails", "x-litellm-guardrail-scan-id", + "x-litellm-guardrail-scan-metadata", "x-litellm-cache-key", ] diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index 770963a1f24..561a53409f4 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -1,10 +1,12 @@ import copy +import json import os from collections.abc import Callable, Iterable, Mapping from dataclasses import dataclass +from itertools import accumulate from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn, Optional, TypeAlias -from typing_extensions import assert_never +from typing_extensions import ReadOnly, TypedDict, assert_never import litellm from litellm import get_secret @@ -12,6 +14,7 @@ from litellm._logging import verbose_proxy_logger from litellm.constants import ( CLIENT_OUTPUT_CEILING_METADATA_KEY, CONSUMED_REQUEST_TAGS_METADATA_KEY, + MAX_GUARDRAIL_SCAN_METADATA_HEADER_LENGTH, PRE_CALL_EXECUTED_GUARDRAILS_KEY, ROUTING_REQUEST_TAGS_METADATA_KEY, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, @@ -28,6 +31,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( encrypt_value_helper, ) from litellm.proxy.types_utils.utils import get_instance_fn +from litellm.types.guardrails import GuardrailEventHooks from litellm.types.utils import ( StandardLoggingGuardrailInformation, StandardLoggingPayload, @@ -52,6 +56,15 @@ reset_color_code: Final = "\033[0m" TRUSTED_PILLAR_RESPONSE_HEADERS_METADATA_KEY: Final = "_pillar_response_headers_trusted" GUARDRAIL_SCAN_IDS_METADATA_KEY: Final = "guardrail_scan_ids" +GUARDRAIL_SCAN_METADATA_METADATA_KEY: Final = "guardrail_scan_metadata" + + +class GuardrailScanMetadata(TypedDict): + guardrail: ReadOnly[str | None] + stage: ReadOnly[str] + provider: ReadOnly[str] + scan_id: ReadOnly[str] + if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging @@ -450,6 +463,16 @@ def get_remaining_tokens_and_requests_from_request_data(data: dict) -> dict[str, return headers +def _serialize_scan_metadata_header(entries: Iterable[object], *, max_length: int) -> str | None: + """Compact JSON list of scan metadata entries, dropping trailing entries so the header fits in max_length.""" + encoded: Final = tuple(json.dumps(entry, separators=(",", ":")) for entry in entries) + lengths: Final = tuple(accumulate(len(item) + 1 for item in encoded)) + kept: Final = sum(1 for length in lengths if length + 1 <= max_length) + if kept == 0: + return None + return f"[{','.join(encoded[:kept])}]" + + def get_logging_caching_headers(request_data: dict) -> dict | None: _metadata: Final[dict] = {} metadata_bucket: Final = request_data.get("metadata") @@ -468,6 +491,15 @@ def get_logging_caching_headers(request_data: dict) -> dict | None: if scan_ids: headers["x-litellm-guardrail-scan-id"] = ",".join(scan_ids) + scan_metadata: Final = _metadata.get(GUARDRAIL_SCAN_METADATA_METADATA_KEY) + scan_metadata_header: Final = ( + _serialize_scan_metadata_header(scan_metadata, max_length=MAX_GUARDRAIL_SCAN_METADATA_HEADER_LENGTH) + if isinstance(scan_metadata, (list, tuple)) + else None + ) + if scan_metadata_header: + headers["x-litellm-guardrail-scan-metadata"] = scan_metadata_header + if "applied_policies" in _metadata: headers["x-litellm-applied-policies"] = ",".join(_metadata["applied_policies"]) @@ -501,6 +533,7 @@ LITELLM_PROXY_INTERNAL_METADATA_KEYS: Final = frozenset( "applied_policies", "applied_guardrails", GUARDRAIL_SCAN_IDS_METADATA_KEY, + GUARDRAIL_SCAN_METADATA_METADATA_KEY, "policy_sources", "guardrails", "guardrail_config", @@ -565,21 +598,40 @@ def add_guardrail_to_applied_guardrails_header(request_data: dict, guardrail_nam _metadata["applied_guardrails"] = [guardrail_name] -def add_guardrail_scan_id(request_data: dict, scan_id: str | None) -> None: +def add_guardrail_scan_id( + request_data: dict[str, object], + scan_id: str | None, + *, + guardrail_name: str | None, + provider: str, + stage: GuardrailEventHooks, +) -> None: """ - Record a provider scan id so it can be surfaced to the caller. + Record a provider scan id, keyed to the guardrail execution that produced it, so it can be surfaced to the caller. Guardrails only return scan details to the client when they block, so allowed requests carry no - audit trail. Ids recorded here become the x-litellm-guardrail-scan-id response header. + audit trail. Ids recorded here become the x-litellm-guardrail-scan-id response header, and the + (guardrail, stage, provider, scan_id) entries become the x-litellm-guardrail-scan-metadata header. """ if not scan_id: return _, _metadata = get_or_create_metadata_bucket(request_data) existing: Final = _metadata.get(GUARDRAIL_SCAN_IDS_METADATA_KEY) - scan_ids: Final = tuple(existing) if isinstance(existing, (list, tuple)) else () + scan_ids: Final[tuple[object, ...]] = tuple(existing) if isinstance(existing, (list, tuple)) else () if scan_id not in scan_ids: _metadata[GUARDRAIL_SCAN_IDS_METADATA_KEY] = (*scan_ids, scan_id) + entry: Final[GuardrailScanMetadata] = { + "guardrail": guardrail_name, + "stage": stage.value, + "provider": provider, + "scan_id": scan_id, + } + existing_entries: Final = _metadata.get(GUARDRAIL_SCAN_METADATA_METADATA_KEY) + entries: Final[tuple[object, ...]] = tuple(existing_entries) if isinstance(existing_entries, (list, tuple)) else () + if entry not in entries: + _metadata[GUARDRAIL_SCAN_METADATA_METADATA_KEY] = (*entries, entry) + def add_policy_to_applied_policies_header(request_data: dict, policy_name: str | None): """ diff --git a/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py b/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py index 10683550f85..c22d35509c1 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py +++ b/litellm/proxy/guardrails/guardrail_hooks/openai/moderations.py @@ -17,7 +17,8 @@ from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, ) -from litellm.types.guardrails import GuardrailEventHooks +from litellm.proxy.common_utils.callback_utils import add_guardrail_scan_id +from litellm.types.guardrails import GuardrailEventHooks, SupportedGuardrailIntegrations from litellm.types.utils import ( GenericGuardrailAPIInputs, GuardrailStatus, @@ -218,6 +219,13 @@ class OpenAIModerationGuardrail(OpenAIGuardrailBase, CustomGuardrail): metadata: Final = request_data.get("metadata") or {} request_data["metadata"] = metadata metadata["_openai_moderation_response"] = moderation_response.model_dump() + add_guardrail_scan_id( + request_data=request_data, + scan_id=moderation_response.id, + guardrail_name=self.guardrail_name, + provider=SupportedGuardrailIntegrations.OPENAI_MODERATION.value, + stage=GuardrailEventHooks.post_call if input_type == "response" else GuardrailEventHooks.pre_call, + ) # Check if content is flagged and raise exception if needed self._check_moderation_result(moderation_response) diff --git a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py index b73d3adb99e..3bc0dfabefc 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py +++ b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py @@ -721,10 +721,18 @@ class PanwPrismaAirsHandler(CustomGuardrail): } } - def _record_scan_id(self, request_data: dict[str, object], scan_result: Mapping[str, object]) -> None: + def _record_scan_id( + self, request_data: dict[str, object], scan_result: Mapping[str, object], stage: GuardrailEventHooks + ) -> None: """Surface the AIRS scan id on the response, so allowed calls are auditable too.""" scan_id: Final = scan_result.get("scan_id") - add_guardrail_scan_id(request_data=request_data, scan_id=str(scan_id) if scan_id else None) + add_guardrail_scan_id( + request_data=request_data, + scan_id=str(scan_id) if scan_id else None, + guardrail_name=self.guardrail_name, + provider=self._PROVIDER_NAME, + stage=stage, + ) def _handle_api_error_with_logging( self, @@ -948,7 +956,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): event_type=GuardrailEventHooks.post_call, ) add_guardrail_to_applied_guardrails_header(request_data=request_data, guardrail_name=self.guardrail_name) - self._record_scan_id(request_data, scan_result) + self._record_scan_id(request_data, scan_result, GuardrailEventHooks.post_call) def _check_and_mark_scanned(self, data: dict, scan_type: str) -> bool: """ @@ -1078,7 +1086,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): duration=(end_time - start_time).total_seconds(), event_type=GuardrailEventHooks.pre_call, ) - self._record_scan_id(data, scan_result) + self._record_scan_id(data, scan_result, GuardrailEventHooks.pre_call) action: Final = scan_result.get("action", "block") category: Final = scan_result.get("category", "unknown") @@ -1199,7 +1207,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): duration=(end_time - start_time).total_seconds(), event_type=GuardrailEventHooks.post_call, ) - self._record_scan_id(data, scan_result) + self._record_scan_id(data, scan_result, GuardrailEventHooks.post_call) action: Final = scan_result.get("action", "block") category: Final = scan_result.get("category", "unknown") @@ -1401,7 +1409,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): duration=(end_time - start_time).total_seconds(), event_type=GuardrailEventHooks.post_call, ) - self._record_scan_id(request_data, scan_result) + self._record_scan_id(request_data, scan_result, GuardrailEventHooks.post_call) # Add guardrail to applied guardrails header for observability add_guardrail_to_applied_guardrails_header( @@ -1475,7 +1483,11 @@ class PanwPrismaAirsHandler(CustomGuardrail): ) continue - self._record_scan_id(request_data, scan_result) + self._record_scan_id( + request_data, + scan_result, + GuardrailEventHooks.post_call if is_response else GuardrailEventHooks.pre_call, + ) action = scan_result.get("action", "block") masked_args = self._masked_tool_call_arguments( @@ -1829,7 +1841,11 @@ class PanwPrismaAirsHandler(CustomGuardrail): new_texts.append(text) continue - self._record_scan_id(request_data, scan_result) + self._record_scan_id( + request_data, + scan_result, + GuardrailEventHooks.post_call if is_response else GuardrailEventHooks.pre_call, + ) action = scan_result.get("action", "block") masked_text = self._get_masked_text(scan_result, is_response=is_response) @@ -1901,7 +1917,7 @@ class PanwPrismaAirsHandler(CustomGuardrail): ) # If we reach here, fallback_on_error="allow" else: - self._record_scan_id(request_data, mcp_scan_result) + self._record_scan_id(request_data, mcp_scan_result, GuardrailEventHooks.pre_call) action = mcp_scan_result.get("action", "block") masked_text = self._get_masked_text(mcp_scan_result, is_response=False) if action == "allow": diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 924f84be5f4..3250ae5cca9 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -235,6 +235,7 @@ _UNTRUSTED_ROOT_CONTROL_FIELDS: Final = ( "applied_policies", "policy_sources", "guardrail_scan_ids", + "guardrail_scan_metadata", "routing_decision", GATEWAY_INJECTED_CACHE_METADATA_KEY, "pillar_response_headers", @@ -291,6 +292,7 @@ _UNTRUSTED_METADATA_CONTROL_FIELDS: Final = ( "applied_policies", "policy_sources", "guardrail_scan_ids", + "guardrail_scan_metadata", "routing_decision", GATEWAY_INJECTED_CACHE_METADATA_KEY, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, diff --git a/tests/test_litellm/proxy/common_utils/test_callback_utils.py b/tests/test_litellm/proxy/common_utils/test_callback_utils.py index 66f77db6da9..ecb2375d495 100644 --- a/tests/test_litellm/proxy/common_utils/test_callback_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_callback_utils.py @@ -1,30 +1,33 @@ import copy +import json import sys from types import ModuleType, SimpleNamespace +from typing import Final +from unittest.mock import patch import pytest - +import litellm +from litellm.caching.caching import DualCache +from litellm.constants import MAX_GUARDRAIL_SCAN_METADATA_HEADER_LENGTH +from litellm.integrations.custom_logger import CustomLogger +from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_utils.callback_utils import ( + _serialize_scan_metadata_header, add_guardrail_scan_id, add_policy_to_applied_policies_header, decrypt_callback_vars, encrypt_callback_vars, get_logging_caching_headers, - initialize_callbacks_on_proxy, get_remaining_tokens_and_requests_from_request_data, + initialize_callbacks_on_proxy, normalize_callback_names, + process_callback, sanitize_openai_provider_metadata, strip_callback_config, ) -import litellm -from litellm.caching.caching import DualCache -from litellm.integrations.custom_logger import CustomLogger -from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.utils import ProxyLogging - -from unittest.mock import patch -from litellm.proxy.common_utils.callback_utils import process_callback +from litellm.types.guardrails import GuardrailEventHooks def test_get_remaining_tokens_and_requests_from_request_data(): @@ -189,20 +192,109 @@ def test_get_logging_caching_headers_merges_metadata_and_litellm_metadata(): assert headers["x-litellm-policy-sources"] == "global-baseline=team_default" +def _record( + request_data: dict[str, object], + scan_id: str | None, + guardrail_name: str = "airs", + provider: str = "panw_prisma_airs", + stage: GuardrailEventHooks = GuardrailEventHooks.pre_call, +) -> None: + add_guardrail_scan_id( + request_data=request_data, scan_id=scan_id, guardrail_name=guardrail_name, provider=provider, stage=stage + ) + + def test_add_guardrail_scan_id_dedupes_and_becomes_response_header(): request_data = {"litellm_metadata": {}} - add_guardrail_scan_id(request_data=request_data, scan_id="scan-1") - add_guardrail_scan_id(request_data=request_data, scan_id="scan-1") - add_guardrail_scan_id(request_data=request_data, scan_id="scan-2") - add_guardrail_scan_id(request_data=request_data, scan_id=None) + _record(request_data, "scan-1") + _record(request_data, "scan-1") + _record(request_data, "scan-2") + _record(request_data, None) assert request_data["litellm_metadata"]["guardrail_scan_ids"] == ("scan-1", "scan-2") assert get_logging_caching_headers(request_data)["x-litellm-guardrail-scan-id"] == "scan-1,scan-2" -def test_get_logging_caching_headers_omits_scan_id_header_without_scans(): - assert "x-litellm-guardrail-scan-id" not in get_logging_caching_headers({"litellm_metadata": {}}) +def test_scan_metadata_header_maps_each_id_to_its_guardrail_stage_and_provider(): + request_data: Final[dict[str, object]] = {"litellm_metadata": {}} + + _record( + request_data, "scan-1", guardrail_name="airs", provider="panw_prisma_airs", stage=GuardrailEventHooks.pre_call + ) + _record( + request_data, "mod-1", guardrail_name="mod", provider="openai_moderation", stage=GuardrailEventHooks.pre_call + ) + _record( + request_data, "scan-2", guardrail_name="airs", provider="panw_prisma_airs", stage=GuardrailEventHooks.post_call + ) + _record( + request_data, "scan-2", guardrail_name="airs", provider="panw_prisma_airs", stage=GuardrailEventHooks.post_call + ) + _record(request_data, None, guardrail_name="mod", provider="openai_moderation", stage=GuardrailEventHooks.post_call) + + headers: Final = get_logging_caching_headers(request_data) + assert headers is not None + assert headers["x-litellm-guardrail-scan-id"] == "scan-1,mod-1,scan-2" + assert json.loads(headers["x-litellm-guardrail-scan-metadata"]) == [ + {"guardrail": "airs", "stage": "pre_call", "provider": "panw_prisma_airs", "scan_id": "scan-1"}, + {"guardrail": "mod", "stage": "pre_call", "provider": "openai_moderation", "scan_id": "mod-1"}, + {"guardrail": "airs", "stage": "post_call", "provider": "panw_prisma_airs", "scan_id": "scan-2"}, + ] + + +def test_scan_metadata_keeps_same_id_reused_across_stages(): + request_data: Final[dict[str, object]] = {"metadata": {}} + + _record(request_data, "scan-1", stage=GuardrailEventHooks.pre_call) + _record(request_data, "scan-1", stage=GuardrailEventHooks.post_call) + + headers: Final = get_logging_caching_headers(request_data) + assert headers is not None + assert headers["x-litellm-guardrail-scan-id"] == "scan-1" + assert [entry["stage"] for entry in json.loads(headers["x-litellm-guardrail-scan-metadata"])] == [ + "pre_call", + "post_call", + ] + + +def test_scan_metadata_header_drops_trailing_entries_to_stay_within_length_limit(): + request_data: Final[dict[str, object]] = {"litellm_metadata": {}} + scan_ids: Final = tuple(f"0f9c4b7e-3d2a-4c1b-9e8f-{index:012d}" for index in range(40)) + for scan_id in scan_ids: + _record(request_data, scan_id, stage=GuardrailEventHooks.post_call) + + headers: Final = get_logging_caching_headers(request_data) + assert headers is not None + assert headers["x-litellm-guardrail-scan-id"] == ",".join(scan_ids) + header: Final = headers["x-litellm-guardrail-scan-metadata"] + assert len(header) <= MAX_GUARDRAIL_SCAN_METADATA_HEADER_LENGTH + kept: Final = json.loads(header) + assert 1 < len(kept) < len(scan_ids) + assert [entry["scan_id"] for entry in kept] == list(scan_ids[: len(kept)]) + + +def test_serialize_scan_metadata_header_keeps_exactly_the_entries_that_fit(): + entries: Final = ({"scan_id": "a"}, {"scan_id": "b"}, {"scan_id": "c"}) + two_entries: Final = '[{"scan_id":"a"},{"scan_id":"b"}]' + + assert _serialize_scan_metadata_header(entries, max_length=len(two_entries)) == two_entries + assert _serialize_scan_metadata_header(entries, max_length=len(two_entries) - 1) == '[{"scan_id":"a"}]' + assert _serialize_scan_metadata_header(entries, max_length=len(two_entries) + 1) == two_entries + assert _serialize_scan_metadata_header(entries, max_length=1000) == json.dumps(entries, separators=(",", ":")) + assert _serialize_scan_metadata_header(entries, max_length=5) is None + assert _serialize_scan_metadata_header((), max_length=1000) is None + + +def test_scan_metadata_is_an_internal_metadata_key(): + assert sanitize_openai_provider_metadata({"guardrail_scan_metadata": "x", "keep": "y"}) == {"keep": "y"} + + +def test_get_logging_caching_headers_omits_scan_headers_without_scans(): + headers: Final = get_logging_caching_headers({"litellm_metadata": {}}) + assert headers is not None + assert "x-litellm-guardrail-scan-id" not in headers + assert "x-litellm-guardrail-scan-metadata" not in headers def test_initialize_callbacks_on_proxy_instantiates_compression_interception( diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_moderations.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_moderations.py index 2b43720a126..615d06b0f42 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_moderations.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/openai/test_moderations.py @@ -3,14 +3,19 @@ Test OpenAI Moderation Guardrail """ +import json import os +from typing import Final from unittest.mock import MagicMock, patch +import httpx import pytest +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.common_utils.callback_utils import get_logging_caching_headers from litellm.proxy.guardrails.guardrail_hooks.openai.moderations import ( OpenAIModerationGuardrail, ) @@ -989,3 +994,30 @@ async def test_openai_moderation_initialize_guardrail_forwards_streaming_flags() assert guardrail.streaming_sampling_rate == 2 finally: litellm.logging_callback_manager._reset_all_callbacks() + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("input_type", "stage"), [("request", "pre_call"), ("response", "post_call")]) +async def test_openai_moderation_records_moderation_id_as_scan_metadata(input_type: str, stage: str): + """Each moderation call's id is exposed with the guardrail name, stage and provider that produced it.""" + payload: Final = { + "id": f"modr-{stage}", + "model": "omni-moderation-latest", + "results": [{"flagged": False, "categories": {}, "category_scores": {}, "category_applied_input_types": {}}], + } + http_client: Final = AsyncHTTPHandler() + http_client.client = httpx.AsyncClient(transport=httpx.MockTransport(lambda _: httpx.Response(200, json=payload))) + + with patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"}): + guardrail: Final = OpenAIModerationGuardrail(guardrail_name="openai-mod") + guardrail.async_handler = http_client + request_data: Final[dict[str, object]] = {"metadata": {}} + + await guardrail.apply_guardrail(inputs={"texts": ["hello"]}, request_data=request_data, input_type=input_type) + + headers: Final = get_logging_caching_headers(request_data) + assert headers is not None + assert headers["x-litellm-guardrail-scan-id"] == f"modr-{stage}" + assert json.loads(headers["x-litellm-guardrail-scan-metadata"]) == [ + {"guardrail": "openai-mod", "stage": stage, "provider": "openai_moderation", "scan_id": f"modr-{stage}"} + ] diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py index 8f29ba66814..3d7c6e06d94 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_panw_prisma_airs.py @@ -12,6 +12,7 @@ This test file follows LiteLLM's testing patterns and covers: import copy import json from datetime import datetime +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -5785,7 +5786,14 @@ class TestPanwAirsScanIdExposure: headers = get_logging_caching_headers(data) assert headers["x-litellm-guardrail-scan-id"] == "scan-abc-123" - assert "x-litellm-guardrail-scan-metadata" not in headers + assert json.loads(headers["x-litellm-guardrail-scan-metadata"]) == [ + { + "guardrail": handler.guardrail_name, + "stage": "pre_call", + "provider": "panw_prisma_airs", + "scan_id": "scan-abc-123", + } + ] @pytest.mark.asyncio async def test_request_and_response_scan_ids_are_both_exposed(self, user_api_key_dict): @@ -5809,6 +5817,26 @@ class TestPanwAirsScanIdExposure: headers = get_logging_caching_headers(data) assert headers["x-litellm-guardrail-scan-id"] == "scan-abc-123,scan-response-456" + assert [(e["stage"], e["scan_id"]) for e in json.loads(headers["x-litellm-guardrail-scan-metadata"])] == [ + ("pre_call", "scan-abc-123"), + ("post_call", "scan-response-456"), + ] + + @pytest.mark.asyncio + async def test_apply_guardrail_response_scan_is_tagged_post_call(self): + from litellm.proxy.common_utils.callback_utils import get_logging_caching_headers + + handler: Final = self._handler(self.ALLOW_SCAN_RESULT) + request_data: Final[dict[str, object]] = {"litellm_call_id": "test-call-id", "model": "gpt-4", "metadata": {}} + + await handler.apply_guardrail( + inputs={"texts": ["Hello world"]}, request_data=request_data, input_type="response" + ) + + headers: Final = get_logging_caching_headers(request_data) + assert headers is not None + entries: Final = json.loads(headers["x-litellm-guardrail-scan-metadata"]) + assert [(e["stage"], e["provider"]) for e in entries] == [("post_call", "panw_prisma_airs")] @pytest.mark.asyncio async def test_repeated_scan_id_is_not_duplicated(self, user_api_key_dict): @@ -5850,6 +5878,8 @@ class TestPanwAirsScanIdExposure: assert "guardrail_scan_ids" in _UNTRUSTED_METADATA_CONTROL_FIELDS assert "guardrail_scan_ids" in _UNTRUSTED_ROOT_CONTROL_FIELDS + assert "guardrail_scan_metadata" in _UNTRUSTED_METADATA_CONTROL_FIELDS + assert "guardrail_scan_metadata" in _UNTRUSTED_ROOT_CONTROL_FIELDS class TestPanwAirsBlockedErrorDetailPassthrough: """Regression tests for the full AIRS scan response on blocks. From 9d90a544916c6f38397862dd18b6a4c12de98827 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 9 Sep 2026 00:06:26 -0700 Subject: [PATCH 10/11] revert(model-management): roll back #40047 This reverts commit e8e3172d7d70558929f32f057ebe4c7471c8c352 Restore the previous model update and router cost registration behavior while pricing compatibility is investigated --- .../model_management_endpoints.py | 70 +--- litellm/router.py | 26 +- tests/e2e/coverage_registry/mgmt.yaml | 2 - .../management/test_model_lifecycle_e2e.py | 365 ------------------ tests/e2e/models.py | 74 +--- tests/e2e/proxy_client.py | 199 +--------- tests/e2e/test_proxy_client.py | 57 +-- tests/e2e/transport.py | 1 - .../test_model_management_endpoints.py | 178 +-------- .../test_router_model_cost_isolation.py | 192 --------- ui/litellm-dashboard/src/lib/http/schema.d.ts | 5 +- 11 files changed, 39 insertions(+), 1130 deletions(-) delete mode 100644 tests/e2e/management/test_model_lifecycle_e2e.py diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 742b9d9817f..0f19e9ce149 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -119,7 +119,6 @@ from litellm.types.router import ( SPECIAL_MODEL_INFO_PARAMS, Deployment, GenericLiteLLMParams, - LiteLLM_Params, ModelInfo, updateDeployment, ) @@ -729,44 +728,6 @@ def _ptu_priced_deployment(model_params: Deployment) -> Deployment: ) -_OWNERSHIP_FIELDS: Final = frozenset( - { - "db_model", - "team_id", - "team_public_model_name", - "access_groups", - "created_at", - "created_by", - "updated_at", - "updated_by", - "blocked", - } -) - -_STORED_REQUIRED_FIELDS: Final = frozenset( - name for model in (LiteLLM_Params, ModelInfo) for name, field in model.model_fields.items() if field.is_required() -) - -_NULL_CLEAR_IGNORED_FIELDS: Final = _OWNERSHIP_FIELDS | _STORED_REQUIRED_FIELDS | frozenset(PTU_MODEL_INFO_FIELDS) - - -def _explicitly_cleared_fields(patch: BaseModel | None) -> frozenset[str]: - """The keys a patch sends as an explicit null, which update_db_model removes from the - stored blob (JSON Merge Patch). Ownership keys are left alone, as are the keys the stored - models require, since clearing one writes a row no reload can rebuild through - LiteLLM_Params / ModelInfo. The PTU keys are handled by _explicitly_cleared_ptu_fields, - whose clear is gated on the feature flag. Applied after both blobs merge, so a model_info blob - the UI echoes back cannot resurrect a pricing key the litellm_params patch clears. - """ - if patch is None: - return frozenset() - return frozenset( - field - for field in patch.model_fields_set - if field not in _NULL_CLEAR_IGNORED_FIELDS and getattr(patch, field) is None - ) - - def update_db_model(db_model: Deployment, updated_patch: updateDeployment) -> PrismaCompatibleUpdateDBModel: if updated_patch.model_info is not None: _raise_if_ptu_cost_attribution_disabled(updated_patch.model_info.model_dump(exclude_none=True)) @@ -787,15 +748,25 @@ def update_db_model(db_model: Deployment, updated_patch: updateDeployment) -> Pr if updated_patch.model_info: merged_model_info.update(updated_patch.model_info.model_dump(exclude_none=True)) - for field in _explicitly_cleared_fields(updated_patch.litellm_params): - merged_litellm_params.pop(field, None) - if field in SPECIAL_MODEL_INFO_PARAMS: - merged_model_info.pop(field, None) - for field in _explicitly_cleared_fields(updated_patch.model_info): - merged_model_info.pop(field, None) - if field in SPECIAL_MODEL_INFO_PARAMS: - merged_litellm_params.pop(field, None) + # Honor explicit-null clears LAST, after both merges, so a model_info blob the UI + # passes through (which today re-sends the OLD pricing on every save) cannot + # silently undo a litellm_params clear via .update(). + # + # Restricted to SPECIAL_MODEL_INFO_PARAMS (input/output cost per token/character + # and cache read/write costs) so this path cannot be used to null out privileged + # model_info fields like team_id or access groups. SPECIAL_MODEL_INFO_PARAMS are + # mirrored between litellm_params and model_info by Deployment.__init__, so the + # clear propagates to both blobs. + if updated_patch.litellm_params: + for field in updated_patch.litellm_params.model_fields_set: + if field in SPECIAL_MODEL_INFO_PARAMS and getattr(updated_patch.litellm_params, field) is None: + merged_litellm_params.pop(field, None) + merged_model_info.pop(field, None) if updated_patch.model_info: + for field in updated_patch.model_info.model_fields_set: + if field in SPECIAL_MODEL_INFO_PARAMS and getattr(updated_patch.model_info, field) is None: + merged_model_info.pop(field, None) + merged_litellm_params.pop(field, None) for field in _explicitly_cleared_ptu_fields(updated_patch.model_info): merged_model_info.pop(field, None) @@ -845,9 +816,8 @@ async def patch_model( """ PATCH Endpoint for partial model updates. - JSON Merge Patch semantics over `litellm_params` and `model_info`: a key absent from the - body is unchanged, a key sent as null is removed from the stored row, and a key sent with a - value is set (identity and ownership keys such as `id` and `team_id` ignore a null). + Only updates the fields specified in the request while preserving other existing values. + Follows proper PATCH semantics by only modifying provided fields. Args: model_id: The ID of the model to update diff --git a/litellm/router.py b/litellm/router.py index 1252d7e7487..934a4ac86a9 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -628,15 +628,6 @@ RETRY_BREADCRUMB_EXCLUDED_KWARGS: Final = frozenset( RETRY_BREADCRUMB_LIMIT: Final = 4 -# Cost-map keys created by _register_deployment_in_model_cost, which shares one flat -# namespace with the built-in model catalog. Only a key it created may be evicted, or a -# deployment whose id names a real model would strip that model's pricing and -# capabilities for every other deployment of it. delete_deployment gives a key back once no -# live router still serves that id, so a later catalog refresh that starts serving the name -# is not treated as a deployment's own. -_DEPLOYMENT_COST_MAP_KEYS: Final[set[str]] = set() # mutable-ok: ownership of shared cost-map keys - - class FallbackAwareStreamWrapper(CustomStreamWrapper): """Base for the Router's chat-completion stream wrappers, which are built around the attempt the Router picked first and have to repoint themselves when a fallback takes over.""" @@ -9770,7 +9761,6 @@ class Router: """ idx: Final = len(self.model_list) self.model_list.append(model) - _live_routers.add(self) # mutable-ok: track dynamic routers without extending their lifetimes self._invalidate_model_group_info_cache() self._invalidate_access_groups_cache() @@ -9939,12 +9929,7 @@ class Router: """Write a deployment's metadata into ``litellm.model_cost``. Runs when a deployment is added and again after a price data reload, so - the entries a refresh rebuilds are the ones a fresh boot would produce. An - entry this function created is replaced rather than merged, so a price cleared - from the deployment does not linger from an earlier registration and keep - billing at the old rate. An entry it did not create is left to merge, because - a deployment id that collides with a catalog model name shares that model's - entry with every other deployment of it. + the entries a refresh rebuilds are the ones a fresh boot would produce. Nothing is recorded for replay: a refresh walks the live routers instead, so a deleted, repointed or never-added deployment, and a discarded router, drop out of the rebuild on their own. @@ -9961,10 +9946,6 @@ class Router: } if model_id is not None: - if model_id in _DEPLOYMENT_COST_MAP_KEYS: - litellm.model_cost.pop(model_id, None) # mutable-ok: remove cleared prices from the shared entry - elif model_id not in litellm.model_cost: - _DEPLOYMENT_COST_MAP_KEYS.add(model_id) # mutable-ok: retain shared ownership across reloads litellm.register_model( model_cost={model_id: model_info}, persist_across_reloads=False, @@ -10061,11 +10042,6 @@ class Router: _budget_limiter: Final = self._get_router_deployment_budget_limiter() if _budget_limiter is not None: _budget_limiter.unregister_deployment_budget(model_id=id) - if not any( - router is not self and id in router.model_id_to_deployment_index_map - for router in tuple(_live_routers) - ): - _DEPLOYMENT_COST_MAP_KEYS.discard(id) # mutable-ok: the last owning router released this key try: self._unregister_pre_routing_strategy_for_deployment( deployment=item if isinstance(item, Deployment) else Deployment(**item) diff --git a/tests/e2e/coverage_registry/mgmt.yaml b/tests/e2e/coverage_registry/mgmt.yaml index 83f1711a245..c8d7037d2fd 100644 --- a/tests/e2e/coverage_registry/mgmt.yaml +++ b/tests/e2e/coverage_registry/mgmt.yaml @@ -76,8 +76,6 @@ - {id: mgmt.credential_migration.check.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "key_management_endpoints.py:4252", rationale: "Encryption migration (smoke)"} - {id: mgmt.credential.new.serves_request, module: mgmt, tier: P1, surface: api, assertions: [serves_request], source: "credential_endpoints/endpoints.py:42", rationale: "Stored credential resolves into a deployment and serves a live /messages request"} - {id: mgmt.model.test_connection.happy_path, module: mgmt, tier: P0, surface: api, assertions: [happy_path], source: "_health_endpoints.py:1785", rationale: "Test Connection for a responses-mode Bedrock Mantle deployment reaches the live provider and reports success; this exact shape 500ed on an acompletion partial before v1.91.0", fail_before_fix: proven} -- {id: mgmt.model.update.preserves_unrelated_fields, module: mgmt, tier: P0, surface: api, assertions: [preserves_unrelated_fields], source: "model_management_endpoints.py:731", rationale: "A partial PATCH changes only the keys it names; every other stored key reads back byte-for-byte, and the new rate reaches billing"} -- {id: mgmt.model.update.clear_persists, module: mgmt, tier: P0, surface: api, assertions: [clear_persists], source: "model_management_endpoints.py:731", fail_before_fix: proven, rationale: "An explicit null on PATCH removes the key from the stored row and billing falls back to the cost map; before the fix only mirrored pricing keys could be cleared"} - {id: mgmt.mcp_server.new.persists, module: mgmt, tier: P0, surface: api, assertions: [persists], source: "mcp_management_endpoints.py:1577", rationale: "Every field of an admin-created MCP server reads back verbatim, by id and in the list, on every replica"} - {id: mgmt.mcp_server.list.persists, module: mgmt, tier: P1, surface: api, assertions: [persists], source: "mcp_management_endpoints.py:1112", rationale: "The MCP page grid lists a created server with the same field values its detail view reports"} - {id: mgmt.mcp_server.update.preserves_unrelated_fields, module: mgmt, tier: P0, surface: api, assertions: [preserves_unrelated_fields], source: "mcp_management_endpoints.py:2665", rationale: "A dashboard edit of one field leaves the others intact and is visible on every replica after one save; edits that took several saves to stick were a customer defect"} diff --git a/tests/e2e/management/test_model_lifecycle_e2e.py b/tests/e2e/management/test_model_lifecycle_e2e.py deleted file mode 100644 index 99044efeb46..00000000000 --- a/tests/e2e/management/test_model_lifecycle_e2e.py +++ /dev/null @@ -1,365 +0,0 @@ -"""Live e2e: the lifecycle of a DB-stored deployment through the model management -routes, read back on every gateway replica. - -Each test registers its own gpt-4o-mini mock deployment through /model/new (deleted -on teardown) with non-default pricing, context window, mode, and api_base pinned, then -walks the lifecycle up to the step it proves: the create reads back field for field, -a partial PATCH changes only the key it names, an explicit null on PATCH removes the -key from the stored row (JSON Merge Patch), a call after the price clear is billed at -the cost map's rate rather than the cleared override, and a delete removes the -deployment from /model/info and makes the model name unknown to /chat/completions. - -The stored row is read back from /model/info, a control-plane route with one answer -behind it. What every gateway must agree on is which models it serves, so the create -and delete steps poll /v1/models on every URL in PROXY_REPLICA_URLS through -ProxyClient.read_model_back_everywhere, failing by name on the gateway that never converged. -""" - -from __future__ import annotations - -import math -import time -from collections.abc import Callable -from dataclasses import dataclass -from typing import Final - -import pytest -from pydantic import BaseModel - -from e2e_config import unique_marker -from e2e_http import unwrap -from lifecycle import ResourceManager -from management_client import ManagementClient -from models import ( - ChatBody, - ChatMessage, - Clear, - LiteLLMParamsBody, - LiteLLMParamsPatch, - ModelInfoBody, - ModelInfoEntry, - ModelInfoResponse, - ModelNewBody, - ModelPatchBody, - ModelsListResponse, - SpendLogRow, -) - -pytestmark = pytest.mark.e2e - -BACKEND_MODEL: Final = "gpt-4o-mini" -PINNED_API_BASE: Final = "https://pinned.example.invalid/v1" -PINNED_MAX_INPUT_TOKENS: Final = 4096 -PINNED_INPUT_RATE: Final = 1e-05 -UPDATED_INPUT_RATE: Final = 2e-05 -PINNED_OUTPUT_RATE: Final = 3e-05 - -# A PATCH lands on the control plane, and each gateway picks it up on its own config -# reload, so the first call after the write can still be billed at the old rate. There -# is no price on the gateway's data-plane surface to poll, so the billing steps drive -# calls until the new rate shows up in the spend row and let the deadline be what fails. -BILLING_CONVERGENCE_TIMEOUT: Final = 90.0 -BILLING_CONVERGENCE_INTERVAL: Final = 5.0 - - -@dataclass(frozen=True, slots=True) -class Registered: - model_name: str - model_id: str - - -class _ErrorDetail(BaseModel): - message: str - - -class _ErrorEnvelope(BaseModel): - error: _ErrorDetail - - -def _register(client: ManagementClient, resources: ResourceManager) -> Registered: - """Register a mock gpt-4o-mini deployment with every field under test pinned to a - non-default value, deleted on teardown. max_input_tokens is pinned in - litellm_params only: a value in model_info is copied into the shared cost-map - entry for the backend model, which would leak into every other gpt-4o-mini - deployment on the proxy.""" - model_name: Final = f"e2e-lifecycle-{unique_marker()}" - model_id: Final = client.proxy.register_model( - ModelNewBody( - model_name=model_name, - litellm_params=LiteLLMParamsBody( - model=BACKEND_MODEL, - mock_response="ok", - api_base=PINNED_API_BASE, - input_cost_per_token=PINNED_INPUT_RATE, - output_cost_per_token=PINNED_OUTPUT_RATE, - max_input_tokens=PINNED_MAX_INPUT_TOKENS, - ), - model_info=ModelInfoBody(mode="chat"), - ) - ) - resources.defer(lambda: client.proxy.delete_model(model_id)) - return Registered(model_name=model_name, model_id=model_id) - - -def _entry(body: ModelInfoResponse, model_name: str) -> ModelInfoEntry | None: - return next((entry for entry in body.data if entry.model_name == model_name), None) - - -def _stored_entry( - client: ManagementClient, - model_name: str, - *, - converged: Callable[[ModelInfoEntry], bool], -) -> ModelInfoEntry: - """The stored /model/info row for `model_name`, once it satisfies `converged`. - - /model/info is a control-plane route: the gateways named in PROXY_REPLICA_URLS - serve the LLM surface only, so the stored row has one answer, not one per - gateway. What every gateway must agree on is which models it serves, and - `_assert_served_everywhere` / `_assert_absent_everywhere` poll /v1/models for - that.""" - - def has_converged(body: ModelInfoResponse) -> bool: - entry: Final = _entry(body, model_name) - return entry is not None and converged(entry) - - body: Final = client.proxy.read_model_back("/model/info", ModelInfoResponse, predicate=has_converged) - entry: Final = _entry(body, model_name) - assert entry is not None, f"/model/info stopped listing {model_name!r} between the poll and the read" - return entry - - -def _serves(body: ModelsListResponse, model_name: str) -> bool: - return any(entry.id == model_name for entry in body.data) - - -def _assert_served_everywhere(client: ManagementClient, model_name: str) -> None: - _ = client.proxy.read_model_back_everywhere( - "/v1/models", ModelsListResponse, predicate=lambda body: _serves(body, model_name) - ) - - -def _assert_absent_everywhere(client: ManagementClient, model_name: str) -> None: - _ = client.proxy.read_model_back_everywhere( - "/v1/models", ModelsListResponse, predicate=lambda body: not _serves(body, model_name) - ) - - -def _assert_untouched_keys_as_created(entry: ModelInfoEntry, replica: str) -> None: - """The keys no later step names read back byte-for-byte as /model/new wrote them.""" - params: Final = entry.litellm_params - assert params.model == BACKEND_MODEL, f"{replica}: litellm_params.model {params.model!r} != {BACKEND_MODEL!r}" - assert params.api_base == PINNED_API_BASE, f"{replica}: api_base {params.api_base!r} != {PINNED_API_BASE!r}" - assert params.output_cost_per_token == PINNED_OUTPUT_RATE, ( - f"{replica}: output_cost_per_token {params.output_cost_per_token} != {PINNED_OUTPUT_RATE}" - ) - assert entry.model_info.mode == "chat", f"{replica}: model_info.mode {entry.model_info.mode!r} != 'chat'" - - -def _approx_equal(actual: float, expected: float) -> bool: - return math.isclose(actual, expected, rel_tol=1e-2, abs_tol=1e-9) - - -def _priced(rows: list[SpendLogRow]) -> bool: - return any(row.metadata and row.metadata.cost_breakdown and row.metadata.cost_breakdown.input_cost for row in rows) - - -def _billed_input_cost(client: ManagementClient, model_name: str, key: str) -> tuple[int, float]: - """Drive one chat completion through `model_name` and return the prompt tokens and - input cost its spend row recorded, so a test can assert the rate the gateway actually - billed rather than only the rate it stored.""" - chat: Final = unwrap( - client.proxy.chat( - key, - ChatBody( - model=model_name, - messages=[ChatMessage(role="user", content=f"reply with one word {unique_marker()}")], - max_tokens=16, - ), - ) - ) - assert chat.id is not None, f"chat completion carried no id to find its spend row by: {chat}" - - rows: Final = client.proxy.poll_logs_for_request_id(chat.id, predicate=_priced) - row: Final = next((row for row in rows if row.request_id == chat.id), None) - assert row is not None and row.metadata and row.metadata.cost_breakdown, ( - f"no priced spend row for request {chat.id} before the deadline: {rows}" - ) - prompt_tokens: Final = row.prompt_tokens or 0 - input_cost: Final = row.metadata.cost_breakdown.input_cost - assert prompt_tokens > 0 and input_cost is not None, f"spend row logged no prompt tokens or input cost: {row}" - return prompt_tokens, input_cost - - -def _await_billed_input_cost( - client: ManagementClient, model_name: str, key: str, *, expected_rate: float -) -> tuple[int, float]: - """Drive calls through `model_name` until one is billed at `expected_rate`, and - return the prompt tokens and input cost of the last spend row either way. - - Only the deadline ends the wait unsatisfied: a rate that never reaches the gateway - comes back as the stale cost for the caller to assert on, so the rate the caller - expects is still what decides the test.""" - deadline: Final = time.monotonic() + BILLING_CONVERGENCE_TIMEOUT - while True: - prompt_tokens, input_cost = _billed_input_cost(client, model_name, key) - if _approx_equal(input_cost, prompt_tokens * expected_rate) or time.monotonic() >= deadline: - return prompt_tokens, input_cost - time.sleep(BILLING_CONVERGENCE_INTERVAL) - - -class TestModelLifecycle: - @pytest.mark.covers("mgmt.model.add.persists") - def test_create_reads_back_every_field_and_serves_on_every_replica( - self, client: ManagementClient, resources: ResourceManager - ) -> None: - registered = _register(client, resources) - - entry = _stored_entry(client, registered.model_name, converged=lambda _entry: True) - stored = "/model/info" - - _assert_untouched_keys_as_created(entry, stored) - assert entry.litellm_params.input_cost_per_token == PINNED_INPUT_RATE, ( - f"{stored}: input_cost_per_token {entry.litellm_params.input_cost_per_token} != {PINNED_INPUT_RATE}" - ) - assert entry.litellm_params.max_input_tokens == PINNED_MAX_INPUT_TOKENS, ( - f"{stored}: max_input_tokens {entry.litellm_params.max_input_tokens} != {PINNED_MAX_INPUT_TOKENS}" - ) - assert entry.model_info.id == registered.model_id, ( - f"{stored}: model_info.id {entry.model_info.id!r} != {registered.model_id!r}" - ) - - _assert_served_everywhere(client, registered.model_name) - - @pytest.mark.covers("mgmt.model.update.preserves_unrelated_fields") - def test_partial_update_changes_only_the_named_key( - self, client: ManagementClient, resources: ResourceManager, scoped_key: str - ) -> None: - registered = _register(client, resources) - - stored = client.proxy.patch_model( - registered.model_id, - ModelPatchBody(litellm_params=LiteLLMParamsPatch(input_cost_per_token=UPDATED_INPUT_RATE)), - ) - assert stored.litellm_params.input_cost_per_token == UPDATED_INPUT_RATE, ( - f"PATCH response stores input_cost_per_token {stored.litellm_params.input_cost_per_token}, " - f"sent {UPDATED_INPUT_RATE}" - ) - - entry = _stored_entry( - client, - registered.model_name, - converged=lambda entry: entry.litellm_params.input_cost_per_token == UPDATED_INPUT_RATE, - ) - stored = "/model/info" - - _assert_untouched_keys_as_created(entry, stored) - assert entry.litellm_params.max_input_tokens == PINNED_MAX_INPUT_TOKENS, ( - f"{stored}: max_input_tokens {entry.litellm_params.max_input_tokens} != {PINNED_MAX_INPUT_TOKENS}" - ) - assert entry.model_info.input_cost_per_token == UPDATED_INPUT_RATE, ( - f"{stored}: model_info.input_cost_per_token {entry.model_info.input_cost_per_token} " - f"did not mirror the updated {UPDATED_INPUT_RATE}" - ) - - prompt_tokens, input_cost = _await_billed_input_cost( - client, registered.model_name, scoped_key, expected_rate=UPDATED_INPUT_RATE - ) - assert _approx_equal(input_cost, prompt_tokens * UPDATED_INPUT_RATE), ( - f"input_cost {input_cost} != {prompt_tokens} tokens * updated rate {UPDATED_INPUT_RATE} " - f"= {prompt_tokens * UPDATED_INPUT_RATE}; the partial update did not reach billing" - ) - - @pytest.mark.covers("mgmt.model.update.clear_persists") - def test_explicit_null_removes_the_key_from_the_stored_row( - self, client: ManagementClient, resources: ResourceManager - ) -> None: - registered = _register(client, resources) - - stored = client.proxy.patch_model( - registered.model_id, - ModelPatchBody(litellm_params=LiteLLMParamsPatch(max_input_tokens=Clear(), input_cost_per_token=Clear())), - ) - stored_params = stored.litellm_params.model_fields_set - assert "max_input_tokens" not in stored_params, ( - f"stored litellm_params still carries max_input_tokens " - f"{stored.litellm_params.max_input_tokens} after an explicit null" - ) - assert "input_cost_per_token" not in stored_params, ( - f"stored litellm_params still carries input_cost_per_token " - f"{stored.litellm_params.input_cost_per_token} after an explicit null" - ) - assert "max_input_tokens" not in stored.model_info.model_fields_set, ( - f"stored model_info carries max_input_tokens {stored.model_info.max_input_tokens} after the clear" - ) - assert "input_cost_per_token" not in stored.model_info.model_fields_set, ( - f"stored model_info still mirrors input_cost_per_token {stored.model_info.input_cost_per_token}" - ) - - cost_map_input_rate = client.proxy.model_cost_map()[BACKEND_MODEL].input_cost_per_token - assert cost_map_input_rate is not None, f"cost map has no input rate for {BACKEND_MODEL}" - entry = _stored_entry( - client, - registered.model_name, - converged=lambda entry: "max_input_tokens" not in entry.litellm_params.model_fields_set, - ) - stored = "/model/info" - - _assert_untouched_keys_as_created(entry, stored) - served = entry.litellm_params.model_fields_set - assert "max_input_tokens" not in served, ( - f"{stored}: litellm_params still serves max_input_tokens {entry.litellm_params.max_input_tokens}" - ) - assert "input_cost_per_token" not in served, ( - f"{stored}: litellm_params still serves input_cost_per_token {entry.litellm_params.input_cost_per_token}" - ) - assert entry.model_info.input_cost_per_token == cost_map_input_rate, ( - f"{stored}: model_info.input_cost_per_token {entry.model_info.input_cost_per_token} is not the " - f"cost map's {cost_map_input_rate}; the cleared override {PINNED_INPUT_RATE} still resolves" - ) - - @pytest.mark.covers("mgmt.model.update.clear_persists") - def test_cleared_price_is_billed_at_the_cost_map_rate( - self, client: ManagementClient, resources: ResourceManager, scoped_key: str - ) -> None: - registered = _register(client, resources) - _ = client.proxy.patch_model( - registered.model_id, - ModelPatchBody(litellm_params=LiteLLMParamsPatch(max_input_tokens=Clear(), input_cost_per_token=Clear())), - ) - _ = _stored_entry( - client, - registered.model_name, - converged=lambda entry: "input_cost_per_token" not in entry.litellm_params.model_fields_set, - ) - cost_map_input_rate = client.proxy.model_cost_map()[BACKEND_MODEL].input_cost_per_token - assert cost_map_input_rate is not None, f"cost map has no input rate for {BACKEND_MODEL}" - - prompt_tokens, input_cost = _await_billed_input_cost( - client, registered.model_name, scoped_key, expected_rate=cost_map_input_rate - ) - - assert _approx_equal(input_cost, prompt_tokens * cost_map_input_rate), ( - f"input_cost {input_cost} != {prompt_tokens} tokens * cost map rate {cost_map_input_rate} " - f"= {prompt_tokens * cost_map_input_rate}" - ) - assert not _approx_equal(input_cost, prompt_tokens * PINNED_INPUT_RATE), ( - f"input_cost {input_cost} is still billed at the cleared override {PINNED_INPUT_RATE}" - ) - - @pytest.mark.covers("mgmt.model.delete.persists") - def test_delete_removes_the_deployment_everywhere( - self, client: ManagementClient, resources: ResourceManager, scoped_key: str - ) -> None: - registered = _register(client, resources) - _ = _stored_entry(client, registered.model_name, converged=lambda _entry: True) - - client.delete_model_strict(registered.model_id) - - _assert_absent_everywhere(client, registered.model_name) - refused = client.chat_status(scoped_key, registered.model_name, "hi this is a test") - assert refused.status_code == 400, ( - f"chat against the deleted model must be rejected 400, got {refused.status_code}: {refused.body[:300]}" - ) - envelope = _ErrorEnvelope.model_validate_json(refused.body) - assert envelope.error.message, f"400 body must carry an error message: {refused.body[:300]}" diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 8db37bd25a5..62810e6cfd9 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -655,11 +655,6 @@ class OcrResponse(BaseModel): # ---------- spend logs ---------- -class CostBreakdown(BaseModel): - input_cost: float | None = None - output_cost: float | None = None - - class GuardrailEntityMatch(BaseModel): entity_type: str score: float @@ -677,7 +672,6 @@ class GuardrailRunRecord(BaseModel): class SpendLogMetadata(BaseModel): - cost_breakdown: CostBreakdown | None = None applied_guardrails: list[str] | None = None guardrail_information: list[GuardrailRunRecord] | None = None @@ -821,42 +815,15 @@ class CustomPricing(BaseModel): return prompt_tokens * self.input_cost_per_token + completion_tokens * self.output_cost_per_token -class DeploymentParams(CustomPricing): - """The litellm_params half of a /model/info row: the stored deployment as written, - credentials scrubbed. Unlike model_info it is never back-filled from the cost map, - so a key the store dropped is absent here (check `model_fields_set`).""" - - model: str | None = None - api_base: str | None = None - max_input_tokens: int | None = None - - -class DeploymentModelInfo(CustomPricing): - id: str | None = None - max_input_tokens: int | None = None - - class ModelInfoEntry(BaseModel): """One /model/info row. `litellm_params` is the configured deployment (carries any custom-pricing override); `model_info` is the price the proxy resolved for - it - the override merged over the cost-map defaults, so a key cleared from the - stored blob reads as the cost-map default here.""" + it - the override merged over the cost-map defaults.""" model_config = ConfigDict(protected_namespaces=()) model_name: str - litellm_params: DeploymentParams = DeploymentParams() - model_info: DeploymentModelInfo = DeploymentModelInfo() - - -class StoredDeployment(BaseModel): - """PATCH /model/{model_id}/update answers with the row as stored: both blobs raw, - nothing back-filled, so a cleared key is absent from `model_fields_set` of the - blob it was cleared from.""" - - model_config = ConfigDict(protected_namespaces=()) - model_name: str - litellm_params: DeploymentParams - model_info: DeploymentModelInfo + litellm_params: CustomPricing = CustomPricing() + model_info: CustomPricing = CustomPricing() class ModelInfoResponse(BaseModel): @@ -957,10 +924,9 @@ class LiteLLMParamsBody(BaseModel): timeout: float | None = None tpm: int | None = None weight: int | None = None - max_input_tokens: int | None = None -ModelMode = Literal["chat", "batch", "realtime", "image_generation"] +ModelMode = Literal["batch", "realtime", "image_generation"] class ModelInfoBody(BaseModel): @@ -970,7 +936,6 @@ class ModelInfoBody(BaseModel): # constraint when a prior run's teardown had not removed the row. id: str | None = None mode: ModelMode | None = None - max_input_tokens: int | None = None access_groups: list[str] | None = None team_id: str | None = None allowed_fails_policy: dict[str, int] | None = None @@ -999,37 +964,6 @@ class ModelUpdateBody(BaseModel): model_info: ModelInfoBody -class Clear(BaseModel): - """Serializes to JSON null. The transport dumps every body with exclude_none, so a - field set to this is how a patch carries the explicit null that removes a stored key.""" - - @model_serializer - def _as_null(self) -> None: - return None - - -class LiteLLMParamsPatch(BaseModel): - api_base: str | Clear | None = None - max_input_tokens: int | Clear | None = None - input_cost_per_token: float | Clear | None = None - output_cost_per_token: float | Clear | None = None - - -class ModelInfoPatch(BaseModel): - mode: ModelMode | Clear | None = None - max_input_tokens: int | Clear | None = None - - -class ModelPatchBody(BaseModel): - """PATCH /model/{model_id}/update body, JSON Merge Patch over the stored deployment: - a field left None is dropped from the body and unchanged, a field set to `Clear()` - is sent as null and removed, a field with a value is set.""" - - model_config = ConfigDict(protected_namespaces=()) - litellm_params: LiteLLMParamsPatch | None = None - model_info: ModelInfoPatch | None = None - - class ModelListEntry(BaseModel): id: str diff --git a/tests/e2e/proxy_client.py b/tests/e2e/proxy_client.py index fa1b06fe7ed..1bac5116a9d 100644 --- a/tests/e2e/proxy_client.py +++ b/tests/e2e/proxy_client.py @@ -10,10 +10,10 @@ from __future__ import annotations import time import warnings -from collections.abc import Callable, Iterator, Mapping +from collections.abc import Callable, Mapping from dataclasses import dataclass -from datetime import datetime from functools import reduce +from datetime import datetime from types import MappingProxyType from typing import Final @@ -62,7 +62,6 @@ from models import ( ModelMode, ModelNewBody, ModelNewResponse, - ModelPatchBody, ModelsListParams, ModelsListResponse, ModelUpdateBody, @@ -73,7 +72,6 @@ from models import ( SpendLogsPage, SpendLogsPageParams, SpendLogsParams, - StoredDeployment, ToolsetCreateBody, ToolsetRow, ToolsetUpdateBody, @@ -134,103 +132,6 @@ class NotServableOn: last_result: Result[ModelsListResponse] | None -type BodyReader[R: BaseModel] = Callable[[float], Result[R]] - - -@dataclass(frozen=True, slots=True) -class BodyNotConverged[R: BaseModel]: - """The deadline passed without a read the predicate accepted; `last_result` is the - final read, so the caller can tell a body that never matched from a read that - failed.""" - - last_result: Result[R] | None - - -@dataclass(frozen=True, slots=True) -class BodyConverged[R: BaseModel]: - """Every replica answered a body the predicate accepted; `bodies` is the last read - per replica.""" - - bodies: Mapping[str, R] - - -@dataclass(frozen=True, slots=True) -class BodyNeverConvergedOn[R: BaseModel]: - """`BodyNotConverged` labeled with the replica whose reads never satisfied the predicate.""" - - replica: str - last_result: Result[R] | None - - -def await_body_converged[R: BaseModel]( - read: BodyReader[R], - *, - predicate: Callable[[R], bool], - timeout: float, - interval: float, - request_timeout: float, - now: Callable[[], float], - sleep: Callable[[float], None], -) -> Success[R] | BodyNotConverged[R]: - """Poll `read` until it answers a body `predicate` accepts, or `timeout` passes. - - Each read's request timeout is clamped to the remaining budget, and the sleep - between reads to the time left, so the last read before the deadline is never - skipped. Clock and sleep are injected.""" - deadline: Final = now() + timeout - - def reads() -> Iterator[Result[R]]: - while (remaining := deadline - now()) > 0: - yield read(min(request_timeout, remaining)) - sleep(min(interval, max(deadline - now(), 0.0))) - - def attempts() -> Iterator[Success[R] | BodyNotConverged[R]]: - for result in reads(): - if isinstance(result, Success) and predicate(result.data): - yield result - return - yield BodyNotConverged(last_result=result) - - initial: Final[Success[R] | BodyNotConverged[R]] = BodyNotConverged(last_result=None) - return reduce(lambda _previous, result: result, attempts(), initial) - - -def await_body_converged_everywhere[R: BaseModel]( - readers: Mapping[str, BodyReader[R]], - *, - predicate: Callable[[R], bool], - timeout: float, - interval: float, - request_timeout: float, - now: Callable[[], float], - sleep: Callable[[float], None], -) -> BodyConverged[R] | BodyNeverConvergedOn[R]: - """`await_body_converged` against every replica in turn, each with the full budget, so a - write counts as landed only once every replica serves it.""" - def read_replica( - outcome: BodyConverged[R] | BodyNeverConvergedOn[R], - item: tuple[str, BodyReader[R]], - ) -> BodyConverged[R] | BodyNeverConvergedOn[R]: - if isinstance(outcome, BodyNeverConvergedOn): - return outcome - replica, read = item - match await_body_converged( - read, - predicate=predicate, - timeout=timeout, - interval=interval, - request_timeout=request_timeout, - now=now, - sleep=sleep, - ): - case Success(data=data): - return BodyConverged(bodies=MappingProxyType({**outcome.bodies, replica: data})) - case BodyNotConverged(last_result=last_result): - return BodyNeverConvergedOn(replica=replica, last_result=last_result) - initial: Final[BodyConverged[R] | BodyNeverConvergedOn[R]] = BodyConverged(bodies=MappingProxyType({})) - return reduce(read_replica, readers.items(), initial) - - def await_servable( list_models: Callable[[float], Result[ModelsListResponse]], *, @@ -757,102 +658,6 @@ class ProxyClient: ) ) - def patch_model(self, model_id: str, body: ModelPatchBody) -> StoredDeployment: - """JSON Merge Patch the deployment `model_id` via PATCH /model/{model_id}/update: - a field the body omits is unchanged, one sent as null is removed from the stored - row, one sent with a value is set. See ModelPatchBody for how a null is sent. - Returns the row as stored after the write.""" - return unwrap( - self.transport.patch( - f"/model/{model_id}/update", - headers=self.transport.master, - json=body, - response_type=StoredDeployment, - ) - ) - - def read_model_back_everywhere[R: BaseModel]( - self, path: str, response_type: type[R], *, predicate: Callable[[R], bool] - ) -> Mapping[str, R]: - """GET `path` on every replica until each answers a body `predicate` accepts, - polling to poll_timeout, and return the last body per replica. - - Fails naming the replica that never converged, so a write that reached one - gateway but not the others is caught instead of passing on whichever gateway - the balancer answered from. Falls back to the single proxy address when no - replica list is configured. - - `path` must be a data-plane route. The replicas are gateways, which serve only - the LLM surface, so a control-plane path answers on exactly one service and - 404s on every replica in a split deployment: asking each replica for one is - never the question the caller means. Read those through `self.transport` - instead, which routes them to the control plane.""" - if is_control_plane_path(path): - raise AssertionError( - f"read_model_back_everywhere({path!r}) asks every data-plane replica for a control-plane route. " - "The replicas are gateways and do not serve it; poll a data-plane path such as /v1/models " - "here, and read the control plane through the shared transport." - ) - readers: Final = { - url: self._body_reader(transport, path, response_type) - for url, transport in self._read_back_replicas().items() - } - outcome: Final = await_body_converged_everywhere( - readers, - predicate=predicate, - timeout=self.poll_timeout, - interval=self.poll_interval, - request_timeout=REQUEST_TIMEOUT, - now=time.monotonic, - sleep=time.sleep, - ) - match outcome: - case BodyConverged(bodies=bodies): - return bodies - case BodyNeverConvergedOn(replica=replica, last_result=last_result): - raise AssertionError( - f"GET {path} on {replica} never answered the expected body within " - f"{self.poll_timeout}s; last read: {last_result}" - ) - - def read_model_back[R: BaseModel](self, path: str, response_type: type[R], *, predicate: Callable[[R], bool]) -> R: - """GET `path` through the shared transport until the body satisfies `predicate`, - polling to poll_timeout, and return that body. - - The counterpart to `read_model_back_everywhere` for a control-plane route such as - /model/info: the stored row lives in one database behind one control plane, so - there is a single answer to converge on rather than one per gateway.""" - outcome: Final = await_body_converged_everywhere( - {CONTROL_PLANE_BASE_URL: self._body_reader(self.transport, path, response_type)}, - predicate=predicate, - timeout=self.poll_timeout, - interval=self.poll_interval, - request_timeout=REQUEST_TIMEOUT, - now=time.monotonic, - sleep=time.sleep, - ) - match outcome: - case BodyConverged(bodies=bodies): - return bodies[CONTROL_PLANE_BASE_URL] - case BodyNeverConvergedOn(last_result=last_result): - raise AssertionError( - f"GET {path} never answered the expected body within " - f"{self.poll_timeout}s; last read: {last_result}" - ) - - def _read_back_replicas(self) -> Mapping[str, Transport]: - return self.replicas or MappingProxyType({CONTROL_PLANE_BASE_URL: self.transport}) - - @staticmethod - def _body_reader[R: BaseModel](transport: Transport, path: str, response_type: type[R]) -> BodyReader[R]: - return lambda timeout: transport.get( - path, - headers=transport.master, - params=NoBody(), - response_type=response_type, - timeout=timeout, - ) - def delete_model(self, model_id: str) -> None: result = self.transport.post( "/model/delete", diff --git a/tests/e2e/test_proxy_client.py b/tests/e2e/test_proxy_client.py index cbf7f5648d4..3b84a47e3cc 100644 --- a/tests/e2e/test_proxy_client.py +++ b/tests/e2e/test_proxy_client.py @@ -20,12 +20,8 @@ from typing import Final, cast import pytest from e2e_config import parse_replica_urls from e2e_http import Result, Success -from models import KeyInfo, KeyInfoResponse, ModelInfoEntry, ModelInfoResponse, ModelListEntry, ModelsListResponse +from models import KeyInfo, KeyInfoResponse, ModelListEntry, ModelsListResponse from proxy_client import ( - BodyReader, - BodyConverged, - BodyNeverConvergedOn, - await_body_converged_everywhere, ConvergeOutcome, Converged, EverywhereConverged, @@ -278,54 +274,3 @@ class TestReplicasFor: client: Final = ProxyClient(transport=_NO_TRANSPORTS, replicas={}, control_replicas={}) with pytest.raises(AssertionError, match="no replica is configured"): _ = client.replicas_for("/v1/models") - - -def _info(*model_names: str) -> Success[ModelInfoResponse]: - entries: Final = [ModelInfoEntry(model_name=model_name) for model_name in model_names] - return Success(status_code=200, data=ModelInfoResponse(data=entries)) - - -def _reader(results: Iterable[Success[ModelInfoResponse]]) -> BodyReader[ModelInfoResponse]: - it: Final = iter(results) - return lambda _timeout: next(it) - - -def _lists_model(body: ModelInfoResponse) -> bool: - return any(entry.model_name == MODEL for entry in body.data) - - -def _read_back( - readers: Mapping[str, BodyReader[ModelInfoResponse]], -) -> tuple[BodyConverged[ModelInfoResponse] | BodyNeverConvergedOn[ModelInfoResponse], FakeClock]: - clock: Final = FakeClock() - outcome: Final = await_body_converged_everywhere( - readers, - predicate=_lists_model, - timeout=TIMEOUT, - interval=INTERVAL, - request_timeout=5.0, - now=clock.now, - sleep=clock.sleep, - ) - return outcome, clock - - -class TestAwaitBodyConvergedEverywhere: - def test_waits_for_the_lagging_replica_and_returns_every_body(self) -> None: - readers: Final = { - "gateway-1": _reader(repeat(_info(MODEL))), - "gateway-2": _reader(chain(repeat(_info(), 2), repeat(_info(MODEL)))), - } - outcome, clock = _read_back(readers) - assert outcome == BodyConverged(bodies={"gateway-1": _info(MODEL).data, "gateway-2": _info(MODEL).data}) - assert clock.elapsed == 2 * INTERVAL - - @pytest.mark.parametrize("lagging", ["gateway-1", "gateway-2"]) - def test_fails_naming_the_replica_that_never_converges(self, lagging: str) -> None: - readers: Final = { - "gateway-1": _reader(repeat(_info(MODEL))), - "gateway-2": _reader(repeat(_info(MODEL))), - } | {lagging: _reader(repeat(_info()))} - outcome, clock = _read_back(readers) - assert outcome == BodyNeverConvergedOn(replica=lagging, last_result=_info()) - assert clock.elapsed >= TIMEOUT diff --git a/tests/e2e/transport.py b/tests/e2e/transport.py index 804e073a4a0..44fdbaa3e41 100644 --- a/tests/e2e/transport.py +++ b/tests/e2e/transport.py @@ -306,7 +306,6 @@ CONTROL_PLANE_PREFIXES: tuple[str, ...] = ( "/config", "/guardrails", "/openapi.json", - "/public/", ) diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index 1376727e296..c02f886fc31 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -3115,6 +3115,9 @@ class TestUpdateDBModelClearPricing: """Sending an explicit `null` for a pricing field must remove it from both `litellm_params` and `model_info` (SPECIAL_MODEL_INFO_PARAMS are mirrored between the two by Deployment.__init__). + + Restricted to SPECIAL_MODEL_INFO_PARAMS so non-pricing fields (e.g. team_id) + cannot be cleared via this path. """ def test_clear_input_cost_removes_from_both_blobs(self): @@ -3190,10 +3193,10 @@ class TestUpdateDBModelClearPricing: assert params["input_cost_per_token"] == 0.000001 assert params["output_cost_per_token"] == 0.000007 - def test_null_on_one_field_leaves_other_fields_alone(self): - """A null clears only the key it names: pricing the patch never mentions and - the ownership key team_id stay put, so a team admin can't ungate a - team-scoped model through the clear path. + def test_null_on_non_pricing_field_does_not_clear(self): + """Security guard: only SPECIAL_MODEL_INFO_PARAMS can be cleared via null. + Privileged or unrelated model_info fields (e.g. team_id) must be unaffected + by the null-clearing path so a team admin can't ungate a team-scoped model. """ from litellm.proxy.management_endpoints.model_management_endpoints import ( update_db_model, @@ -3214,6 +3217,8 @@ class TestUpdateDBModelClearPricing: model_info=ModelInfo(id="dep-pricing-1", team_id="team-keep-me"), ) + # Patch sends a null for api_base (non-SPECIAL field). Must NOT clear team_id + # or any other non-pricing field from the merged dict. result = update_db_model( db_model=db_model, updated_patch=updateDeployment( @@ -3389,171 +3394,6 @@ class TestUpdateDBModelClearPricing: assert info["cache_creation_input_token_cost"] == 0.000003 -_PROTECTED_MODEL_INFO_VALUES = { - "team_id": "team-keep-me", - "team_public_model_name": "team-facing-name", - "access_groups": ["group-a"], - "created_at": "2026-01-01T00:00:00+00:00", - "created_by": "creator", - "updated_at": "2026-01-02T00:00:00+00:00", - "updated_by": "updater", - "blocked": True, -} - - -def _build_db_model_with_pinned_model_info(): - """Deployment whose stored blobs pin non-pricing keys an earlier save wrote, next to a - pricing override, so a clear can be checked key by key.""" - from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo - - return Deployment( - model_name="pinned-gpt-4o-mini", - litellm_params=LiteLLM_Params( - model="gpt-4o-mini", input_cost_per_token=0.000001, max_input_tokens=4096 - ), - model_info=ModelInfo( - id="dep-pinned-0", - max_input_tokens=4096, - mode="chat", - supports_vision=True, - **_PROTECTED_MODEL_INFO_VALUES, - ), - ) - - -class TestUpdateDBModelNullClearsAnyKey: - """JSON Merge Patch on PATCH /model/{id}/update: a key sent as null is removed from the - stored blob it was sent in, whatever the key, except the identity and ownership keys, - whose nulls are ignored.""" - - def test_model_info_nulls_remove_pinned_non_pricing_keys(self): - from litellm.proxy.management_endpoints.model_management_endpoints import ( - update_db_model, - ) - - result = update_db_model( - db_model=_build_db_model_with_pinned_model_info(), - updated_patch=updateDeployment.model_validate( - {"model_info": {"max_input_tokens": None, "mode": None}} - ), - ) - - info = json.loads(result["model_info"]) - assert "max_input_tokens" not in info - assert "mode" not in info - assert info["supports_vision"] is True - assert info["input_cost_per_token"] == 0.000001 - - def test_litellm_params_null_removes_pinned_non_pricing_key(self): - from litellm.proxy.management_endpoints.model_management_endpoints import ( - update_db_model, - ) - - result = update_db_model( - db_model=_build_db_model_with_pinned_model_info(), - updated_patch=updateDeployment.model_validate( - {"litellm_params": {"max_input_tokens": None}} - ), - ) - - params = json.loads(result["litellm_params"]) - info = json.loads(result["model_info"]) - assert "max_input_tokens" not in params - assert params["model"] == "gpt-4o-mini" - assert params["input_cost_per_token"] == 0.000001 - assert info["max_input_tokens"] == 4096 - - def test_omitted_key_is_untouched_by_a_null_elsewhere(self): - from litellm.proxy.management_endpoints.model_management_endpoints import ( - update_db_model, - ) - - result = update_db_model( - db_model=_build_db_model_with_pinned_model_info(), - updated_patch=updateDeployment.model_validate( - {"model_info": {"mode": None, "supports_vision": False}} - ), - ) - - info = json.loads(result["model_info"]) - assert "mode" not in info - assert info["supports_vision"] is False - assert info["max_input_tokens"] == 4096 - - @pytest.mark.parametrize("field", sorted(_PROTECTED_MODEL_INFO_VALUES)) - def test_null_on_protected_key_is_ignored(self, field): - from litellm.proxy.management_endpoints.model_management_endpoints import ( - update_db_model, - ) - - result = update_db_model( - db_model=_build_db_model_with_pinned_model_info(), - updated_patch=updateDeployment.model_validate({"model_info": {field: None}}), - ) - - info = json.loads(result["model_info"]) - assert info[field] == _PROTECTED_MODEL_INFO_VALUES[field] - assert info["max_input_tokens"] == 4096 - - def test_echoing_the_read_back_blob_preserves_every_stored_key(self): - """The Admin UI edit form submits the whole /model/info row back, and that read reports - every key the deployment never stored as an explicit null. Those nulls have to stay - no-ops: a write drops None before storing, so a null in the echoed blob always names a - key the stored row does not carry. - """ - from litellm.proxy.management_endpoints.model_management_endpoints import ( - update_db_model, - ) - - db_model = _build_db_model_with_pinned_model_info() - echoed = { - "id": "dep-pinned-0", - "max_input_tokens": 4096, - "mode": "chat", - "supports_vision": True, - "input_cost_per_token": 0.000001, - "team_id": "team-keep-me", - "base_model": None, - "tier": None, - "max_output_tokens": None, - "supports_function_calling": None, - "cache_read_input_token_cost": None, - } - - result = update_db_model( - db_model=db_model, - updated_patch=updateDeployment.model_validate({"model_info": echoed}), - ) - - info = json.loads(result["model_info"]) - assert info["max_input_tokens"] == 4096 - assert info["mode"] == "chat" - assert info["supports_vision"] is True - assert info["input_cost_per_token"] == 0.000001 - assert info["team_id"] == "team-keep-me" - for never_stored in ("base_model", "tier", "max_output_tokens", "supports_function_calling"): - assert never_stored not in info - - def test_null_on_pricing_key_still_clears_both_blobs(self): - from litellm.proxy.management_endpoints.model_management_endpoints import ( - update_db_model, - ) - - result = update_db_model( - db_model=_build_db_model_with_pinned_model_info(), - updated_patch=updateDeployment.model_validate( - {"model_info": {"input_cost_per_token": None}} - ), - ) - - params = json.loads(result["litellm_params"]) - info = json.loads(result["model_info"]) - assert "input_cost_per_token" not in params - assert "input_cost_per_token" not in info - assert params["max_input_tokens"] == 4096 - assert info["max_input_tokens"] == 4096 - - class TestGetModelInfoWithIdBlocked: """`ProxyConfig.get_model_info_with_id` must propagate the DB-level `blocked` column into the in-memory `model_info` dict so the router filter can read it.""" diff --git a/tests/test_litellm/test_router_model_cost_isolation.py b/tests/test_litellm/test_router_model_cost_isolation.py index d22ec60e61a..30b265905f3 100644 --- a/tests/test_litellm/test_router_model_cost_isolation.py +++ b/tests/test_litellm/test_router_model_cost_isolation.py @@ -220,198 +220,6 @@ def test_should_store_full_pricing_under_deployment_model_id(): assert entry["output_cost_per_token"] == 0.0 -def test_should_drop_a_price_the_deployment_no_longer_carries(): - """Re-registering a deployment must replace its model_id entry, not merge onto it. - - A merge left the old rate in the cost map after an operator cleared the override, so - the deployment kept billing at a price its config no longer had. - """ - backend_model = "vertex_ai/gemini-2.5-flash" - model_id = "deployment-cleared-price" - original = {model_id: litellm.model_cost.get(model_id)} - - try: - Router._register_deployment_in_model_cost( - model_id=model_id, - model_info={"input_cost_per_token": 0.005, "output_cost_per_token": 0.01}, - model=backend_model, - custom_llm_provider="vertex_ai", - ) - assert litellm.model_cost[model_id]["input_cost_per_token"] == 0.005 - - Router._register_deployment_in_model_cost( - model_id=model_id, - model_info={"mode": "chat"}, - model=backend_model, - custom_llm_provider="vertex_ai", - ) - - entry = litellm.model_cost[model_id] - assert entry.get("input_cost_per_token") != 0.005, ( - "the cleared override survived re-registration, so the deployment still bills at it" - ) - assert entry.get("output_cost_per_token") != 0.01 - finally: - _restore_model_cost_entries(original) - - -def test_should_not_strip_a_builtin_entry_when_a_deployment_id_collides_with_it(): - """Deployments are keyed into the same cost map as the built-in catalog, so a deployment - whose id happens to name a real model must not evict that model's entry. - - Stripping it would take the pricing and capability flags every other deployment of that - model reads, process-wide, until the next price-map reload. Registering twice, because - the first registration is what would mark the entry as this deployment's own. - """ - colliding_id = "gpt-4o" - original = {colliding_id: litellm.model_cost.get(colliding_id)} - builtin_max_tokens = litellm.model_cost[colliding_id]["max_tokens"] - - try: - for _ in range(2): - Router._register_deployment_in_model_cost( - model_id=colliding_id, - model_info={"id": colliding_id, "db_model": True, "mode": "chat"}, - model="gpt-4o-mini", - custom_llm_provider="openai", - ) - - entry = litellm.model_cost[colliding_id] - assert entry["max_tokens"] == builtin_max_tokens, ( - "registering a deployment under a catalog model's name wiped that model's context window" - ) - assert entry["litellm_provider"] == "openai" - assert entry["supports_vision"] is True - finally: - _restore_model_cost_entries(original) - - -def test_should_drop_a_stale_price_even_when_the_deployment_declares_a_provider(): - """A deployment may carry `litellm_provider` in its own model_info, which must not be - read as "this is a catalog entry" and stop the stale price from being dropped.""" - model_id = "deployment-provider-tagged" - original = {model_id: litellm.model_cost.get(model_id)} - - try: - Router._register_deployment_in_model_cost( - model_id=model_id, - model_info={"id": model_id, "litellm_provider": "openai", "input_cost_per_token": 0.005}, - model="gpt-4o-mini", - custom_llm_provider="openai", - ) - assert litellm.model_cost[model_id]["input_cost_per_token"] == 0.005 - - Router._register_deployment_in_model_cost( - model_id=model_id, - model_info={"id": model_id, "litellm_provider": "openai", "mode": "chat"}, - model="gpt-4o-mini", - custom_llm_provider="openai", - ) - - assert litellm.model_cost[model_id].get("input_cost_per_token") != 0.005, ( - "a deployment that declares its provider kept billing at the price it no longer carries" - ) - finally: - _restore_model_cost_entries(original) - - -def test_should_give_a_cost_map_key_back_when_the_deployment_is_deleted(): - """Deleting a deployment releases its claim on the shared cost-map key. - - Held forever, a later catalog refresh that starts publishing a model under that same - name would be treated as the deleted deployment's own entry and evicted. - """ - from litellm.router import _DEPLOYMENT_COST_MAP_KEYS - - model_id = "deployment-to-delete" - original = {model_id: litellm.model_cost.get(model_id)} - router = Router( - model_list=[ - { - "model_name": "to-delete", - "litellm_params": {"model": "gpt-4o-mini", "mock_response": "ok"}, - "model_info": {"id": model_id, "input_cost_per_token": 0.005}, - } - ] - ) - - try: - assert model_id in _DEPLOYMENT_COST_MAP_KEYS - - assert router.delete_deployment(id=model_id) is not None - - assert model_id not in _DEPLOYMENT_COST_MAP_KEYS, ( - "a deleted deployment kept its claim on the shared cost-map key" - ) - finally: - _DEPLOYMENT_COST_MAP_KEYS.discard(model_id) - _restore_model_cost_entries(original) - - -def test_should_keep_the_cost_map_key_while_another_router_still_serves_it(): - """Two live routers can serve the same deployment id, and the claim is process-wide. - - Releasing it when only one of them drops the deployment would put the survivor back on - merging, so the price it just cleared would keep billing. - """ - from litellm.router import _DEPLOYMENT_COST_MAP_KEYS - - model_id = "deployment-served-twice" - original = {model_id: litellm.model_cost.get(model_id)} - entry = { - "model_name": "served-twice", - "litellm_params": {"model": "gpt-4o-mini", "mock_response": "ok"}, - "model_info": {"id": model_id, "input_cost_per_token": 0.005}, - } - first = Router(model_list=[entry]) - second = Router(model_list=[entry]) - - try: - assert model_id in _DEPLOYMENT_COST_MAP_KEYS - - assert first.delete_deployment(id=model_id) is not None - - assert model_id in _DEPLOYMENT_COST_MAP_KEYS, ( - "the claim was released while another router still served the deployment" - ) - - assert second.delete_deployment(id=model_id) is not None - assert model_id not in _DEPLOYMENT_COST_MAP_KEYS - finally: - _DEPLOYMENT_COST_MAP_KEYS.discard(model_id) - _restore_model_cost_entries(original) - - -def test_should_keep_the_cost_map_key_while_a_dynamically_built_router_serves_it(): - """A router built with no model_list still serves whatever add_deployment gives it, so it - counts when deciding whether the shared cost-map claim can be released.""" - from litellm.router import _DEPLOYMENT_COST_MAP_KEYS - - model_id = "deployment-added-dynamically" - original = {model_id: litellm.model_cost.get(model_id)} - entry = { - "model_name": "added-dynamically", - "litellm_params": {"model": "gpt-4o-mini", "mock_response": "ok"}, - "model_info": {"id": model_id, "input_cost_per_token": 0.005}, - } - configured = Router(model_list=[entry]) - dynamic = Router() - dynamic.add_deployment(deployment=Deployment(**entry)) - - try: - assert configured.delete_deployment(id=model_id) is not None - - assert model_id in _DEPLOYMENT_COST_MAP_KEYS, ( - "the claim was released while a dynamically built router still served the deployment" - ) - - assert dynamic.delete_deployment(id=model_id) is not None - assert model_id not in _DEPLOYMENT_COST_MAP_KEYS - finally: - _DEPLOYMENT_COST_MAP_KEYS.discard(model_id) - _restore_model_cost_entries(original) - - def test_should_preserve_builtin_pricing_regardless_of_deployment_order(): """ The built-in pricing should be preserved no matter which deployment diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 93aaf3ca58c..83b0d58f2b2 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -9064,9 +9064,8 @@ export interface paths { * Patch Model * @description PATCH Endpoint for partial model updates. * - * JSON Merge Patch semantics over `litellm_params` and `model_info`: a key absent from the - * body is unchanged, a key sent as null is removed from the stored row, and a key sent with a - * value is set (identity and ownership keys such as `id` and `team_id` ignore a null). + * Only updates the fields specified in the request while preserving other existing values. + * Follows proper PATCH semantics by only modifying provided fields. * * Args: * model_id: The ID of the model to update From 1183b2abc6645a92f9a852daa7843662b6fa6f20 Mon Sep 17 00:00:00 2001 From: yujonglee Date: Wed, 9 Sep 2026 09:46:37 -0700 Subject: [PATCH 11/11] fix(integrations): pass original request object to post-call guardrail hooks (#40414) --- litellm/integrations/custom_guardrail.py | 30 ++++--- .../integrations/test_custom_guardrail.py | 90 ++++++++++++++++++- 2 files changed, 106 insertions(+), 14 deletions(-) diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 37d6a7e793d..77bf4820a1a 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -850,20 +850,24 @@ class CustomGuardrail(CustomLogger): if self.should_run_guardrail(data=request_data, event_type=GuardrailEventHooks.post_call) is not True: return None - # CHECK IF GUARDRAIL REJECTS THE REQUEST target: Final = self._deployment_hook_target() - hook_request_data: Final = {**request_data, "guardrail_to_apply": self} if target is not self else request_data - result: Final = await target.async_post_call_success_hook( - user_api_key_dict=UserAPIKeyAuth( - user_id=request_data.get("user_api_key_user_id"), - team_id=request_data.get("user_api_key_team_id"), - end_user_id=request_data.get("user_api_key_end_user_id"), - api_key=request_data.get("user_api_key_hash"), - request_route=request_data.get("user_api_key_request_route"), - ), - data=hook_request_data, - response=response, - ) + try: + if target is not self: + request_data["guardrail_to_apply"] = self # rebind-ok: dispatch consumes this key + result: Final = await target.async_post_call_success_hook( + user_api_key_dict=UserAPIKeyAuth( + user_id=request_data.get("user_api_key_user_id"), + team_id=request_data.get("user_api_key_team_id"), + end_user_id=request_data.get("user_api_key_end_user_id"), + api_key=request_data.get("user_api_key_hash"), + request_route=request_data.get("user_api_key_request_route"), + ), + data=request_data, + response=response, + ) + finally: + if target is not self: + request_data.pop("guardrail_to_apply", None) if not self._is_valid_response_type(result): return None diff --git a/tests/test_litellm/integrations/test_custom_guardrail.py b/tests/test_litellm/integrations/test_custom_guardrail.py index cd8d609cf71..ddc8439a83a 100644 --- a/tests/test_litellm/integrations/test_custom_guardrail.py +++ b/tests/test_litellm/integrations/test_custom_guardrail.py @@ -1842,12 +1842,14 @@ class _ApplyStyleGuardrail(CustomGuardrail): self.block = block self.apply_called = False self.seen_texts = None + self.seen_request_data = None async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): from fastapi import HTTPException self.apply_called = True self.seen_texts = inputs.get("texts") + self.seen_request_data = request_data if self.block: raise HTTPException(status_code=400, detail={"error": "Violated moderation policy"}) return inputs @@ -2646,6 +2648,91 @@ class TestCustomGuardrailPostCallSuccessDeploymentHook: which starved every later callback in litellm.callbacks (notably the lazily-appended VectorStorePreCallHook that attaches provider_specific_fields["search_results"]).""" + @pytest.mark.asyncio + async def test_apply_guardrail_retains_request_identity(self) -> None: + from litellm.types.guardrails import GuardrailEventHooks + from litellm.types.utils import Choices, Message, ModelResponse + + guardrail: Final = _ApplyStyleGuardrail(block=False) + guardrail.event_hook = GuardrailEventHooks.post_call + request_data: Final = {"guardrails": ["apply-style-guardrail"]} + response: Final = ModelResponse(choices=[Choices(message=Message(content="review me"))]) + + await guardrail.async_post_call_success_deployment_hook( + request_data=request_data, response=response, call_type=CallTypes.acompletion + ) + + assert guardrail.seen_request_data is request_data + assert guardrail.seen_texts == ["review me"] + assert "guardrail_to_apply" not in request_data + + @pytest.mark.asyncio + @pytest.mark.parametrize("call_type", (None, CallTypes.acompletion)) + async def test_apply_guardrail_masks_response_and_records_metadata(self, call_type: CallTypes | None) -> None: + from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ( + ContentFilterGuardrail, + ) + from litellm.types.guardrails import BlockedWord, ContentFilterAction, GuardrailEventHooks + from litellm.types.utils import Choices, Message, ModelResponse + + guardrail: Final = ContentFilterGuardrail( + guardrail_name="response-filter", + event_hook=GuardrailEventHooks.post_call, + blocked_words=[BlockedWord(keyword="secret", action=ContentFilterAction.MASK)], + ) + request_data: Final = {"guardrails": ["response-filter"]} + response: Final = ModelResponse(choices=[Choices(message=Message(content="a secret"))]) + + result: Final = await guardrail.async_post_call_success_deployment_hook( + request_data=request_data, response=response, call_type=call_type + ) + + assert isinstance(result, ModelResponse) + assert result.choices[0].message.content == f"a {guardrail.keyword_redaction_tag}" + entries: Final = _guardrail_entries(request_data) + assert len(entries) == 1 + assert entries[0]["guardrail_name"] == "response-filter" + assert entries[0]["guardrail_mode"] == "post_call" + assert "guardrail_to_apply" not in request_data + + @pytest.mark.asyncio + @pytest.mark.parametrize("error_type", (None, RuntimeError, asyncio.CancelledError)) + async def test_dispatch_cleans_up_request_on_every_exit(self, error_type: type[BaseException] | None) -> None: + from contextlib import nullcontext + + from litellm.integrations.custom_logger import CustomLogger + from litellm.types.utils import LLMResponseTypes, ModelResponse + + error: Final = error_type("dispatch interrupted") if error_type is not None else None + + class Dispatch(CustomLogger): + request_data: dict[str, object] | None = None + + async def async_post_call_success_hook( + self, data: dict[str, object], user_api_key_dict: UserAPIKeyAuth, response: LLMResponseTypes + ) -> LLMResponseTypes: + self.request_data = data + if error is not None: + raise error + return response + + dispatch: Final = Dispatch() + + class Guardrail(_ApplyStyleGuardrail): + def _deployment_hook_target(self) -> CustomLogger: + return dispatch + + guardrail: Final = Guardrail(block=False) + guardrail.event_hook = GuardrailEventHooks.post_call + request_data: Final = {"guardrails": ["apply-style-guardrail"]} + with pytest.raises(error_type) if error_type is not None else nullcontext(): + await guardrail.async_post_call_success_deployment_hook( + request_data=request_data, response=ModelResponse(), call_type=CallTypes.acompletion + ) + + assert dispatch.request_data is request_data + assert "guardrail_to_apply" not in request_data + @pytest.mark.asyncio async def test_returns_none_when_request_has_no_guardrails(self): from litellm.types.utils import ModelResponse @@ -2740,4 +2827,5 @@ class TestCustomGuardrailPostCallSuccessDeploymentHook: assert result is response assert response.choices[0].message.content == "filtered response" - assert request_data == {"guardrails": ["test-guardrail"]} + assert "guardrail_to_apply" not in request_data + assert len(_guardrail_entries(request_data)) == 1