From 6a8e568f8ae360a525ac4f75c643eea204e0d13d Mon Sep 17 00:00:00 2001 From: Armaan Sandhu <74664101+Ar-maan05@users.noreply.github.com> Date: Mon, 8 Jun 2026 18:06:51 +0530 Subject: [PATCH 01/12] feat(proxy): add disable_budget_reservation general setting (#27639) (#29493) * feat(proxy): add disable_budget_reservation general setting (#27639) * feat(proxy): register disable_budget_reservation in ConfigGeneralSettings (#27639) * docs(proxy): document disable_budget_reservation concurrency tradeoff (#27639) * ci: re-trigger flaky docker build (prisma generate ECONNRESET) * fix(proxy): warn and document budget enforcement tradeoff when disable_budget_reservation is set (#27639) Provenance: #29493 merged into litellm_oss_staging_080626, which was promoted to litellm_internal_staging by the aggregator squash 32c88ca74f (#29932). There is no discrete #29493 commit on internal_staging; this pick is the original PR squash 1032dd75, content-verified identical to the disable_budget_reservation hunks that 32c88ca74f introduced on internal_staging. (cherry picked from commit 1032dd751fe612333412fd6673083148b4bd7e09) --- litellm/proxy/_types.py | 18 ++++++ litellm/proxy/auth/user_api_key_auth.py | 12 ++++ .../proxy/auth/test_user_api_key_auth.py | 60 +++++++++++++++++++ 3 files changed, 90 insertions(+) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 751f855ea34..c80455cabb6 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2533,6 +2533,24 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): None, description="List of MCP server fields that must be filled in for a submission to pass standards checks (e.g. ['description', 'source_url', 'alias']).", ) + disable_budget_reservation: Optional[bool] = Field( + None, + description=( + "If True, disables the optimistic per-request budget reservation " + "introduced in v1.84.0. " + "WARNING: This weakens hard budget enforcement. Without the reservation, " + "a burst of concurrent requests from a single key can each pass the " + "read-time spend check before any of them is charged, allowing a " + "configured budget to be exceeded under high concurrency. " + "Budgets are still evaluated on every request at read time, so " + "an already-exhausted budget is still rejected. " + "Enable only if your deployment is experiencing phantom " + "BudgetExceededError responses caused by leaked reservations " + "(see GitHub issue #27639). " + "A proxy-level WARNING is logged on every request while this flag " + "is active as a reminder that hard enforcement is relaxed." + ), + ) class ConfigYAML(LiteLLMPydanticObjectBase): diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 2e6cd1f8e70..37809e043a8 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -2108,6 +2108,7 @@ async def _run_centralized_common_checks( # noqa: PLR0915 user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, skip_budget_checks=skip_budget_checks, + general_settings=general_settings, ) @@ -2128,12 +2129,23 @@ async def _reserve_budget_after_common_checks( user_api_key_cache: UserApiKeyCache, proxy_logging_obj: ProxyLogging, skip_budget_checks: bool, + general_settings: dict, end_user_id: Optional[str] = None, end_user_object: Optional[LiteLLM_EndUserTable] = None, ) -> None: user_api_key_auth_obj.budget_reservation = None if skip_budget_checks: return + if general_settings.get("disable_budget_reservation") is True: + verbose_proxy_logger.warning( + "disable_budget_reservation is enabled: skipping optimistic budget " + "reservation. Budget enforcement is read-time only — concurrent " + "requests can each pass the spend check before their cost is recorded, " + "so a configured budget may be briefly exceeded under high concurrency. " + "Set disable_budget_reservation to False or remove it to restore " + "hard per-request budget enforcement." + ) + return from litellm.proxy.spend_tracking.budget_reservation import ( reserve_budget_for_request, diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index a3452ac8024..f6848dac6fd 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -110,11 +110,71 @@ async def test_should_clear_stale_budget_reservation_when_budget_checks_skip(): user_api_key_cache=MagicMock(), proxy_logging_obj=MagicMock(), skip_budget_checks=True, + general_settings={}, ) assert user_api_key_auth_obj.budget_reservation is None +@pytest.mark.asyncio +async def test_disable_budget_reservation_skips_reservation(): + """#27639: general_settings.disable_budget_reservation turns off the optimistic Redis + reservation so operators hit by phantom BudgetExceededError can opt out of it.""" + user_api_key_auth_obj = UserAPIKeyAuth(token="test_token") + + with patch( + "litellm.proxy.spend_tracking.budget_reservation.reserve_budget_for_request", + new=AsyncMock(return_value={"reserved_cost": 0.5, "entries": []}), + ) as mock_reserve: + await _reserve_budget_after_common_checks( + user_api_key_auth_obj=user_api_key_auth_obj, + request_data={"model": "gpt-4o"}, + route="/v1/chat/completions", + llm_router=None, + team_object=None, + user_object=None, + prisma_client=None, + user_api_key_cache=MagicMock(), + proxy_logging_obj=MagicMock(), + skip_budget_checks=False, + general_settings={"disable_budget_reservation": True}, + ) + + mock_reserve.assert_not_called() + assert user_api_key_auth_obj.budget_reservation is None + + +@pytest.mark.asyncio +async def test_budget_reservation_runs_when_not_disabled(): + """Control for #27639: with the flag absent, the reservation still runs and is stored.""" + user_api_key_auth_obj = UserAPIKeyAuth(token="test_token") + reservation = { + "reserved_cost": 0.5, + "entries": [{"counter_key": "spend:key:test_token"}], + } + + with patch( + "litellm.proxy.spend_tracking.budget_reservation.reserve_budget_for_request", + new=AsyncMock(return_value=reservation), + ) as mock_reserve: + await _reserve_budget_after_common_checks( + user_api_key_auth_obj=user_api_key_auth_obj, + request_data={"model": "gpt-4o"}, + route="/v1/chat/completions", + llm_router=None, + team_object=None, + user_object=None, + prisma_client=None, + user_api_key_cache=MagicMock(), + proxy_logging_obj=MagicMock(), + skip_budget_checks=False, + general_settings={}, + ) + + mock_reserve.assert_awaited_once() + assert user_api_key_auth_obj.budget_reservation == reservation + + @pytest.mark.asyncio async def test_should_not_reuse_cached_key_object_for_request_state(): key_cache = DualCache() From dac0f13ad99a406bfc6eda3db70a294966e7e016 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 2 Jun 2026 17:45:28 -0700 Subject: [PATCH 02/12] test(proxy/utils): pin PrismaClient get_data behavior (subset of #29488) Scaffolding dependency for the #29983 and #30327 regression tests, which modify test_prisma_client_get_data.py. Only conftest.py and test_prisma_client_get_data.py are taken from #29488; the other twelve test files in that PR pin behavior unrelated to this backport and are dropped. test_prisma_client_get_data.py and conftest.py are self-contained (no imports from the dropped siblings) and pass on this line. (cherry picked from commit 457f65eff9) --- .../proxy/utils/prisma_and_spend/__init__.py | 0 .../proxy/utils/prisma_and_spend/conftest.py | 387 +++++++++++++++++ .../test_prisma_client_get_data.py | 400 ++++++++++++++++++ 3 files changed, 787 insertions(+) create mode 100644 tests/test_litellm/proxy/utils/prisma_and_spend/__init__.py create mode 100644 tests/test_litellm/proxy/utils/prisma_and_spend/conftest.py create mode 100644 tests/test_litellm/proxy/utils/prisma_and_spend/test_prisma_client_get_data.py diff --git a/tests/test_litellm/proxy/utils/prisma_and_spend/__init__.py b/tests/test_litellm/proxy/utils/prisma_and_spend/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/proxy/utils/prisma_and_spend/conftest.py b/tests/test_litellm/proxy/utils/prisma_and_spend/conftest.py new file mode 100644 index 00000000000..2305a88b6dd --- /dev/null +++ b/tests/test_litellm/proxy/utils/prisma_and_spend/conftest.py @@ -0,0 +1,387 @@ +"""Shared fixtures for tests/test_litellm/proxy/utils/prisma_and_spend/. + +All fixtures used by PR2 test files live here. Do NOT add fixtures inside +individual test files; if a fixture is missing, add it here and update the +Notion plan. + +The PrismaClient is exercised against a fully-mocked Prisma stack: the +``prisma.Prisma`` constructor and the writer/reader wrappers are patched +before PrismaClient.__init__ runs so the init code paths execute without +needing a generated Prisma client or a real database. +""" + +from __future__ import annotations + +import asyncio +import sys +from dataclasses import dataclass, field +from email.message import EmailMessage +from pathlib import Path +from typing import Any, Callable, Dict, Iterator, List, Optional +from unittest.mock import AsyncMock, MagicMock + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[5])) + + +VOLATILE_KEYS = frozenset( + { + "created_at", + "updated_at", + "checked_at", + "started_at", + "request_id", + "id", + "token", + "expires", + "expires_at", + "litellm_call_id", + "created", + "spend", + "last_refreshed_at", + "startTime", + "endTime", + "salt", + } +) + + +def normalize(data: Any, volatile: frozenset = VOLATILE_KEYS) -> Any: + """Recursively replace values for volatile keys with ''.""" + if isinstance(data, dict): + return { + k: ("" if k in volatile else normalize(v, volatile)) + for k, v in data.items() + } + if isinstance(data, list): + return [normalize(v, volatile) for v in data] + return data + + +_PRISMA_TABLES: List[str] = [ + "litellm_verificationtoken", + "litellm_teamtable", + "litellm_usertable", + "litellm_endusertable", + "litellm_organizationtable", + "litellm_proxymodeltable", + "litellm_modeltable", + "litellm_budgettable", + "litellm_spendlogs", + "litellm_config", + "litellm_usernotifications", + "litellm_healthchecktable", + "litellm_dailyuserspend", + "litellm_dailyteamspend", + "litellm_dailytagspend", + "litellm_managed_object_table", + "litellm_credentialstable", + "litellm_mcpservertable", + "litellm_audit_log", + "litellm_invitationlink", + "litellm_session_token_table", + "litellm_passthrough_endpoint_table", + "litellm_cron_job", + "litellm_passthrough_logs", + "litellm_promptstable", + "litellm_guardrailstable", + "litellm_managed_files", + "litellm_mcpusercredentials", + "litellm_objectpermissiontable", + "litellm_organizationmembership", +] + + +def _make_table_mock() -> MagicMock: + table = MagicMock() + table.find_unique = AsyncMock(return_value=None) + table.find_many = AsyncMock(return_value=[]) + table.find_first = AsyncMock(return_value=None) + table.create = AsyncMock() + table.create_many = AsyncMock() + table.update = AsyncMock() + table.update_many = AsyncMock() + table.upsert = AsyncMock() + table.delete = AsyncMock() + table.delete_many = AsyncMock() + table.count = AsyncMock(return_value=0) + table.group_by = AsyncMock(return_value=[]) + table.aggregate = AsyncMock(return_value={}) + return table + + +@pytest.fixture +def mock_prisma_client() -> MagicMock: + """Bare ``db`` mock with all common LiteLLM_* tables stubbed. + + Override individual return values in a test:: + + mock_prisma_client.db.litellm_usertable.find_unique.return_value = user + """ + client = MagicMock(name="MockPrismaClient") + client.db = MagicMock(name="MockPrismaDB") + client.connect = AsyncMock() + client.disconnect = AsyncMock() + client.health_check = AsyncMock(return_value=[{"?column?": 1}]) + client.proxy_logging_obj = MagicMock() + client.proxy_logging_obj.failure_handler = AsyncMock() + client.spend_log_transactions = [] + client._spend_log_transactions_lock = asyncio.Lock() + client.jsonify_object = lambda data: dict(data) + client.db.is_connected = MagicMock(return_value=False) + client.db.connect = AsyncMock() + client.db.disconnect = AsyncMock() + client.db.query_raw = AsyncMock(return_value=[{"?column?": 1}]) + client.db.execute_raw = AsyncMock() + client.db.tx = MagicMock() + client.db.batch_ = MagicMock() + for table_name in _PRISMA_TABLES: + setattr(client.db, table_name, _make_table_mock()) + return client + + +@pytest.fixture +def mock_dual_cache() -> MagicMock: + """In-memory DualCache stand-in. + + Sync and async get/set wired against a private dict. Override or read + ``cache._store`` directly in a test for assertion convenience. + """ + cache = MagicMock(name="MockDualCache") + cache._store: Dict[str, Any] = {} + + def _sync_get(key: str, **_: Any) -> Any: + return cache._store.get(key) + + def _sync_set(key: str, value: Any, **_: Any) -> None: + cache._store[key] = value + + async def _async_get(key: str, **_: Any) -> Any: + return cache._store.get(key) + + async def _async_set(key: str, value: Any, **_: Any) -> None: + cache._store[key] = value + + async def _async_delete(key: str, **_: Any) -> None: + cache._store.pop(key, None) + + cache.get_cache = MagicMock(side_effect=_sync_get) + cache.set_cache = MagicMock(side_effect=_sync_set) + cache.async_get_cache = AsyncMock(side_effect=_async_get) + cache.async_set_cache = AsyncMock(side_effect=_async_set) + cache.async_delete_cache = AsyncMock(side_effect=_async_delete) + return cache + + +@pytest.fixture +def patched_prisma_import(monkeypatch: pytest.MonkeyPatch) -> Iterator[MagicMock]: + """Replace ``prisma.Prisma`` and ``PrismaWrapper`` so PrismaClient.__init__ + runs without a generated client. Yields the fake Prisma instance. + + ``prisma`` raises RuntimeError (not AttributeError) for the missing + ``Prisma`` attribute, so ``monkeypatch.setattr`` can't probe it; assign + directly and restore in teardown. + """ + import prisma as _prisma_pkg + import litellm.proxy.utils as _utils_mod + + fake_prisma = MagicMock(name="FakePrisma") + fake_prisma.is_connected = MagicMock(return_value=False) + fake_prisma.connect = AsyncMock() + fake_prisma.disconnect = AsyncMock() + + fake_prisma_factory = MagicMock(name="FakePrismaFactory", return_value=fake_prisma) + had_prisma_attr = "Prisma" in _prisma_pkg.__dict__ + previous_prisma_attr = _prisma_pkg.__dict__.get("Prisma") + _prisma_pkg.Prisma = fake_prisma_factory # type: ignore[attr-defined] + + fake_wrapper = MagicMock(name="FakePrismaWrapper") + fake_wrapper.is_connected = MagicMock(return_value=False) + fake_wrapper.connect = AsyncMock() + fake_wrapper.disconnect = AsyncMock() + fake_wrapper.query_raw = AsyncMock(return_value=[{"?column?": 1}]) + + def _fake_wrapper_ctor(*args: Any, **kwargs: Any) -> MagicMock: + return fake_wrapper + + monkeypatch.setattr(_utils_mod, "PrismaWrapper", _fake_wrapper_ctor) + fake_prisma.__wrapper__ = fake_wrapper + try: + yield fake_prisma + finally: + if had_prisma_attr: + _prisma_pkg.Prisma = previous_prisma_attr # type: ignore[attr-defined] + else: + try: + del _prisma_pkg.Prisma # type: ignore[attr-defined] + except AttributeError: + pass + + +@pytest.fixture +def prisma_client( + patched_prisma_import: MagicMock, + mock_prisma_client: MagicMock, + monkeypatch: pytest.MonkeyPatch, +) -> Any: + """Wired ``PrismaClient`` whose ``db`` attribute is the table mock. + + The init runs through the real code path (testing the constructor's + config-attribute setup) and is then snapped to the easier-to-assert + table mock for downstream behavior pinning. + """ + monkeypatch.delenv("DATABASE_URL_READ_REPLICA", raising=False) + monkeypatch.delenv("IAM_TOKEN_DB_AUTH", raising=False) + from litellm.proxy.utils import PrismaClient + + proxy_logging_obj = MagicMock(name="MockProxyLogging") + proxy_logging_obj.failure_handler = AsyncMock() + pc = PrismaClient( + database_url="postgresql://test:test@localhost:5432/test", + proxy_logging_obj=proxy_logging_obj, + ) + pc.db = mock_prisma_client.db + return pc + + +@dataclass +class FakeClock: + """Monotonic-time controller for the spend monitor loop. + + Tests advance time via ``clock.advance(seconds)`` while asyncio.sleep + is replaced with a clock-driven no-op. + """ + + now: float = 0.0 + sleep_calls: List[float] = field(default_factory=list) + + def advance(self, seconds: float) -> None: + self.now += seconds + + def time(self) -> float: + return self.now + + async def sleep(self, seconds: float) -> None: + self.sleep_calls.append(seconds) + self.now += seconds + + +@pytest.fixture +def fake_clock(monkeypatch: pytest.MonkeyPatch) -> FakeClock: + """Install a controllable clock + asyncio.sleep replacement.""" + clock = FakeClock() + monkeypatch.setattr("time.time", clock.time) + monkeypatch.setattr("time.monotonic", clock.time) + + async def _fast_sleep(seconds: float, *_: Any, **__: Any) -> None: + clock.sleep_calls.append(seconds) + clock.now += seconds + + monkeypatch.setattr("asyncio.sleep", _fast_sleep) + return clock + + +@pytest.fixture +def make_spend_log_row() -> Callable[..., Dict[str, Any]]: + """Factory for fake LiteLLM_SpendLogs rows.""" + + def _make( + request_id: str = "req-1", + spend: float = 0.01, + model: str = "gpt-4o-mini", + **overrides: Any, + ) -> Dict[str, Any]: + row = { + "request_id": request_id, + "spend": spend, + "model": model, + "user": "user-1", + "team_id": "team-1", + "api_key": "hashed-key", + "startTime": "2026-06-02T00:00:00Z", + "endTime": "2026-06-02T00:00:01Z", + "metadata": {}, + } + row.update(overrides) + return row + + return _make + + +@dataclass +class _SentMessage: + from_addr: Optional[str] + to_addrs: Any + subject: Optional[str] + body: Optional[str] + starttls_called: bool + login_args: Optional[tuple] + + +@dataclass +class InMemorySMTP: + """Captures outbound SMTP traffic for ``send_email`` tests.""" + + sent: List[_SentMessage] = field(default_factory=list) + raise_on_send: Optional[Exception] = None + + def server_factory(self) -> Callable[..., Any]: + outer = self + + class _Conn: + def __init__(self) -> None: + self._starttls_called = False + self._login_args: Optional[tuple] = None + + def __enter__(self) -> "_Conn": + return self + + def __exit__(self, *exc: Any) -> None: + return None + + def starttls(self) -> None: + self._starttls_called = True + + def login(self, user: str, password: str) -> None: + self._login_args = (user, password) + + def send_message( + self, + msg: EmailMessage, + from_addr: Optional[str] = None, + to_addrs: Any = None, + ) -> None: + if outer.raise_on_send is not None: + raise outer.raise_on_send + body = "" + for part in msg.walk(): + if part.get_content_type() == "text/html": + body = part.get_payload(decode=False) or "" + break + outer.sent.append( + _SentMessage( + from_addr=from_addr, + to_addrs=to_addrs, + subject=msg["Subject"], + body=body, + starttls_called=self._starttls_called, + login_args=self._login_args, + ) + ) + + def _factory(*args: Any, **kwargs: Any) -> _Conn: + return _Conn() + + return _factory + + +@pytest.fixture +def in_memory_smtp(monkeypatch: pytest.MonkeyPatch) -> InMemorySMTP: + """Patch ``smtplib.SMTP`` to capture sends in memory. + + Override ``smtp.raise_on_send`` to test the SMTP error path. + """ + smtp = InMemorySMTP() + monkeypatch.setattr("smtplib.SMTP", smtp.server_factory()) + return smtp diff --git a/tests/test_litellm/proxy/utils/prisma_and_spend/test_prisma_client_get_data.py b/tests/test_litellm/proxy/utils/prisma_and_spend/test_prisma_client_get_data.py new file mode 100644 index 00000000000..7e7e98d1360 --- /dev/null +++ b/tests/test_litellm/proxy/utils/prisma_and_spend/test_prisma_client_get_data.py @@ -0,0 +1,400 @@ +"""Pin ``PrismaClient`` read-side data operations. + +Symbols pinned here: + - ``PrismaClient.hash_token`` + - ``PrismaClient.jsonify_object`` + - ``PrismaClient.jsonify_team_object`` + - ``PrismaClient.check_view_exists`` + - ``PrismaClient.get_request_status`` + - ``PrismaClient.get_generic_data`` + - ``PrismaClient._query_first_with_cached_plan_fallback`` + - ``PrismaClient.get_data`` +""" + +from __future__ import annotations + +import hashlib +import json +from types import SimpleNamespace +from typing import Any +from unittest.mock import AsyncMock, MagicMock + +import pytest +from fastapi import HTTPException + +from litellm.proxy.utils import PrismaClient + + +def test_hash_token_method_returns_sha256(prisma_client: PrismaClient) -> None: + token = "sk-token-xyz" + actual = { + "result": prisma_client.hash_token(token), + "len": len(prisma_client.hash_token(token)), + "expected": hashlib.sha256(token.encode()).hexdigest(), + "deterministic": prisma_client.hash_token(token) + == prisma_client.hash_token(token), + } + assert actual == { + "result": hashlib.sha256(token.encode()).hexdigest(), + "len": 64, + "expected": hashlib.sha256(token.encode()).hexdigest(), + "deterministic": True, + } + + +def test_hash_token_method_error_on_non_string(prisma_client: PrismaClient) -> None: + with pytest.raises(AttributeError): + prisma_client.hash_token(None) # type: ignore[arg-type] + + +def test_jsonify_object_serializes_nested_dicts(prisma_client: PrismaClient) -> None: + data = { + "metadata": {"a": 1, "b": [2, 3]}, + "models": ["gpt-4o", "gpt-4o-mini"], + "token": "abc", + "spend": 1.23, + } + result = prisma_client.jsonify_object(data) + parsed_meta = json.loads(result["metadata"]) + assert result == { + "metadata": json.dumps(data["metadata"]), + "models": ["gpt-4o", "gpt-4o-mini"], + "token": "abc", + "spend": 1.23, + } + assert parsed_meta == {"a": 1, "b": [2, 3]} + + +def test_jsonify_object_fallback_for_unserializable_dict( + prisma_client: PrismaClient, +) -> None: + class _Bad: + pass + + data = {"metadata": {"x": _Bad()}, "label": "ok", "n": 1} + result = prisma_client.jsonify_object(data) + assert result == { + "metadata": "failed-to-serialize-json", + "label": "ok", + "n": 1, + } + + +def test_jsonify_object_error_on_non_dict(prisma_client: PrismaClient) -> None: + with pytest.raises(AttributeError): + prisma_client.jsonify_object(None) # type: ignore[arg-type] + + +def test_jsonify_team_object_converts_members_to_json_string( + prisma_client: PrismaClient, +) -> None: + data = { + "team_id": "t1", + "members_with_roles": [{"role": "admin", "user_id": "u1"}], + "metadata": {"foo": "bar"}, + "models": ["gpt-4"], + } + result = prisma_client.jsonify_team_object(data) + assert result == { + "team_id": "t1", + "members_with_roles": json.dumps(data["members_with_roles"]), + "metadata": json.dumps(data["metadata"]), + "models": ["gpt-4"], + } + + +def test_jsonify_team_object_error_on_non_dict(prisma_client: PrismaClient) -> None: + with pytest.raises(AttributeError): + prisma_client.jsonify_team_object(None) # type: ignore[arg-type] + + +@pytest.mark.parametrize( + "metadata,expected", + [ + ({"status": "failure"}, "failure"), + ({"status": "success"}, "success"), + ({}, "success"), + ("not-json", "success"), + (json.dumps({"status": "failure"}), "failure"), + ], +) +def test_get_request_status_pins_status_resolution( + prisma_client: PrismaClient, metadata: Any, expected: str +) -> None: + assert prisma_client.get_request_status({"metadata": metadata}) == expected + + +def test_get_request_status_error_returns_success_default( + prisma_client: PrismaClient, +) -> None: + """``get_request_status`` swallows AttributeError / JSONDecodeError and + defaults to ``success`` to avoid blocking the request pipeline. + """ + + class _Broken: + def get(self, *_: Any, **__: Any) -> Any: + raise AttributeError("broken metadata") + + actual = prisma_client.get_request_status({"metadata": _Broken()}) + assert actual == "success" + + +@pytest.mark.asyncio +async def test_get_generic_data_dispatches_by_table( + prisma_client: PrismaClient, +) -> None: + row = SimpleNamespace(user_id="u1", spend=0.5, name="Alice") + prisma_client.db.litellm_usertable.find_first = AsyncMock(return_value=row) + result = await prisma_client.get_generic_data( + key="user_id", value="u1", table_name="users" + ) + actual = { + "result_is_row": result is row, + "find_first_count": prisma_client.db.litellm_usertable.find_first.await_count, + "where_kwarg": prisma_client.db.litellm_usertable.find_first.await_args.kwargs[ + "where" + ], + "user_attr": result.user_id, + } + assert actual == { + "result_is_row": True, + "find_first_count": 1, + "where_kwarg": {"user_id": "u1"}, + "user_attr": "u1", + } + + +@pytest.mark.asyncio +async def test_get_generic_data_unknown_table_returns_none( + prisma_client: PrismaClient, +) -> None: + result = await prisma_client.get_generic_data( + key="x", value="y", table_name="bogus" # type: ignore[arg-type] + ) + assert result is None + + +@pytest.mark.asyncio +async def test_get_generic_data_logs_failure_handler_and_raises_on_error( + prisma_client: PrismaClient, +) -> None: + prisma_client.db.litellm_usertable.find_first = AsyncMock( + side_effect=RuntimeError("db boom") + ) + with pytest.raises(RuntimeError, match="db boom"): + await prisma_client.get_generic_data( + key="user_id", value="x", table_name="users" + ) + + +@pytest.mark.asyncio +async def test_query_first_with_cached_plan_fallback_happy_returns_row( + prisma_client: PrismaClient, +) -> None: + expected = {"token": "abc", "team_spend": 1.0, "team_max_budget": 5.0} + prisma_client.db.query_first = AsyncMock(return_value=expected) + result = await prisma_client._query_first_with_cached_plan_fallback( + "SELECT * FROM x WHERE token = $1", "abc" + ) + actual = { + "result": result, + "call_count": prisma_client.db.query_first.await_count, + "args": prisma_client.db.query_first.await_args.args, + "matches": result == expected, + } + assert actual == { + "result": expected, + "call_count": 1, + "args": ("SELECT * FROM x WHERE token = $1", "abc"), + "matches": True, + } + + +@pytest.mark.asyncio +async def test_query_first_with_cached_plan_fallback_retries_on_cached_plan_error( + prisma_client: PrismaClient, +) -> None: + expected = {"token": "abc", "team_spend": 1.0, "team_max_budget": 5.0} + prisma_client.db.query_first = AsyncMock( + side_effect=[ + RuntimeError("cached plan must not change result type"), + expected, + ] + ) + result = await prisma_client._query_first_with_cached_plan_fallback( + "SELECT * FROM x WHERE token = $1", "abc" + ) + assert result == expected + assert prisma_client.db.query_first.await_count == 2 + second_call_sql = prisma_client.db.query_first.await_args_list[1].args[0] + assert "cache_invalidated_" in second_call_sql + + +@pytest.mark.asyncio +async def test_query_first_with_cached_plan_fallback_reraises_non_plan_errors( + prisma_client: PrismaClient, +) -> None: + prisma_client.db.query_first = AsyncMock(side_effect=RuntimeError("totally unrelated")) + with pytest.raises(RuntimeError, match="totally unrelated"): + await prisma_client._query_first_with_cached_plan_fallback("SELECT 1") + + +@pytest.mark.asyncio +async def test_check_view_exists_noop_when_all_views_present( + prisma_client: PrismaClient, +) -> None: + prisma_client.db.query_raw = AsyncMock( + return_value=[ + { + "view_count": 8, + "view_names": [ + "LiteLLM_VerificationTokenView", + "MonthlyGlobalSpend", + "Last30dKeysBySpend", + "Last30dModelsBySpend", + "MonthlyGlobalSpendPerKey", + "MonthlyGlobalSpendPerUserPerKey", + "Last30dTopEndUsersSpend", + "DailyTagSpend", + ], + } + ] + ) + prisma_client.db.execute_raw = AsyncMock() + result = await prisma_client.check_view_exists() + actual = { + "result": result, + "query_raw_calls": prisma_client.db.query_raw.await_count, + "execute_raw_calls": prisma_client.db.execute_raw.await_count, + "view_query_contains_token_view": "LiteLLM_VerificationTokenView" + in prisma_client.db.query_raw.await_args.args[0], + } + assert actual == { + "result": None, + "query_raw_calls": 1, + "execute_raw_calls": 0, + "view_query_contains_token_view": True, + } + + +@pytest.mark.asyncio +async def test_check_view_exists_creates_token_view_when_missing( + prisma_client: PrismaClient, +) -> None: + prisma_client.db.query_raw = AsyncMock( + return_value=[ + { + "view_count": 1, + "view_names": ["DailyTagSpend"], + } + ] + ) + prisma_client.db.execute_raw = AsyncMock() + prisma_client.health_check = AsyncMock(return_value=[{"?column?": 1}]) + result = await prisma_client.check_view_exists() + actual = { + "result": result, + "create_called": prisma_client.db.execute_raw.await_count, + "create_sql_starts_with_create_view": prisma_client.db.execute_raw.await_args.args[ + 0 + ] + .strip() + .startswith('CREATE VIEW "LiteLLM_VerificationTokenView"'), + } + assert actual == { + "result": None, + "create_called": 1, + "create_sql_starts_with_create_view": True, + } + + +@pytest.mark.asyncio +async def test_check_view_exists_raises_when_query_raw_fails( + prisma_client: PrismaClient, +) -> None: + prisma_client.db.query_raw = AsyncMock(side_effect=RuntimeError("db down")) + with pytest.raises(RuntimeError, match="db down"): + await prisma_client.check_view_exists() + + +@pytest.mark.asyncio +async def test_get_data_token_find_unique_returns_record( + prisma_client: PrismaClient, +) -> None: + token = "sk-key-1" + hashed = hashlib.sha256(token.encode()).hexdigest() + record = SimpleNamespace(token=hashed, user_id="u1", expires=None, spend=0.5) + prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + return_value=record + ) + + result = await prisma_client.get_data(token=token, table_name="key") + actual = { + "result_is_record": result is record, + "where_arg": prisma_client.db.litellm_verificationtoken.find_unique.await_args.kwargs[ + "where" + ], + "include_arg": prisma_client.db.litellm_verificationtoken.find_unique.await_args.kwargs[ + "include" + ], + "token_field_matches": result.token == hashed, + } + assert actual == { + "result_is_record": True, + "where_arg": {"token": hashed}, + "include_arg": {"litellm_budget_table": True}, + "token_field_matches": True, + } + + +@pytest.mark.asyncio +async def test_get_data_token_find_unique_missing_token_raises_401( + prisma_client: PrismaClient, +) -> None: + prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=None) + with pytest.raises(HTTPException) as excinfo: + await prisma_client.get_data(token="sk-missing", table_name="key") + err = excinfo.value + assert "invalid user key" in err.detail + assert err.status_code == 401 + + +@pytest.mark.asyncio +async def test_get_data_user_find_unique_returns_user_row( + prisma_client: PrismaClient, +) -> None: + row = SimpleNamespace( + user_id="u-7", + spend=1.5, + max_budget=10.0, + organization_memberships=[], + ) + prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=row) + result = await prisma_client.get_data(user_id="u-7", table_name="user") + actual = { + "result_is_row": result is row, + "where_arg": prisma_client.db.litellm_usertable.find_unique.await_args.kwargs[ + "where" + ], + "include_arg": prisma_client.db.litellm_usertable.find_unique.await_args.kwargs[ + "include" + ], + "spend": row.spend, + } + assert actual == { + "result_is_row": True, + "where_arg": {"user_id": "u-7"}, + "include_arg": {"organization_memberships": True}, + "spend": 1.5, + } + + +@pytest.mark.asyncio +async def test_get_data_logs_and_raises_on_db_error( + prisma_client: PrismaClient, +) -> None: + prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + side_effect=RuntimeError("network split") + ) + with pytest.raises(RuntimeError, match="network split"): + await prisma_client.get_data(token="sk-broken", table_name="key") From c4a7b27e9185f2759a64d03b9b20f88e7c33fe8c Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Wed, 10 Jun 2026 16:06:01 -0700 Subject: [PATCH 03/12] fix(proxy): recover from cached-plan errors by reconnecting the Prisma client (#29983) (cherry picked from commit 3bd3951e37a0b3201eac9eb1f858d2253aff56e5) --- litellm/proxy/utils.py | 59 +++++++----- .../test_prisma_client_get_data.py | 92 +++++++++++++++++-- 2 files changed, 119 insertions(+), 32 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 0e72f47e224..157303689d8 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -3159,40 +3159,49 @@ class PrismaClient: self, sql_query: str, *args ) -> Optional[dict]: """ - Execute a query with automatic fallback for PostgreSQL cached plan errors. + Execute a query, recovering once from PostgreSQL's "cached plan must not + change result type" error. - This handles the "cached plan must not change result type" error that occurs - during rolling deployments when schema changes are applied while old pods - still have cached query plans expecting the old schema. + That error surfaces during rolling deployments when a schema change + invalidates the prepared-statement plans that pooled connections still + hold. Clearing only the server-side plans with DEALLOCATE ALL makes + things worse: Prisma's query engine keeps a per-connection client-side + cache of prepared-statement names, so once the server drops a plan the + engine re-sends a name PostgreSQL no longer recognizes and the + connection breaks with `prepared statement "sN" does not exist`. With a + small pool that connection stays poisoned and every auth lookup fails. - Args: - sql_query: SQL query string to execute + Recreating the Prisma client kills the engine subprocess and drops the + server-side plans and the engine's client-side name cache together, so + the retried query is prepared fresh. We reconnect through + `attempt_db_reconnect`, which is singleflight: when a schema change + poisons every pooled connection at once, the first cached-plan error + recreates the client and the concurrent waiters reuse that single + recreate instead of racing to kill each other's fresh engine. We then + retry the identical query exactly once. - Returns: - Query result or None + The retry reuses the original query byte-for-byte. Mutating the SQL + (e.g. injecting a unique comment) would defeat PostgreSQL's plan cache, + forcing a fresh plan on every request and pegging the database CPU. - Raises: - Original exception if not a cached plan error + If the reconnect is skipped because a recent reconnect is still within + its cooldown, the retry runs against the same connection and may fail + again; the get_data backoff decorator re-runs the lookup and a later + attempt reconnects once the cooldown elapses. """ try: return await self.db.query_first(sql_query, *args) except Exception as e: - error_str = str(e) - if "cached plan must not change result type" in error_str: - # Force PostgreSQL to re-plan by invalidating the cache - # Add a unique comment to make the query different - sql_query_retry = sql_query.replace( - "SELECT", - f"SELECT /* cache_invalidated_{int(time.time() * 1000)} */", - ) - verbose_proxy_logger.warning( - "PostgreSQL cached plan error detected for token lookup, " - "retrying with fresh plan. This may occur during rolling deployments " - "when schema changes are applied." - ) - return await self.db.query_first(sql_query_retry, *args) - else: + if "cached plan must not change result type" not in str(e): raise + verbose_proxy_logger.warning( + "PostgreSQL cached plan error detected for token lookup; " + "recreating the database connection and retrying with the same " + "query. This may occur during rolling deployments when schema " + "changes are applied." + ) + await self.attempt_db_reconnect(reason="postgres_cached_plan_error") + return await self.db.query_first(sql_query, *args) @backoff.on_exception( backoff.expo, diff --git a/tests/test_litellm/proxy/utils/prisma_and_spend/test_prisma_client_get_data.py b/tests/test_litellm/proxy/utils/prisma_and_spend/test_prisma_client_get_data.py index 7e7e98d1360..437984d9273 100644 --- a/tests/test_litellm/proxy/utils/prisma_and_spend/test_prisma_client_get_data.py +++ b/tests/test_litellm/proxy/utils/prisma_and_spend/test_prisma_client_get_data.py @@ -193,6 +193,7 @@ async def test_query_first_with_cached_plan_fallback_happy_returns_row( ) -> None: expected = {"token": "abc", "team_spend": 1.0, "team_max_budget": 5.0} prisma_client.db.query_first = AsyncMock(return_value=expected) + prisma_client.attempt_db_reconnect = AsyncMock(return_value=True) result = await prisma_client._query_first_with_cached_plan_fallback( "SELECT * FROM x WHERE token = $1", "abc" ) @@ -208,35 +209,110 @@ async def test_query_first_with_cached_plan_fallback_happy_returns_row( "args": ("SELECT * FROM x WHERE token = $1", "abc"), "matches": True, } + prisma_client.attempt_db_reconnect.assert_not_awaited() @pytest.mark.asyncio -async def test_query_first_with_cached_plan_fallback_retries_on_cached_plan_error( +async def test_query_first_with_cached_plan_fallback_reconnects_then_retries_identical_query( prisma_client: PrismaClient, ) -> None: + original_query = 'SELECT * FROM "LiteLLM_VerificationToken" WHERE v.token = $1' expected = {"token": "abc", "team_spend": 1.0, "team_max_budget": 5.0} + manager = MagicMock() + query_first = AsyncMock( + side_effect=[ + RuntimeError("cached plan must not change result type"), + expected, + ] + ) + reconnect = AsyncMock(return_value=True) + manager.attach_mock(query_first, "query_first") + manager.attach_mock(reconnect, "attempt_db_reconnect") + prisma_client.db.query_first = query_first + prisma_client.attempt_db_reconnect = reconnect + + result = await prisma_client._query_first_with_cached_plan_fallback( + original_query, "abc" + ) + + assert result == expected + assert query_first.await_count == 2 + first_call, retry_call = query_first.await_args_list + assert retry_call.args == first_call.args == (original_query, "abc") + reconnect.assert_awaited_once() + assert reconnect.await_args.kwargs.get("force", False) is False + assert [name for name, *_ in manager.mock_calls] == [ + "query_first", + "attempt_db_reconnect", + "query_first", + ] + + +@pytest.mark.asyncio +async def test_query_first_with_cached_plan_fallback_never_deallocates( + prisma_client: PrismaClient, +) -> None: + expected = {"token": "abc"} prisma_client.db.query_first = AsyncMock( side_effect=[ RuntimeError("cached plan must not change result type"), expected, ] ) - result = await prisma_client._query_first_with_cached_plan_fallback( - "SELECT * FROM x WHERE token = $1", "abc" + prisma_client.db.execute_raw = AsyncMock(return_value=0) + prisma_client.attempt_db_reconnect = AsyncMock(return_value=True) + + await prisma_client._query_first_with_cached_plan_fallback("SELECT 1") + + prisma_client.db.execute_raw.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_query_first_with_cached_plan_fallback_propagates_when_retry_also_fails( + prisma_client: PrismaClient, +) -> None: + plan_error = RuntimeError("cached plan must not change result type") + prisma_client.db.query_first = AsyncMock(side_effect=[plan_error, plan_error]) + prisma_client.attempt_db_reconnect = AsyncMock(return_value=True) + + with pytest.raises(RuntimeError, match="cached plan must not change result type"): + await prisma_client._query_first_with_cached_plan_fallback("SELECT 1") + + assert prisma_client.db.query_first.await_count == 2 + prisma_client.attempt_db_reconnect.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_query_first_with_cached_plan_fallback_retries_when_reconnect_returns_false( + prisma_client: PrismaClient, +) -> None: + expected = {"token": "abc"} + prisma_client.db.query_first = AsyncMock( + side_effect=[ + RuntimeError("cached plan must not change result type"), + expected, + ] ) + prisma_client.attempt_db_reconnect = AsyncMock(return_value=False) + + result = await prisma_client._query_first_with_cached_plan_fallback("SELECT 1") + assert result == expected assert prisma_client.db.query_first.await_count == 2 - second_call_sql = prisma_client.db.query_first.await_args_list[1].args[0] - assert "cache_invalidated_" in second_call_sql @pytest.mark.asyncio async def test_query_first_with_cached_plan_fallback_reraises_non_plan_errors( prisma_client: PrismaClient, ) -> None: - prisma_client.db.query_first = AsyncMock(side_effect=RuntimeError("totally unrelated")) + prisma_client.db.query_first = AsyncMock( + side_effect=RuntimeError("totally unrelated") + ) + prisma_client.attempt_db_reconnect = AsyncMock(return_value=True) with pytest.raises(RuntimeError, match="totally unrelated"): await prisma_client._query_first_with_cached_plan_fallback("SELECT 1") + assert prisma_client.db.query_first.await_count == 1 + prisma_client.attempt_db_reconnect.assert_not_awaited() @pytest.mark.asyncio @@ -351,7 +427,9 @@ async def test_get_data_token_find_unique_returns_record( async def test_get_data_token_find_unique_missing_token_raises_401( prisma_client: PrismaClient, ) -> None: - prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=None) + prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + return_value=None + ) with pytest.raises(HTTPException) as excinfo: await prisma_client.get_data(token="sk-missing", table_name="key") err = excinfo.value From f8c22cdd8a9eef85d7f48a638a076f42d40675c7 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Wed, 10 Jun 2026 16:06:32 -0700 Subject: [PATCH 04/12] feat(proxy): add option to disable server-side prepared statements for DB lookups (#29984) Adaptation: the generated ui/litellm-dashboard/src/lib/http/schema.d.ts hunk is dropped; that file is absent on this line and is regenerated at UI build, with no Python dependency. Source changes apply verbatim. (cherry picked from commit dff25fef449bc3e2051ee2638e323d09d05a850a) --- litellm/proxy/_types.py | 11 ++ litellm/proxy/proxy_cli.py | 25 ++++- tests/test_litellm/proxy/test_proxy_cli.py | 121 +++++++++++++++++++++ 3 files changed, 154 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index c80455cabb6..aeb52a3d239 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2397,6 +2397,17 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): "`statement_cache_size`). Keys here override any default LiteLLM sets." ), ) + database_disable_prepared_statements: Optional[bool] = Field( + None, + description=( + "Disable server-side prepared statements by setting Prisma's " + "`pgbouncer=true` URL param. Use this for pgbouncer transaction-pooling " + "deployments, or to prevent the 'cached plan must not change result " + "type' error that pooled connections hit during rolling schema " + "migrations. An explicit `pgbouncer` in `database_extra_connection_params` " + "takes precedence." + ), + ) database_type: Optional[Literal["dynamo_db"]] = Field( None, description="to use dynamodb instead of postgres db" ) diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index c0246f234a8..0d60806da54 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -44,15 +44,19 @@ def _build_db_connection_url_params( pool_timeout: Optional[Union[int, float]], connect_timeout: Optional[Union[int, float]] = None, socket_timeout: Optional[Union[int, float]] = None, + disable_prepared_statements: bool = False, extra_params: Optional[dict] = None, ) -> dict: """Build the Prisma DATABASE_URL query params controlling connection pool behavior. `connect_timeout` / `socket_timeout` map to the Prisma URL params of the same name (https://www.prisma.io/docs/orm/overview/databases/postgresql) and are - omitted when None so Prisma's defaults apply. `extra_params` is an - untyped passthrough — keys it provides win over the named arguments above, - so it can be used to override any default we set here. + omitted when None so Prisma's defaults apply. `disable_prepared_statements` + sets `pgbouncer=true`, which makes Prisma stop using server-side prepared + statements (pgbouncer transaction-pool compatible; also sidesteps the + "cached plan must not change result type" error during rolling migrations). + `extra_params` is an untyped passthrough — keys it provides win over the + named arguments above, so it can be used to override any default we set here. """ params: dict = { "connection_limit": connection_limit, @@ -63,6 +67,8 @@ def _build_db_connection_url_params( params["connect_timeout"] = connect_timeout if socket_timeout is not None: params["socket_timeout"] = socket_timeout + if disable_prepared_statements: + params["pgbouncer"] = "true" if extra_params: params.update(extra_params) return params @@ -925,6 +931,7 @@ def run_server( # noqa: PLR0915 db_connection_timeout: Optional[Union[int, float]] = 60 db_connect_timeout: Optional[Union[int, float]] = None db_socket_timeout: Optional[Union[int, float]] = None + db_disable_prepared_statements: bool = False db_extra_connection_params: Optional[dict] = None general_settings = {} ### GET DB TOKEN FOR IAM AUTH ### @@ -1045,6 +1052,17 @@ def run_server( # noqa: PLR0915 ) db_connect_timeout = general_settings.get("database_connect_timeout") db_socket_timeout = general_settings.get("database_socket_timeout") + _disable_prepared_statements = general_settings.get( + "database_disable_prepared_statements", False + ) + if isinstance(_disable_prepared_statements, str): + from litellm.secret_managers.main import str_to_bool + + db_disable_prepared_statements = ( + str_to_bool(_disable_prepared_statements) is True + ) + else: + db_disable_prepared_statements = bool(_disable_prepared_statements) db_extra_connection_params = general_settings.get( "database_extra_connection_params" ) @@ -1092,6 +1110,7 @@ def run_server( # noqa: PLR0915 pool_timeout=db_connection_timeout, connect_timeout=db_connect_timeout, socket_timeout=db_socket_timeout, + disable_prepared_statements=db_disable_prepared_statements, extra_params=db_extra_connection_params, ) if os.getenv("DATABASE_URL", None) is not None: diff --git a/tests/test_litellm/proxy/test_proxy_cli.py b/tests/test_litellm/proxy/test_proxy_cli.py index 580ed95062b..3b3b4387bc7 100644 --- a/tests/test_litellm/proxy/test_proxy_cli.py +++ b/tests/test_litellm/proxy/test_proxy_cli.py @@ -707,6 +707,127 @@ class TestProxyInitializationHelpers: assert appended_params["pgbouncer"] == "true" assert appended_params["statement_cache_size"] == 0 + def test_build_db_connection_url_params_disable_prepared_statements(self): + from litellm.proxy.proxy_cli import _build_db_connection_url_params + + params = _build_db_connection_url_params( + connection_limit=10, + pool_timeout=60, + disable_prepared_statements=True, + ) + assert params["pgbouncer"] == "true" + + def test_build_db_connection_url_params_no_pgbouncer_by_default(self): + from litellm.proxy.proxy_cli import _build_db_connection_url_params + + params = _build_db_connection_url_params( + connection_limit=10, + pool_timeout=60, + ) + assert "pgbouncer" not in params + + def test_build_db_connection_url_params_extra_pgbouncer_overrides_flag(self): + from litellm.proxy.proxy_cli import _build_db_connection_url_params + + params = _build_db_connection_url_params( + connection_limit=10, + pool_timeout=60, + disable_prepared_statements=True, + extra_params={"pgbouncer": "false"}, + ) + assert params["pgbouncer"] == "false" + + @pytest.mark.parametrize( + "config_value, expect_pgbouncer", + [ + (True, True), + (False, False), + ("true", True), + ("false", False), + ("not-a-bool", False), + ], + ) + @patch("subprocess.run") + @patch("atexit.register") + @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") + @patch( + "litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False + ) + def test_disable_prepared_statements_forwarded_to_url( + self, + mock_should_update, + mock_setup_db, + mock_atexit_register, + mock_subprocess_run, + config_value, + expect_pgbouncer, + ): + from click.testing import CliRunner + + from litellm.proxy.proxy_cli import run_server + + runner = CliRunner() + mock_subprocess_run.return_value = MagicMock(returncode=0) + + mock_proxy_module = MagicMock( + app=MagicMock(), + ProxyConfig=MagicMock(), + KeyManagementSettings=MagicMock(), + save_worker_config=MagicMock(), + ) + mock_proxy_module.ProxyConfig.return_value.get_config = AsyncMock( + return_value={ + "general_settings": { + "database_url": "postgresql://test:test@localhost:5432/test", + "database_disable_prepared_statements": config_value, + } + } + ) + + clean_env = { + k: v + for k, v in os.environ.items() + if k not in ("DATABASE_URL", "DIRECT_URL") + } + + with ( + patch.dict(os.environ, clean_env, clear=True), + patch.dict( + "sys.modules", + { + "proxy_server": mock_proxy_module, + "litellm.proxy.proxy_server": mock_proxy_module, + }, + ), + patch( + "litellm.proxy.proxy_cli.ProxyInitializationHelpers._get_default_unvicorn_init_args" + ) as mock_get_args, + patch( + "litellm.proxy.proxy_cli.append_query_params", + side_effect=lambda url, params: str(url), + ) as mock_append_query_params, + ): + mock_get_args.return_value = { + "app": "litellm.proxy.proxy_server:app", + "host": "localhost", + "port": 8000, + } + + result = runner.invoke( + run_server, + ["--local", "--config", "test-config.yaml", "--skip_server_startup"], + ) + + assert ( + result.exit_code == 0 + ), f"exit_code={result.exit_code}, output={result.output}" + mock_append_query_params.assert_called() + appended_params = mock_append_query_params.call_args.args[1] + if expect_pgbouncer: + assert appended_params["pgbouncer"] == "true" + else: + assert "pgbouncer" not in appended_params + @patch("uvicorn.run") @patch("atexit.register") @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") From c58c59e6b36c5b4d4c0799bfd10aff91fcc01b46 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Wed, 10 Jun 2026 16:48:11 -0700 Subject: [PATCH 05/12] fix(proxy): return 5xx on DB infra errors during auth; reserve 401 for genuine auth failures (#29986) Adaptation: this line's auth_exception_handler.py does not import seed_request_identity. The OTEL request-identity feature has its call site in user_api_key_auth.py here; the second call site staging added in the exception handler is not on this line. The picked tests mock auth_exception_handler.seed_request_identity to keep the failure path off OTEL, so those four patch entries (three in test_auth_exception_handler.py, one in the test_user_api_key_auth.py builder helper) are removed because the symbol is absent and the handler never calls it here. The 503/401 behavioral assertions are unchanged and pass. Separately, the one-line context hunk that reformats an assert inside test_auto_register_passes_validated_org_context_to_generated_key is dropped; that test is not present on this line. Source files apply verbatim. (cherry picked from commit da9d64b4de4b6927d3496f89fa402490a98bfb10) --- litellm/proxy/auth/auth_exception_handler.py | 10 + litellm/proxy/db/exception_handler.py | 86 ++++++++ .../proxy/auth/test_auth_exception_handler.py | 151 ++++++++++++++ .../proxy/auth/test_user_api_key_auth.py | 111 ++++++++++ .../proxy/db/test_exception_handler.py | 195 ++++++++++++++++++ 5 files changed, 553 insertions(+) diff --git a/litellm/proxy/auth/auth_exception_handler.py b/litellm/proxy/auth/auth_exception_handler.py index 431db4254eb..f276a1350f1 100644 --- a/litellm/proxy/auth/auth_exception_handler.py +++ b/litellm/proxy/auth/auth_exception_handler.py @@ -134,6 +134,16 @@ class UserAPIKeyAuthExceptionHandler: ) elif isinstance(e, ProxyException): raise e + if PrismaDBExceptionHandler.is_database_service_unavailable_error(e): + raise ProxyException( + message=( + "Service Unavailable, the authentication database is " + "temporarily unreachable. Please retry shortly." + ), + type=ProxyErrorTypes.no_db_connection, + param="None", + code=status.HTTP_503_SERVICE_UNAVAILABLE, + ) raise ProxyException( message="Authentication Error, " + str(e), type=ProxyErrorTypes.auth_error, diff --git a/litellm/proxy/db/exception_handler.py b/litellm/proxy/db/exception_handler.py index ab9d341aa51..c500e727595 100644 --- a/litellm/proxy/db/exception_handler.py +++ b/litellm/proxy/db/exception_handler.py @@ -109,6 +109,92 @@ class PrismaDBExceptionHandler: return True return False + @staticmethod + def is_prisma_engine_internal_error(e: Exception) -> bool: + """True iff ``e`` is a non-``PrismaError`` exception raised from inside + prisma-client-py's query-engine layer. + + During the instant a DB connection is torn down, the query engine can + return a malformed error payload (``user_facing_error.meta`` is + ``null``). prisma-client-py's ``handle_response_errors`` then crashes + with ``AttributeError: 'NoneType' object has no attribute 'get'`` + before it can raise the proper P1001 "can't reach database server" + error. That AttributeError carries no connection keyword, so it can't + be matched by message; identify it by its ``prisma.engine`` origin + instead. + + Recognized ``PrismaError`` subclasses are excluded: connectivity ones + are already classified by type/keyword above, and data-layer ones + (the DB IS reachable) must stay 401. + """ + import prisma + + if isinstance(e, prisma.errors.PrismaError): + return False + tb = getattr(e, "__traceback__", None) + while tb is not None: + if tb.tb_frame.f_globals.get("__name__", "").startswith("prisma.engine"): + return True + tb = tb.tb_next + return False + + @staticmethod + def is_database_service_unavailable_error(e: Exception) -> bool: + """True iff the exception means the database could not answer at the + infrastructure level (connection refused, socket/interface failure, + timeout) rather than a genuine auth failure (key not found) or a + data-layer error (the DB IS reachable and rejected the data). + + Auth must answer 401 only for a key the DB confirms is invalid. When + the DB itself is unreachable, the request has to surface as 503 so + callers retry instead of treating valid keys as invalid during an + outage. + + Note: prisma-client-py mislabels the P1001 "can't reach database + server" connectivity failure as a ``DataError`` (a data-layer type), + so a type-only check misses real outages. ``is_database_transport_error`` + keyword-matches the connection message and catches that masquerade, + while genuine data errors (no connection keyword) correctly stay 401. + + The Postgres "cached plan must not change result type" error is matched + here, not in ``is_database_transport_error``: it is a transient stale-DB- + state condition (not an invalid key), but the connection is healthy so it + must not trigger a reconnect. + + A non-``PrismaError`` raised from inside the prisma query engine (e.g. + the ``AttributeError`` from ``handle_response_errors`` when the engine + returns a malformed error payload mid-tear-down) is also treated as + unavailable; see ``is_prisma_engine_internal_error``. + """ + import asyncio + + if PrismaDBExceptionHandler.is_database_connection_error(e): + return True + if PrismaDBExceptionHandler.is_database_transport_error(e): + return True + if PrismaDBExceptionHandler.is_prisma_engine_internal_error(e): + return True + if "cached plan must not change result type" in str(e).lower(): + return True + + # OSError already covers ConnectionError and (Py3.3+) TimeoutError. + # asyncio.TimeoutError is a distinct class before Py3.11. + if isinstance(e, (OSError, asyncio.TimeoutError)): + return True + + try: + import asyncpg + except ImportError: + return False + + return isinstance( + e, + ( + asyncpg.exceptions.PostgresConnectionError, + asyncpg.exceptions.InterfaceError, + ), + ) + @staticmethod def handle_db_exception(e: Exception): """ diff --git a/tests/test_litellm/proxy/auth/test_auth_exception_handler.py b/tests/test_litellm/proxy/auth/test_auth_exception_handler.py index 4ccde85dae2..fae315fb474 100644 --- a/tests/test_litellm/proxy/auth/test_auth_exception_handler.py +++ b/tests/test_litellm/proxy/auth/test_auth_exception_handler.py @@ -112,6 +112,157 @@ async def test_handle_authentication_error_data_layer_errors_do_not_fall_back( ) +@pytest.mark.asyncio +@pytest.mark.parametrize( + "db_error", + [ + ConnectionError("connection refused"), + TimeoutError("timed out"), + asyncio.TimeoutError(), + OSError("network is unreachable"), + HTTPClientClosedError(), + PrismaError("can't reach database server"), + RawQueryError( + data={ + "user_facing_error": { + "message": "cached plan must not change result type", + "meta": {"table": "t"}, + } + } + ), + ], +) +async def test_handle_authentication_error_db_infra_error_returns_503(db_error): + """Regression for the outage where valid keys got 401 for 4 hours: an + infrastructure-level DB failure during auth must surface as 503 (the DB + could not confirm the key), never as 401 ("Invalid API key").""" + handler = UserAPIKeyAuthExceptionHandler() + + with ( + patch( + "litellm.proxy.proxy_server.proxy_logging_obj.post_call_failure_hook", + new_callable=AsyncMock, + return_value=None, + ), + patch( + "litellm.proxy.proxy_server.general_settings", + {"allow_requests_on_db_unavailable": False}, + ), + ): + with pytest.raises(ProxyException) as exc_info: + await handler._handle_authentication_error( + db_error, + MagicMock(), + {}, + "/v1/chat/completions", + None, + "sk-valid-but-db-down", + ) + + assert int(exc_info.value.code) == status.HTTP_503_SERVICE_UNAVAILABLE + assert exc_info.value.type == ProxyErrorTypes.no_db_connection + assert "Invalid API key" not in str(exc_info.value.message) + + +@pytest.mark.asyncio +async def test_handle_authentication_error_prisma_engine_teardown_returns_503(): + """Regression for the first-request-of-an-outage edge case: at the instant + the DB socket drops, the prisma query engine returns a malformed error + payload and prisma-client-py crashes with a bare + ``AttributeError: 'NoneType' object has no attribute 'get'`` before it can + raise P1001. That AttributeError reached auth and fell through to 401. It + must surface as 503 like every other infra failure during the outage.""" + from prisma.engine import utils as prisma_engine_utils + + malformed_payload = [ + { + "error": "Can't reach database server", + "user_facing_error": { + "error_code": "P1001", + "message": "Can't reach database server at `localhost`:`5503`", + "meta": None, + }, + } + ] + try: + prisma_engine_utils.handle_response_errors(None, malformed_payload) + raise AssertionError("expected prisma to raise AttributeError") + except AttributeError as e: + teardown_error = e + + handler = UserAPIKeyAuthExceptionHandler() + + with ( + patch( + "litellm.proxy.proxy_server.proxy_logging_obj.post_call_failure_hook", + new_callable=AsyncMock, + return_value=None, + ), + patch( + "litellm.proxy.proxy_server.general_settings", + {"allow_requests_on_db_unavailable": False}, + ), + ): + with pytest.raises(ProxyException) as exc_info: + await handler._handle_authentication_error( + teardown_error, + MagicMock(), + {}, + "/v1/chat/completions", + None, + "sk-valid-but-db-down", + ) + + assert int(exc_info.value.code) == status.HTTP_503_SERVICE_UNAVAILABLE + assert exc_info.value.type == ProxyErrorTypes.no_db_connection + assert "Invalid API key" not in str(exc_info.value.message) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "auth_error", + [ + # DB returned no row -> get_key_object raises this exact 401. + ProxyException( + message="Authentication Error, Invalid proxy server token passed.", + type=ProxyErrorTypes.token_not_found_in_db, + param="key", + code=status.HTTP_401_UNAUTHORIZED, + ), + # A bare auth failure raised as a plain Exception (e.g. master-key-only + # route) must keep returning 401, not get reclassified as 503. + Exception("Invalid proxy server token passed"), + ], +) +async def test_handle_authentication_error_genuine_auth_failure_stays_401(auth_error): + """Guard against the 503 conversion being too broad: a genuine auth + failure (missing key / wrong key) must still be 401.""" + handler = UserAPIKeyAuthExceptionHandler() + + with ( + patch( + "litellm.proxy.proxy_server.proxy_logging_obj.post_call_failure_hook", + new_callable=AsyncMock, + return_value=None, + ), + patch( + "litellm.proxy.proxy_server.general_settings", + {"allow_requests_on_db_unavailable": False}, + ), + ): + with pytest.raises(ProxyException) as exc_info: + await handler._handle_authentication_error( + auth_error, + MagicMock(), + {}, + "/v1/chat/completions", + None, + "sk-bad-key", + ) + + assert int(exc_info.value.code) == status.HTTP_401_UNAUTHORIZED + + @pytest.mark.asyncio async def test_handle_authentication_error_budget_exceeded(): handler = UserAPIKeyAuthExceptionHandler() diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index f6848dac6fd..37f34f82afa 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -3519,3 +3519,114 @@ async def test_user_api_key_auth_does_not_overwrite_end_user_id_set_by_builder() finally: for k, v in originals.items(): setattr(_proxy_server_mod, k, v) + + +def _proxy_attrs_for_db_lookup(): + """Minimal proxy_server attributes for driving the real + ``_user_api_key_auth_builder`` down to the DB key lookup.""" + proxy_logging_obj = MagicMock() + proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None) + return { + "prisma_client": MagicMock(), + "user_api_key_cache": DualCache(), + "proxy_logging_obj": proxy_logging_obj, + "master_key": "sk-test-master", + "general_settings": {"allow_requests_on_db_unavailable": False}, + "llm_model_list": [], + "llm_router": None, + "open_telemetry_logger": None, + "model_max_budget_limiter": MagicMock(), + "user_custom_auth": None, + "jwt_handler": None, + "litellm_proxy_admin_name": "admin", + } + + +async def _run_builder_with_key_lookup(get_key_object_mock): + """Drive the real auth builder with ``get_key_object`` replaced by the + given mock. Returns the builder result.""" + from fastapi import Request + from starlette.datastructures import URL + + import litellm.proxy.proxy_server as _proxy_server_mod + from litellm.proxy.auth.user_api_key_auth import _user_api_key_auth_builder + + attrs = _proxy_attrs_for_db_lookup() + originals = {a: getattr(_proxy_server_mod, a, None) for a in attrs} + try: + for k, v in attrs.items(): + setattr(_proxy_server_mod, k, v) + request = Request(scope={"type": "http"}) + request._url = URL(url="/chat/completions") + with ( + patch( + "litellm.proxy.auth.user_api_key_auth.get_key_object", + get_key_object_mock, + ), + ): + return await _user_api_key_auth_builder( + request=request, + api_key="Bearer sk-db-lookup-test", + azure_api_key_header="", + anthropic_api_key_header=None, + google_ai_studio_api_key_header=None, + azure_apim_header=None, + request_data={}, + ) + finally: + for k, v in originals.items(): + setattr(_proxy_server_mod, k, v) + + +@pytest.mark.asyncio +async def test_builder_returns_503_when_db_lookup_raises_infra_error(): + """End-to-end: a DB infrastructure failure during the key lookup must + propagate past the ``except ProxyException`` guard and surface as 503, + not the 401 that masked the 4-hour outage. Killing the new 503 branch + flips this to 401 and fails the test.""" + get_key_object = AsyncMock(side_effect=ConnectionError("connection refused")) + + with pytest.raises(ProxyException) as exc_info: + await _run_builder_with_key_lookup(get_key_object) + + assert int(exc_info.value.code) == status.HTTP_503_SERVICE_UNAVAILABLE + assert exc_info.value.type == ProxyErrorTypes.no_db_connection + assert "Invalid API key" not in str(exc_info.value.message) + + +@pytest.mark.asyncio +async def test_builder_returns_401_when_db_lookup_reports_missing_key(): + """Regression guard: a genuinely missing key (DB returned no row, which + ``get_key_object`` raises as a 401 ProxyException) must still be 401.""" + missing_key_error = ProxyException( + message="Authentication Error, Invalid proxy server token passed. key=..., not found in db.", + type=ProxyErrorTypes.token_not_found_in_db, + param="key", + code=status.HTTP_401_UNAUTHORIZED, + ) + get_key_object = AsyncMock(side_effect=missing_key_error) + + with pytest.raises(ProxyException) as exc_info: + await _run_builder_with_key_lookup(get_key_object) + + assert int(exc_info.value.code) == status.HTTP_401_UNAUTHORIZED + + +@pytest.mark.asyncio +async def test_builder_succeeds_when_db_lookup_returns_valid_token(): + """Regression guard: a valid key still authenticates. Proves the 503 + conversion only fires on the failure path and never intercepts success.""" + valid_token = UserAPIKeyAuth(api_key="sk-db-lookup-test", token="hashed-valid") + get_key_object = AsyncMock(return_value=valid_token) + + with patch( + "litellm.proxy.auth.user_api_key_auth._return_user_api_key_auth_obj", + new_callable=AsyncMock, + return_value=valid_token, + ) as mock_return: + result = await _run_builder_with_key_lookup(get_key_object) + + assert isinstance(result, UserAPIKeyAuth) + # Reaching the success-assembly return (never the exception handler) + # proves a valid key is unaffected by the 503 conversion. + mock_return.assert_awaited_once() diff --git a/tests/test_litellm/proxy/db/test_exception_handler.py b/tests/test_litellm/proxy/db/test_exception_handler.py index 9dcf5df4aeb..6021c221426 100644 --- a/tests/test_litellm/proxy/db/test_exception_handler.py +++ b/tests/test_litellm/proxy/db/test_exception_handler.py @@ -107,6 +107,201 @@ def test_is_database_connection_generic_errors(): ) +@pytest.mark.parametrize( + "error", + [ + ConnectionError("connection refused"), + TimeoutError("timed out"), + OSError("network is unreachable"), + asyncio.TimeoutError(), + HTTPClientClosedError(), + ClientNotConnectedError(), + PrismaError("can't reach database server"), + PrismaError(), + ], +) +def test_is_database_service_unavailable_error_infra_failures(error): + """Infrastructure-level failures (socket/connection/timeout, prisma + transport, unknown PrismaError) mean the DB could not answer, so auth + must surface 503 instead of treating a valid key as invalid.""" + assert PrismaDBExceptionHandler.is_database_service_unavailable_error(error) is True + + +def test_is_database_service_unavailable_error_prisma_p1001_masquerades_as_dataerror(): + """Real-world regression: prisma-client-py raises the P1001 "can't reach + database server" connectivity failure as a DataError (a data-layer type). + A type-only check would miss it and return 401 during a genuine outage; + the message keyword must still classify it as service-unavailable -> 503.""" + p1001_as_dataerror = DataError( + data={ + "user_facing_error": { + "message": "Can't reach database server at `127.0.0.1`:`5499`", + "meta": {"table": "t"}, + } + } + ) + assert ( + PrismaDBExceptionHandler.is_database_service_unavailable_error( + p1001_as_dataerror + ) + is True + ) + + +def test_is_database_service_unavailable_error_cached_plan_escapes_as_503(): + """Composes with the cached-plan retry: when that recovery fails and the + Postgres "cached plan must not change result type" error escapes (raised by + prisma as a data-layer RawQueryError), it is a transient stale-DB-state + condition, not an invalid key, so it must classify as service-unavailable + -> 503 rather than fall through to 401.""" + cached_plan_error = RawQueryError( + data={ + "user_facing_error": { + "message": "cached plan must not change result type", + "meta": {"table": "t"}, + } + } + ) + assert ( + PrismaDBExceptionHandler.is_database_service_unavailable_error( + cached_plan_error + ) + is True + ) + + +def test_is_database_service_unavailable_error_prisma_engine_malformed_payload(): + """Real-world regression: at the instant the DB socket drops, the prisma + query engine returns a malformed error payload (``user_facing_error.meta`` + is ``null``). prisma-client-py's ``handle_response_errors`` then crashes + with ``AttributeError: 'NoneType' object has no attribute 'get'`` before it + can raise the proper P1001 error. That bare AttributeError has no + connection keyword, so without the prisma-engine-origin check it falls + through to 401 on the first request of an outage. Reproduce the exact + prisma crash and assert it classifies as service-unavailable -> 503.""" + from prisma.engine import utils as prisma_engine_utils + + malformed_payload = [ + { + "error": "Can't reach database server", + "user_facing_error": { + "error_code": "P1001", + "message": "Can't reach database server at `localhost`:`5503`", + "meta": None, + }, + } + ] + with pytest.raises(AttributeError) as exc_info: + prisma_engine_utils.handle_response_errors(None, malformed_payload) + + assert "no attribute 'get'" in str(exc_info.value) + assert ( + PrismaDBExceptionHandler.is_database_service_unavailable_error(exc_info.value) + is True + ) + + +def test_is_prisma_engine_internal_error_excludes_application_attributeerror(): + """The prisma-engine-origin check must stay narrow: a genuine AttributeError + raised by application code (a real bug) must NOT be classified as + service-unavailable, otherwise real bugs would silently become 503s.""" + + def application_bug(): + none_value = None + return none_value.get("oops") + + with pytest.raises(AttributeError) as exc_info: + application_bug() + + assert ( + PrismaDBExceptionHandler.is_prisma_engine_internal_error(exc_info.value) + is False + ) + assert ( + PrismaDBExceptionHandler.is_database_service_unavailable_error(exc_info.value) + is False + ) + + +def test_is_prisma_engine_internal_error_excludes_data_layer_prisma_error(): + """A data-layer ``PrismaError`` (the DB IS reachable and rejected the data) + must stay 401. These are always raised from prisma internals, so the check + excludes any ``PrismaError`` by type before inspecting the traceback.""" + data_layer_error = UniqueViolationError( + data={"user_facing_error": {"meta": {"table": "t"}}} + ) + try: + raise data_layer_error + except UniqueViolationError as e: + assert PrismaDBExceptionHandler.is_prisma_engine_internal_error(e) is False + + +@pytest.mark.parametrize( + "error", + [ + DataError(data={"user_facing_error": {"meta": {"table": "t"}}}), + UniqueViolationError(data={"user_facing_error": {"meta": {"table": "t"}}}), + RecordNotFoundError(data={"user_facing_error": {"meta": {"table": "t"}}}), + Exception("some unrelated error"), + ValueError("bad value"), + ], +) +def test_is_database_service_unavailable_error_excludes_non_infra(error): + """Data-layer errors (the DB IS reachable and answered) and generic + non-DB errors must NOT be classified as service-unavailable, otherwise a + genuine 401 would be masked as a transient 503.""" + assert ( + PrismaDBExceptionHandler.is_database_service_unavailable_error(error) is False + ) + + +def test_is_database_service_unavailable_error_asyncpg(monkeypatch): + """asyncpg connection/interface errors map to service-unavailable. asyncpg + is not a hard dependency, so inject a stand-in module to exercise the + branch deterministically regardless of the install environment.""" + import sys + import types + + fake_asyncpg = types.ModuleType("asyncpg") + fake_exceptions = types.ModuleType("asyncpg.exceptions") + + class PostgresConnectionError(Exception): + pass + + class InterfaceError(Exception): + pass + + class UniqueViolationError(Exception): # data-layer, must stay False + pass + + fake_exceptions.PostgresConnectionError = PostgresConnectionError + fake_exceptions.InterfaceError = InterfaceError + fake_exceptions.UniqueViolationError = UniqueViolationError + fake_asyncpg.exceptions = fake_exceptions + + monkeypatch.setitem(sys.modules, "asyncpg", fake_asyncpg) + monkeypatch.setitem(sys.modules, "asyncpg.exceptions", fake_exceptions) + + assert ( + PrismaDBExceptionHandler.is_database_service_unavailable_error( + PostgresConnectionError("connection reset") + ) + is True + ) + assert ( + PrismaDBExceptionHandler.is_database_service_unavailable_error( + InterfaceError("connection was closed") + ) + is True + ) + assert ( + PrismaDBExceptionHandler.is_database_service_unavailable_error( + UniqueViolationError("duplicate key") + ) + is False + ) + + # Test should_allow_request_on_db_unavailable method @patch( "litellm.proxy.proxy_server.general_settings", From 82a971a730064dda53d2ab2f18fcf0dbdb3de04e Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Thu, 11 Jun 2026 14:26:55 -0700 Subject: [PATCH 06/12] fix(passthrough): resolve costing model when body model is unknown (#30160) Adaptation: in the test file, #30160's six own tests (cost-calculation model resolution + extract_model) are taken; the trailing test_passthrough_logging_sets_response_cost_with_server_tool_use_dict, which is diff context present at the PR's base but not on this line, is dropped. Source handler applies verbatim. (cherry picked from commit 1828a7c6f03528b4c174d983bf29d1ca6299439b) --- .../anthropic_passthrough_logging_handler.py | 49 ++++ ...t_anthropic_passthrough_logging_handler.py | 264 ++++++++++++++++++ 2 files changed, 313 insertions(+) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py index a94672f9487..82d90be3bec 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py @@ -100,6 +100,42 @@ class AnthropicPassthroughLoggingHandler: return get_end_user_id_from_request_body(request_body) return None + @staticmethod + def _resolve_costing_model(model: str, logging_obj: LiteLLMLoggingObj) -> str: + if model and model != "unknown": + return model + litellm_params = (getattr(logging_obj, "model_call_details", {}) or {}).get( + "litellm_params", {} + ) or {} + deployment_model = litellm_params.get("model") + if deployment_model and deployment_model != "unknown": + return deployment_model + model_group = (litellm_params.get("metadata", {}) or {}).get("model_group") + if model_group: + return model_group.removeprefix("passthrough/") + return model + + @staticmethod + def _extract_model_from_anthropic_chunks( + all_chunks: Sequence[Union[str, bytes]], + ) -> Optional[str]: + for raw in all_chunks: + text = raw.decode("utf-8") if isinstance(raw, bytes) else raw + for line in text.splitlines(): + if not line.startswith("data:"): + continue + try: + data = json.loads(line[len("data:") :].strip()) + except (json.JSONDecodeError, ValueError): + continue + if not isinstance(data, dict): + continue + if data.get("type") == "message_start": + model = (data.get("message") or {}).get("model") + if model: + return model + return None + @staticmethod def _create_anthropic_response_logging_payload( litellm_model_response: Union[ModelResponse, TextCompletionResponse], @@ -127,6 +163,10 @@ class AnthropicPassthroughLoggingHandler: "custom_llm_provider" ) + model = AnthropicPassthroughLoggingHandler._resolve_costing_model( + model, logging_obj + ) + # Prepend custom_llm_provider to model if not already present model_for_cost = model if custom_llm_provider and not model.startswith(f"{custom_llm_provider}/"): @@ -213,6 +253,15 @@ class AnthropicPassthroughLoggingHandler: ): model = cast(str, litellm_logging_obj.model_call_details.get("model")) + if not model or model == "unknown": + chunk_model = ( + AnthropicPassthroughLoggingHandler._extract_model_from_anthropic_chunks( + all_chunks + ) + ) + if chunk_model: + model = chunk_model + complete_streaming_response = ( AnthropicPassthroughLoggingHandler._build_complete_streaming_response( all_chunks=all_chunks, diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py index f8b6fbde3dc..da786222d30 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py @@ -321,6 +321,270 @@ class TestAzureAnthropicCostCalculation: assert call_kwargs["model"] == "azure_ai/claude-sonnet-4-5_gb_20250929" assert call_kwargs["custom_llm_provider"] == "azure_ai" + @patch("litellm.completion_cost") + def test_cost_calculation_resolves_unknown_model_from_litellm_params( + self, mock_completion_cost + ): + """When the body model is the "unknown" sentinel, the deployment model + from litellm_params must be used for costing, not "unknown" (which makes + completion_cost raise and the cost silently fall back to $0).""" + from datetime import datetime + + from litellm.types.utils import ModelResponse + + mock_completion_cost.return_value = 0.001 + + logging_obj = self._create_mock_logging_obj(model="unknown") + logging_obj.model_call_details["litellm_params"] = { + "model": "anthropic/claude-3-5-haiku-20241022", + "metadata": { + "model_group": "passthrough/anthropic/claude-3-5-haiku-20241022" + }, + } + logging_obj.litellm_params = logging_obj.model_call_details["litellm_params"] + + mock_response = MagicMock(spec=ModelResponse) + mock_response.id = "test-id" + mock_response.model = "unknown" + + kwargs = AnthropicPassthroughLoggingHandler._create_anthropic_response_logging_payload( + litellm_model_response=mock_response, + model="unknown", + kwargs={}, + start_time=datetime.now(), + end_time=datetime.now(), + logging_obj=logging_obj, + ) + + mock_completion_cost.assert_called_once() + assert ( + mock_completion_cost.call_args[1]["model"] + == "anthropic/claude-3-5-haiku-20241022" + ) + assert kwargs["response_cost"] == 0.001 + assert kwargs["model"] == "anthropic/claude-3-5-haiku-20241022" + + @patch("litellm.completion_cost") + def test_cost_calculation_resolves_unknown_model_from_model_group( + self, mock_completion_cost + ): + """With only model_group available (no deployment litellm_params.model), + the leading passthrough/ prefix must be stripped so the cost map can + resolve the model.""" + from datetime import datetime + + from litellm.types.utils import ModelResponse + + mock_completion_cost.return_value = 0.002 + + logging_obj = self._create_mock_logging_obj(model="unknown") + logging_obj.model_call_details["litellm_params"] = { + "metadata": { + "model_group": "passthrough/anthropic/claude-3-5-haiku-20241022" + } + } + logging_obj.litellm_params = logging_obj.model_call_details["litellm_params"] + + mock_response = MagicMock(spec=ModelResponse) + mock_response.id = "test-id" + mock_response.model = "unknown" + + kwargs = AnthropicPassthroughLoggingHandler._create_anthropic_response_logging_payload( + litellm_model_response=mock_response, + model="unknown", + kwargs={}, + start_time=datetime.now(), + end_time=datetime.now(), + logging_obj=logging_obj, + ) + + mock_completion_cost.assert_called_once() + assert ( + mock_completion_cost.call_args[1]["model"] + == "anthropic/claude-3-5-haiku-20241022" + ) + assert kwargs["response_cost"] == 0.002 + + @patch("litellm.completion_cost") + def test_cost_calculation_skips_unknown_litellm_params_model_for_model_group( + self, mock_completion_cost + ): + """When litellm_params.model is itself the "unknown" sentinel, the + deployment-model branch must not short-circuit; resolution falls through + to model_group so costing still prices the real model instead of "unknown".""" + from datetime import datetime + + from litellm.types.utils import ModelResponse + + mock_completion_cost.return_value = 0.003 + + logging_obj = self._create_mock_logging_obj(model="unknown") + logging_obj.model_call_details["litellm_params"] = { + "model": "unknown", + "metadata": { + "model_group": "passthrough/anthropic/claude-3-5-haiku-20241022" + }, + } + logging_obj.litellm_params = logging_obj.model_call_details["litellm_params"] + + mock_response = MagicMock(spec=ModelResponse) + mock_response.id = "test-id" + mock_response.model = "unknown" + + kwargs = AnthropicPassthroughLoggingHandler._create_anthropic_response_logging_payload( + litellm_model_response=mock_response, + model="unknown", + kwargs={}, + start_time=datetime.now(), + end_time=datetime.now(), + logging_obj=logging_obj, + ) + + mock_completion_cost.assert_called_once() + assert ( + mock_completion_cost.call_args[1]["model"] + == "anthropic/claude-3-5-haiku-20241022" + ) + assert kwargs["response_cost"] == 0.003 + assert kwargs["model"] == "anthropic/claude-3-5-haiku-20241022" + + @patch("litellm.completion_cost") + def test_streaming_cost_calculation_resolves_model_from_message_start_chunk( + self, mock_completion_cost + ): + """On the bare /anthropic passthrough path litellm_params carries no model + or model_group and the body model is the "unknown" sentinel; the model + must be recovered from the message_start SSE event so completion_cost + prices the real model instead of failing on "unknown" and logging $0.""" + from datetime import datetime + + from litellm.litellm_core_utils.litellm_logging import ( + Logging as RealLoggingObj, + ) + from litellm.proxy.pass_through_endpoints.streaming_handler import ( + PassThroughStreamingHandler, + ) + + mock_completion_cost.return_value = 0.001 + + def _sse(event, data): + return f"event: {event}\ndata: {json.dumps(data)}\n\n".encode() + + frames = [ + _sse( + "message_start", + { + "type": "message_start", + "message": { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-3-5-haiku-20241022", + "content": [], + "stop_reason": None, + "stop_sequence": None, + "usage": {"input_tokens": 10, "output_tokens": 0}, + }, + }, + ), + _sse( + "content_block_start", + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "text", "text": ""}, + }, + ), + _sse( + "content_block_delta", + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": "hi"}, + }, + ), + _sse("content_block_stop", {"type": "content_block_stop", "index": 0}), + _sse( + "message_delta", + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": 1}, + }, + ), + _sse("message_stop", {"type": "message_stop"}), + ] + all_chunks = list( + PassThroughStreamingHandler._convert_raw_bytes_to_str_lines(frames) + ) + + logging_obj = RealLoggingObj( + model="unknown", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="pass_through_endpoint", + start_time=datetime.now(), + litellm_call_id="test-call-id", + function_id="1", + ) + logging_obj.model_call_details["model"] = "unknown" + logging_obj.model_call_details["stream"] = True + logging_obj.model_call_details["litellm_params"] = {} + logging_obj.litellm_params = {} + + result = AnthropicPassthroughLoggingHandler._handle_logging_anthropic_collected_chunks( + litellm_logging_obj=logging_obj, + passthrough_success_handler_obj=MagicMock(), + url_route="/anthropic/v1/messages", + request_body={"stream": True}, + endpoint_type="messages", + start_time=datetime.now(), + all_chunks=all_chunks, + end_time=datetime.now(), + ) + + assert result["result"] is not None + mock_completion_cost.assert_called_once() + assert mock_completion_cost.call_args[1]["model"] == "claude-3-5-haiku-20241022" + assert result["kwargs"]["response_cost"] == 0.001 + assert result["kwargs"]["model"] == "claude-3-5-haiku-20241022" + + def test_extract_model_skips_non_dict_data_payload(self): + """A scalar data: payload (e.g. `data: null`) must be skipped, not crash + the streaming log handler with AttributeError, which would propagate out + and break spend logging for the whole request.""" + chunks = [ + "event: ping\ndata: null\n\n", + 'event: message_start\ndata: {"type": "message_start", "message": ' + '{"model": "claude-3-5-haiku-20241022"}}\n\n', + ] + + assert ( + AnthropicPassthroughLoggingHandler._extract_model_from_anthropic_chunks( + chunks + ) + == "claude-3-5-haiku-20241022" + ) + + def test_extract_model_parses_per_line_not_first_data_substring(self): + """A raw multi-line SSE event whose non-data line contains the substring + "data:" must not derail parsing: matching only lines that start with + "data:" recovers the message_start model, whereas a first-substring slice + would consume the wrong offset, fail to parse JSON, and return None.""" + raw_event = ( + "event: ping data: not-json\n" + 'data: {"type": "message_start", "message": ' + '{"model": "claude-3-5-haiku-20241022"}}\n\n' + ) + + assert ( + AnthropicPassthroughLoggingHandler._extract_model_from_anthropic_chunks( + [raw_event] + ) + == "claude-3-5-haiku-20241022" + ) + + class TestAnthropicBatchPassthroughCostTracking: """Test cases for Anthropic batch passthrough cost tracking functionality""" From f59192b87acf6c00fcd5c04ac363098f00343059 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 12 Jun 2026 16:45:31 -0700 Subject: [PATCH 07/12] fix(passthrough): skip [DONE] sentinels and non-JSON SSE frames in Anthropic streaming logging Targeted subset of staging commit cfcdf8714a6968f8248e56f00f37f6f232aa41cd (#30202): only the anthropic_passthrough_logging_handler.py hardening hunks and their four tests are taken; the rest of that staging batch is intentionally excluded. Backport adaptation (stable/1.88.x): the new TestBuildCompleteStreamingResponseRobustness class is unioned with this line's existing test_parity_*/test_collapse_* tests, which the 1.84.x base of this commit did not have. Source handler applies verbatim. (cherry picked from commit cfcdf8714a6968f8248e56f00f37f6f232aa41cd) (cherry picked from commit 973c7eb8d62b76f1f2724e2ac0ce8b64637089cc) --- .../anthropic_passthrough_logging_handler.py | 15 +++++ ...t_anthropic_passthrough_logging_handler.py | 66 +++++++++++++++++++ 2 files changed, 81 insertions(+) diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py index 82d90be3bec..a912a88a993 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py @@ -517,6 +517,13 @@ class AnthropicPassthroughLoggingHandler: # Process each individual event for event_str in individual_events: try: + # Skip OpenAI-style [DONE] sentinels some Anthropic-compatible + # providers emit. Match the whole SSE line so a valid chunk whose + # text payload happens to contain "[DONE]" is not dropped. + if any( + line.strip() == "data: [DONE]" for line in event_str.split("\n") + ): + continue transformed_openai_chunk = anthropic_model_response_iterator.convert_str_chunk_to_generic_chunk( chunk=event_str ) @@ -525,6 +532,14 @@ class AnthropicPassthroughLoggingHandler: except (StopIteration, StopAsyncIteration): break + except json.JSONDecodeError: + # Some upstreams emit non-JSON SSE lines; skip them so the + # logging pipeline is not broken by a single bad frame. + verbose_proxy_logger.debug( + "Skipping non-JSON SSE event: %s", + event_str[:200], + ) + continue complete_streaming_response = litellm.stream_chunk_builder( chunks=all_openai_chunks, diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py index da786222d30..78816f9ff2d 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py @@ -1309,6 +1309,72 @@ class TestPureTextFastPathParity: ) +class TestBuildCompleteStreamingResponseRobustness: + """_build_complete_streaming_response must tolerate non-standard SSE frames.""" + + def _build(self, chunks: List[str]): + return AnthropicPassthroughLoggingHandler._build_complete_streaming_response( + all_chunks=chunks, + litellm_logging_obj=MagicMock(), + model="claude-3-sonnet-20240229", + ) + + def test_done_frame_is_skipped(self): + """A bare 'data: [DONE]' control frame must not break reconstruction.""" + chunks = [ + 'event: message_start\ndata: {"type":"message_start","message":{"id":"msg_1","type":"message","role":"assistant","content":[],"model":"claude-3-sonnet-20240229","stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":10,"output_tokens":1}}}', + 'event: content_block_start\ndata: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}', + 'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hi"}}', + 'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}', + 'event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":2}}', + 'event: message_stop\ndata: {"type":"message_stop"}', + "data: [DONE]", + ] + result = self._build(chunks) + assert result is not None + assert result.choices[0].message.content == "Hi" + + def test_non_json_sse_line_is_skipped(self): + """Non-JSON SSE lines (comments, keep-alive pings) must be skipped.""" + chunks = [ + ": ping", + 'event: message_start\ndata: {"type":"message_start","message":{"id":"msg_1","type":"message","role":"assistant","content":[],"model":"claude-3-sonnet-20240229","stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":10,"output_tokens":1}}}', + "this is not json at all", + ] + # Must not raise; a malformed stream simply yields no usable response. + result = self._build(chunks) + assert result is None or hasattr(result, "choices") + + def test_mixed_valid_and_invalid_frames(self): + """Valid events are still collected when interleaved with invalid ones.""" + chunks = [ + 'event: message_start\ndata: {"type":"message_start","message":{"id":"msg_1","type":"message","role":"assistant","content":[],"model":"claude-3-sonnet-20240229","stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":10,"output_tokens":1}}}', + "data: [DONE]", + ": keep-alive", + "not-json", + 'event: content_block_start\ndata: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}', + 'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hello"}}', + 'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}', + 'event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":2}}', + 'event: message_stop\ndata: {"type":"message_stop"}', + ] + result = self._build(chunks) + assert result is not None + assert result.choices[0].message.content == "Hello" + + def test_done_in_text_payload_is_not_dropped(self): + """A valid event whose text content contains '[DONE]' must NOT be skipped.""" + chunks = [ + 'event: message_start\ndata: {"type":"message_start","message":{"id":"msg_1","type":"message","role":"assistant","content":[],"model":"claude-3-sonnet-20240229","stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":10,"output_tokens":1}}}', + 'event: content_block_start\ndata: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}', + 'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"The stream ends with [DONE]"}}', + 'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}', + 'event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":8}}', + 'event: message_stop\ndata: {"type":"message_stop"}', + ] + result = self._build(chunks) + assert result is not None + assert result.choices[0].message.content == "The stream ends with [DONE]" class TestStreamFalseDeduplication: """ Regression tests for the duplicate-callback bug where a streaming pass-through From 44ff751e54c37d9a416b052e87a42f0e7a817828 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 12 Jun 2026 17:44:04 -0700 Subject: [PATCH 08/12] fix(proxy): return deprecated-key lookup result directly in get_data combined view (#30327) The grace-period branch assigned the recursive get_data result (a finished LiteLLM_VerificationTokenView) back into the variable that the combined-view dict normalization then subscripts, raising TypeError on every request made with a rotated key inside its grace window; auth surfaced that as a 401. Return the recursive result directly instead. Regression test drives the full get_data flow: old hash misses the view, deprecated table resolves to the active token, and the call must return the view object (cherry picked from commit 5047eaf7f0e7151891d7edbf19f92eb0004ff274) --- litellm/proxy/utils.py | 8 +++- .../test_prisma_client_get_data.py | 38 +++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 157303689d8..0720389e1f2 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -3557,7 +3557,10 @@ class PrismaClient: db=self.db, hashed_token=hashed_token ) if active_token_id: - response = await self.get_data( + # The recursive call returns a finished + # LiteLLM_VerificationTokenView; the dict + # normalization below would crash subscripting it. + deprecated_response = await self.get_data( token=active_token_id, table_name="combined_view", query_type="find_unique", @@ -3565,10 +3568,11 @@ class PrismaClient: proxy_logging_obj=proxy_logging_obj, check_deprecated=False, ) - if response is not None: + if deprecated_response is not None: verbose_proxy_logger.debug( "Deprecated key used during grace period" ) + return deprecated_response if response is not None: if response["team_models"] is None: diff --git a/tests/test_litellm/proxy/utils/prisma_and_spend/test_prisma_client_get_data.py b/tests/test_litellm/proxy/utils/prisma_and_spend/test_prisma_client_get_data.py index 437984d9273..08d1ef619a7 100644 --- a/tests/test_litellm/proxy/utils/prisma_and_spend/test_prisma_client_get_data.py +++ b/tests/test_litellm/proxy/utils/prisma_and_spend/test_prisma_client_get_data.py @@ -15,6 +15,7 @@ from __future__ import annotations import hashlib import json +from datetime import datetime, timedelta, timezone from types import SimpleNamespace from typing import Any from unittest.mock import AsyncMock, MagicMock @@ -22,6 +23,7 @@ from unittest.mock import AsyncMock, MagicMock import pytest from fastapi import HTTPException +from litellm.proxy._types import LiteLLM_VerificationTokenView from litellm.proxy.utils import PrismaClient @@ -476,3 +478,39 @@ async def test_get_data_logs_and_raises_on_db_error( ) with pytest.raises(RuntimeError, match="network split"): await prisma_client.get_data(token="sk-broken", table_name="key") + + +@pytest.mark.asyncio +async def test_get_data_combined_view_returns_view_for_deprecated_key( + prisma_client: PrismaClient, +) -> None: + """Grace-period rotation, full get_data flow: the old hash misses the + combined view, the deprecated-key table resolves it to the active token, + and get_data must return the recursive lookup's finished view instead of + re-running dict normalization on it (which raised TypeError and turned + every grace-period request into a 401).""" + old_hash = "hashed-old-token-grace-e2e" + active_hash = "hashed-active-token-grace-e2e" + active_row = { + "token": active_hash, + "team_models": None, + "team_blocked": None, + "team_members_with_roles": None, + "user_id": None, + "expires": None, + } + prisma_client.db.query_first = AsyncMock(side_effect=[None, active_row]) + prisma_client.db.litellm_deprecatedverificationtoken = MagicMock() + prisma_client.db.litellm_deprecatedverificationtoken.find_first = AsyncMock( + return_value=SimpleNamespace( + active_token_id=active_hash, + revoke_at=datetime.now(timezone.utc) + timedelta(hours=1), + ) + ) + + response = await prisma_client.get_data( + token=old_hash, table_name="combined_view", query_type="find_unique" + ) + + assert isinstance(response, LiteLLM_VerificationTokenView) + assert response.token == active_hash From c86bf9b4d9d08db6f2639767504d11bcbe458aca Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 12 Jun 2026 17:48:00 -0700 Subject: [PATCH 09/12] chore(deps): bump vitest, brace-expansion, pypdf and tornado (#30220) * chore(deps): bump aiohttp to 3.14.1 and vitest to 3.2.6 Lockfile-only bump for aiohttp (3.13.5 -> 3.14.1, within the existing pyproject constraint) and dashboard devDependency bumps for vitest, @vitest/coverage-v8, @vitest/ui (3.2.4 -> 3.2.6) plus transitive brace-expansion (5.0.5 -> 5.0.6). Clears the currently published advisories flagged by osv.dev against uv.lock and the dashboard lockfile. Verified: 154 custom_httpx unit tests and all 3943 dashboard vitest tests pass; live proxy completion and streaming calls succeed on the bumped venv * chore(deps): raise aiohttp floor to 3.14.0 The lockfile bump alone only protects environments built from uv.lock. Raising the pyproject floor extends the same minimum to package consumers installing litellm from PyPI, and prevents a future lockfile regeneration from resolving below 3.14.0 * Revert "chore(deps): raise aiohttp floor to 3.14.0" This reverts commit d6c1c9dc0c8664c015a5dabbde2469539bd247fd. * revert(deps): roll back aiohttp to 3.13.5 vcrpy is incompatible with aiohttp >= 3.14 (the aiohttp_stubs module imports a symbol removed in 3.14) and the upstream fix is merged but unreleased, so every cassette-based test suite fails on 3.14. Hold aiohttp at 3.13.5 until a vcrpy release ships; the vitest and brace-expansion bumps stay * chore(deps): bump pypdf to 6.13.1 and tornado to 6.5.7 Lockfile-only bumps clearing the advisories published for both since this branch was opened * chore(deps): add regression guards for the bumped versions Raise the pypdf floor to 6.12.0 (direct dependency, applies to package consumers too) and add uv constraint-dependencies for the transitive pins: tornado >= 6.5.6, and aiohttp held in [3.13.5, 3.14) so a lockfile regeneration can neither fall back below the current version nor move onto 3.14 while vcrpy is incompatible. Constraints live in [tool.uv] and only affect this repo's resolution, not published metadata. Verified: uv lock -P with each out-of-range version fails to resolve; in-range resolutions unchanged (pypdf 6.13.1, tornado 6.5.7, aiohttp 3.13.5) Backport handling (stable/1.88.x): the manifest hunks (pyproject pypdf floor 6.10.2->6.12.0 plus the [tool.uv] tornado/aiohttp constraints, and the package.json vitest 3.2.4->3.2.6 bumps) are taken as-is; the staging lockfiles are not. uv.lock and package-lock.json are regenerated on this line so the closure stays minimal: uv.lock moves only pypdf 6.10.2->6.13.2 and tornado 6.5.5->6.5.7 (aiohttp held at 3.13.5 by the new constraint), and package-lock.json moves vitest and @vitest to 3.2.6. brace-expansion reached 5.0.6 transitively on staging but resolves to 5.0.5 on this line's tree, so a brace-expansion 5.0.6 override is added to clear the same advisory. No version bump: 1.88.2 is bumped but unreleased, so these ride the pending 1.88.2. (cherry picked from commit d96ab467f1dba5e4dbe02de3d5af62ec710c44fd) --- pyproject.toml | 6 +- ui/litellm-dashboard/package-lock.json | 116 ++++++++++++------------- ui/litellm-dashboard/package.json | 7 +- uv.lock | 36 ++++---- 4 files changed, 87 insertions(+), 78 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c7c28583808..bd7e002b717 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -125,7 +125,7 @@ proxy-runtime = [ "mangum>=0.17.0,<1.0", "azure-ai-contentsafety>=1.0.0,<2.0", "azure-storage-file-datalake>=12.20.0,<13.0", - "pypdf>=6.10.2,<7.0; python_version < '3.14'", + "pypdf>=6.12.0,<7.0; python_version < '3.14'", "llm-sandbox>=0.3.39,<1.0", "detect-secrets>=1.5.0,<2.0", ] @@ -231,6 +231,10 @@ requires = ["uv_build==0.11.8"] build-backend = "uv_build" [tool.uv] +constraint-dependencies = [ + "tornado>=6.5.6", + "aiohttp>=3.13.5,<3.14", +] default-groups = ["dev"] required-version = ">=0.10.9" exclude-newer = "3 days" diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index 18418f7bc78..726f1949d86 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -51,8 +51,8 @@ "@types/react-dom": "18.3.7", "@types/react-syntax-highlighter": "15.5.13", "@types/uuid": "10.0.0", - "@vitest/coverage-v8": "3.2.4", - "@vitest/ui": "3.2.4", + "@vitest/coverage-v8": "3.2.6", + "@vitest/ui": "3.2.6", "autoprefixer": "10.4.24", "dotenv": "17.2.3", "eslint": "9.39.2", @@ -66,7 +66,7 @@ "tailwindcss": "3.4.19", "typescript": "5.9.3", "vite": "7.3.2", - "vitest": "3.2.4" + "vitest": "3.2.6" }, "engines": { "node": ">=20.9.0", @@ -3998,9 +3998,9 @@ ] }, "node_modules/@vitest/coverage-v8": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-3.2.4.tgz", - "integrity": "sha512-EyF9SXU6kS5Ku/U82E259WSnvg6c8KTjppUncuNdm5QHpe17mwREHnjDzozC8x9MZ0xfBUFSaLkRv4TMA75ALQ==", + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-3.2.6.tgz", + "integrity": "sha512-LsAdmUapA0qSN306d8+zOyawM0hFm2m2Hg9IwVNIKBm+qJV8cijiq2c+gxKZcB1HCfIWAy+0qEZDCUQA58A1cw==", "dev": true, "license": "MIT", "dependencies": { @@ -4022,8 +4022,8 @@ "url": "https://opencollective.com/vitest" }, "peerDependencies": { - "@vitest/browser": "3.2.4", - "vitest": "3.2.4" + "@vitest/browser": "3.2.6", + "vitest": "3.2.6" }, "peerDependenciesMeta": { "@vitest/browser": { @@ -4032,15 +4032,15 @@ } }, "node_modules/@vitest/expect": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-3.2.4.tgz", - "integrity": "sha512-Io0yyORnB6sikFlt8QW5K7slY4OjqNX9jmJQ02QDda8lyM6B5oNgVWoSoKPac8/kgnCUzuHQKrSLtu/uOqqrig==", + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-3.2.6.tgz", + "integrity": "sha512-1+7q9BtaKzEmO+fmNT3kYvoNn5Y71XWAx2Q5HRim4tTVRQVRv4uJFAQ5FbK0OPUeNP/WmVCpxYxoJdvuHVjzBQ==", "dev": true, "license": "MIT", "dependencies": { "@types/chai": "^5.2.2", - "@vitest/spy": "3.2.4", - "@vitest/utils": "3.2.4", + "@vitest/spy": "3.2.6", + "@vitest/utils": "3.2.6", "chai": "^5.2.0", "tinyrainbow": "^2.0.0" }, @@ -4049,13 +4049,13 @@ } }, "node_modules/@vitest/mocker": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-3.2.4.tgz", - "integrity": "sha512-46ryTE9RZO/rfDd7pEqFl7etuyzekzEhUbTW3BvmeO/BcCMEgq59BKhek3dXDWgAj4oMK6OZi+vRr1wPW6qjEQ==", + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-3.2.6.tgz", + "integrity": "sha512-EZOrpDbkKotFAP7wPAQV1UIyoGOk4oX7ynWhBhLB7v+meMHbQhU16oPpIYGTTe4oFlhpryGpgpcZP/sin3hYuw==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/spy": "3.2.4", + "@vitest/spy": "3.2.6", "estree-walker": "^3.0.3", "magic-string": "^0.30.17" }, @@ -4076,9 +4076,9 @@ } }, "node_modules/@vitest/pretty-format": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-3.2.4.tgz", - "integrity": "sha512-IVNZik8IVRJRTr9fxlitMKeJeXFFFN0JaB9PHPGQ8NKQbGpfjlTx9zO4RefN8gp7eqjNy8nyK3NZmBzOPeIxtA==", + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-3.2.6.tgz", + "integrity": "sha512-lb7XXXzmm2h2ASzFnRvQpDo6onT1NmMJA3tkGTWiBFtRJ9lxGY3d3mm/Apt36gej2bkkOVLL/yTOtufDaFa/jA==", "dev": true, "license": "MIT", "dependencies": { @@ -4089,13 +4089,13 @@ } }, "node_modules/@vitest/runner": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-3.2.4.tgz", - "integrity": "sha512-oukfKT9Mk41LreEW09vt45f8wx7DordoWUZMYdY/cyAk7w5TWkTRCNZYF7sX7n2wB7jyGAl74OxgwhPgKaqDMQ==", + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-3.2.6.tgz", + "integrity": "sha512-HYcoSj1w5tcgUnzoF0HcyaAQjpA1gj9ftUJ7iSJSuipc02jW9gKkigwZbjFldAfYHA1fa8UZVRftdMY5msWM9Q==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/utils": "3.2.4", + "@vitest/utils": "3.2.6", "pathe": "^2.0.3", "strip-literal": "^3.0.0" }, @@ -4104,13 +4104,13 @@ } }, "node_modules/@vitest/snapshot": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-3.2.4.tgz", - "integrity": "sha512-dEYtS7qQP2CjU27QBC5oUOxLE/v5eLkGqPE0ZKEIDGMs4vKWe7IjgLOeauHsR0D5YuuycGRO5oSRXnwnmA78fQ==", + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-3.2.6.tgz", + "integrity": "sha512-H+ZjNTWGpObenh0YnlBctAPnJSI20P81PL8BPzWpx54YXLLTm8hEsWawtcYLMrwvpK48hGxLLbCS+1KRXhsKhw==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "3.2.4", + "@vitest/pretty-format": "3.2.6", "magic-string": "^0.30.17", "pathe": "^2.0.3" }, @@ -4119,9 +4119,9 @@ } }, "node_modules/@vitest/spy": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-3.2.4.tgz", - "integrity": "sha512-vAfasCOe6AIK70iP5UD11Ac4siNUNJ9i/9PZ3NKx07sG6sUxeag1LWdNrMWeKKYBLlzuK+Gn65Yd5nyL6ds+nw==", + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-3.2.6.tgz", + "integrity": "sha512-oq6BbH68WzcWmwtBrU9nqLeaXTR4XwJF7FSLkKEZo4i6eoXcrxjcwSuTvWBIRUTC6VC72nXYunzqgZA+IKdtxg==", "dev": true, "license": "MIT", "dependencies": { @@ -4132,13 +4132,13 @@ } }, "node_modules/@vitest/ui": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/@vitest/ui/-/ui-3.2.4.tgz", - "integrity": "sha512-hGISOaP18plkzbWEcP/QvtRW1xDXF2+96HbEX6byqQhAUbiS5oH6/9JwW+QsQCIYON2bI6QZBF+2PvOmrRZ9wA==", + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/@vitest/ui/-/ui-3.2.6.tgz", + "integrity": "sha512-mATfG3zVdhobE9U1rIpvtYD3DGuSSxqZ3Aj/8ityGqKXy8YDJ9BoAjZmAz6dZ1IZ1xI5V+MerkCczvVa+3QK9Q==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/utils": "3.2.4", + "@vitest/utils": "3.2.6", "fflate": "^0.8.2", "flatted": "^3.3.3", "pathe": "^2.0.3", @@ -4150,17 +4150,17 @@ "url": "https://opencollective.com/vitest" }, "peerDependencies": { - "vitest": "3.2.4" + "vitest": "3.2.6" } }, "node_modules/@vitest/utils": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-3.2.4.tgz", - "integrity": "sha512-fB2V0JFrQSMsCo9HiSq3Ezpdv4iYaXRG1Sx8edX3MwxfyNn83mKiGzOcH+Fkxt4MHxr3y42fQi1oeAInqgX2QA==", + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-3.2.6.tgz", + "integrity": "sha512-lI23nIs4bnT3T8NIoh+vFaz5s2/DdP0Jgt2jxwgWljvwn82cLJtyi/If+fjFyoLMGIOz0U/fKvWE0d4jsNQEfg==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "3.2.4", + "@vitest/pretty-format": "3.2.6", "loupe": "^3.1.4", "tinyrainbow": "^2.0.0" }, @@ -4739,9 +4739,9 @@ } }, "node_modules/brace-expansion": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", - "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.6.tgz", + "integrity": "sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g==", "dev": true, "license": "MIT", "dependencies": { @@ -6512,9 +6512,9 @@ } }, "node_modules/fflate": { - "version": "0.8.2", - "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.2.tgz", - "integrity": "sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==", + "version": "0.8.3", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.3.tgz", + "integrity": "sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA==", "dev": true, "license": "MIT" }, @@ -13002,20 +13002,20 @@ } }, "node_modules/vitest": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-3.2.4.tgz", - "integrity": "sha512-LUCP5ev3GURDysTWiP47wRRUpLKMOfPh+yKTx3kVIEiu5KOMeqzpnYNsKyOoVrULivR8tLcks4+lga33Whn90A==", + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-3.2.6.tgz", + "integrity": "sha512-xejya+bT/j/+R/AGa1XOfRxLmNUlLtlwjRsFUILF+xHfzElmGcmFydy2gqqIrd62ptIEfwVMofd19uNWD9L7Nw==", "dev": true, "license": "MIT", "dependencies": { "@types/chai": "^5.2.2", - "@vitest/expect": "3.2.4", - "@vitest/mocker": "3.2.4", - "@vitest/pretty-format": "^3.2.4", - "@vitest/runner": "3.2.4", - "@vitest/snapshot": "3.2.4", - "@vitest/spy": "3.2.4", - "@vitest/utils": "3.2.4", + "@vitest/expect": "3.2.6", + "@vitest/mocker": "3.2.6", + "@vitest/pretty-format": "^3.2.6", + "@vitest/runner": "3.2.6", + "@vitest/snapshot": "3.2.6", + "@vitest/spy": "3.2.6", + "@vitest/utils": "3.2.6", "chai": "^5.2.0", "debug": "^4.4.1", "expect-type": "^1.2.1", @@ -13045,8 +13045,8 @@ "@edge-runtime/vm": "*", "@types/debug": "^4.1.12", "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", - "@vitest/browser": "3.2.4", - "@vitest/ui": "3.2.4", + "@vitest/browser": "3.2.6", + "@vitest/ui": "3.2.6", "happy-dom": "*", "jsdom": "*" }, diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index 6c37996ac09..ce793f97715 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -63,8 +63,8 @@ "@types/react-dom": "18.3.7", "@types/react-syntax-highlighter": "15.5.13", "@types/uuid": "10.0.0", - "@vitest/coverage-v8": "3.2.4", - "@vitest/ui": "3.2.4", + "@vitest/coverage-v8": "3.2.6", + "@vitest/ui": "3.2.6", "autoprefixer": "10.4.24", "dotenv": "17.2.3", "eslint": "9.39.2", @@ -78,7 +78,7 @@ "tailwindcss": "3.4.19", "typescript": "5.9.3", "vite": "7.3.2", - "vitest": "3.2.4" + "vitest": "3.2.6" }, "overrides": { "prismjs": "1.30.0", @@ -88,6 +88,7 @@ "lodash": "4.18.1", "ws": "8.20.1", "braces": "3.0.3", + "brace-expansion": "5.0.6", "axios": "1.13.6", "postcss": "8.5.13" }, diff --git a/uv.lock b/uv.lock index 932318504b6..e254ead54e1 100644 --- a/uv.lock +++ b/uv.lock @@ -9,7 +9,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-06-05T23:54:44.890497Z" +exclude-newer = "2026-06-11T00:01:45.852753Z" exclude-newer-span = "P3D" [manifest] @@ -18,6 +18,10 @@ members = [ "litellm-enterprise", "litellm-proxy-extras", ] +constraints = [ + { name = "aiohttp", specifier = ">=3.13.5,<3.14" }, + { name = "tornado", specifier = ">=6.5.6" }, +] [[package]] name = "a2a-sdk" @@ -3524,7 +3528,7 @@ requires-dist = [ { name = "pydantic-settings", marker = "extra == 'proxy'", specifier = ">=2.14.1,<3.0" }, { name = "pyjwt", marker = "extra == 'proxy'", specifier = ">=2.13.0,<3.0" }, { name = "pynacl", marker = "extra == 'proxy'", specifier = ">=1.6.2,<2.0" }, - { name = "pypdf", marker = "python_full_version < '3.14' and extra == 'proxy-runtime'", specifier = ">=6.10.2,<7.0" }, + { name = "pypdf", marker = "python_full_version < '3.14' and extra == 'proxy-runtime'", specifier = ">=6.12.0,<7.0" }, { name = "pyroscope-io", marker = "sys_platform != 'win32' and extra == 'proxy'", specifier = ">=0.8.16,<1.0" }, { name = "python-dotenv", specifier = ">=1.0.0,<2.0" }, { name = "python-multipart", marker = "extra == 'proxy'", specifier = ">=0.0.27,<1.0" }, @@ -6051,14 +6055,14 @@ wheels = [ [[package]] name = "pypdf" -version = "6.10.2" +version = "6.13.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/7b/3f/9f2167401c2e94833ca3b69535bad89e533b5de75fefe4197a2c224baec2/pypdf-6.10.2.tar.gz", hash = "sha256:7d09ce108eff6bf67465d461b6ef352dcb8d84f7a91befc02f904455c6eea11d", size = 5315679, upload-time = "2026-04-15T16:37:36.978Z" } +sdist = { url = "https://files.pythonhosted.org/packages/99/0a/48fe05c6bb3aa4bb4d2a4079a383d33c0dfec1edf613a642f07d8b8b5c2e/pypdf-6.13.2.tar.gz", hash = "sha256:5a96a17dbdfbf9c2ab24c0a13fa0aba182be22ba6f283098712c16fc242f509f", size = 6479250, upload-time = "2026-06-10T16:42:34.5Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0c/d6/1d5c60cc17bbdf37c1552d9c03862fc6d32c5836732a0415b2d637edc2d0/pypdf-6.10.2-py3-none-any.whl", hash = "sha256:aa53be9826655b51c96741e5d7983ca224d898ac0a77896e64636810517624aa", size = 336308, upload-time = "2026-04-15T16:37:34.851Z" }, + { url = "https://files.pythonhosted.org/packages/cb/17/378943705992f74e451a06de3401ce68e3213763c81e44d0614559c45599/pypdf-6.13.2-py3-none-any.whl", hash = "sha256:6eeb9e57693f29d41bd01255d02660cbbb41fd7fc818a982677389a35e4f2083", size = 346555, upload-time = "2026-06-10T16:42:32.37Z" }, ] [[package]] @@ -7574,19 +7578,19 @@ wheels = [ [[package]] name = "tornado" -version = "6.5.5" +version = "6.5.7" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f8/f1/3173dfa4a18db4a9b03e5d55325559dab51ee653763bb8745a75af491286/tornado-6.5.5.tar.gz", hash = "sha256:192b8f3ea91bd7f1f50c06955416ed76c6b72f96779b962f07f911b91e8d30e9", size = 516006, upload-time = "2026-03-10T21:31:02.067Z" } +sdist = { url = "https://files.pythonhosted.org/packages/64/24/95ec527ad67b76d59299e5465b3935d05e4294b7e0290a3924b7487df30b/tornado-6.5.7.tar.gz", hash = "sha256:66c513a76cda70d53907bc27cf1447557699c2e95aa48ba27a442ff61c3ddfc2", size = 519252, upload-time = "2026-06-08T17:34:51.232Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/59/8c/77f5097695f4dd8255ecbd08b2a1ed8ba8b953d337804dd7080f199e12bf/tornado-6.5.5-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:487dc9cc380e29f58c7ab88f9e27cdeef04b2140862e5076a66fb6bb68bb1bfa", size = 445983, upload-time = "2026-03-10T21:30:44.28Z" }, - { url = "https://files.pythonhosted.org/packages/ab/5e/7625b76cd10f98f1516c36ce0346de62061156352353ef2da44e5c21523c/tornado-6.5.5-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:65a7f1d46d4bb41df1ac99f5fcb685fb25c7e61613742d5108b010975a9a6521", size = 444246, upload-time = "2026-03-10T21:30:46.571Z" }, - { url = "https://files.pythonhosted.org/packages/b2/04/7b5705d5b3c0fab088f434f9c83edac1573830ca49ccf29fb83bf7178eec/tornado-6.5.5-cp39-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:e74c92e8e65086b338fd56333fb9a68b9f6f2fe7ad532645a290a464bcf46be5", size = 447229, upload-time = "2026-03-10T21:30:48.273Z" }, - { url = "https://files.pythonhosted.org/packages/34/01/74e034a30ef59afb4097ef8659515e96a39d910b712a89af76f5e4e1f93c/tornado-6.5.5-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:435319e9e340276428bbdb4e7fa732c2d399386d1de5686cb331ec8eee754f07", size = 448192, upload-time = "2026-03-10T21:30:51.22Z" }, - { url = "https://files.pythonhosted.org/packages/be/00/fe9e02c5a96429fce1a1d15a517f5d8444f9c412e0bb9eadfbe3b0fc55bf/tornado-6.5.5-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:3f54aa540bdbfee7b9eb268ead60e7d199de5021facd276819c193c0fb28ea4e", size = 448039, upload-time = "2026-03-10T21:30:53.52Z" }, - { url = "https://files.pythonhosted.org/packages/82/9e/656ee4cec0398b1d18d0f1eb6372c41c6b889722641d84948351ae19556d/tornado-6.5.5-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:36abed1754faeb80fbd6e64db2758091e1320f6bba74a4cf8c09cd18ccce8aca", size = 447445, upload-time = "2026-03-10T21:30:55.541Z" }, - { url = "https://files.pythonhosted.org/packages/5a/76/4921c00511f88af86a33de770d64141170f1cfd9c00311aea689949e274e/tornado-6.5.5-cp39-abi3-win32.whl", hash = "sha256:dd3eafaaeec1c7f2f8fdcd5f964e8907ad788fe8a5a32c4426fbbdda621223b7", size = 448582, upload-time = "2026-03-10T21:30:57.142Z" }, - { url = "https://files.pythonhosted.org/packages/2c/23/f6c6112a04d28eed765e374435fb1a9198f73e1ec4b4024184f21faeb1ad/tornado-6.5.5-cp39-abi3-win_amd64.whl", hash = "sha256:6443a794ba961a9f619b1ae926a2e900ac20c34483eea67be4ed8f1e58d3ef7b", size = 448990, upload-time = "2026-03-10T21:30:58.857Z" }, - { url = "https://files.pythonhosted.org/packages/b7/c8/876602cbc96469911f0939f703453c1157b0c826ecb05bdd32e023397d4e/tornado-6.5.5-cp39-abi3-win_arm64.whl", hash = "sha256:2c9a876e094109333f888539ddb2de4361743e5d21eece20688e3e351e4990a6", size = 448016, upload-time = "2026-03-10T21:31:00.43Z" }, + { url = "https://files.pythonhosted.org/packages/02/dc/c7043cab6fed8ae159fc1923ce829ada35c4dbd797d408a43858ffaf9639/tornado-6.5.7-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:148b2eb15c2c765a50796172c1e499649b35f30d2e3c3d3e15913cfa56bfb163", size = 448543, upload-time = "2026-06-08T17:34:38.052Z" }, + { url = "https://files.pythonhosted.org/packages/92/4f/090b1431e5a43df696feceffc268c5383cc079ecb5f08ce58f917109aafe/tornado-6.5.7-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9da38de27f1da3b78a966f0dae12b5a1ea9afe72ca805d84ff06508272ddf100", size = 446707, upload-time = "2026-06-08T17:34:39.594Z" }, + { url = "https://files.pythonhosted.org/packages/37/d8/ef374952fd5da67d4463122c2b8e5a96536ec10b4b339254c6dcde81d01c/tornado-6.5.7-cp39-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8d759e71906ee783f8867b93bf26a265743da4c1e2f4a018464c1ba019862972", size = 449774, upload-time = "2026-06-08T17:34:41.204Z" }, + { url = "https://files.pythonhosted.org/packages/35/37/d434c73f4c6e014b745b9b37085f34f40c022f007efff3d7fe65991899f3/tornado-6.5.7-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a46347a18f23fb92b396beebe0fb78f61dda0cc302445202c16203d8a18848b", size = 450745, upload-time = "2026-06-08T17:34:42.531Z" }, + { url = "https://files.pythonhosted.org/packages/b6/2b/56b9aff361d7f1ab728a805ec7d7ea835f8807afa9f5cc690ea0e630efb9/tornado-6.5.7-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7778b30bef919231265e91c69963ce0f49a1e9c07ac900bbe75b19ce2575ba92", size = 450578, upload-time = "2026-06-08T17:34:43.787Z" }, + { url = "https://files.pythonhosted.org/packages/02/30/a7444fb23aa76860a14198fab96ac79f1866b0a6e19e26c4381b0938e50f/tornado-6.5.7-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e726f0c75da7726eec023aa62751ff8878bd2737e34fbdd33b1ae5897d2200f5", size = 449985, upload-time = "2026-06-08T17:34:45.326Z" }, + { url = "https://files.pythonhosted.org/packages/5c/42/5f0e56c01e8d9d36f4e23f367b85ae6cae0c1ecddd5e6977d8388ad27488/tornado-6.5.7-cp39-abi3-win32.whl", hash = "sha256:f8de3bf12d3efdd0cbe7c8887868198f8a91415e3f29fcf258d9b8eb7b1d9ae4", size = 451047, upload-time = "2026-06-08T17:34:46.784Z" }, + { url = "https://files.pythonhosted.org/packages/c9/a4/b393076ffb21b469eec5b328a0534cf03a3b90bfc6b1f09507cdd075d938/tornado-6.5.7-cp39-abi3-win_amd64.whl", hash = "sha256:de942f843533a039ef9fa3d9c88c7cd8a7c94553fb5ad0154270989b3d99a2c4", size = 451485, upload-time = "2026-06-08T17:34:48.248Z" }, + { url = "https://files.pythonhosted.org/packages/71/2e/7b1c769803121b809112cf9a00681c472eae1d80e32d7ec0e0bd61d0d0e1/tornado-6.5.7-cp39-abi3-win_arm64.whl", hash = "sha256:ff934fce95643af5f11efdae618eaa73d469dc588641e5c8d19295a0c65c4796", size = 450506, upload-time = "2026-06-08T17:34:49.702Z" }, ] [[package]] From 91b2013644a95fb06ea58bdadf5a47b74a1a43b9 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 4 Jun 2026 20:40:40 -0700 Subject: [PATCH 10/12] fix(proxy): stop team BYOK model name corruption on model edit (#29731) * fix(proxy): stop team model name corruption on edit (#28382) (#29001) Team-scoped ("Team-BYOK") models store an internal routing key model_name_{team_id}_{uuid} in the model_name column and the user-facing name in model_info.team_public_model_name. The internal name leaked into /v1, /v2, and /model/info responses; the dashboard bound its edit form to it, so any non-rename save (e.g. a TPM tweak) PATCHed the internal name back. The update path then treated it as a rename, overwriting team_public_model_name and rewriting the team's models[] ACL with the mangled string -- breaking team key calls with team_model_access_denied. Two-layer fix: - Read path (root cause): add _translate_model_name_for_response and apply it in model_info_v2 and _get_proxy_model_info so /v1, /v2, and /model/info surface the public name for team-scoped rows. The DB column and router index keep the internal name as the routing key; this is a presentation-layer swap on a shallow copy (never mutates input). - Write path (defense in depth): harden _get_public_model_name so a value matching the internal shape, or a no-op against the current DB column, is never treated as a rename -- for both the top-level model_name and an explicit model_info.team_public_model_name. Tests: regression for the reported scenario, full branch coverage of _get_public_model_name, two internal-shape guard cases, an end-to-end PATCH through _update_team_model_in_db (asserts the team ACL is untouched), and four response-translation cases. 60 passed (model management), 181 passed (proxy server). * fix(ui): key Agent Builder agent selection on model_info.id (#29729) * fix(ui): key Agent Builder agent selection on model_info.id Once team-scoped BYOK models can share a public name (the backend now returns the public name on /model/info instead of the internal routing key), selecting agents by model_name collides. Key selection, create, update and delete on the stable model_info.id instead, falling back to model_name only for config-defined agents that have no id. * fix(ui): add name-match fallback to post-create agent selection If the just-created agent's id is not yet present in the re-fetched list, try matching by name before falling back to the first agent. Addresses greptile review on #29729. --------- Co-authored-by: tushar8408 <32977767+tushar8408@users.noreply.github.com> (cherry picked from commit 56aa55b991373632fec89cd5b408605457aa7686) --- .../model_management_endpoints.py | 42 ++- litellm/proxy/proxy_server.py | 39 ++- .../test_model_management_endpoints.py | 301 ++++++++++++++++++ .../test_team_model_name_translation.py | 178 +++++++++++ .../playground/chat_ui/AgentBuilderView.tsx | 154 +++++---- 5 files changed, 648 insertions(+), 66 deletions(-) create mode 100644 tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 722fcd30033..404ed4491e5 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -490,9 +490,45 @@ def _get_public_model_name( patch_data: updateDeployment, db_model: Deployment, ) -> str: - """Determine the public model name from patch or existing model.""" - if patch_data.model_name: - return patch_data.model_name + """Determine the public model name from patch or existing model. + + The top-level ``model_name`` is the rename channel. For team-scoped rows + the DB ``model_name`` column holds an internal routing key + (``model_name_{team_id}_{uuid}``), and ``/model/info`` historically leaked + it into the dashboard edit form, so a non-rename save (e.g. a TPM tweak) + would PATCH the internal name and the update path would treat it as a + rename -- overwriting ``team_public_model_name`` and rewriting the team ACL + (see issue #28382). + + Guard against that by ignoring an incoming ``model_name`` that matches the + internal shape, or is a no-op against the current DB column. Anything else + is a genuine rename and wins. We deliberately do NOT read + ``patch_data.model_info.team_public_model_name``: the dashboard passes the + existing ``model_info`` blob through untouched on a rename, so honoring it + would return the OLD public name and silently drop the rename. + + Precedence (highest first): + 1. patch_data.model_name -- a genuine rename: not internal-shape and not a + no-op against db_model.model_name. + 2. db_model.model_info.team_public_model_name -- existing public name. + 3. db_model.model_name -- last-resort fallback for legacy rows. + """ + team_id = (patch_data.model_info.team_id if patch_data.model_info else None) or ( + db_model.model_info.team_id if db_model.model_info else None + ) + + def _is_internal_shape(name: Optional[str]) -> bool: + if team_id is None or not name: + return False + return name.startswith(f"model_name_{team_id}_") + + incoming = patch_data.model_name + if ( + incoming + and not _is_internal_shape(incoming) + and incoming != db_model.model_name + ): + return incoming if db_model.model_info and db_model.model_info.team_public_model_name: return db_model.model_info.team_public_model_name diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b296792cd09..5587d37f1f7 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -11875,6 +11875,9 @@ async def model_info_v2( # Update total count to include agents search_total_count = len(all_models) + # Translate `model_name` to the public name for team-scoped rows. + all_models = [_translate_model_name_for_response(m) for m in all_models] + return _paginate_models_response( all_models=all_models, page=page, @@ -12309,6 +12312,33 @@ async def model_metrics_exceptions( return {"data": response, "exception_types": list(exception_types)} +def _translate_model_name_for_response(model: dict) -> dict: + """For team-scoped DB rows, replace `model_name` with the public name + in `model_info.team_public_model_name` before returning. The DB column + and the in-memory router index keep the internal mangled name + (`model_name_{team_id}_{uuid}`) as the routing key -- this swap is a + presentation-layer concern. Returns a shallow copy; never mutates. + + Without this swap the internal name leaks into `/v1/model/info` and + `/v2/model/info`, the dashboard binds its edit form to it, and a + non-rename save round-trips the internal name back -- corrupting + `team_public_model_name` and the team ACL (see issue #28382). + """ + if not isinstance(model, dict): + return model + model_info = model.get("model_info") or {} + if not isinstance(model_info, dict): + return model + team_public = model_info.get("team_public_model_name") + team_id = model_info.get("team_id") + if not team_public or not team_id: + return model + current = model.get("model_name") or "" + if not current.startswith(f"model_name_{team_id}_"): + return model + return {**model, "model_name": team_public} + + def _get_proxy_model_info(model: dict) -> dict: # provided model_info in config.yaml model_info = model.get("model_info", {}) @@ -12349,7 +12379,7 @@ def _get_proxy_model_info(model: dict) -> dict: deployment_dict=model, excluded_keys={"litellm_credential_name"} ) - return model + return _translate_model_name_for_response(model) @router.get( @@ -12489,8 +12519,11 @@ async def model_info_v1( # noqa: PLR0915 else: all_models = [] - for in_place_model in all_models: - in_place_model = _get_proxy_model_info(model=in_place_model) + # Reassign each entry: _get_proxy_model_info returns a (possibly new) + # dict via _translate_model_name_for_response, which does NOT mutate in + # place. Binding only the loop variable would drop the public-name swap + # for team-scoped rows and leak the internal routing key (#28382). + all_models = [_get_proxy_model_info(model=model) for model in all_models] verbose_proxy_logger.debug("all_models: %s", all_models) return {"data": all_models} diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index 85c7c130b36..f16074a049b 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -1129,6 +1129,307 @@ class TestTeamModelUpdate: ) assert "403" in str(exc_info.value) + def test_get_public_model_name_28382_dashboard_echo_preserves_public_name(self): + """Regression for #28382 - a non-rename dashboard PATCH echoes the + internal generated model_name (model_name_{team}_{uuid}) at the top + level. That internal-shape value must be ignored (not treated as a + rename), so _get_public_model_name falls through to the existing public + name instead of overwriting it with the internal one.""" + from litellm.proxy.management_endpoints.model_management_endpoints import ( + _get_public_model_name, + ) + from litellm.types.router import ModelInfo + + db_model = Deployment( + model_name="model_name_test-team_abc123", + litellm_params=LiteLLM_Params(model="azure/gpt-5.2-low-rpm-testing"), + model_info=ModelInfo( + team_id="test-team", + team_public_model_name="gpt-5.2-low-rpm-testing", + ), + ) + patch_data = updateDeployment( + model_name="model_name_test-team_abc123", + model_info=ModelInfo( + team_id="test-team", + team_public_model_name="gpt-5.2-low-rpm-testing", + ), + ) + + assert ( + _get_public_model_name(patch_data=patch_data, db_model=db_model) + == "gpt-5.2-low-rpm-testing" + ) + + def test_get_public_model_name_preserves_db_public_name_when_internal_name_unchanged( + self, + ): + """If patch_data.model_info has no team_public_model_name and + patch_data.model_name equals db_model.model_name (dashboard re-sending + the internal name without touching the public-name field), the + existing db_model.model_info.team_public_model_name must be preserved.""" + from litellm.proxy.management_endpoints.model_management_endpoints import ( + _get_public_model_name, + ) + from litellm.types.router import ModelInfo + + db_model = Deployment( + model_name="model_name_test-team_abc123", + litellm_params=LiteLLM_Params(model="azure/gpt-5.2-low-rpm-testing"), + model_info=ModelInfo( + team_id="test-team", + team_public_model_name="gpt-5.2-low-rpm-testing", + ), + ) + patch_data = updateDeployment( + model_name="model_name_test-team_abc123", + model_info=ModelInfo(team_id="test-team"), + ) + + assert ( + _get_public_model_name(patch_data=patch_data, db_model=db_model) + == "gpt-5.2-low-rpm-testing" + ) + + def test_get_public_model_name_allows_top_level_rename(self): + """A genuine rename via the top-level model_name field (no + patch_data.model_info.team_public_model_name supplied, and the new + name differs from the existing internal db model_name) must still + return the new name.""" + from litellm.proxy.management_endpoints.model_management_endpoints import ( + _get_public_model_name, + ) + from litellm.types.router import ModelInfo + + db_model = Deployment( + model_name="model_name_test-team_abc123", + litellm_params=LiteLLM_Params(model="azure/gpt-5.2-low-rpm-testing"), + model_info=ModelInfo( + team_id="test-team", + team_public_model_name="old-public-name", + ), + ) + patch_data = updateDeployment( + model_name="new-public-name", + model_info=ModelInfo(team_id="test-team"), + ) + + assert ( + _get_public_model_name(patch_data=patch_data, db_model=db_model) + == "new-public-name" + ) + + def test_get_public_model_name_top_level_rename_wins_over_stale_model_info(self): + """Regression (codex review): on a dashboard rename the UI sends the new + name in model_name but passes the existing model_info blob through + untouched -- so it still carries the OLD team_public_model_name. The + top-level rename must win; otherwise _update_existing_team_model_assignment + sees no change, never updates the team ACL, and the rename is silently + dropped while the UI optimistically shows the new name.""" + from litellm.proxy.management_endpoints.model_management_endpoints import ( + _get_public_model_name, + ) + from litellm.types.router import ModelInfo + + db_model = Deployment( + model_name="model_name_team-a_abc123", + litellm_params=LiteLLM_Params(model="azure/gpt-4.1"), + model_info=ModelInfo( + team_id="team-a", team_public_model_name="old-public-name" + ), + ) + patch_data = updateDeployment( + model_name="new-public-name", + model_info=ModelInfo( + team_id="team-a", + team_public_model_name="old-public-name", # stale, untouched by UI + ), + ) + + assert ( + _get_public_model_name(patch_data=patch_data, db_model=db_model) + == "new-public-name" + ) + + def test_get_public_model_name_falls_back_to_db_public_name(self): + """When patch_data carries no name hints at all (neither model_name + nor model_info.team_public_model_name), fall back to the existing + db_model.model_info.team_public_model_name.""" + from litellm.proxy.management_endpoints.model_management_endpoints import ( + _get_public_model_name, + ) + from litellm.types.router import ModelInfo + + db_model = Deployment( + model_name="model_name_test-team_abc123", + litellm_params=LiteLLM_Params(model="azure/gpt-5.2-low-rpm-testing"), + model_info=ModelInfo( + team_id="test-team", + team_public_model_name="gpt-5.2-low-rpm-testing", + ), + ) + patch_data = updateDeployment( + model_info=ModelInfo(team_id="test-team"), + ) + + assert ( + _get_public_model_name(patch_data=patch_data, db_model=db_model) + == "gpt-5.2-low-rpm-testing" + ) + + def test_get_public_model_name_last_resort_returns_db_model_name(self): + """Legacy rows may have no team_public_model_name anywhere; the + function must still return a string (the existing db_model.model_name) + rather than raising.""" + from litellm.proxy.management_endpoints.model_management_endpoints import ( + _get_public_model_name, + ) + from litellm.types.router import ModelInfo + + db_model = Deployment( + model_name="legacy-model", + litellm_params=LiteLLM_Params(model="azure/legacy"), + model_info=ModelInfo(team_id="test-team"), + ) + patch_data = updateDeployment( + model_info=ModelInfo(team_id="test-team"), + ) + + assert ( + _get_public_model_name(patch_data=patch_data, db_model=db_model) + == "legacy-model" + ) + + def test_get_public_model_name_ignores_different_internal_shape_name(self): + """A stale client may PATCH an internal-shaped model_name that does not + equal the current DB column (e.g. a different uuid). It must NOT be + treated as a rename -- fall through to the existing public name.""" + from litellm.proxy.management_endpoints.model_management_endpoints import ( + _get_public_model_name, + ) + from litellm.types.router import ModelInfo + + db_model = Deployment( + model_name="model_name_test-team_realuuid", + litellm_params=LiteLLM_Params(model="azure/gpt-5.2-low-rpm-testing"), + model_info=ModelInfo( + team_id="test-team", + team_public_model_name="gpt-5.2-low-rpm-testing", + ), + ) + patch_data = updateDeployment( + model_name="model_name_test-team_differentuuid", + model_info=ModelInfo(team_id="test-team"), + ) + + assert ( + _get_public_model_name(patch_data=patch_data, db_model=db_model) + == "gpt-5.2-low-rpm-testing" + ) + + def test_get_public_model_name_ignores_internal_shape_patch_public(self): + """If a corrupted row round-trips an internal-shaped value in + model_info.team_public_model_name, it must not be accepted as the + public name -- fall through to the existing db public name.""" + from litellm.proxy.management_endpoints.model_management_endpoints import ( + _get_public_model_name, + ) + from litellm.types.router import ModelInfo + + db_model = Deployment( + model_name="model_name_test-team_realuuid", + litellm_params=LiteLLM_Params(model="azure/gpt-5.2-low-rpm-testing"), + model_info=ModelInfo( + team_id="test-team", + team_public_model_name="gpt-5.2-low-rpm-testing", + ), + ) + patch_data = updateDeployment( + model_info=ModelInfo( + team_id="test-team", + team_public_model_name="model_name_test-team_realuuid", + ), + ) + + assert ( + _get_public_model_name(patch_data=patch_data, db_model=db_model) + == "gpt-5.2-low-rpm-testing" + ) + + @pytest.mark.asyncio + async def test_dashboard_edit_preserves_public_name_and_acl(self): + """End-to-end regression for #28382: PATCH payload shaped like the + dashboard's model-edit form (top-level model_name = internal generated + name, model_info.team_public_model_name = public name) must NOT trigger + a public-name rename, must NOT touch the team ACL, and must serialize + the public name back into model_info.""" + from litellm.proxy.management_endpoints.model_management_endpoints import ( + _update_team_model_in_db, + ) + from litellm.types.router import ModelInfo + + db_model = Deployment( + model_name="model_name_test-team_abc123", + litellm_params=LiteLLM_Params( + model="azure/gpt-5.2-low-rpm-testing", + custom_llm_provider="azure", + ), + model_info=ModelInfo( + id="model-id-123", + team_id="test-team", + team_public_model_name="gpt-5.2-low-rpm-testing", + ), + ) + patch_data = updateDeployment( + model_name="model_name_test-team_abc123", + litellm_params=None, + model_info=ModelInfo( + id="model-id-123", + team_id="test-team", + team_public_model_name="gpt-5.2-low-rpm-testing", + ), + ) + user_api_key_dict = UserAPIKeyAuth( + user_id="test_user", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + prisma_client = MockPrismaClient(team_exists=True) + + with ( + patch( + "litellm.proxy.proxy_server.premium_user", + True, + ), + patch( + "litellm.proxy.management_endpoints.model_management_endpoints.team_model_add" + ) as mock_team_model_add, + patch( + "litellm.proxy.management_endpoints.model_management_endpoints.team_model_delete" + ) as mock_team_model_delete, + ): + result = await _update_team_model_in_db( + db_model=db_model, + patch_data=patch_data, + user_api_key_dict=user_api_key_dict, + prisma_client=prisma_client, # type: ignore + ) + + # team ACL must not be touched on a no-op edit + mock_team_model_add.assert_not_called() + mock_team_model_delete.assert_not_called() + + # the merged model_info written to the DB must keep the public name + model_info_json = result.get("model_info", "") + parsed_model_info = json.loads(model_info_json) + assert ( + parsed_model_info.get("team_public_model_name") == "gpt-5.2-low-rpm-testing" + ) + + # the internal model_name must not have been overwritten (caller + # intentionally clears patch_data.model_name so the DB row's name + # column is left alone) + assert result.get("model_name") == "model_name_test-team_abc123" + class TestModelInfoEndpoint: """Test the model_info endpoint for retrieving individual model information""" diff --git a/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py b/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py new file mode 100644 index 00000000000..97e5c494916 --- /dev/null +++ b/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py @@ -0,0 +1,178 @@ +"""Coverage for team-scoped model-name translation in /model/info responses. + +These live in tests/test_litellm/proxy/proxy_server/ (not the top-level +test_proxy_server.py) because the CI coverage job collects this directory. +They exercise the read-path fix for issue #28382: `/v1`, `/v2`, and +`/model/info` must surface `model_info.team_public_model_name` for team-scoped +rows instead of the internal routing key `model_name_{team_id}_{uuid}`. +""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock + +import pytest + +import litellm.proxy.proxy_server as ps +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.proxy_server import ( + _get_proxy_model_info, + _translate_model_name_for_response, +) + + +def _team_row() -> dict: + return { + "model_name": "model_name_team-abc-123_4a6b8", + "litellm_params": {"model": "azure/gpt-5.2-low-rpm-testing"}, + "model_info": { + "id": "byok-id-1", + "team_id": "team-abc-123", + "team_public_model_name": "team-claude-sonnet", + "db_model": True, + }, + } + + +def test_translate_swaps_internal_name_for_public(): + """Team-scoped row: model_name is swapped to the public name.""" + result = _translate_model_name_for_response(_team_row()) + assert result["model_name"] == "team-claude-sonnet" + + +def test_translate_leaves_global_row_untouched(): + """No team_id / team_public_model_name -> pass through unchanged.""" + model = { + "model_name": "gpt-4o", + "litellm_params": {"model": "gpt-4o"}, + "model_info": {"id": "normal-id-1", "db_model": False}, + } + assert _translate_model_name_for_response(model)["model_name"] == "gpt-4o" + + +def test_translate_leaves_non_internal_shape_untouched(): + """Team row whose model_name is not the internal routing key is not rewritten.""" + model = _team_row() + model["model_name"] = "already-public-name" + assert ( + _translate_model_name_for_response(model)["model_name"] == "already-public-name" + ) + + +def test_translate_handles_missing_or_non_dict_model_info(): + """Missing / None / non-dict model_info, and a non-dict model, must not raise.""" + # missing model_info + assert _translate_model_name_for_response({"model_name": "x"})["model_name"] == "x" + # model_info is None -> coerced to {} -> no team fields + assert ( + _translate_model_name_for_response({"model_name": "x", "model_info": None})[ + "model_name" + ] + == "x" + ) + # model_info is a truthy non-dict (e.g. a stray string) -> early return + assert ( + _translate_model_name_for_response( + {"model_name": "x", "model_info": "garbage"} + )["model_name"] + == "x" + ) + # model itself is not a dict + assert _translate_model_name_for_response("not-a-dict") == "not-a-dict" # type: ignore[arg-type] + + +def test_translate_does_not_mutate_input(): + """Returns a shallow copy; the router's in-memory list keeps the routing key.""" + model = _team_row() + result = _translate_model_name_for_response(model) + assert result is not model + assert model["model_name"] == "model_name_team-abc-123_4a6b8" + + +def test_get_proxy_model_info_returns_public_name_for_team_row(): + """`_get_proxy_model_info` must return the public name for a team-scoped + row. Because _translate_model_name_for_response returns a shallow copy + (it does not mutate), callers MUST use the return value -- the + `/v1/model/info` list path historically discarded it, leaking the internal + routing key (#28382).""" + # Mirror the (fixed) /v1/model/info list path: assign the return back. + all_models = [_get_proxy_model_info(model=m) for m in [_team_row()]] + assert all_models[0]["model_name"] == "team-claude-sonnet" + + +@pytest.mark.asyncio +async def test_model_info_v2_translates_team_model_name(monkeypatch): + """/v2/model/info must surface the public name for team-scoped rows. + Covers the translation step in model_info_v2 (the read-path call site).""" + router = MagicMock() + router.model_list = [_team_row()] + + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr(ps, "prisma_client", MagicMock()) + monkeypatch.setattr(ps.proxy_config, "get_config", AsyncMock(return_value={})) + monkeypatch.setattr( + ps, + "_apply_search_filter_to_models", + AsyncMock(side_effect=lambda all_models, **kw: (all_models, len(all_models))), + ) + monkeypatch.setattr( + ps, "_enrich_model_info_with_litellm_data", lambda model, **kw: model + ) + import litellm.proxy.agent_endpoints.model_list_helpers as mlh + + monkeypatch.setattr( + mlh, + "append_agents_to_model_info", + AsyncMock(side_effect=lambda models, **kw: models), + ) + + admin = UserAPIKeyAuth(user_id="u", user_role=LitellmUserRoles.PROXY_ADMIN) + # Pass every query param explicitly: called directly (not through FastAPI), + # the fastapi.Query(...) defaults are Query objects, not their values. + resp = await ps.model_info_v2( + user_api_key_dict=admin, + model=None, + user_models_only=False, + include_team_models=False, + debug=False, + page=1, + size=50, + search=None, + modelId=None, + teamId=None, + sortBy=None, + sortOrder="asc", + ) + + names = [m["model_name"] for m in resp["data"]] + assert "team-claude-sonnet" in names + assert "model_name_team-abc-123_4a6b8" not in names + + +@pytest.mark.asyncio +async def test_model_info_v1_list_path_translates_team_model_name(monkeypatch): + """/v1/model/info list path (no litellm_model_id) must surface the public + name. Covers the list comprehension that assigns _get_proxy_model_info's + return back into all_models (#28382 review).""" + router = MagicMock() + router.get_model_names.return_value = ["team-claude-sonnet"] + router.get_model_access_groups.return_value = {} + router.get_model_list.return_value = [_team_row()] + + monkeypatch.setattr(ps, "user_model", None) + monkeypatch.setattr(ps, "llm_model_list", [_team_row()]) + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr(ps, "get_key_models", lambda **kw: []) + monkeypatch.setattr(ps, "get_team_models", lambda **kw: []) + monkeypatch.setattr( + ps, "get_complete_model_list", lambda **kw: ["team-claude-sonnet"] + ) + + admin = UserAPIKeyAuth( + user_id="u", user_role=LitellmUserRoles.PROXY_ADMIN, team_models=[] + ) + resp = await ps.model_info_v1(user_api_key_dict=admin, litellm_model_id=None) + + names = [m["model_name"] for m in resp["data"]] + assert "team-claude-sonnet" in names + assert "model_name_team-abc-123_4a6b8" not in names diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/AgentBuilderView.tsx b/ui/litellm-dashboard/src/components/playground/chat_ui/AgentBuilderView.tsx index c47c201d074..8b0de5dda7c 100644 --- a/ui/litellm-dashboard/src/components/playground/chat_ui/AgentBuilderView.tsx +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/AgentBuilderView.tsx @@ -1,6 +1,14 @@ "use client"; -import { CommentOutlined, DeleteOutlined, ExperimentOutlined, LinkOutlined, PlusOutlined, RobotOutlined, SaveOutlined } from "@ant-design/icons"; +import { + CommentOutlined, + DeleteOutlined, + ExperimentOutlined, + LinkOutlined, + PlusOutlined, + RobotOutlined, + SaveOutlined, +} from "@ant-design/icons"; import { Button, Input, Modal, Select, Spin, Tabs } from "antd"; import React, { useCallback, useEffect, useState } from "react"; import CodeBlock from "@/app/(dashboard)/api-reference/components/CodeBlock"; @@ -64,9 +72,10 @@ function ConnectTabContent({ onCreateKey, }: ConnectTabContentProps) { const baseUrl = proxyBaseUrl ?? getConnectTabBaseUrl(proxySettings, customProxyBaseUrl); - const apiKeyForCurl = - createdKeyValue ? - createdKeyValue.startsWith("Bearer ") ? createdKeyValue : `Bearer ${createdKeyValue}` + const apiKeyForCurl = createdKeyValue + ? createdKeyValue.startsWith("Bearer ") + ? createdKeyValue + : `Bearer ${createdKeyValue}` : "Bearer sk-1234"; const curlExample = `curl -L -X POST '${baseUrl}/v1/chat/completions' \\ -H 'x-litellm-api-key: ${apiKeyForCurl}' \\ @@ -101,12 +110,7 @@ function ConnectTabContent({ Create a virtual key that can only call this agent. The key will be scoped to you (user_id) and restricted to the model {agentName}.

- {disabledPersonalKeyCreation && ( @@ -127,6 +131,14 @@ function getAgentModelId(agent: AgentModel): string | null { return info?.id ?? null; } +// Selection key that always resolves to a non-null string. Prefers the DB +// id (stable across renames and unique across teams) but falls back to +// `model_name` so config-file-defined agents — which have no `model_info.id` +// — remain selectable. +function getAgentSelectionKey(agent: AgentModel): string { + return getAgentModelId(agent) ?? agent.model_name; +} + function parseUnderlyingModel(litellmModel: string | undefined): string | undefined { if (!litellmModel || !litellmModel.startsWith("litellm_agent/")) return undefined; return litellmModel.slice("litellm_agent/".length) || undefined; @@ -191,22 +203,25 @@ export default function AgentBuilderView({ const [deleting, setDeleting] = useState(false); const effectiveApiKey = apiKey || accessToken || ""; - const selectedAgent = selectedId === NEW_AGENT_ID ? null : agentModels.find((a) => a.model_name === selectedId) ?? null; + const selectedAgent = + selectedId === NEW_AGENT_ID ? null : agentModels.find((a) => getAgentSelectionKey(a) === selectedId) ?? null; const isNewAgent = selectedId === NEW_AGENT_ID; const selectedAgentModelId = selectedAgent ? getAgentModelId(selectedAgent) : null; - const loadAgents = useCallback(async () => { - if (!accessToken || !userID || !userRole) return; + const loadAgents = useCallback(async (): Promise => { + if (!accessToken || !userID || !userRole) return []; setLoadingAgents(true); try { const list = await fetchAvailableAgentModels(accessToken, userID, userRole); setAgentModels(list); - if (!selectedId || (selectedId !== NEW_AGENT_ID && !list.some((a) => a.model_name === selectedId))) { - setSelectedId(list.length > 0 ? list[0].model_name : null); + if (!selectedId || (selectedId !== NEW_AGENT_ID && !list.some((a) => getAgentSelectionKey(a) === selectedId))) { + setSelectedId(list.length > 0 ? getAgentSelectionKey(list[0]) : null); } + return list; } catch (e) { console.error(e); NotificationsManager.fromBackend("Failed to load agents"); + return []; } finally { setLoadingAgents(false); } @@ -267,7 +282,13 @@ export default function AgentBuilderView({ setDraftMaxTokens(typeof p?.max_tokens === "number" ? p.max_tokens : 4096); const rawTools = selectedAgent.litellm_params?.tools; const tools: MCPToolEntry[] = Array.isArray(rawTools) - ? rawTools.filter((t): t is MCPToolEntry => t && typeof t === "object" && (t as MCPToolEntry).type === "mcp" && typeof (t as MCPToolEntry).server_url === "string") + ? rawTools.filter( + (t): t is MCPToolEntry => + t && + typeof t === "object" && + (t as MCPToolEntry).type === "mcp" && + typeof (t as MCPToolEntry).server_url === "string", + ) : []; setDraftTools(tools); } @@ -297,7 +318,7 @@ export default function AgentBuilderView({ } setSaving(true); try { - await modelCreateCall(accessToken, { + const response = await modelCreateCall(accessToken, { model_name: draftName.trim(), litellm_params: { model: `litellm_agent/${draftUnderlyingModel}`, @@ -308,9 +329,15 @@ export default function AgentBuilderView({ }, model_info: {}, }); - const newName = draftName.trim(); - await loadAgents(); - setSelectedId(newName); + // /model/new returns the row with `model_id` at the top level. + // Prefer that id over name-matching so we land on the just-created + // agent even when its public name collides with another team's. + const createdId: string | null = response?.model_id ?? response?.model_info?.id ?? null; + const list = await loadAgents(); + const created = createdId + ? list.find((a) => getAgentModelId(a) === createdId) ?? list.find((a) => a.model_name === draftName.trim()) + : list.find((a) => a.model_name === draftName.trim()); + setSelectedId(created ? getAgentSelectionKey(created) : list[0] ? getAgentSelectionKey(list[0]) : null); setActiveTab("chat"); } catch (e) { NotificationsManager.fromBackend("Failed to save agent"); @@ -342,8 +369,10 @@ export default function AgentBuilderView({ selectedAgentModelId, ); NotificationsManager.success("Agent updated successfully"); - await loadAgents(); - setSelectedId(draftName.trim()); + const list = await loadAgents(); + const stillSelected = list.find((a) => getAgentModelId(a) === selectedAgentModelId); + const target = stillSelected ?? list[0]; + setSelectedId(target ? getAgentSelectionKey(target) : null); } catch (e) { NotificationsManager.fromBackend("Failed to update agent"); } finally { @@ -387,9 +416,9 @@ export default function AgentBuilderView({ try { await modelDeleteCall(accessToken, selectedAgentModelId); NotificationsManager.success("Agent deleted"); - await loadAgents(); - const remaining = agentModels.filter((a) => a.model_name !== selectedAgent.model_name); - setSelectedId(remaining.length > 0 ? remaining[0].model_name : null); + const list = await loadAgents(); + const remaining = list.filter((a) => getAgentModelId(a) !== selectedAgentModelId); + setSelectedId(remaining.length > 0 ? getAgentSelectionKey(remaining[0]) : null); } catch (e) { NotificationsManager.fromBackend("Failed to delete agent"); } finally { @@ -401,9 +430,7 @@ export default function AgentBuilderView({ if (!accessToken || !userID || !userRole) { return ( -
- Sign in to use Agent Builder. -
+
Sign in to use Agent Builder.
); } @@ -412,24 +439,25 @@ export default function AgentBuilderView({
Agent Builder - {isNewAgent ? ( - - ) : ( - Build Agents that pass your compliance requirements. - )} + {isNewAgent ? ( + + ) : ( + Build Agents that pass your compliance requirements. + )}
- Agent Builder is experimental and may change or be removed without notice. We’d love your feedback—email us at{" "} + Agent Builder is experimental and may change or be removed without notice. We’d love your feedback—email us + at{" "} product@berri.ai @@ -452,21 +480,24 @@ export default function AgentBuilderView({
) : ( <> - {agentModels.map((agent) => ( - - ))} + {agentModels.map((agent) => { + const key = getAgentSelectionKey(agent); + return ( + + ); + })}